Scraping LinkedIn Company Data

const axios = require('axios');

const apiKey = 'sf_your_api_key';
const apiUrl = 'https://api.scrapingforge.com/api/v1/scraper';

// LinkedIn company URL (public page)
const companyUrl = 'https://www.linkedin.com/company/microsoft';

const payload = {
  url: companyUrl,
  render_js: true,
  premium_proxy: true,
  country: 'US',
  wait_for: 'h1',
  window_width: 1920,
  window_height: 1080,
  custom_headers: {
    'Spf-User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
  },
  extract_rules: {
    name: {
      selector: 'h1.org-top-card-summary__title',
      type: 'text'
    },
    description: {
      selector: 'p.org-top-card-summary__tagline',
      type: 'text'
    },
    website: {
      selector: 'a.org-top-card-primary-actions__action',
      type: 'attr',
      attr: 'href'
    },
    industry: {
      selector: '.org-top-card-summary-info-list__info-item:nth-child(1)',
      type: 'text'
    },
    company_size: {
      selector: '.org-top-card-summary-info-list__info-item:nth-child(2)',
      type: 'text'
    },
    headquarters: {
      selector: '.org-top-card-summary-info-list__info-item:nth-child(3)',
      type: 'text'
    },
    about: {
      selector: 'section.org-page-details-module__card-spacing p',
      type: 'text'
    }
  }
};

axios.post(apiUrl, payload, {
  headers: {
    'Authorization': `Bearer ${apiKey}`,
    'Content-Type': 'application/json'
  }
})
.then(response => {
  console.log(JSON.stringify(response.data, null, 2));
})
.catch(error => {
  console.error('Error:', error.response?.data || error.message);
});

{
  "name": "Microsoft",
  "description": "Every company has a mission. What's ours? To empower every person and every organization to achieve more.",
  "website": "https://www.microsoft.com",
  "industry": "Software Development",
  "company_size": "10,001+ employees",
  "headquarters": "Redmond, Washington",
  "about": "At Microsoft, our mission is to empower every person and every organization on the planet to achieve more. Our mission is grounded in both the world in which we live and the future we strive to create...",
  "specialties": "Business Software, Developer Tools, Cloud Computing, Artificial Intelligence, Gaming, Hardware"
}

import requests
import time

api_key = "sf_your_api_key"
base_url = "https://api.scrapingforge.com/api/v1/scraper"

companies = [
    "microsoft",
    "google",
    "apple",
    "amazon",
    "meta"
]

# Submit jobs with delays
job_ids = []
for company_slug in companies:
    company_url = f"https://www.linkedin.com/company/{company_slug}"

    payload = {
        "url": company_url,
        "render_js": True,
        "premium_proxy": True,
        "country": "US",
        "ai_query": "Extract company name, description, industry, size, and location"
    }

    response = requests.post(
        f"{base_url}/jobs",
        json=payload,
        headers={"Authorization": f"Bearer {api_key}"}
    )

    job_id = response.json()["job_id"]
    job_ids.append((company_slug, job_id))
    print(f"Submitted job for {company_slug}: {job_id}")

    # Rate limiting: wait between submissions
    time.sleep(5)

# Wait for all jobs to complete
print("\nWaiting for jobs to complete...")
time.sleep(60)

# Fetch results
companies_data = []
for company_slug, job_id in job_ids:
    try:
        # Check status
        status_response = requests.get(
            f"{base_url}/jobs/{job_id}",
            headers={"Authorization": f"Bearer {api_key}"}
        )
        status = status_response.json()["status"]

        if status == "completed":
            # Get result
            result_response = requests.get(
                f"{base_url}/jobs/{job_id}/result",
                headers={"Authorization": f"Bearer {api_key}"}
            )
            company_data = result_response.json()
            company_data["slug"] = company_slug
            companies_data.append(company_data)
            print(f"✓ Scraped {company_slug}")
        else:
            print(f"✗ Job {job_id} for {company_slug} is {status}")

    except Exception as e:
        print(f"✗ Error fetching {company_slug}: {e}")

# Save to file
import json
with open("companies.json", "w") as f:
    json.dump(companies_data, f, indent=2)

print(f"\nScraped {len(companies_data)}/{len(companies)} companies")

payload = {
    "url": company_url,
    "render_js": True,
    "premium_proxy": True,
    "wait": 5000,  # Wait 5 seconds after page load
    "wait_for": "h1.org-top-card-summary__title",  # Wait for specific element
    "window_width": 1920,
    "window_height": 1080
}