Concurrency

import time, requests

API_KEY = "YOUR_API_KEY"

def submit_job(prompt, country="US"):
    resp = requests.post(
        "https://api.scrapellm.com/scrapers/chatgpt/jobs",
        headers={"X-API-Key": API_KEY},
        params={"prompt": prompt, "country": country},
    )
    return resp.json()["job_id"]

def poll_job(job_id, interval=3):
    while True:
        job = requests.get(
            f"https://api.scrapellm.com/jobs/{job_id}"
        ).json()
        if job["status"] in ("done", "failed"):
            return job
        time.sleep(interval)

prompts = [
    "Best CRM for small business?",
    "Top email marketing tools?",
    "Leading project management software?",
]

job_ids = [submit_job(p) for p in prompts]
results = [poll_job(jid) for jid in job_ids]

for result in results:
    if result["status"] == "done":
        print(result["result"]["result"][:200])

const API_KEY = "YOUR_API_KEY";
const BASE = "https://api.scrapellm.com";

async function submitJob(prompt, country = "US") {
  const params = new URLSearchParams({ prompt, country });
  const resp = await fetch(`${BASE}/scrapers/chatgpt/jobs?${params}`, {
    method: "POST",
    headers: { "X-API-Key": API_KEY },
  });
  const { job_id } = await resp.json();
  return job_id;
}

async function pollJob(jobId, interval = 3000) {
  while (true) {
    const resp = await fetch(`${BASE}/jobs/${jobId}`);
    const job = await resp.json();
    if (job.status === "done" || job.status === "failed") return job;
    await new Promise(r => setTimeout(r, interval));
  }
}

const prompts = [
  "Best CRM for small business?",
  "Top email marketing tools?",
  "Leading project management software?",
];

const jobIds = await Promise.all(prompts.map(p => submitJob(p)));
const results = await Promise.all(jobIds.map(id => pollJob(id)));

results.forEach(r => {
  if (r.status === "done") console.log(r.result.result.slice(0, 200));
});

import asyncio, aiohttp

API_KEY = "YOUR_API_KEY"
MAX_CONCURRENT = 5  # Stay within your plan's rate limit

async def scrape(session, prompt, semaphore):
    async with semaphore:
        params = {"prompt": prompt, "country": "US"}
        async with session.get(
            "https://api.scrapellm.com/scrapers/chatgpt",
            headers={"X-API-Key": API_KEY},
            params=params,
        ) as resp:
            return await resp.json()

async def main(prompts):
    semaphore = asyncio.Semaphore(MAX_CONCURRENT)
    async with aiohttp.ClientSession() as session:
        tasks = [scrape(session, p, semaphore) for p in prompts]
        return await asyncio.gather(*tasks)

prompts = ["Prompt 1", "Prompt 2", "Prompt 3"]
results = asyncio.run(main(prompts))

const API_KEY = "YOUR_API_KEY";
const MAX_CONCURRENT = 5;

async function scrape(prompt) {
  const params = new URLSearchParams({ prompt, country: "US" });
  const resp = await fetch(
    `https://api.scrapellm.com/scrapers/chatgpt?${params}`,
    { headers: { "X-API-Key": API_KEY } }
  );
  return resp.json();
}

async function runWithConcurrency(prompts, limit) {
  const results = [];
  const pool = [];

  for (const prompt of prompts) {
    const p = scrape(prompt).then(r => { results.push(r); pool.splice(pool.indexOf(p), 1); });
    pool.push(p);
    if (pool.length >= limit) await Promise.race(pool);
  }

  await Promise.all(pool);
  return results;
}

const prompts = ["Prompt 1", "Prompt 2", "Prompt 3"];
const results = await runWithConcurrency(prompts, MAX_CONCURRENT);