Concurrency

import time, requests

API_KEY = "YOUR_API_KEY"

def submit_job(prompt, country="US"):
    resp = requests.post(
        "https://api.scrapellm.com/scrapers/chatgpt/jobs",
        headers={"X-API-Key": API_KEY},
        params={"prompt": prompt, "country": country},
    )
    return resp.json()["job_id"]

def poll_job(job_id, interval=3):
    while True:
        job = requests.get(
            f"https://api.scrapellm.com/jobs/{job_id}"
        ).json()
        if job["status"] in ("done", "failed"):
            return job
        time.sleep(interval)

prompts = [
    "Best CRM for small business?",
    "Top email marketing tools?",
    "Leading project management software?",
]

job_ids = [submit_job(p) for p in prompts]
results = [poll_job(jid) for jid in job_ids]

for result in results:
    if result["status"] == "done":
        print(result["result"]["result"][:200])

import asyncio, aiohttp

API_KEY = "YOUR_API_KEY"
MAX_CONCURRENT = 5  # Stay within your plan's rate limit

async def scrape(session, prompt, semaphore):
    async with semaphore:
        params = {"prompt": prompt, "country": "US"}
        async with session.get(
            "https://api.scrapellm.com/scrapers/chatgpt",
            headers={"X-API-Key": API_KEY},
            params=params,
        ) as resp:
            return await resp.json()

async def main(prompts):
    semaphore = asyncio.Semaphore(MAX_CONCURRENT)
    async with aiohttp.ClientSession() as session:
        tasks = [scrape(session, p, semaphore) for p in prompts]
        return await asyncio.gather(*tasks)

prompts = ["Prompt 1", "Prompt 2", "Prompt 3"]
results = asyncio.run(main(prompts))