Best Apify Alternative for Agentic Web Scraping

import os

import requests
from apify_client import ApifyClient  # apify-client 3.x needs Python 3.11+

PAGE_URL = os.environ["PAGE_URL"]  # the web page URL to turn into Markdown

# Step 1: Apify - pick an Actor, start a run, wait, read the dataset
apify = ApifyClient(os.environ["APIFY_API_TOKEN"])
run = apify.actor("apify/website-content-crawler").call(
    run_input={
        "startUrls": [{"url": PAGE_URL}],
        "maxCrawlDepth": 0,  # crawl only the start URL
        "maxCrawlPages": 1,
        "saveMarkdown": True,
    }
)
if run is None or run.status != "SUCCEEDED":
    raise RuntimeError(f"Apify run did not succeed for {PAGE_URL}")

items = apify.dataset(run.default_dataset_id).list_items().items
if not items:
    raise RuntimeError(f"Apify returned no dataset items for {PAGE_URL}")
apify_markdown = items[0]["markdown"]  # field name is specific to this Actor

# Apify also offers a synchronous endpoint that returns dataset items
# in one HTTP call (300 second limit), but Actor selection and per-Actor
# input/output field names still apply.

# Step 2: EnConvert - one POST to Perceive, then download the Markdown
response = requests.post(
    "https://api.enconvert.com/v2/perceive",
    headers={
        "X-API-Key": os.environ["ENCONVERT_API_KEY"],
        "Content-Type": "application/json",
    },
    json={
        "url": PAGE_URL,
        "outputs": ["markdown"],
        "only_main_content": True,
    },
    timeout=60,
)
response.raise_for_status()
perceived = response.json()

# A blocked page or failed render comes back without a markdown output
if perceived.get("error") or "markdown" not in perceived.get("outputs", {}):
    raise RuntimeError(
        f"Perceive failed for {PAGE_URL}: "
        f"{perceived.get('error') or perceived.get('warnings')}"
    )

# The Markdown is a signed artifact URL (valid 15 minutes). Plain GET, no API key.
content_response = requests.get(perceived["outputs"]["markdown"]["url"], timeout=60)
content_response.raise_for_status()
enconvert_markdown = content_response.text

# Both results are plain Markdown strings, ready for an agent context or vector store
print(f"Apify: {len(apify_markdown)} characters of Markdown")
print(f"EnConvert: {len(enconvert_markdown)} characters of Markdown")
print(enconvert_markdown)