Best Firecrawl Alternative for RAG Pipeline Ingestion

import os

import requests

ENCONVERT_API_KEY = os.environ["ENCONVERT_API_KEY"]
PAGE_URL = os.environ["PAGE_URL"]  # the web page URL to extract

response = requests.post(
    "https://api.enconvert.com/v2/perceive",
    headers={
        "X-API-Key": ENCONVERT_API_KEY,
        "Content-Type": "application/json",
    },
    json={
        "url": PAGE_URL,
        "outputs": ["markdown"],
        "only_main_content": True,
    },
    timeout=60,
)
response.raise_for_status()
perceived = response.json()

if perceived.get("error") or "markdown" not in perceived.get("outputs", {}):
    raise RuntimeError(f"Perceive failed for {PAGE_URL}: {perceived.get('error')}")

# The markdown is a signed artifact URL (valid 15 minutes). Plain GET, no API key.
content_response = requests.get(perceived["outputs"]["markdown"]["url"], timeout=60)
content_response.raise_for_status()
clean_content = content_response.text

print(clean_content)