Best Reworkd Alternative for Multi-Format File Ingestion

import json
import os
import time

import requests

HEADERS = {"X-API-Key": os.environ["ENCONVERT_API_KEY"]}
PAGE_URL = os.environ["PAGE_URL"]  # the web page URL to ingest


def load_chunks(job_id: str, max_wait_seconds: int = 600) -> list[dict]:
    """Wait for an ingest job (web or file) and return its JSONL chunks."""
    deadline = time.monotonic() + max_wait_seconds
    while time.monotonic() < deadline:
        poll = requests.get(
            f"https://api.enconvert.com/v2/ingest/{job_id}",
            headers=HEADERS,
            timeout=60,
        )
        poll.raise_for_status()
        job = poll.json()
        if job["status"] == "completed":
            break
        if job["status"] in ("failed", "canceled"):
            raise RuntimeError(
                job.get("error_message") or f"Ingest job {job_id} was {job['status']}"
            )
        time.sleep(2)
    else:
        raise TimeoutError(f"Ingest job {job_id} did not finish in {max_wait_seconds}s")

    # output_url is a signed URL (valid 15 minutes). Plain GET, no API key.
    jsonl = requests.get(job["output_url"], timeout=60)
    jsonl.raise_for_status()

    # One chunk per line: {"id", "content", "metadata": {"source_url", "title",
    # "headings_path", "section", "word_count", "chunk_index"}}. Web pages also
    # carry render_quality and deductions in metadata. The file is UTF-8, so split
    # the raw bytes instead of relying on the charset guessed from the headers.
    return [json.loads(line) for line in jsonl.content.splitlines() if line.strip()]


# Step 1: submit a web page (mode "urls") and a PDF; both answer 202 + job_id
page_job = requests.post(
    "https://api.enconvert.com/v2/ingest",
    headers=HEADERS,
    json={"mode": "urls", "urls": [PAGE_URL]},
    timeout=60,
)
page_job.raise_for_status()

with open("/path/to/your/report.pdf", "rb") as f:
    pdf_job = requests.post(
        "https://api.enconvert.com/v2/ingest/files",
        headers=HEADERS,
        files={"files": ("report.pdf", f, "application/pdf")},
        timeout=60,
    )
pdf_job.raise_for_status()

# Step 2: same poll, same JSONL parse, same error handling for both jobs
chunks = load_chunks(page_job.json()["job_id"]) + load_chunks(pdf_job.json()["job_id"])

# metadata.source_url is the page URL for web pages and the filename for uploads
print(f"Loaded {len(chunks)} chunks from the page and the PDF, ready for embedding")

# Next: embed chunk["content"] and store chunk["metadata"] alongside it

# Step 3 (optional): the Reworkd side, only if you still have a Reworkd group.
# reworkd.ai says the product was scheduled to be sunset on February 6, 2025.
# Groups and schemas are set up in its dashboard; the public API reads their
# outputs (GET /v1/outputs/{group_id}) instead of taking a URL and a schema, so
# this call needs an existing group and a valid Reworkd API key. It returns
# structured records scraped from websites only; the public API has no file upload.
if os.environ.get("REWORKD_GROUP_ID"):
    reworkd_response = requests.get(
        f"https://api.reworkd.dev/v1/outputs/{os.environ['REWORKD_GROUP_ID']}",
        headers={"Authorization": f"Bearer {os.environ['REWORKD_API_KEY']}"},
        params={"limit": 100},  # first page; metadata.next_cursor pages through the rest
        timeout=60,
    )
    reworkd_response.raise_for_status()
    records = [item["data"] for item in reworkd_response.json()["items"]]
    print(f"Reworkd returned {len(records)} structured records")