Best Tavily Alternative for RAG Pipeline Ingestion

import json
import os
import time

import requests
from langchain_core.documents import Document

ENCONVERT_API_KEY = os.environ["ENCONVERT_API_KEY"]
HEADERS = {"X-API-Key": ENCONVERT_API_KEY}

# Step 1: Upload the PDF and submit an ingestion job (the form field is "files")
with open("/path/to/your/report.pdf", "rb") as f:
    upload = requests.post(
        "https://api.enconvert.com/v2/ingest/files",
        headers=HEADERS,
        files={"files": ("report.pdf", f, "application/pdf")},
        timeout=60,
    )
upload.raise_for_status()
job_id = upload.json()["job_id"]

# Step 2: Poll until the job completes
while True:
    poll = requests.get(
        f"https://api.enconvert.com/v2/ingest/{job_id}",
        headers=HEADERS,
        timeout=30,
    )
    poll.raise_for_status()
    job = poll.json()
    if job["status"] == "completed":
        output_url = job["output_url"]
        break
    if job["status"] in ("failed", "canceled"):
        raise RuntimeError(job.get("error_message") or f"Ingest job {job['status']}")
    time.sleep(2)

# Step 3: Download the JSONL (a signed URL valid for 15 minutes; plain GET, no API key)
jsonl = requests.get(output_url, timeout=60)
jsonl.raise_for_status()

# Step 4: Load each chunk as a LangChain document
documents = []
for line in jsonl.content.decode("utf-8").split("\n"):
    if not line.strip():
        continue
    chunk = json.loads(line)
    metadata = chunk["metadata"]
    # headings_path is a list; most vector stores accept only scalar metadata values
    metadata["headings_path"] = " > ".join(metadata["headings_path"])
    documents.append(Document(page_content=chunk["content"], metadata=metadata))

print(f"Loaded {len(documents)} chunks ready for embedding")

# vector_store.add_documents(documents)