How to Ingest HTML Files into LlamaIndex with EnConvert

import json
import os
import time

import requests
from llama_index.core import VectorStoreIndex
from llama_index.core.schema import TextNode
from llama_index.core.vector_stores import (
    FilterOperator,
    MetadataFilter,
    MetadataFilters,
)
from llama_index.embeddings.openai import OpenAIEmbedding

HEADERS = {"X-API-Key": os.environ["ENCONVERT_API_KEY"]}

# Step 1: Upload the HTML file (the multipart field is named "files")
# Navigation chrome and sidebar content are stripped automatically
with open("/path/to/your/exported-docs-page.html", "rb") as f:
    upload = requests.post(
        "https://api.enconvert.com/v2/ingest/files",
        headers=HEADERS,
        files={"files": ("exported-docs-page.html", f, "text/html")},
        timeout=60,
    )
upload.raise_for_status()
job_id = upload.json()["job_id"]

# Step 2: Poll until the job completes
deadline = time.monotonic() + 900
while True:
    status_response = requests.get(
        f"https://api.enconvert.com/v2/ingest/{job_id}",
        headers=HEADERS,
        timeout=30,
    )
    status_response.raise_for_status()
    job = status_response.json()
    if job["status"] == "completed":
        break
    if job["status"] in ("failed", "canceled"):
        raise RuntimeError(
            job.get("error_message") or f"Ingest job {job_id} {job['status']}"
        )
    if time.monotonic() >= deadline:
        raise TimeoutError(f"Job {job_id} is still running; resume polling later")
    time.sleep(5)

# Step 3: Download the JSONL output, one chunk per line
# The output_url is a signed URL (valid 15 minutes). Plain GET, no API key.
jsonl = requests.get(job["output_url"], timeout=60)
jsonl.raise_for_status()
# Split on "\n" only: splitlines() also breaks on Unicode separators inside content
chunks = [
    json.loads(line)
    for line in jsonl.content.decode("utf-8").split("\n")
    if line.strip()
]
print(f"Loaded {len(chunks)} chunks ready for embedding")

# Step 4: Turn each chunk into a TextNode with its metadata preserved
# Passing nodes (not documents) stops LlamaIndex from re-splitting EnConvert chunks
nodes = []
for chunk in chunks:
    meta = chunk["metadata"]
    nodes.append(
        TextNode(
            id_=chunk["id"],
            text=chunk["content"],
            metadata={
                "source_url": meta["source_url"],
                "title": meta["title"],
                "section": meta["section"],
                # headings_path is a list; store it as one string so any vector store accepts it
                "headings_path": " > ".join(meta["headings_path"]),
                "chunk_index": meta["chunk_index"],
                "word_count": meta["word_count"],
            },
            excluded_embed_metadata_keys=[
                "chunk_index",
                "word_count",
                "source_url",
                "headings_path",
            ],
        )
    )

# The section values in your file that you can filter on
print(sorted({node.metadata["section"] for node in nodes}))

# Step 5: Build the vector index
index = VectorStoreIndex(
    nodes=nodes,
    embed_model=OpenAIEmbedding(
        model="text-embedding-3-small",
        api_key=os.environ["OPENAI_API_KEY"],
    ),
)

# Step 6: Retrieve restricted to one section by a metadata filter
# Replace the value with a section heading from your own HTML file (see the list above)
filters = MetadataFilters(
    filters=[
        MetadataFilter(
            key="section",
            operator=FilterOperator.EQ,
            value="Rotating API keys",
        )
    ]
)

retriever = index.as_retriever(
    similarity_top_k=4,
    filters=filters,
)

results = retriever.retrieve("how do I rotate an API key")
for result in results:
    print(result.node.metadata["section"])
    print(result.node.metadata["headings_path"])
    print(result.node.get_content()[:200])