How to Ingest Web Pages into OpenAI Assistants

import os

import requests
from openai import OpenAI

ENCONVERT_API_KEY = os.environ["ENCONVERT_API_KEY"]
OPENAI_API_KEY = os.environ["OPENAI_API_KEY"]
ASSISTANT_ID = os.environ["ASSISTANT_ID"]  # create one at platform.openai.com/assistants
PAGE_URL = os.environ["PAGE_URL"]

openai_client = OpenAI(api_key=OPENAI_API_KEY)

# Step 1: Fetch clean markdown from any web page via Perceive
response = requests.post(
    "https://api.enconvert.com/v2/perceive",
    headers={
        "X-API-Key": ENCONVERT_API_KEY,
        "Content-Type": "application/json",
    },
    json={
        "url": PAGE_URL,
        "outputs": ["markdown"],
    },
    timeout=120,
)
response.raise_for_status()
perceived = response.json()

if perceived.get("error") or "markdown" not in perceived.get("outputs", {}):
    raise RuntimeError(f"Perceive failed for {PAGE_URL}: {perceived.get('error')}")
if (perceived.get("render_quality") or 0) < 0.5:
    raise RuntimeError(
        f"Low render quality ({perceived.get('render_quality')}): "
        f"{perceived.get('warnings')}"
    )

# The markdown is a signed artifact URL (valid 15 minutes). Plain GET, no API key.
markdown_response = requests.get(perceived["outputs"]["markdown"]["url"], timeout=60)
markdown_response.raise_for_status()
markdown = markdown_response.text

# Optional: preview the extracted markdown before uploading
print(markdown)

# Step 2: Upload the markdown straight from memory to the OpenAI Files API
# (no temp file, so there is nothing to clean up on failure)
uploaded_file = openai_client.files.create(
    file=("page_content.md", markdown.encode("utf-8"), "text/markdown"),
    purpose="assistants",
)

# Step 3: Create a thread and attach the uploaded file to a message
thread = openai_client.beta.threads.create()

openai_client.beta.threads.messages.create(
    thread_id=thread.id,
    role="user",
    content="Read the attached page and summarise its key points.",
    attachments=[
        {
            "file_id": uploaded_file.id,
            "tools": [{"type": "file_search"}],
        }
    ],
)

# Step 4: Run the Assistant and wait for completion
run = openai_client.beta.threads.runs.create_and_poll(
    thread_id=thread.id,
    assistant_id=ASSISTANT_ID,
)
if run.status != "completed":
    raise RuntimeError(f"Run ended with status {run.status}: {run.last_error}")

# Step 5: Read the response (list() returns newest first)
messages = openai_client.beta.threads.messages.list(thread_id=thread.id)
print(messages.data[0].content[0].text.value)