How to Summarise arXiv Papers with AutoGen

import os

import requests
from autogen import AssistantAgent, UserProxyAgent

# Step 1: Define the schema and target arXiv abstract URLs
schema = {
    "title": "the paper title",
    "authors": "a comma-separated list of author names",
    "abstract": "the full abstract text",
    "subjects": "the arXiv subject categories listed for the paper",
}

# Step 2: Call Distill — synchronous, no job_id, no polling, no download step
# Distill requires a paid plan; the free tier returns 402
response = requests.post(
    "https://api.enconvert.com/v2/distill",
    headers={
        "X-API-Key": os.environ["ENCONVERT_API_KEY"],
        "Content-Type": "application/json",
    },
    json={
        "urls": [
            "https://arxiv.org/abs/2005.11401",
            "https://arxiv.org/abs/2005.14165",
            "https://arxiv.org/abs/2201.11903",
        ],
        "schema": schema,
    },
    timeout=120,  # increase for larger batches
)
if response.status_code == 402:
    raise RuntimeError("Distill requires a paid plan")
response.raise_for_status()

# Step 3: Parse results — failed pages arrive as rows, not as request errors
papers = []
for result in response.json()["results"]:
    if result["status"] != "completed":
        print(f"Skipped {result['url']}: {result['status']}")
        continue
    if result.get("render_quality", 1.0) < 0.5:
        # Low render_quality flags anti-bot challenges or login walls
        print(f"Low render quality for {result['url']}: {result['render_quality']}")
        continue
    # result["data"] contains exactly the schema keys above
    papers.append({"url": result["url"], **(result.get("data") or {})})

if not papers:
    raise RuntimeError("No arXiv papers were extracted successfully")

print(f"Extracted {len(papers)} papers")

# Step 4: Format structured records as agent context
context = "\n\n".join(
    f"[{p.get('url')}]\nTitle: {p.get('title')}\nAuthors: {p.get('authors')}\nSubjects: {p.get('subjects')}\nAbstract: {p.get('abstract')}"
    for p in papers
)

# Step 5: Pass the structured context into an AutoGen agent
assistant = AssistantAgent(
    name="paper_analyst",
    llm_config={"config_list": [{"model": "gpt-4.1", "api_key": os.environ["OPENAI_API_KEY"]}]},
    system_message=f"""Answer questions about the following arXiv papers.
Cite the paper URL and title for every claim you make.
Use only the fields provided.

Paper records:
{context}
""",
)

user = UserProxyAgent(
    name="user",
    human_input_mode="NEVER",
    code_execution_config=False,
)
user.initiate_chat(assistant, message="Which of these papers is closest to retrieval augmented generation, and why?")