How to Scrape Confluence with OpenAI Agents SDK

import os
import requests
from agents import Agent, Runner, function_tool

# The OpenAI Agents SDK reads OPENAI_API_KEY from the environment

BASE_URL = "https://api.enconvert.com/v2"
headers = {
    "X-API-Key": os.environ["ENCONVERT_API_KEY"],
    "Content-Type": "application/json",
}

CONFLUENCE_URLS = [url.strip() for url in os.environ["CONFLUENCE_PAGE_URLS"].split(",")]  # public Confluence page URLs

@function_tool
def read_confluence_pages(urls: list[str]) -> str:
    """Read publicly accessible Confluence pages and return the main content
    of each as clean Markdown, labelled with its source URL."""
    if len(urls) > 10:
        raise ValueError("This tool covers inline batches of 10 URLs or fewer")

    response = requests.post(
        f"{BASE_URL}/perceive/batch",
        headers=headers,
        json={
            "urls": urls,
            "options": {"outputs": ["markdown"]},
        },
        timeout=300,
    )

    if response.status_code == 402:
        return "Monthly ops allowance exhausted"
    response.raise_for_status()

    sections = []
    for result in response.json()["items"]:
        page_url = result["url"]

        if result.get("status") != "completed":
            sections.append(f"Source: {page_url}\nRender failed ({result.get('status')}).")
            continue

        if result.get("is_blocked"):
            sections.append(f"Source: {page_url}\nBlocked page. No content returned.")
            continue

        render_quality = result.get("render_quality", 1.0)
        if render_quality < 0.40:
            sections.append(f"Source: {page_url}\nRender failed (quality {render_quality:.2f}).")
            continue

        # Markdown is a pre-signed URL; fetch the content in a second request
        markdown_url = result["outputs"]["markdown"]["url"]
        markdown = requests.get(markdown_url, timeout=60)
        markdown.raise_for_status()

        sections.append(f"Source: {page_url}\n\n{markdown.text}")

    return "\n\n---\n\n".join(sections) or "No pages were returned."

agent = Agent(
    name="Confluence Assistant",
    instructions=(
        "You answer questions about Confluence documentation. "
        "Use read_confluence_pages to read the pages you need, "
        "answer only from what they say, and cite the source URL for every statement."
    ),
    tools=[read_confluence_pages],
    model="gpt-4.1",
)

result = Runner.run_sync(
    agent,
    f"Read these Confluence pages: {', '.join(CONFLUENCE_URLS)} "
    "and summarise the deployment steps.",
)
print(result.final_output)