A developer building an AutoGen pipeline that reads Medium articles needs clean page text before passing it into agent context or a tool. Medium pages mix article content with publication headers, clap counts, author bios, member-only banners, related story carousels, and response sections, and scraping them by hand requires CSS selectors that break when Medium updates its layout. EnConvert Perceive renders up to 10 Medium pages in a single batch call and returns clean Markdown for each one, with navigation and sidebars stripped by default. The extracted Markdown is ready to pass into an AutoGen tool with no HTML parsing.
How to Scrape Medium with AutoGen
Scrape Medium articles into an AutoGen pipeline with EnConvert Perceive, returning clean Markdown with navigation and sidebars stripped in a single batch call
Get API keyExample
python
import os
import requests
from autogen import AssistantAgent, UserProxyAgent, register_function
BASE_URL = "https://api.enconvert.com/v2"
enconvert_headers = {
"X-API-Key": os.environ["ENCONVERT_API_KEY"],
"Content-Type": "application/json",
}
ARTICLE_URLS = [url.strip() for url in os.environ["MEDIUM_ARTICLE_URLS"].split(",")] # public Medium article URLs
def fetch_medium_articles(urls: list[str]) -> str:
"""Fetch clean Markdown from up to 10 Medium articles."""
if len(urls) > 10:
return "Submit no more than 10 URLs per call for inline batch processing."
response = requests.post(
f"{BASE_URL}/perceive/batch",
headers=enconvert_headers,
json={"urls": urls, "options": {"outputs": ["markdown"]}},
timeout=300,
)
if response.status_code == 402:
return "Monthly ops allowance exhausted"
response.raise_for_status()
sections = []
for result in response.json()["items"]:
page_url = result["url"]
if result.get("status") != "completed":
sections.append(f"Source: {page_url}\nRender failed ({result.get('status')}).")
continue
if result.get("is_blocked"):
sections.append(f"Source: {page_url}\nBlocked page. No content returned.")
continue
render_quality = result.get("render_quality", 1.0)
if render_quality < 0.40:
sections.append(f"Source: {page_url}\nRender failed (quality {render_quality:.2f}).")
continue
# Markdown is a pre-signed URL; fetch the content in a second request
markdown_url = result["outputs"]["markdown"]["url"]
markdown = requests.get(markdown_url, timeout=60)
markdown.raise_for_status()
sections.append(f"Source: {page_url}\n\n{markdown.text}")
return "\n\n---\n\n".join(sections) or "No articles were returned."
llm_config = {
"config_list": [
{
"model": "gpt-4.1",
"api_key": os.environ["OPENAI_API_KEY"],
}
]
}
assistant = AssistantAgent(
name="medium_researcher",
system_message=(
"You analyse Medium articles using the fetch_medium_articles tool. "
"Answer only from the retrieved article content and cite the source URL for every statement."
),
llm_config=llm_config,
)
user_proxy = UserProxyAgent(
name="user_proxy",
human_input_mode="NEVER",
max_consecutive_auto_reply=5,
code_execution_config=False,
)
register_function(
fetch_medium_articles,
caller=assistant,
executor=user_proxy,
name="fetch_medium_articles",
description="Retrieve clean Markdown from up to 10 Medium article URLs.",
)
chat_result = user_proxy.initiate_chat(
assistant,
message=(
f"Read these Medium articles: {', '.join(ARTICLE_URLS)}. "
"Compare their approaches to the topic, citing each article."
),
)
print(chat_result.summary)What you get back
Perceive returns clean Markdown for each URL in the batch, with navigation and sidebars stripped by default, delivered as pre-signed URLs that you fetch once per article. Scraping Medium pages by hand needs CSS selectors that break when Medium updates its layout, and passing raw HTML into an AutoGen agent tool fills the agent’s context with publication chrome and boilerplate that degrades the quality of the agent’s output.