A developer building a CrewAI pipeline that reads Twilio documentation needs clean page text before passing it into agent context or a tool. Twilio docs pages mix reference content with navigation, version selectors, sidebar menus, and related resource links, and scraping them by hand requires CSS selectors that break when Twilio updates its docs layout. EnConvert Perceive renders up to 10 Twilio docs pages in a single batch call and returns clean Markdown for each one, with navigation and sidebars stripped by default. The extracted Markdown is ready to pass into a CrewAI tool with no HTML parsing.
How to Scrape Twilio Docs with CrewAI
Scrape Twilio documentation pages into a CrewAI pipeline with EnConvert Perceive, returning clean Markdown with navigation and sidebars stripped in a batch call
Get API keyExample
python
import os
import requests
from crewai import Agent, Crew, Process, Task
from crewai.tools import BaseTool
from pydantic import BaseModel, Field
# CrewAI reads OPENAI_API_KEY from the environment
BASE_URL = "https://api.enconvert.com/v2"
enconvert_headers = {
"X-API-Key": os.environ["ENCONVERT_API_KEY"],
"Content-Type": "application/json",
}
TWILIO_DOCS_URLS = [url.strip() for url in os.environ["TWILIO_DOCS_URLS"].split(",")] # public Twilio documentation URLs
class TwilioDocsInput(BaseModel):
urls: list[str] = Field(description="Twilio documentation page URLs to read, up to 10 per call")
class TwilioDocsTool(BaseTool):
name: str = "read_twilio_docs"
description: str = (
"Reads publicly accessible Twilio documentation pages and returns the main content "
"of each as clean Markdown, labelled with its source URL."
)
args_schema: type[BaseModel] = TwilioDocsInput
def _run(self, urls: list[str]) -> str:
if len(urls) > 10:
return "Submit no more than 10 URLs per call for inline batch processing."
response = requests.post(
f"{BASE_URL}/perceive/batch",
headers=enconvert_headers,
json={"urls": urls, "options": {"outputs": ["markdown"]}},
timeout=300,
)
if response.status_code == 402:
return "Monthly ops allowance exhausted"
response.raise_for_status()
sections = []
for result in response.json()["items"]:
page_url = result["url"]
if result.get("status") != "completed":
sections.append(f"Source: {page_url}\nRender failed ({result.get('status')}).")
continue
if result.get("is_blocked"):
sections.append(f"Source: {page_url}\nBlocked page. No content returned.")
continue
render_quality = result.get("render_quality", 1.0)
if render_quality < 0.40:
sections.append(f"Source: {page_url}\nRender failed (quality {render_quality:.2f}).")
continue
# Markdown is a pre-signed URL; fetch the content in a second request
markdown_url = result["outputs"]["markdown"]["url"]
markdown = requests.get(markdown_url, timeout=60)
markdown.raise_for_status()
sections.append(f"Source: {page_url}\n\n{markdown.text}")
return "\n\n---\n\n".join(sections) or "No pages were returned."
docs_analyst = Agent(
role="Twilio Documentation Analyst",
goal="Answer questions about the Twilio API using only the documentation pages you read",
backstory="You read API reference pages closely and cite the source URL for every statement.",
tools=[TwilioDocsTool()],
llm="gpt-4.1",
)
task = Task(
description=(
f"Read these Twilio documentation pages: {', '.join(TWILIO_DOCS_URLS)}. "
"Explain the main configuration steps, citing each page."
),
expected_output="A short list of key parameters and configuration steps, with source URLs.",
agent=docs_analyst,
)
crew = Crew(
agents=[docs_analyst],
tasks=[task],
process=Process.sequential,
)
result = crew.kickoff()
print(result)What you get back
Perceive returns clean Markdown for each URL in the batch, with navigation and sidebars stripped by default, delivered as pre-signed URLs that you fetch once per page. Scraping Twilio docs pages by hand needs CSS selectors that break when Twilio updates its layout, and passing raw HTML into a CrewAI tool fills the agent’s context with navigation and boilerplate that degrades the quality of the agent’s output.