> ## Documentation Index
> Fetch the complete documentation index at: https://docs.agno.com/llms.txt
> Use this file to discover all available pages before exploring further.

# Web Readers: Website, YouTube, ArXiv, Firecrawl

> Readers for web-based content sources.

```python web.py theme={null}
"""
Web Readers: Website, YouTube, ArXiv, Firecrawl
=================================================
Readers for web-based content sources.

Supported web sources:
- WebsiteReader: Crawls web pages and extracts content
- YouTubeReader: Extracts transcripts from YouTube videos
- ArxivReader: Fetches academic papers from ArXiv
- FirecrawlReader: Advanced web scraping via Firecrawl API

See also: 01_documents.py for PDF/DOCX, 02_data.py for CSV/JSON.
"""

import asyncio

from agno.agent import Agent
from agno.knowledge.embedder.openai import OpenAIEmbedder
from agno.knowledge.knowledge import Knowledge
from agno.knowledge.reader.website_reader import WebsiteReader
from agno.models.openai import OpenAIResponses
from agno.vectordb.qdrant import Qdrant
from agno.vectordb.search import SearchType

# ---------------------------------------------------------------------------
# Setup
# ---------------------------------------------------------------------------

qdrant_url = "http://localhost:6333"

knowledge = Knowledge(
    vector_db=Qdrant(
        collection="web_readers",
        url=qdrant_url,
        search_type=SearchType.hybrid,
        embedder=OpenAIEmbedder(id="text-embedding-3-small"),
    ),
)

agent = Agent(
    model=OpenAIResponses(id="gpt-5.2"),
    knowledge=knowledge,
    search_knowledge=True,
    markdown=True,
)

# ---------------------------------------------------------------------------
# Run Demo
# ---------------------------------------------------------------------------

if __name__ == "__main__":

    async def main():
        # --- Website: crawl and extract content ---
        print("\n" + "=" * 60)
        print("READER: Website (crawl and extract)")
        print("=" * 60 + "\n")

        # WebsiteReader crawls pages up to max_depth and max_links
        website_reader = WebsiteReader(max_depth=1, max_links=5)
        await knowledge.ainsert(
            name="Agno Docs",
            url="https://docs.agno.com/introduction",
            reader=website_reader,
        )
        agent.print_response("What is Agno?", stream=True)

        # --- URL: direct URL loading (auto-detected) ---
        print("\n" + "=" * 60)
        print("READER: Direct URL (auto-detected)")
        print("=" * 60 + "\n")

        # URLs ending in .pdf, .md, .txt etc. are auto-detected
        await knowledge.ainsert(
            name="Recipes",
            url="https://agno-public.s3.amazonaws.com/recipes/ThaiRecipes.pdf",
        )
        agent.print_response("What Thai recipes are available?", stream=True)

    asyncio.run(main())
```

## Run the Example

<Steps>
  <Snippet file="create-venv-step.mdx" />

  <Step title="Install dependencies">
    ```bash theme={null}
    uv pip install -U agno beautifulsoup4 fastembed openai qdrant-client
    ```
  </Step>

  <Step title="Export your OpenAI API key">
    <CodeGroup>
      ```bash Mac/Linux theme={null}
      export OPENAI_API_KEY="your_openai_api_key_here"
      ```

      ```bash Windows theme={null}
      $Env:OPENAI_API_KEY="your_openai_api_key_here"
      ```
    </CodeGroup>
  </Step>

  <Step title="Run Qdrant">
    ```bash theme={null}
    docker run -d --name qdrant -p 6333:6333 qdrant/qdrant:latest
    ```
  </Step>

  <Step title="Run the example">
    Save the code above as `web.py`, then run:

    ```bash theme={null}
    python web.py
    ```
  </Step>
</Steps>

Full source: [cookbook/07\_knowledge/05\_integrations/readers/03\_web.py](https://github.com/agno-agi/agno/blob/main/cookbook/07_knowledge/05_integrations/readers/03_web.py)