pip install langchain==0.3.0 langchain-openai==0.2.0 \
    requests==2.32.3 beautifulsoup4==4.12.3 \
    playwright==1.47.0 python-dotenv==1.0.1 \
    urllib3==2.2.3 lxml==5.3.0 fake-useragent==1.5.1

playwright install chromium

# fetchers/static_fetcher.py
import time
import random
import urllib.robotparser
from urllib.parse import urljoin, urlparse
import requests
from bs4 import BeautifulSoup
from fake_useragent import UserAgent

ua = UserAgent()
_robots_cache: dict[str, urllib.robotparser.RobotFileParser] = {}

def check_robots(url: str, user_agent: str = "*") -> bool:
    """Check if scraping the URL is permitted by robots.txt."""
    parsed = urlparse(url)
    base = f"{parsed.scheme}://{parsed.netloc}"
    if base not in _robots_cache:
        rp = urllib.robotparser.RobotFileParser()
        rp.set_url(f"{base}/robots.txt")
        try:
            rp.read()
        except Exception:
            rp = None  # If robots.txt is inaccessible, assume allowed
        _robots_cache[base] = rp
    rp = _robots_cache[base]
    return rp is None or rp.can_fetch(user_agent, url)

def fetch_static(url: str, delay_range: tuple = (1.0, 3.0)) -> dict:
    """Fetch a static HTML page with rate limiting and robots.txt compliance."""
    if not check_robots(url):
        return {"url": url, "html": None, "error": "Blocked by robots.txt"}

    # Respectful rate limiting
    time.sleep(random.uniform(*delay_range))

    headers = {
        "User-Agent": ua.random,
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": "en-US,en;q=0.5",
    }
    try:
        response = requests.get(url, headers=headers, timeout=15)
        response.raise_for_status()
        return {"url": url, "html": response.text, "status": response.status_code, "error": None}
    except requests.RequestException as e:
        return {"url": url, "html": None, "error": str(e)}

def clean_html(html: str, url: str = "") -> str:
    """Strip navigation, scripts, ads, and boilerplate from HTML."""
    soup = BeautifulSoup(html, "lxml")

    # Remove non-content elements
    for tag in soup(["script", "style", "nav", "footer", "header",
                      "aside", "iframe", "form", "button", "noscript"]):
        tag.decompose()

    # Remove common ad/nav class patterns
    for cls in ["nav", "menu", "header", "footer", "sidebar", "ad", "ads",
                 "advertisement", "cookie", "popup", "modal", "banner"]:
        for el in soup.find_all(class_=lambda c: c and cls in c.lower()):
            el.decompose()

    # Extract main content areas first if available
    main = soup.find("main") or soup.find("article") or soup.find(id="content")
    if main:
        text = main.get_text(separator="\n", strip=True)
    else:
        text = soup.get_text(separator="\n", strip=True)

    # Collapse excessive whitespace
    import re
    text = re.sub(r'\n{3,}', '\n\n', text)
    return text.strip()

# fetchers/dynamic_fetcher.py
import asyncio
from playwright.async_api import async_playwright

async def fetch_dynamic_async(url: str, wait_for: str = None) -> dict:
    """Fetch a JavaScript-rendered page using Playwright."""
    async with async_playwright() as p:
        browser = await p.chromium.launch(headless=True)
        context = await browser.new_context(
            user_agent="Mozilla/5.0 (compatible; ResearchBot/1.0)",
            java_script_enabled=True,
        )
        page = await context.new_page()

        try:
            await page.goto(url, wait_until="networkidle", timeout=30000)
            if wait_for:
                await page.wait_for_selector(wait_for, timeout=5000)
            html = await page.content()
            return {"url": url, "html": html, "error": None}
        except Exception as e:
            return {"url": url, "html": None, "error": str(e)}
        finally:
            await browser.close()

def fetch_dynamic(url: str, wait_for: str = None) -> dict:
    """Sync wrapper for the async Playwright fetcher."""
    return asyncio.run(fetch_dynamic_async(url, wait_for))

# extractor.py
import json
from langchain_openai import ChatOpenAI
from langchain_core.prompts import ChatPromptTemplate
from pydantic import BaseModel

EXTRACTOR_SYSTEM = """You are a data extraction specialist.
Extract structured data from the provided web page text according to the schema description.
Return ONLY valid JSON matching the schema. Do not add commentary.
If a field is not found, use null. If a field is a list and nothing is found, use [].
Extract ALL instances if multiple records are present."""

extractor_llm = ChatOpenAI(model="gpt-4o", temperature=0)
extract_chain = ChatPromptTemplate.from_messages([
    ("system", EXTRACTOR_SYSTEM),
    ("human", "Schema to extract:\n{schema}\n\nPage content:\n{content}"),
]) | extractor_llm

def extract_structured_data(content: str, schema_description: str) -> dict | list:
    """
    Extract structured data from page text using the given schema description.

    schema_description examples:
    - "List of products with fields: name, price, rating, in_stock"
    - "Article with fields: title, author, published_date, content_summary, tags"
    - "Job listing with fields: title, company, location, salary_range, requirements"
    """
    # Truncate content to avoid token limits
    truncated = content[:6000] if len(content) > 6000 else content
    result = extract_chain.invoke({
        "schema": schema_description,
        "content": truncated,
    })
    try:
        return json.loads(result.content)
    except json.JSONDecodeError:
        # Try to find JSON in the response
        import re
        json_match = re.search(r'(\[.*\]|\{.*\})', result.content, re.DOTALL)
        if json_match:
            return json.loads(json_match.group())
        return {"error": "Could not parse JSON", "raw": result.content}

# agent.py
import json
from langchain.tools import tool
from langchain_openai import ChatOpenAI
from langchain.agents import create_openai_tools_agent, AgentExecutor
from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
from fetchers.static_fetcher import fetch_static, clean_html
from fetchers.dynamic_fetcher import fetch_dynamic
from extractor import extract_structured_data
from bs4 import BeautifulSoup

@tool
def scrape_page(url: str, javascript_required: bool = False) -> str:
    """Fetch and clean a web page. Set javascript_required=True for SPAs and dynamic sites."""
    if javascript_required:
        result = fetch_dynamic(url)
    else:
        result = fetch_static(url)

    if result.get("error"):
        return f"Error fetching {url}: {result['error']}"

    cleaned = clean_html(result["html"], url)
    return cleaned[:5000]  # Return first 5000 chars for the agent to inspect

@tool
def extract_data_from_page(url: str, schema_description: str, javascript_required: bool = False) -> str:
    """
    Fetch a page and extract structured data according to a schema description.
    Returns JSON string with the extracted data.
    """
    if javascript_required:
        result = fetch_dynamic(url)
    else:
        result = fetch_static(url)

    if result.get("error"):
        return f"Error: {result['error']}"

    cleaned = clean_html(result["html"], url)
    extracted = extract_structured_data(cleaned, schema_description)
    return json.dumps(extracted, indent=2)

@tool
def find_pagination_links(url: str, javascript_required: bool = False) -> str:
    """Find pagination links on a page (next page, page numbers)."""
    if javascript_required:
        result = fetch_dynamic(url)
    else:
        result = fetch_static(url)

    if result.get("error") or not result.get("html"):
        return "Could not load page to find pagination."

    soup = BeautifulSoup(result["html"], "lxml")
    pagination_links = []

    # Common pagination patterns
    for selector in ["a[rel='next']", ".next a", ".pagination a", "[aria-label='Next']"]:
        links = soup.select(selector)
        for link in links[:3]:
            href = link.get("href", "")
            if href:
                from urllib.parse import urljoin
                pagination_links.append(urljoin(url, href))

    if pagination_links:
        return f"Pagination links found: {json.dumps(pagination_links)}"
    return "No pagination links found on this page."

def build_scraping_agent() -> AgentExecutor:
    llm = ChatOpenAI(model="gpt-4o", temperature=0)
    tools = [scrape_page, extract_data_from_page, find_pagination_links]
    prompt = ChatPromptTemplate.from_messages([
        ("system", """You are a web scraping agent.
Given a URL and extraction goal, fetch pages and extract structured data.
Guidelines:
- Always check page content before extracting to understand the structure
- For pagination, extract all pages up to the requested limit
- Respect rate limits — don't fetch the same URL twice
- If extraction fails, try adjusting the schema description and retry once
- Return final data as a JSON array"""),
        MessagesPlaceholder(variable_name="chat_history"),
        ("human", "{input}"),
        MessagesPlaceholder(variable_name="agent_scratchpad"),
    ])
    agent = create_openai_tools_agent(llm, tools, prompt)
    return AgentExecutor(agent=agent, tools=tools, verbose=True, max_iterations=15)

# cli.py
import argparse
import json
from agent import build_scraping_agent

def main():
    parser = argparse.ArgumentParser(description="Web Scraping AI Agent")
    parser.add_argument("url", help="Starting URL to scrape")
    parser.add_argument("schema", help="Schema description: e.g. 'list of products with name, price, rating'")
    parser.add_argument("--pages", type=int, default=1, help="Number of pages to crawl")
    parser.add_argument("--js", action="store_true", help="Use JavaScript rendering (Playwright)")
    parser.add_argument("--output", help="Output JSON file path")
    args = parser.parse_args()

    executor = build_scraping_agent()
    task = (
        f"Scrape {args.url} "
        f"{'using JavaScript rendering ' if args.js else ''}"
        f"and extract: {args.schema}. "
        f"{'Crawl up to ' + str(args.pages) + ' pages using pagination.' if args.pages > 1 else ''}"
        f" Return all results as a JSON array."
    )

    result = executor.invoke({"input": task, "chat_history": []})
    print("\nExtracted Data:")
    print(result["output"])

    if args.output:
        with open(args.output, "w") as f:
            f.write(result["output"])
        print(f"\nSaved to {args.output}")

if __name__ == "__main__":
    main()

# Scrape static site
python cli.py "https://books.toscrape.com" \
  "list of books with title, price, rating, availability" \
  --pages 3 --output books.json

# Scrape JavaScript-rendered site
python cli.py "https://example-spa.com/products" \
  "product listings with name, price, SKU" \
  --js --output products.json