%%{init: {'themeVariables': { 'fontSize': '14px'}}}%%
graph TD
    subgraph A1[Advanced Crawling Systems 🔧]
        A["`
        • Graph Crawler ✓
        • Question-Based Crawler
        • Knowledge-Optimal Crawler
        • Agentic Crawler
        `"]
    end

    subgraph A2[Specialized Features 🛠️]
        B["`
        • Automated Schema Generator
        • Domain-Specific Scrapers
        • 
        • 
        `"]
    end

    subgraph A3[Development Tools 🔨]
        C["`
        • Interactive Playground
        • Performance Monitor
        • Cloud Integration
        • 
        `"]
    end

    subgraph A4[Community & Growth 🌱]
        D["`
        • Sponsorship Program
        • Educational Content
        • 
        • 
        `"]
    end

    classDef default fill:#f9f9f9,stroke:#333,stroke-width:2px
    classDef section fill:#f0f0f0,stroke:#333,stroke-width:4px,rx:10
    class A1,A2,A3,A4 section

    %% Layout hints
    A1 --> A2[" "]
    A3 --> A4[" "]
    linkStyle 0,1 stroke:none

from crawl4ai import AsyncWebCrawler
from crawl4ai.discovery import QuestionBasedDiscovery

async with AsyncWebCrawler() as crawler:
    discovery = QuestionBasedDiscovery(crawler)
    results = await discovery.arun(
        question="What are the system requirements for major cloud providers' GPU instances?",
        max_urls=5,
        relevance_threshold=0.7
    )
    
    for result in results:
        print(f"Source: {result.url} (Relevance: {result.relevance_score})")
        print(f"Content: {result.markdown}\n")

from crawl4ai import AsyncWebCrawler
from crawl4ai.optimization import KnowledgeOptimizer

async with AsyncWebCrawler() as crawler:
    optimizer = KnowledgeOptimizer(
        objective="Understand GPU instance pricing and limitations across cloud providers",
        required_knowledge=[
            "pricing structure",
            "GPU specifications",
            "usage limits",
            "availability zones"
        ],
        confidence_threshold=0.85
    )
    
    result = await crawler.arun(
        urls=[
            "https://aws.amazon.com/ec2/pricing/",
            "https://cloud.google.com/gpu",
            "https://azure.microsoft.com/pricing/"
        ],
        optimizer=optimizer,
        optimization_mode="minimal_extraction"
    )
    
    print(f"Knowledge Coverage: {result.knowledge_coverage}")
    print(f"Data Efficiency: {result.efficiency_ratio}")
    print(f"Extracted Content: {result.optimal_content}")

from crawl4ai import AsyncWebCrawler
from crawl4ai.agents import CrawlerAgent

async with AsyncWebCrawler() as crawler:
    agent = CrawlerAgent(crawler)
    
    # Automatic planning and execution
    result = await agent.arun(
        goal="Find research papers about quantum computing published in 2023 with more than 50 citations",
        auto_retry=True
    )
    print("Generated Plan:", result.executed_steps)
    print("Extracted Data:", result.data)
    
    # Using custom steps with automatic execution
    result = await agent.arun(
        goal="Extract conference deadlines from ML conferences",
        custom_plan=[
            "Navigate to conference page",
            "Find important dates section",
            "Extract submission deadlines",
            "Verify dates are for 2024"
        ]
    )
    
    # Monitoring execution
    print("Step Completion:", result.step_status)
    print("Execution Time:", result.execution_time)
    print("Success Rate:", result.success_rate)

from crawl4ai import AsyncWebCrawler
from crawl4ai.schema import SchemaGenerator

# Generate schema from natural language description
generator = SchemaGenerator()
schema = await generator.generate(
    url="https://news-website.com",
    description="For each news article on the page, I need the headline, publication date, and main image"
)

# Use generated schema with crawler
async with AsyncWebCrawler() as crawler:
    result = await crawler.arun(
        url="https://news-website.com",
        extraction_strategy=schema
    )

# Example of generated schema:
"""
{
    "name": "News Article Extractor",
    "baseSelector": "article.news-item",
    "fields": [
        {
            "name": "headline",
            "selector": "h2.article-title",
            "type": "text"
        },
        {
            "name": "date",
            "selector": "span.publish-date",
            "type": "text"
        },
        {
            "name": "image",
            "selector": "img.article-image",
            "type": "attribute",
            "attribute": "src"
        }
    ]
}
"""

from crawl4ai import AsyncWebCrawler
from crawl4ai.extractors import AcademicExtractor, EcommerceExtractor

async with AsyncWebCrawler() as crawler:
    # Academic paper extraction
    papers = await crawler.arun(
        url="https://arxiv.org/list/cs.AI/recent",
        extractor="academic",  # Built-in extractor type
        site_type="arxiv",     # Specific site optimization
        extract_fields=[
            "title", 
            "authors", 
            "abstract", 
            "citations"
        ]
    )
    
    # E-commerce product data
    products = await crawler.arun(
        url="https://store.example.com/products",
        extractor="ecommerce",
        extract_fields=[
            "name",
            "price",
            "availability",
            "reviews"
        ]
    )

from crawl4ai import AsyncWebCrawler
from crawl4ai.indexing import WebIndex

# Initialize and build index
index = WebIndex(model="efficient-mini")

async with AsyncWebCrawler() as crawler:
    # Crawl and index content
    await index.build(
        urls=["https://docs.example.com"],
        crawler=crawler,
        options={
            "chunk_method": "semantic",
            "update_policy": "incremental",
            "embedding_batch_size": 100
        }
    )

    # Search through indexed content
    results = await index.search(
        query="How to implement OAuth authentication?",
        filters={
            "content_type": "technical",
            "recency": "6months"
        },
        top_k=5
    )

    # Get similar content
    similar = await index.find_similar(
        url="https://docs.example.com/auth/oauth",
        threshold=0.85
    )

from crawl4ai import AsyncWebCrawler
from crawl4ai.monitor import CrawlMonitor

# Initialize monitoring
monitor = CrawlMonitor()

# Start monitoring with CLI interface
await monitor.start(
    mode="cli",  # or "gui"
    refresh_rate="1s",
    metrics={
        "resources": ["cpu", "memory", "network"],
        "crawls": ["active", "queued", "completed"],
        "performance": ["success_rate", "response_times"]
    }
)

# Example CLI output:
"""
Crawl4AI Monitor (Live) - Press Q to exit
────────────────────────────────────────
System Usage:
 ├─ CPU: ███████░░░ 70%
 └─ Memory: ████░░░░░ 2.1GB/8GB

Active Crawls:
ID    URL                   Status    Progress
001   docs.example.com     🟢 Active   75%
002   api.service.com      🟡 Queue    -

Metrics (Last 5min):
 ├─ Success Rate: 98%
 ├─ Avg Response: 0.6s
 └─ Pages/sec: 8.5
"""

from crawl4ai import AsyncWebCrawler
from crawl4ai.deploy import CloudDeployer

# Initialize deployer
deployer = CloudDeployer()

# Deploy crawler service
deployment = await deployer.deploy(
    service_name="crawler-cluster",
    platform="aws",  # or "gcp", "azure"
    config={
        "instance_type": "compute-optimized",
        "auto_scaling": {
            "min_instances": 2,
            "max_instances": 10,
            "scale_based_on": "cpu_usage"
        },
        "region": "us-east-1",
        "monitoring": True
    }
)

# Get deployment status and endpoints
print(f"Service Status: {deployment.status}")
print(f"API Endpoint: {deployment.endpoint}")
print(f"Monitor URL: {deployment.monitor_url}")

Sponsorship Tiers:

🥉 Bronze Supporter
- GitHub Sponsor badge
- Priority issue response
- Community Discord role

🥈 Silver Supporter
- All Bronze benefits
- Technical support channel
- Vote on roadmap priorities
- Early access to beta features

🥇 Gold Supporter
- All Silver benefits
- Custom feature requests
- Direct developer access
- Private support sessions

💎 Diamond Partner
- All Gold benefits
- Custom development
- On-demand consulting
- Integration support