# Install Crawl4AI and dependencies
!pip install crawl4ai
!playwright install
!pip install nest_asyncio

# Import nest_asyncio and apply it to allow asyncio in Colab
import nest_asyncio
nest_asyncio.apply()

print('Setup complete!')

import asyncio
from crawl4ai import AsyncWebCrawler

async def fit_markdown_demo():
    async with AsyncWebCrawler() as crawler:
        result = await crawler.arun(url="https://janineintheworld.com/places-to-visit-in-central-mexico")
        print(result.fit_markdown)  # Shows main content in Markdown format

# Run the demo
await fit_markdown_demo()

async def magic_mode_demo():
    async with AsyncWebCrawler() as crawler:  # Enables anti-bot detection bypass
        result = await crawler.arun(
            url="https://www.reuters.com/markets/us/global-markets-view-usa-pix-2024-08-29/",
            magic=True  # Enables magic mode
        )
        print(result.markdown)  # Shows the full content in Markdown format

# Run the demo
await magic_mode_demo()

async def multi_browser_demo():
    async with AsyncWebCrawler(browser_type="firefox") as crawler:  # Using Firefox instead of default Chromium
        result = await crawler.arun(url="https://crawl4i.com")
        print(result.markdown)  # Shows content extracted using Firefox

# Run the demo
await multi_browser_demo()

from crawl4ai import LLMExtractionStrategy
from pydantic import BaseModel
import json, os
from typing import List

# Define classes for the knowledge graph structure
class Landmark(BaseModel):
    name: str
    description: str
    activities: list[str]  # E.g., visiting, sightseeing, relaxing

class City(BaseModel):
    name: str
    description: str
    landmarks: list[Landmark]
    cultural_highlights: list[str]  # E.g., food, music, traditional crafts

class TravelKnowledgeGraph(BaseModel):
    cities: list[City]  # Central Mexican cities to visit

async def combined_demo():
    # Define the knowledge graph extraction strategy
    strategy = LLMExtractionStrategy(
        # provider="ollama/nemotron",
        provider='openai/gpt-4o-mini', # Or any other provider, including Ollama and open source models
        pi_token=os.getenv('OPENAI_API_KEY'), # In case of Ollama just pass "no-token"
        schema=TravelKnowledgeGraph.schema(),
        instruction=(
            "Extract cities, landmarks, and cultural highlights for places to visit in Central Mexico. "
            "For each city, list main landmarks with descriptions and activities, as well as cultural highlights."
        )
    )

    # Set up the AsyncWebCrawler with multi-browser support, Magic Mode, and Fit Markdown
    async with AsyncWebCrawler(browser_type="firefox") as crawler:
        result = await crawler.arun(
            url="https://janineintheworld.com/places-to-visit-in-central-mexico",
            extraction_strategy=strategy,
            bypass_cache=True,
            magic=True
        )
        
        # Display main article content in Fit Markdown format
        print("Extracted Main Content:\n", result.fit_markdown)
        
        # Display extracted knowledge graph of cities, landmarks, and cultural highlights
        if result.extracted_content:
            travel_graph = json.loads(result.extracted_content)
            print("\nExtracted Knowledge Graph:\n", json.dumps(travel_graph, indent=2))

# Run the combined demo
await combined_demo()