flowchart TB
    subgraph Entry["入口层"]
        main["main.py
程序入口"]
        cmdarg["cmd_arg
命令行参数"]
        config["config
配置管理"]
    end

    subgraph Core["核心爬虫层"]
        factory["CrawlerFactory
爬虫工厂"]
        base["AbstractCrawler
爬虫基类"]

        subgraph Platforms["平台实现"]
            xhs["XiaoHongShuCrawler"]
            dy["DouYinCrawler"]
            ks["KuaishouCrawler"]
            bili["BilibiliCrawler"]
            wb["WeiboCrawler"]
            tieba["TieBaCrawler"]
            zhihu["ZhihuCrawler"]
        end
    end

    subgraph Client["API客户端层"]
        absClient["AbstractApiClient
客户端基类"]
        xhsClient["XiaoHongShuClient"]
        dyClient["DouYinClient"]
        ksClient["KuaiShouClient"]
        biliClient["BilibiliClient"]
        wbClient["WeiboClient"]
        tiebaClient["BaiduTieBaClient"]
        zhihuClient["ZhiHuClient"]
    end

    subgraph Storage["数据存储层"]
        storeFactory["StoreFactory
存储工厂"]
        csv["CSV存储"]
        json["JSON存储"]
        sqlite["SQLite存储"]
        mysql["MySQL存储"]
        mongodb["MongoDB存储"]
        excel["Excel存储"]
    end

    subgraph Infra["基础设施层"]
        browser["浏览器管理
Playwright/CDP"]
        proxy["代理IP池"]
        cache["缓存系统"]
        login["登录管理"]
    end

    main --> factory
    cmdarg --> main
    config --> main
    factory --> base
    base --> Platforms
    Platforms --> Client
    Client --> Storage
    Client --> Infra
    Storage --> storeFactory
    storeFactory --> csv & json & sqlite & mysql & mongodb & excel

flowchart LR
    subgraph Input["输入"]
        keywords["关键词/ID"]
        config["配置参数"]
    end

    subgraph Process["处理流程"]
        browser["启动浏览器"]
        login["登录认证"]
        search["搜索/爬取"]
        parse["数据解析"]
        comment["获取评论"]
    end

    subgraph Output["输出"]
        content["内容数据"]
        comments["评论数据"]
        creator["创作者数据"]
        media["媒体文件"]
    end

    subgraph Storage["存储"]
        file["文件存储
CSV/JSON/Excel"]
        db["数据库
SQLite/MySQL"]
        nosql["NoSQL
MongoDB"]
    end

    keywords --> browser
    config --> browser
    browser --> login
    login --> search
    search --> parse
    parse --> comment
    parse --> content
    comment --> comments
    parse --> creator
    parse --> media
    content & comments & creator --> file & db & nosql
    media --> file

MediaCrawler/
├── main.py                 # 程序入口
├── var.py                  # 全局上下文变量
├── pyproject.toml          # 项目配置
│
├── base/                   # 基础抽象类
│   └── base_crawler.py     # 爬虫、登录、存储、客户端基类
│
├── config/                 # 配置管理
│   ├── base_config.py      # 核心配置
│   ├── db_config.py        # 数据库配置
│   └── {platform}_config.py # 平台特定配置
│
├── media_platform/         # 平台爬虫实现
│   ├── xhs/                # 小红书
│   ├── douyin/             # 抖音
│   ├── kuaishou/           # 快手
│   ├── bilibili/           # B站
│   ├── weibo/              # 微博
│   ├── tieba/              # 百度贴吧
│   └── zhihu/              # 知乎
│
├── store/                  # 数据存储
│   ├── excel_store_base.py # Excel存储基类
│   └── {platform}/         # 各平台存储实现
│
├── database/               # 数据库层
│   ├── models.py           # ORM模型定义
│   ├── db_session.py       # 数据库会话管理
│   └── mongodb_store_base.py # MongoDB基类
│
├── proxy/                  # 代理管理
│   ├── proxy_ip_pool.py    # IP池管理
│   ├── proxy_mixin.py      # 代理刷新混入
│   └── providers/          # 代理提供商
│
├── cache/                  # 缓存系统
│   ├── abs_cache.py        # 缓存抽象类
│   ├── local_cache.py      # 本地缓存
│   └── redis_cache.py      # Redis缓存
│
├── tools/                  # 工具模块
│   ├── app_runner.py       # 应用运行管理
│   ├── browser_launcher.py # 浏览器启动
│   ├── cdp_browser.py      # CDP浏览器管理
│   ├── crawler_util.py     # 爬虫工具
│   └── async_file_writer.py # 异步文件写入
│
├── model/                  # 数据模型
│   └── m_{platform}.py     # Pydantic模型
│
├── libs/                   # JS脚本库
│   └── stealth.min.js      # 反检测脚本
│
└── cmd_arg/                # 命令行参数
    └── arg.py              # 参数定义

classDiagram
    class AbstractCrawler {
        <<abstract>>
        +start()* 启动爬虫
        +search()* 搜索功能
        +launch_browser() 启动浏览器
        +launch_browser_with_cdp() CDP模式启动
    }

    class AbstractLogin {
        <<abstract>>
        +begin()* 开始登录
        +login_by_qrcode()* 二维码登录
        +login_by_mobile()* 手机号登录
        +login_by_cookies()* Cookie登录
    }

    class AbstractStore {
        <<abstract>>
        +store_content()* 存储内容
        +store_comment()* 存储评论
        +store_creator()* 存储创作者
        +store_image()* 存储图片
        +store_video()* 存储视频
    }

    class AbstractApiClient {
        <<abstract>>
        +request()* HTTP请求
        +update_cookies()* 更新Cookies
    }

    class ProxyRefreshMixin {
        +init_proxy_pool() 初始化代理池
        +_refresh_proxy_if_expired() 刷新过期代理
    }

    class XiaoHongShuCrawler {
        +xhs_client: XiaoHongShuClient
        +start()
        +search()
        +get_specified_notes()
        +get_creators_and_notes()
    }

    class XiaoHongShuClient {
        +playwright_page: Page
        +cookie_dict: Dict
        +request()
        +pong() 检查登录状态
        +get_note_by_keyword()
        +get_note_by_id()
    }

    AbstractCrawler <|-- XiaoHongShuCrawler
    AbstractApiClient <|-- XiaoHongShuClient
    ProxyRefreshMixin <|-- XiaoHongShuClient

sequenceDiagram
    participant Main as main.py
    participant Factory as CrawlerFactory
    participant Crawler as XiaoHongShuCrawler
    participant Browser as Playwright/CDP
    participant Login as XiaoHongShuLogin
    participant Client as XiaoHongShuClient
    participant Store as StoreFactory

    Main->>Factory: create_crawler("xhs")
    Factory-->>Main: crawler实例

    Main->>Crawler: start()

    alt 启用IP代理
        Crawler->>Crawler: create_ip_pool()
    end

    alt CDP模式
        Crawler->>Browser: launch_browser_with_cdp()
    else 标准模式
        Crawler->>Browser: launch_browser()
    end
    Browser-->>Crawler: browser_context

    Crawler->>Crawler: create_xhs_client()
    Crawler->>Client: pong() 检查登录状态

    alt 未登录
        Crawler->>Login: begin()
        Login->>Login: login_by_qrcode/mobile/cookie
        Login-->>Crawler: 登录成功
    end

    alt search模式
        Crawler->>Client: get_note_by_keyword()
        Client-->>Crawler: 搜索结果
        loop 获取详情
            Crawler->>Client: get_note_by_id()
            Client-->>Crawler: 笔记详情
        end
    else detail模式
        Crawler->>Client: get_note_by_id()
    else creator模式
        Crawler->>Client: get_creator_info()
    end

    Crawler->>Store: store_content/comment/creator
    Store-->>Crawler: 存储完成

    Main->>Crawler: cleanup()
    Crawler->>Browser: close()

media_platform/{platform}/
├── __init__.py         # 模块导出
├── core.py             # 爬虫主实现类
├── client.py           # API客户端
├── login.py            # 登录实现
├── field.py            # 字段/枚举定义
├── exception.py        # 异常定义
├── help.py             # 辅助函数
└── {特殊实现}.py       # 平台特定逻辑

classDiagram
    class AbstractStore {
        <<abstract>>
        +store_content()*
        +store_comment()*
        +store_creator()*
    }

    class StoreFactory {
        +STORES: Dict
        +create_store() AbstractStore
    }

    class CsvStoreImplement {
        +async_file_writer: AsyncFileWriter
        +store_content()
        +store_comment()
    }

    class JsonStoreImplement {
        +async_file_writer: AsyncFileWriter
        +store_content()
        +store_comment()
    }

    class DbStoreImplement {
        +session: AsyncSession
        +store_content()
        +store_comment()
    }

    class SqliteStoreImplement {
        +session: AsyncSession
        +store_content()
        +store_comment()
    }

    class MongoStoreImplement {
        +mongo_base: MongoDBStoreBase
        +store_content()
        +store_comment()
    }

    class ExcelStoreImplement {
        +excel_base: ExcelStoreBase
        +store_content()
        +store_comment()
    }

    AbstractStore <|-- CsvStoreImplement
    AbstractStore <|-- JsonStoreImplement
    AbstractStore <|-- DbStoreImplement
    AbstractStore <|-- SqliteStoreImplement
    AbstractStore <|-- MongoStoreImplement
    AbstractStore <|-- ExcelStoreImplement
    StoreFactory --> AbstractStore

# 以抖音为例
class DouyinStoreFactory:
    STORES = {
        "csv": DouyinCsvStoreImplement,
        "db": DouyinDbStoreImplement,
        "json": DouyinJsonStoreImplement,
        "sqlite": DouyinSqliteStoreImplement,
        "mongodb": DouyinMongoStoreImplement,
        "excel": DouyinExcelStoreImplement,
    }

    @staticmethod
    def create_store() -> AbstractStore:
        store_class = DouyinStoreFactory.STORES.get(config.SAVE_DATA_OPTION)
        return store_class()

flowchart TB
    subgraph Config["配置"]
        enable["ENABLE_IP_PROXY"]
        provider["IP_PROXY_PROVIDER"]
        count["IP_PROXY_POOL_COUNT"]
    end

    subgraph Pool["代理池管理"]
        pool["ProxyIpPool"]
        load["load_proxies()"]
        validate["_is_valid_proxy()"]
        get["get_proxy()"]
        refresh["get_or_refresh_proxy()"]
    end

    subgraph Providers["代理提供商"]
        kuaidl["快代理
KuaiDaiLiProxy"]
        wandou["万代理
WanDouHttpProxy"]
        jishu["技术IP
JiShuHttpProxy"]
    end

    subgraph Client["API客户端"]
        mixin["ProxyRefreshMixin"]
        request["request()"]
    end

    enable --> pool
    provider --> Providers
    count --> load
    pool --> load
    load --> validate
    validate --> Providers
    pool --> get
    pool --> refresh
    mixin --> refresh
    mixin --> Client
    request --> mixin

flowchart TB
    Start([开始登录]) --> CheckType{登录类型?}

    CheckType -->|qrcode| QR[显示二维码]
    QR --> WaitScan[等待扫描]
    WaitScan --> CheckQR{扫描成功?}
    CheckQR -->|是| SaveCookie[保存Cookie]
    CheckQR -->|否| WaitScan

    CheckType -->|phone| Phone[输入手机号]
    Phone --> SendCode[发送验证码]
    SendCode --> Slider{需要滑块?}
    Slider -->|是| DoSlider[滑动验证]
    DoSlider --> InputCode[输入验证码]
    Slider -->|否| InputCode
    InputCode --> Verify[验证登录]
    Verify --> SaveCookie

    CheckType -->|cookie| LoadCookie[加载已保存Cookie]
    LoadCookie --> VerifyCookie{Cookie有效?}
    VerifyCookie -->|是| SaveCookie
    VerifyCookie -->|否| Fail[登录失败]

    SaveCookie --> UpdateContext[更新浏览器上下文]
    UpdateContext --> End([登录完成])

flowchart LR
    subgraph Mode["启动模式"]
        standard["标准模式
Playwright"]
        cdp["CDP模式
Chrome DevTools"]
    end

    subgraph Standard["标准模式流程"]
        launch["chromium.launch()"]
        context["new_context()"]
        stealth["注入stealth.js"]
    end

    subgraph CDP["CDP模式流程"]
        detect["检测浏览器路径"]
        start["启动浏览器进程"]
        connect["connect_over_cdp()"]
        cdpContext["获取已有上下文"]
    end

    subgraph Features["特性"]
        f1["用户数据持久化"]
        f2["扩展和设置继承"]
        f3["反检测能力增强"]
    end

    standard --> Standard
    cdp --> CDP
    CDP --> Features

classDiagram
    class AbstractCache {
        <<abstract>>
        +get(key)* 获取缓存
        +set(key, value, expire)* 设置缓存
        +keys(pattern)* 获取所有键
    }

    class ExpiringLocalCache {
        -_cache: Dict
        -_expire_times: Dict
        +get(key)
        +set(key, value, expire_time)
        +keys(pattern)
        -_is_expired(key)
    }

    class RedisCache {
        -_client: Redis
        +get(key)
        +set(key, value, expire_time)
        +keys(pattern)
    }

    class CacheFactory {
        +create_cache(type) AbstractCache
    }

    AbstractCache <|-- ExpiringLocalCache
    AbstractCache <|-- RedisCache
    CacheFactory --> AbstractCache

erDiagram
    DouyinAweme {
        int id PK
        string aweme_id UK
        string aweme_type
        string title
        string desc
        int create_time
        int liked_count
        int collected_count
        int comment_count
        int share_count
        string user_id FK
        datetime add_ts
        datetime last_modify_ts
    }

    DouyinAwemeComment {
        int id PK
        string comment_id UK
        string aweme_id FK
        string content
        int create_time
        int sub_comment_count
        string user_id
        datetime add_ts
        datetime last_modify_ts
    }

    DyCreator {
        int id PK
        string user_id UK
        string nickname
        string avatar
        string desc
        int follower_count
        int total_favorited
        datetime add_ts
        datetime last_modify_ts
    }

    DouyinAweme ||--o{ DouyinAwemeComment : "has"
    DyCreator ||--o{ DouyinAweme : "creates"

# config/base_config.py

# 平台选择
PLATFORM = "xhs"  # xhs, dy, ks, bili, wb, tieba, zhihu

# 登录配置
LOGIN_TYPE = "qrcode"  # qrcode, phone, cookie
SAVE_LOGIN_STATE = True

# 爬虫配置
CRAWLER_TYPE = "search"  # search, detail, creator
KEYWORDS = "编程副业,编程兼职"
CRAWLER_MAX_NOTES_COUNT = 15
MAX_CONCURRENCY_NUM = 1

# 评论配置
ENABLE_GET_COMMENTS = True
ENABLE_GET_SUB_COMMENTS = False
CRAWLER_MAX_COMMENTS_COUNT_SINGLENOTES = 10

# 浏览器配置
HEADLESS = False
ENABLE_CDP_MODE = True
CDP_DEBUG_PORT = 9222

# 代理配置
ENABLE_IP_PROXY = False
IP_PROXY_PROVIDER = "kuaidaili"
IP_PROXY_POOL_COUNT = 2

# 存储配置
SAVE_DATA_OPTION = "jsonl"  # csv, db, json, jsonl, sqlite, mongodb, excel, postgres

# config/db_config.py

# MySQL
MYSQL_DB_HOST = "localhost"
MYSQL_DB_PORT = 3306
MYSQL_DB_NAME = "media_crawler"

# Redis
REDIS_DB_HOST = "127.0.0.1"
REDIS_DB_PORT = 6379

# MongoDB
MONGODB_HOST = "localhost"
MONGODB_PORT = 27017

# SQLite
SQLITE_DB_PATH = "database/sqlite_tables.db"

flowchart TB
    Start([程序启动]) --> Run["run(app_main, app_cleanup)"]
    Run --> Main["执行 app_main()"]
    Main --> Running{运行中}

    Running -->|正常完成| Cleanup1["执行 app_cleanup()"]
    Running -->|SIGINT/SIGTERM| Signal["捕获信号"]

    Signal --> First{第一次信号?}
    First -->|是| Cleanup2["启动清理流程"]
    First -->|否| Force["强制退出"]

    Cleanup1 & Cleanup2 --> Cancel["取消其他任务"]
    Cancel --> Wait["等待任务完成
(超时15秒)"]
    Wait --> End([程序退出])
    Force --> End

flowchart TB
    subgraph Entry["入口层"]
        main["main.py"]
        config["config/"]
        cmdarg["cmd_arg/"]
    end

    subgraph Core["核心层"]
        base["base/base_crawler.py"]
        platforms["media_platform/*/"]
    end

    subgraph Client["客户端层"]
        client["*/client.py"]
        login["*/login.py"]
    end

    subgraph Storage["存储层"]
        store["store/"]
        database["database/"]
    end

    subgraph Infra["基础设施"]
        proxy["proxy/"]
        cache["cache/"]
        tools["tools/"]
    end

    subgraph External["外部依赖"]
        playwright["Playwright"]
        httpx["httpx"]
        sqlalchemy["SQLAlchemy"]
        motor["Motor/MongoDB"]
    end

    main --> config
    main --> cmdarg
    main --> Core

    Core --> base
    platforms --> base
    platforms --> Client

    client --> proxy
    client --> httpx
    login --> tools

    platforms --> Storage
    Storage --> sqlalchemy
    Storage --> motor

    client --> playwright
    tools --> playwright

    proxy --> cache

# 启动爬虫
python main.py

# 指定平台
python main.py --platform xhs

# 指定登录方式
python main.py --lt qrcode

# 指定爬虫类型
python main.py --type search