crawlrs 0.2.0

crawlrs - Enterprise-grade web scraping and crawling platform built with Rust
Documentation
// Copyright (c) 2025-2026 Kirky.X🌠
// SPDX-License-Identifier: Apache-2.0

/// 基础 HTTP 引擎模块 (始终可用)
pub mod reqwest;

/// 客户端句柄(tuple 泄漏状态)
pub mod handle;

/// Playwright 浏览器自动化引擎
#[cfg(feature = "engine-playwright")]
pub mod playwright;

/// Playwright 浏览器实例池
#[cfg(feature = "engine-playwright")]
pub mod playwright_pool;

/// UA 客户端提示 metadata 构造(Sec-CH-UA 一致性,纯函数)
///
/// 依赖 chromiumoxide CDP 类型,仅在浏览器引擎启用时可用。
#[cfg(feature = "engine-playwright")]
pub mod ua_metadata;

/// Chrome CDP Tab 池
///
/// Page(tab)级复用,减少 `browser.new_page` 开销。
/// 依赖 chromiumoxide::Page,仅在浏览器引擎启用时可用。
#[cfg(feature = "engine-playwright")]
pub mod tab_pool;

/// FlareSolverr 引擎模块(统一 Full / CDP / TLS 三模式)
///
/// 通过 `FlareSolverrMode` 枚举区分 Full / Cdp / Tls 三种工作模式:
/// - `Full`:完整 FlareSolverr 客户端
/// - `Cdp`:CDP 模式
/// - `Tls`:TLS 指纹模式
#[cfg(feature = "engine-flaresolverr")]
pub mod flare_solverr;

/// Wreq TLS 指纹引擎(feature-gated)
///
/// 基于 `wreq`(BoringSSL 后端,Apache-2.0)实现真实 JA3/JA4 指纹伪装,
/// 为 `needs_tls_fingerprint` 请求提供专业引擎。不依赖 GPL 的 wreq-util。
#[cfg(feature = "engine-tls-fingerprint")]
pub mod wreq_engine;

/// MLLM 自主导航爬取引擎
///
/// 使用视觉大模型分析页面截图,自主决策导航操作,实现 agentic loop 式智能爬取。
/// 依赖 engine-playwright(浏览器)+ llm(视觉模型)。
#[cfg(feature = "engine-mllm")]
pub mod mllm;

// MoliEngine:CDP 控制外部 moli serve 的 JS 渲染引擎(端点模式,PoC GO 后落地)
#[cfg(feature = "engine-moli")]
pub mod moli;

// Re-exports

/// Reqwest 引擎 (始终可用)
pub use self::handle::ClientHandle;
/// Reqwest 引擎 (始终可用)
pub use self::reqwest::ReqwestEngine;

/// Playwright 引擎
#[cfg(feature = "engine-playwright")]
pub use self::playwright::PlaywrightEngine;

/// 浏览器池
#[cfg(feature = "engine-playwright")]
pub use self::playwright_pool::{
    get_global_pool, init_global_pool, shutdown_global_pool, BrowserInstance, BrowserPool,
    BrowserPoolConfig, BrowserPoolStats, PooledPage,
};

/// Chrome CDP Tab 池
#[cfg(feature = "engine-playwright")]
pub use self::tab_pool::TabPool;

/// 统一的 FlareSolverr 引擎(合并原 FireEngineCdp / FireEngineTls / FlareSolverrEngine)
///
/// 原 `FireEngineCdp` / `FireEngineTls` / `FlareSolverrEngine` 三个独立引擎
/// 已合并为 `FlareSolverrEngine` + `FlareSolverrMode` 枚举。原类型别名已删除
/// (无生产代码使用,避免误导用户继续使用旧 API)。
///
/// 使用方式:
/// - `FlareSolverrEngine::new` — Full 模式(原 FlareSolverrEngine)
/// - `FlareSolverrEngine::with_cdp_mode` — Cdp 模式(原 FireEngineCdp)
/// - `FlareSolverrEngine::with_tls_mode` — Tls 模式(原 FireEngineTls)
#[cfg(feature = "engine-flaresolverr")]
pub use self::flare_solverr::{
    FlareSolverrConfig, FlareSolverrEngine, FlareSolverrEngineBuilder, FlareSolverrMode,
};

/// Wreq TLS 指纹引擎
#[cfg(feature = "engine-tls-fingerprint")]
pub use self::wreq_engine::{emulation_provider, WreqEngine};

/// MLLM 自主导航爬取引擎
#[cfg(feature = "engine-mllm")]
pub use self::mllm::MllmEngine;