crawlrs 0.2.0

crawlrs - Enterprise-grade web scraping and crawling platform built with Rust
Documentation
// Copyright (c) 2025-2026 Kirky.X🌠
// SPDX-License-Identifier: Apache-2.0

//! 引擎模块 — 提供 `EngineComponents`

use std::any::TypeId;
use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;

use trait_kit::core::{AsyncAutoBuilder, ModuleMeta};
use trait_kit::kit::AsyncKit;

use super::http_module::HttpModule;
use super::settings_module::SettingsModule;
use super::ModuleBuildError;
use crate::bootstrap::engines::EngineComponents;
use crate::engines::provider::ProxyProvider;
use crate::engines::proxy_pool::ProxyPool;

/// 引擎模块 — 提供 `EngineComponents`
///
/// 依赖 `HttpModule` 和 `SettingsModule`,创建 EngineRouter + EngineClient。
pub struct EngineModule;

impl ModuleMeta for EngineModule {
    const NAME: &'static str = "engines";

    fn dependencies() -> &'static [(&'static str, TypeId)] {
        static DEPS: [(&str, TypeId); 2] = [
            (HttpModule::NAME, TypeId::of::<HttpModule>()),
            (SettingsModule::NAME, TypeId::of::<SettingsModule>()),
        ];
        &DEPS
    }
}

impl AsyncAutoBuilder for EngineModule {
    type Capability = EngineComponents;
    type Error = ModuleBuildError;

    fn build<'a>(
        kit: &'a AsyncKit,
    ) -> Pin<Box<dyn Future<Output = Result<Self::Capability, Self::Error>> + Send + 'a>> {
        Box::pin(async move {
            let settings = kit.require::<SettingsModule>()?;
            let http_client = kit.require::<HttpModule>()?;
            // 构造 ProxyProvider 注入 ReqwestEngine
            // - proxy.enabled=true 且 urls 非空 → Some(Arc<dyn ProxyProvider>)
            // - 否则 → None(ReqwestEngine 不使用代理)
            //
            // sticky_ttl / cooldown 从 settings.proxy 注入(原硬编码 60s/30s)
            //
            // 策略:从 settings.proxy.strategy 注入(H1:RoundRobin / Sticky 路由)
            let proxy_provider: Option<Arc<dyn ProxyProvider>> =
                if settings.proxy.enabled && !settings.proxy.urls.is_empty() {
                    Some(Arc::new(ProxyPool::from_urls(
                        settings.proxy.urls.clone(),
                        std::time::Duration::from_secs(settings.proxy.sticky_ttl_seconds),
                        std::time::Duration::from_secs(settings.proxy.cooldown_seconds),
                    )))
                } else {
                    None
                };
            // FlareSolverr 单代理:取 urls.first 作为 fallback(FlareSolverr 暂未接入 ProxyProvider,
            // 范围外)。proxy.enabled=false 时为 None。
            let proxy_url: Option<String> = if settings.proxy.enabled {
                settings.proxy.urls.first().cloned()
            } else {
                None
            };
            // vision 适配器经 fluxweir 网关承载(engine-mllm 显式依赖 gateway,
            // 无单独编译态);[fluxweir].enabled=false 时为 None,engine-mllm
            // 启用会在引擎装配期 fail-fast
            #[cfg(feature = "gateway")]
            let vision_adapter: Option<
                std::sync::Arc<
                    dyn crate::domain::services::llm::vision_adapter::VisionAdapterTrait,
                >,
            > = {
                use crate::domain::services::llm::vision_adapter::FluxweirVisionAdapter;
                crate::infrastructure::services::fluxweir_providers::build_client(settings.as_ref())
                    .map(|client| {
                        std::sync::Arc::new(FluxweirVisionAdapter::new(Some(client))) as _
                    })
            };
            #[cfg(not(feature = "gateway"))]
            let vision_adapter: Option<
                std::sync::Arc<
                    dyn crate::domain::services::llm::vision_adapter::VisionAdapterTrait,
                >,
            > = None;
            let engines = crate::bootstrap::engines::init_engine_components(
                http_client.clone(),
                proxy_provider,
                settings.proxy.strategy,
                proxy_url,
                &settings.engines,
                // 注入完整 EngineTimeoutSettings(含 default_timeout_seconds + 三个 MRT 字段)
                &settings.timeouts.engines,
                settings.engines.max_response_body_bytes as usize,
                // F1/F5:[politeness] 配置注入 per-host 纪律(AutoThrottle/退避/抖动)
                &settings.politeness,
                // F2:[session] 配置注入 Session 身份三件套(代理 + UA/TLS 档
                // + Cookie 绑定与记分退休;enabled=false 走现状盲轮换)
                &settings.session,
                // F4:[observability] 配置注入失败现场快照器(目录空串 = 零成本)
                &settings.observability,
                vision_adapter,
            );
            Ok(engines)
        })
    }
}