use crate::config::engines::EngineSettings;
use crate::config::settings::{
EngineTimeoutSettings, ObservabilitySettings, PolitenessSettings, ProxyStrategy,
SessionSettings,
};
#[cfg(feature = "engine-flaresolverr")]
use crate::engines::client::flare_solverr::FlareSolverrEngine;
#[cfg(feature = "engine-mllm")]
use crate::engines::client::mllm::MllmEngine;
#[cfg(feature = "engine-playwright")]
use crate::engines::client::playwright::PlaywrightEngine;
use crate::engines::client::reqwest::ReqwestEngine;
#[cfg(feature = "engine-tls-fingerprint")]
use crate::engines::client::wreq_engine::WreqEngine;
use crate::engines::engine_client::EngineClient;
use crate::engines::engine_client::ScraperEngine;
use crate::engines::provider::ProxyProvider;
#[cfg(test)]
use crate::engines::proxy_pool::ProxyPool;
use crate::engines::router::EngineRouter;
use crate::engines::session::SessionManager;
#[cfg(feature = "engine-playwright")]
use crate::engines::xhr_capture::XhrCaptureConfig;
use crate::utils::host_discipline::HostDiscipline;
use crate::utils::ua_pool::UaPool;
use std::sync::Arc;
#[derive(Clone)]
pub struct EngineComponents {
pub engines: Vec<Arc<dyn ScraperEngine>>,
pub router: Arc<EngineRouter>,
pub engine_client: Arc<EngineClient>,
}
#[allow(unused_variables)]
#[allow(clippy::too_many_arguments)]
pub fn init_engines(
http_client: Arc<reqwest::Client>,
proxy_provider: Option<Arc<dyn ProxyProvider>>,
proxy_strategy: ProxyStrategy,
proxy_url: Option<&str>,
engine_config: &EngineSettings,
engine_timeouts: &EngineTimeoutSettings,
max_response_body_bytes: usize,
vision_adapter: Option<
Arc<dyn crate::domain::services::llm::vision_adapter::VisionAdapterTrait>,
>,
) -> Vec<Arc<dyn ScraperEngine>> {
let timeout_seconds = engine_timeouts.default_timeout_seconds;
let flaresolverr_timeout = engine_config.flaresolverr.timeout_seconds;
let fetch_mrt = std::time::Duration::from_secs(engine_timeouts.fetch_seconds);
let tls_mrt = std::time::Duration::from_secs(engine_timeouts.tls_seconds);
let cdp_mrt = std::time::Duration::from_secs(engine_timeouts.cdp_seconds);
#[cfg(feature = "engine-tls-fingerprint")]
let wreq_proxy_provider = proxy_provider.clone();
#[allow(unused_mut)]
let mut engines = build_reqwest_engine(
http_client.clone(),
proxy_provider,
proxy_strategy,
timeout_seconds,
fetch_mrt,
max_response_body_bytes,
);
#[cfg(feature = "engine-playwright")]
push_playwright_engine(&mut engines, engine_config, cdp_mrt);
#[cfg(feature = "engine-flaresolverr")]
push_flaresolverr_engines(
&mut engines,
engine_config,
http_client.clone(),
proxy_url,
tls_mrt,
cdp_mrt,
flaresolverr_timeout,
);
#[cfg(feature = "engine-tls-fingerprint")]
push_wreq_engine(
&mut engines,
engine_config,
wreq_proxy_provider,
proxy_strategy,
tls_mrt,
max_response_body_bytes,
);
#[cfg(feature = "engine-moli")]
push_moli_engine(&mut engines, engine_config);
#[cfg(feature = "engine-mllm")]
push_mllm_engine(&mut engines, engine_config, vision_adapter);
engines
}
fn build_reqwest_engine(
http_client: Arc<reqwest::Client>,
proxy_provider: Option<Arc<dyn ProxyProvider>>,
proxy_strategy: ProxyStrategy,
timeout_seconds: u64,
fetch_mrt: std::time::Duration,
max_response_body_bytes: usize,
) -> Vec<Arc<dyn ScraperEngine>> {
match proxy_provider {
Some(provider) => vec![Arc::new(
ReqwestEngine::with_provider_strategy_timeout_and_mrt(
http_client,
provider,
proxy_strategy,
timeout_seconds,
fetch_mrt,
)
.with_max_response_body_bytes(max_response_body_bytes),
)],
None => vec![Arc::new(
ReqwestEngine::new_with_timeout_and_mrt(http_client, timeout_seconds, fetch_mrt)
.with_max_response_body_bytes(max_response_body_bytes),
)],
}
}
#[cfg(feature = "engine-playwright")]
fn push_playwright_engine(
engines: &mut Vec<Arc<dyn ScraperEngine>>,
engine_config: &EngineSettings,
cdp_mrt: std::time::Duration,
) {
let mut playwright_engine = PlaywrightEngine::with_mrt(cdp_mrt);
if engine_config.playwright.capture_xhr {
let s = &engine_config.playwright;
log::info!(
"{}",
crate::i18n::tr_log_args(
"boot-playwright-capture-enabled",
&[
(
"patterns",
(s.capture_xhr_url_patterns.len() as i64).to_string(),
),
("max_bytes", (s.capture_xhr_max_bytes as i64).to_string(),),
(
"max_responses",
(s.capture_xhr_max_responses as i64).to_string(),
),
],
)
);
playwright_engine = playwright_engine.with_xhr_capture(XhrCaptureConfig {
url_patterns: s.capture_xhr_url_patterns.clone(),
max_bytes: s.capture_xhr_max_bytes,
max_responses: s.capture_xhr_max_responses as usize,
});
}
engines.push(Arc::new(playwright_engine));
}
#[cfg(feature = "engine-flaresolverr")]
fn push_flaresolverr_engines(
engines: &mut Vec<Arc<dyn ScraperEngine>>,
engine_config: &EngineSettings,
http_client: Arc<reqwest::Client>,
proxy_url: Option<&str>,
tls_mrt: std::time::Duration,
cdp_mrt: std::time::Duration,
flaresolverr_timeout: u64,
) {
if engine_config.flaresolverr_tls.enabled {
log::info!(
"FlareSolverr TLS enabled with URL: {}",
engine_config.flaresolverr_tls.url
);
engines.push(Arc::new(FlareSolverrEngine::with_tls_mode_url_and_mrt(
http_client.clone(),
&engine_config.flaresolverr_tls.url,
proxy_url,
tls_mrt,
flaresolverr_timeout,
)));
}
if engine_config.flaresolverr_cdp.enabled {
log::info!(
"FlareSolverr CDP enabled with URL: {}",
engine_config.flaresolverr_cdp.url
);
engines.push(Arc::new(FlareSolverrEngine::with_cdp_mode_url_and_mrt(
http_client.clone(),
&engine_config.flaresolverr_cdp.url,
proxy_url,
cdp_mrt,
flaresolverr_timeout,
)));
}
if engine_config.flaresolverr.enabled {
log::info!(
"FlareSolverr enabled with URL: {}",
engine_config.flaresolverr.url
);
engines.push(Arc::new(FlareSolverrEngine::with_url_and_mrt(
http_client.clone(),
&engine_config.flaresolverr.url,
cdp_mrt,
flaresolverr_timeout,
)));
}
}
#[cfg(feature = "engine-tls-fingerprint")]
fn push_wreq_engine(
engines: &mut Vec<Arc<dyn ScraperEngine>>,
engine_config: &EngineSettings,
wreq_proxy_provider: Option<Arc<dyn ProxyProvider>>,
proxy_strategy: ProxyStrategy,
tls_mrt: std::time::Duration,
max_response_body_bytes: usize,
) {
if engine_config.tls_fingerprint.enabled {
log::info!(
"WreqEngine (TLS fingerprint) enabled with timeout={}s",
engine_config.tls_fingerprint.timeout_seconds
);
let wreq_timeout = u64::from(engine_config.tls_fingerprint.timeout_seconds);
match wreq_proxy_provider {
Some(provider) => match WreqEngine::with_provider(
Arc::new(crate::utils::ua_pool::UaPool::new()),
provider,
proxy_strategy,
tls_mrt,
wreq_timeout,
)
.map(|e| e.with_max_response_body_bytes(max_response_body_bytes))
{
Ok(engine) => engines.push(Arc::new(engine)),
Err(e) => log::error!("WreqEngine init failed, skipping: {e}"),
},
None => match WreqEngine::new(
Arc::new(crate::utils::ua_pool::UaPool::new()),
tls_mrt,
wreq_timeout,
)
.map(|e| e.with_max_response_body_bytes(max_response_body_bytes))
{
Ok(engine) => engines.push(Arc::new(engine)),
Err(e) => log::error!("WreqEngine init failed, skipping: {e}"),
},
}
}
}
#[cfg(feature = "engine-moli")]
fn push_moli_engine(engines: &mut Vec<Arc<dyn ScraperEngine>>, engine_config: &EngineSettings) {
if engine_config.moli.enabled {
log::info!(
"MoliEngine (moli CDP) enabled at {}",
engine_config.moli.endpoint
);
engines.push(Arc::new(crate::engines::client::moli::MoliEngine::new(
crate::engines::client::moli::MoliEngineConfig::from_settings(&engine_config.moli),
)));
}
}
#[cfg(feature = "engine-mllm")]
fn push_mllm_engine(
engines: &mut Vec<Arc<dyn ScraperEngine>>,
engine_config: &EngineSettings,
vision_adapter: Option<
Arc<dyn crate::domain::services::llm::vision_adapter::VisionAdapterTrait>,
>,
) {
if engine_config.mllm.enabled {
log::info!(
"MllmEngine (MLLM autonomous navigation) enabled with model={}",
engine_config.mllm.vision_model
);
let mllm_config = crate::engines::mllm::config::MllmEngineConfig {
vision_model: engine_config.mllm.vision_model.clone(),
max_iterations: engine_config.mllm.max_iterations,
screenshot_quality: engine_config.mllm.screenshot_quality,
system_prompt: crate::engines::mllm::config::MllmEngineConfig::default().system_prompt,
max_token_budget: engine_config.mllm.max_token_budget,
mrt_seconds: engine_config.mllm.mrt_seconds,
ax_tree_enabled: engine_config.mllm.ax_tree_enabled,
};
let vision_adapter = vision_adapter.unwrap_or_else(|| {
panic!(
"engine-mllm enabled but no vision adapter available; \
set [fluxweir].enabled=true (mode embedded/external) — \
vision requests are carried by the fluxweir gateway"
)
});
engines.push(std::sync::Arc::new(MllmEngine::new(
vision_adapter,
mllm_config,
)));
}
}
#[allow(clippy::too_many_arguments)]
pub fn init_engine_components(
http_client: Arc<reqwest::Client>,
proxy_provider: Option<Arc<dyn ProxyProvider>>,
proxy_strategy: ProxyStrategy,
proxy_url: Option<String>,
_engine_config: &EngineSettings,
engine_timeouts: &EngineTimeoutSettings,
max_response_body_bytes: usize,
politeness: &PolitenessSettings,
session: &SessionSettings,
observability: &ObservabilitySettings,
vision_adapter: Option<
Arc<dyn crate::domain::services::llm::vision_adapter::VisionAdapterTrait>,
>,
) -> EngineComponents {
let session_proxy_provider = proxy_provider.clone();
let engines = init_engines(
http_client.clone(),
proxy_provider,
proxy_strategy,
proxy_url.as_deref(),
_engine_config,
engine_timeouts,
max_response_body_bytes,
vision_adapter,
);
let session_manager =
SessionManager::from_settings(session, Arc::new(UaPool::new()), session_proxy_provider);
let mut router = EngineRouter::with_discipline_and_session(
engines.clone(),
HostDiscipline::from_politeness(politeness),
session_manager,
);
router.set_failure_snapshotter(
crate::infrastructure::observability::failure_snapshot::FailureSnapshotter::from_config(
observability,
),
);
#[cfg(feature = "engine-moli")]
install_memory_scheduler(&mut router, _engine_config);
let router = Arc::new(router);
let engine_client = Arc::new(EngineClient::with_router(router.clone()));
EngineComponents {
engines,
router,
engine_client,
}
}
#[cfg(feature = "engine-moli")]
fn install_memory_scheduler(router: &mut EngineRouter, engine_config: &EngineSettings) {
let scheduler = &engine_config.memory_scheduler;
if scheduler.enabled {
let watermark = crate::engines::memory_watermark::MemoryWatermark::new(
scheduler.critical_available_mib,
scheduler.tight_available_mib,
);
log::info!(
"memory scheduler enabled: initial level {:?} (critical<{}MiB, tight<{}MiB, refresh={}ms, fallback_to_static={})",
watermark.level(),
scheduler.critical_available_mib,
scheduler.tight_available_mib,
scheduler.refresh_interval_ms,
scheduler.fallback_to_static
);
crate::engines::memory_watermark::set_global(watermark.clone());
router.set_memory_scheduler(Some(watermark.clone()), scheduler.fallback_to_static);
match tokio::runtime::Handle::try_current() {
Ok(handle) => {
handle.spawn(watermark.run(std::time::Duration::from_millis(
scheduler.refresh_interval_ms,
)));
}
Err(_) => {
log::warn!(
"no tokio runtime at engine assembly; memory watermark sampling not started (level stays at initial)"
);
}
}
}
}
#[cfg(test)]
mod tests {
#[cfg(feature = "engine-moli")]
#[test]
fn test_moli_engine_registration_follows_config() {
let mut settings = crate::config::settings::Settings::default();
settings.engines.moli.enabled = true;
let engines = init_engines(
make_http_client(),
None,
ProxyStrategy::RoundRobin,
None,
&settings.engines,
&default_timeouts(),
10485760,
None,
);
assert!(
engines.iter().any(|e| e.name() == "moli"),
"enabled=true 时 moli 必须注册"
);
let disabled = init_engines(
make_http_client(),
None,
ProxyStrategy::RoundRobin,
None,
&crate::config::settings::Settings::default().engines,
&default_timeouts(),
10485760,
None,
);
assert!(
!disabled.iter().any(|e| e.name() == "moli"),
"默认 enabled=false 时 moli 不得注册"
);
}
use super::*;
use crate::engines::router::EngineRouterTrait;
use std::time::Duration;
#[cfg(feature = "engine-moli")]
fn default_timeouts() -> crate::config::settings::EngineTimeoutSettings {
crate::config::settings::EngineTimeoutSettings::default()
}
fn make_http_client() -> Arc<reqwest::Client> {
Arc::new(reqwest::Client::new())
}
fn make_proxy_provider() -> Arc<dyn ProxyProvider> {
Arc::new(ProxyPool::from_urls(
vec!["http://localhost:10808".to_string()],
Duration::from_secs(60),
Duration::from_secs(30),
))
}
fn make_engine_timeouts() -> EngineTimeoutSettings {
EngineTimeoutSettings {
default_timeout_seconds: 30,
playwright_timeout_seconds: 30,
flaresolverr_timeout_seconds: 30,
fetch_seconds: 5,
tls_seconds: 15,
cdp_seconds: 30,
}
}
#[test]
fn test_init_engines_returns_non_empty_vec() {
let http_client = make_http_client();
let engine_config = EngineSettings::default();
let engines = init_engines(
http_client,
Some(make_proxy_provider()),
ProxyStrategy::RoundRobin,
Some("http://localhost:10808"),
&engine_config,
&make_engine_timeouts(),
engine_config.max_response_body_bytes as usize,
None,
);
assert!(
!engines.is_empty(),
"init_engines should return at least one engine"
);
}
#[test]
fn test_init_engines_default_contains_reqwest_engine() {
let http_client = make_http_client();
let engine_config = EngineSettings::default();
let engines = init_engines(
http_client,
Some(make_proxy_provider()),
ProxyStrategy::RoundRobin,
Some("http://localhost:10808"),
&engine_config,
&make_engine_timeouts(),
engine_config.max_response_body_bytes as usize,
None,
);
let engine_names: Vec<&str> = engines.iter().map(|e| e.name()).collect();
assert!(
engine_names.contains(&"reqwest"),
"init_engines should include the reqwest engine by default, got: {:?}",
engine_names
);
}
#[test]
fn test_init_engines_default_has_at_least_one_engine() {
let http_client = make_http_client();
let engine_config = EngineSettings::default();
let engines = init_engines(
http_client,
Some(make_proxy_provider()),
ProxyStrategy::RoundRobin,
Some("http://localhost:10808"),
&engine_config,
&make_engine_timeouts(),
engine_config.max_response_body_bytes as usize,
None,
);
assert!(
!engines.is_empty(),
"Should have at least 1 engine with default features"
);
}
#[test]
fn test_init_engines_with_no_proxy_provider() {
let http_client = make_http_client();
let engine_config = EngineSettings::default();
let engines = init_engines(
http_client,
None,
ProxyStrategy::RoundRobin,
None,
&engine_config,
&make_engine_timeouts(),
engine_config.max_response_body_bytes as usize,
None,
);
assert!(!engines.is_empty());
}
#[test]
fn test_init_engines_with_sticky_strategy() {
let http_client = make_http_client();
let engine_config = EngineSettings::default();
let engines = init_engines(
http_client,
Some(make_proxy_provider()),
ProxyStrategy::Sticky,
None,
&engine_config,
&make_engine_timeouts(),
engine_config.max_response_body_bytes as usize,
None,
);
assert!(!engines.is_empty());
}
#[test]
fn test_init_engines_tls_fingerprint_disabled_by_default() {
let http_client = make_http_client();
let engine_config = EngineSettings::default();
let engines = init_engines(
http_client,
None,
ProxyStrategy::RoundRobin,
None,
&engine_config,
&make_engine_timeouts(),
engine_config.max_response_body_bytes as usize,
None,
);
assert!(
!engines.iter().any(|e| e.name() == "wreq"),
"tls_fingerprint.enabled=false 时不应注册 WreqEngine"
);
}
#[cfg(feature = "engine-tls-fingerprint")]
#[test]
fn test_init_engines_registers_wreq_when_enabled() {
let http_client = make_http_client();
let mut engine_config = EngineSettings::default();
engine_config.tls_fingerprint.enabled = true;
engine_config.tls_fingerprint.timeout_seconds = 15;
let engines = init_engines(
http_client,
Some(make_proxy_provider()),
ProxyStrategy::Sticky,
None,
&engine_config,
&make_engine_timeouts(),
engine_config.max_response_body_bytes as usize,
None,
);
assert!(
engines.iter().any(|e| e.name() == "wreq"),
"tls_fingerprint.enabled=true 时应注册 WreqEngine"
);
}
#[test]
fn test_init_engine_components_populates_all_fields() {
let http_client = make_http_client();
let engine_config = EngineSettings::default();
let components = init_engine_components(
http_client,
Some(make_proxy_provider()),
ProxyStrategy::RoundRobin,
Some("http://localhost:10808".to_string()),
&engine_config,
&make_engine_timeouts(),
engine_config.max_response_body_bytes as usize,
&PolitenessSettings::default(),
&SessionSettings::default(),
&ObservabilitySettings::default(),
None,
);
assert!(
!components.engines.is_empty(),
"engines vec should be non-empty"
);
let _router = &components.router;
let _engine_client = &components.engine_client;
}
#[test]
fn test_init_engine_components_engines_non_empty() {
let http_client = make_http_client();
let engine_config = EngineSettings::default();
let components = init_engine_components(
http_client,
Some(make_proxy_provider()),
ProxyStrategy::RoundRobin,
Some("http://localhost:10808".to_string()),
&engine_config,
&make_engine_timeouts(),
engine_config.max_response_body_bytes as usize,
&PolitenessSettings::default(),
&SessionSettings::default(),
&ObservabilitySettings::default(),
None,
);
assert!(
!components.engines.is_empty(),
"EngineComponents.engines should be non-empty"
);
}
#[test]
fn test_init_engine_components_router_created() {
let http_client = make_http_client();
let engine_config = EngineSettings::default();
let components = init_engine_components(
http_client,
Some(make_proxy_provider()),
ProxyStrategy::RoundRobin,
Some("http://localhost:10808".to_string()),
&engine_config,
&make_engine_timeouts(),
engine_config.max_response_body_bytes as usize,
&PolitenessSettings::default(),
&SessionSettings::default(),
&ObservabilitySettings::default(),
None,
);
let registered = components.router.registered_engines();
assert!(
!registered.is_empty(),
"router should have registered engines"
);
}
#[test]
fn test_init_engine_components_engine_client_created() {
let http_client = make_http_client();
let engine_config = EngineSettings::default();
let components = init_engine_components(
http_client,
Some(make_proxy_provider()),
ProxyStrategy::RoundRobin,
Some("http://localhost:10808".to_string()),
&engine_config,
&make_engine_timeouts(),
engine_config.max_response_body_bytes as usize,
&PolitenessSettings::default(),
&SessionSettings::default(),
&ObservabilitySettings::default(),
None,
);
assert!(
components.engine_client.engine_count() >= 1,
"engine_client should report at least 1 engine"
);
}
#[test]
fn test_init_engine_components_clone() {
let http_client = make_http_client();
let engine_config = EngineSettings::default();
let components = init_engine_components(
http_client,
Some(make_proxy_provider()),
ProxyStrategy::RoundRobin,
Some("http://localhost:10808".to_string()),
&engine_config,
&make_engine_timeouts(),
engine_config.max_response_body_bytes as usize,
&PolitenessSettings::default(),
&SessionSettings::default(),
&ObservabilitySettings::default(),
None,
);
let cloned = components.clone();
assert_eq!(components.engines.len(), cloned.engines.len());
}
}