use crate::engines::circuit_breaker::CircuitBreaker;
use crate::engines::engine_client::{
EngineError, InternalScrapeRequest, InternalScrapeResponse, ScraperEngine,
};
use crate::engines::session::SessionManager;
use crate::engines::validators::validate_url;
use crate::utils::hedge::HedgeController;
use crate::utils::host_discipline::HostDiscipline;
use crate::utils::retry::{RetryReason, RetryTracker};
use crate::utils::ua_pool::UaPool;
use dashmap::DashMap;
const HOST_BROWSER_HINT_TTL: std::time::Duration = std::time::Duration::from_secs(600);
use log::{info, warn};
use crate::i18n::tr_log_args;
use std::sync::atomic::Ordering;
use std::sync::Arc;
use std::time::{Duration, Instant};
pub use crate::engines::router_metrics::{EngineStats, RouterMetrics};
#[async_trait::async_trait]
pub trait EngineRouterTrait: Send + Sync {
async fn route(
&self,
request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError>;
async fn aggregate(
&self,
request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError>;
fn get_engine_stats(&self) -> std::collections::HashMap<String, EngineStats>;
fn reset_engine_stats(&self, engine_name: &str);
fn registered_engines(&self) -> Vec<String>;
fn set_crawl_delay(&self, _host: &str, _delay: Duration) {}
}
#[derive(Debug, Clone, Copy, PartialEq)]
pub enum LoadBalancingStrategy {
RoundRobin,
WeightedRoundRobin,
LeastConnections,
FastestResponse,
Random,
SmartHybrid,
}
pub struct EngineRouter {
engines: Vec<Arc<dyn ScraperEngine>>,
circuit_breaker: Arc<CircuitBreaker>,
engine_stats: Arc<DashMap<String, EngineStats>>,
host_browser_hint: DashMap<String, Instant>,
host_discipline: HostDiscipline,
round_robin_index: Arc<parking_lot::Mutex<usize>>,
strategy: LoadBalancingStrategy,
metrics: Arc<RouterMetrics>,
max_engine_attempts: usize,
max_retries: usize,
feature_filter_enabled: bool,
race_mode_enabled: bool,
dynamic_threshold_factor: f64,
ua_pool: Arc<UaPool>,
session: SessionManager,
failure_snapshot:
Option<Arc<crate::infrastructure::observability::failure_snapshot::FailureSnapshotter>>,
hedge_controller: HedgeController,
#[cfg(feature = "engine-moli")]
memory: Option<Arc<crate::engines::memory_watermark::MemoryWatermark>>,
#[cfg(feature = "engine-moli")]
memory_fallback_to_static: bool,
}
impl EngineRouter {
pub fn new(engines: Vec<Arc<dyn ScraperEngine>>) -> Self {
Self::with_host_discipline(engines, HostDiscipline::with_defaults())
}
pub fn with_host_discipline(
engines: Vec<Arc<dyn ScraperEngine>>,
host_discipline: HostDiscipline,
) -> Self {
Self::with_discipline_and_session(engines, host_discipline, SessionManager::disabled())
}
pub fn with_discipline_and_session(
engines: Vec<Arc<dyn ScraperEngine>>,
host_discipline: HostDiscipline,
session: SessionManager,
) -> Self {
let engine_stats = DashMap::with_capacity(8);
for engine in &engines {
engine_stats.insert(engine.name().to_string(), EngineStats::default());
}
Self {
engines,
circuit_breaker: Arc::new(CircuitBreaker::new()),
engine_stats: Arc::new(engine_stats),
host_browser_hint: DashMap::new(),
host_discipline,
round_robin_index: Arc::new(parking_lot::Mutex::new(0)),
strategy: LoadBalancingStrategy::SmartHybrid,
metrics: Arc::new(RouterMetrics::new()),
max_engine_attempts: 3,
max_retries: 5, feature_filter_enabled: true, race_mode_enabled: false, dynamic_threshold_factor: 1.0, ua_pool: session.ua_pool().clone(),
session,
failure_snapshot: None, hedge_controller: HedgeController::with_defaults(), #[cfg(feature = "engine-moli")]
memory: None,
#[cfg(feature = "engine-moli")]
memory_fallback_to_static: true,
}
}
pub fn with_circuit_breaker_and_strategy(
engines: Vec<Arc<dyn ScraperEngine>>,
circuit_breaker: Arc<CircuitBreaker>,
strategy: LoadBalancingStrategy,
) -> Self {
let engine_stats = DashMap::with_capacity(8);
for engine in &engines {
engine_stats.insert(engine.name().to_string(), EngineStats::default());
}
Self {
engines,
circuit_breaker,
engine_stats: Arc::new(engine_stats),
host_browser_hint: DashMap::new(),
host_discipline: HostDiscipline::with_defaults(),
round_robin_index: Arc::new(parking_lot::Mutex::new(0)),
strategy,
metrics: Arc::new(RouterMetrics::new()),
max_engine_attempts: 3,
max_retries: 5,
feature_filter_enabled: true,
race_mode_enabled: false,
dynamic_threshold_factor: 1.0,
ua_pool: Arc::new(UaPool::new()), session: SessionManager::disabled(), failure_snapshot: None, hedge_controller: HedgeController::with_defaults(), #[cfg(feature = "engine-moli")]
memory: None,
#[cfg(feature = "engine-moli")]
memory_fallback_to_static: true,
}
}
#[cfg(feature = "engine-moli")]
pub fn set_memory_scheduler(
&mut self,
watermark: Option<Arc<crate::engines::memory_watermark::MemoryWatermark>>,
fallback_to_static: bool,
) {
self.memory = watermark;
self.memory_fallback_to_static = fallback_to_static;
}
pub fn set_max_engine_attempts(&mut self, attempts: usize) {
self.max_engine_attempts = attempts.max(1);
}
pub fn set_failure_snapshotter(
&mut self,
snapshotter: Option<
Arc<crate::infrastructure::observability::failure_snapshot::FailureSnapshotter>,
>,
) {
self.failure_snapshot = snapshotter;
}
pub fn set_max_retries(&mut self, retries: usize) {
self.max_retries = retries.max(1);
}
pub fn set_feature_filter_enabled(&mut self, enabled: bool) {
self.feature_filter_enabled = enabled;
}
pub fn set_race_mode_enabled(&mut self, enabled: bool) {
self.race_mode_enabled = enabled;
}
pub fn set_dynamic_threshold_factor(&mut self, factor: f64) {
self.dynamic_threshold_factor = factor.clamp(0.1, 2.0);
}
pub fn set_strategy(&mut self, strategy: LoadBalancingStrategy) {
self.strategy = strategy;
}
pub fn metrics(&self) -> &Arc<RouterMetrics> {
&self.metrics
}
pub fn hedge_controller(&self) -> &HedgeController {
&self.hedge_controller
}
fn update_engine_stats(&self, engine_name: &str, success: bool, response_time: Duration) {
if let Some(mut stat) = self.engine_stats.get_mut(engine_name) {
let alpha = 0.1; let current_success = if success { 1.0 } else { 0.0 };
stat.success_rate = stat.success_rate * (1.0 - alpha) + current_success * alpha;
let current_avg_ns = stat.avg_response_time.as_nanos() as f64;
let response_ns = response_time.as_nanos() as f64;
let new_avg_ns = current_avg_ns * (1.0 - alpha) + response_ns * alpha;
stat.avg_response_time = Duration::from_nanos(new_avg_ns as u64);
stat.last_used = Some(Instant::now());
stat.usage_count += 1;
}
}
fn get_next_round_robin_index(&self, max_index: usize) -> usize {
let mut index = self.round_robin_index.lock();
let current = *index;
*index = (*index + 1) % max_index;
current
}
pub async fn _route_impl(
&self,
request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
if let Err(e) = validate_url(&request.url).await {
return Err(EngineError::SsrfProtection(e.to_string()));
}
let timeout = request.timeout;
tokio::time::timeout(timeout, self.route_internal(request))
.await
.map_err(|_| EngineError::Timeout(timeout))
.and_then(|result| result)
}
fn should_stop_after_retry_check(
&self,
tracker: &RetryTracker,
reason: RetryReason,
total_attempts: usize,
max_retries: usize,
) -> bool {
if !tracker.should_retry(reason) {
warn!(
"RetryTracker blocked {:?} after total={} (anti_bot={}, feature_toggle={}), stopping",
reason,
tracker.total(),
tracker.anti_bot(),
tracker.feature_toggle()
);
self.metrics.total_requests.fetch_add(1, Ordering::Relaxed);
self.metrics.failed_requests.fetch_add(1, Ordering::Relaxed);
return true;
}
if total_attempts >= max_retries {
warn!(
"Max retries {} reached after {:?}, stopping",
max_retries, reason
);
self.metrics.total_requests.fetch_add(1, Ordering::Relaxed);
self.metrics.failed_requests.fetch_add(1, Ordering::Relaxed);
return true;
}
false
}
#[cfg(feature = "content")]
pub(super) fn record_host_browser_hint(&self, host: &str) {
self.host_browser_hint
.insert(host.to_string(), Instant::now());
}
pub(super) fn host_needs_browser(&self, host: &str) -> bool {
let fresh = match self.host_browser_hint.get(host) {
Some(ts) => ts.elapsed() < HOST_BROWSER_HINT_TTL,
None => return false,
};
if !fresh {
self.host_browser_hint.remove(host);
}
fresh
}
pub async fn _aggregate_impl(
&self,
request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
if let Err(e) = validate_url(&request.url).await {
return Err(EngineError::SsrfProtection(e.to_string()));
}
let candidates = self.select_optimal_engines(request);
if candidates.is_empty() {
return Err(EngineError::AllEnginesFailed(
"No suitable engines available".to_string(),
));
}
let mut results = Vec::new();
let mut errors = Vec::new();
for (_, engine) in candidates {
match engine.scrape(request).await {
Ok(response) => results.push((engine.name().to_string(), response)),
Err(e) => errors.push((engine.name().to_string(), e)),
}
}
if results.is_empty() {
return Err(EngineError::AllEnginesFailed(
"All engines failed in aggregate".to_string(),
));
}
let (primary_name, primary_response) = results.remove(0);
info!(
"Aggregation: Primary result from {}, {} other successes",
primary_name,
results.len()
);
self.circuit_breaker.record_success(&primary_name).await;
for (name, _) in results {
self.circuit_breaker.record_success(&name).await;
}
for (name, error) in errors {
if error.is_retryable() {
self.circuit_breaker.record_failure(&name).await;
}
}
Ok(primary_response)
}
pub fn _get_engine_stats_impl(&self) -> std::collections::HashMap<String, EngineStats> {
self.engine_stats
.iter()
.map(|r| (r.key().clone(), r.value().clone()))
.collect()
}
pub fn _reset_engine_stats_impl(&self, engine_name: &str) {
if let Some(mut stat) = self.engine_stats.get_mut(engine_name) {
*stat = EngineStats::default();
}
}
pub fn register_engine(&mut self, engine: Arc<dyn ScraperEngine>) {
let name = engine.name().to_string();
self.engines.push(engine);
self.engine_stats
.insert(name.clone(), EngineStats::default());
info!(
"{}",
tr_log_args("engine-registered", &[("name", name.as_str().to_string())],)
);
}
pub fn _registered_engines_impl(&self) -> Vec<String> {
self.engines.iter().map(|e| e.name().to_string()).collect()
}
#[doc(hidden)]
pub fn get_engines(&self) -> &Vec<Arc<dyn ScraperEngine>> {
&self.engines
}
}
#[async_trait::async_trait]
impl EngineRouterTrait for EngineRouter {
async fn route(
&self,
request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
self._route_impl(request).await
}
async fn aggregate(
&self,
request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
self._aggregate_impl(request).await
}
fn get_engine_stats(&self) -> std::collections::HashMap<String, EngineStats> {
self._get_engine_stats_impl()
}
fn reset_engine_stats(&self, engine_name: &str) {
self._reset_engine_stats_impl(engine_name)
}
fn registered_engines(&self) -> Vec<String> {
self._registered_engines_impl()
}
fn set_crawl_delay(&self, host: &str, delay: Duration) {
self.host_discipline.set_crawl_delay(host, delay);
}
}
#[cfg(feature = "content")]
pub(super) fn check_antibot_response(
response: &InternalScrapeResponse,
url: &str,
) -> Option<crate::engines::antibot::Detection> {
use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
let mut header_map = HeaderMap::new();
for (k, v) in &response.headers {
if let Ok(name) = HeaderName::from_bytes(k.as_bytes()) {
if let Ok(value) = HeaderValue::from_str(v) {
header_map.append(name, value);
}
}
}
crate::engines::antibot::classify(response.status_code, &response.content, &header_map, url)
}
pub(super) fn check_js_upgrade_probe(
response: &InternalScrapeResponse,
) -> crate::engines::upgrade_probe::ProbeVerdict {
use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
let probe = crate::engines::upgrade_probe::JsUpgradeProbe::default();
let mut header_map = HeaderMap::new();
for (k, v) in &response.headers {
if let Ok(name) = HeaderName::from_bytes(k.as_bytes()) {
if let Ok(value) = HeaderValue::from_str(v) {
header_map.append(name, value);
}
}
}
let body_prefix = crate::common::text_slice::safe_prefix(
&response.content,
crate::engines::upgrade_probe::PROBE_PREFIX_LEN,
);
probe.evaluate(&header_map, body_prefix)
}
#[cfg(test)]
#[path = "../tests/router_test.rs"]
mod tests;
#[cfg(test)]
#[path = "../tests/router_tests_impl.rs"]
mod tests_impl;
mod engine_selector;
mod route_race;
mod route_sequential;