#[cfg(feature = "content")]
mod antibot;
mod attempt;
mod finalize;
mod hints;
mod outcome;
use super::{EngineRouter, LoadBalancingStrategy};
use crate::engines::engine_client::{
EngineError, InternalScrapeRequest, InternalScrapeResponse, ScraperEngine,
};
use crate::utils::retry::{RetryReason, RetryTracker};
use log::{debug, warn};
use outcome::AttemptReport;
use std::sync::atomic::Ordering;
use std::sync::Arc;
use std::time::{Duration, Instant};
enum PreparedRoute {
Ready(RoutePlan),
Immediate(Result<InternalScrapeResponse, EngineError>),
}
struct RoutePlan {
host: Option<String>,
request: InternalScrapeRequest,
candidates: Vec<(f64, Arc<dyn ScraperEngine>)>,
start_time: Instant,
}
struct AttemptContext<'a> {
request: &'a InternalScrapeRequest,
host: Option<&'a str>,
start_time: Instant,
max_retries: usize,
}
struct AttemptState<'a> {
tracker: &'a mut RetryTracker,
last_error: &'a mut Option<EngineError>,
last_reason: &'a mut Option<RetryReason>,
force_needs_js: &'a mut bool,
}
enum AttemptOutcome {
Done(Result<InternalScrapeResponse, EngineError>),
Retry,
}
impl EngineRouter {
#[allow(unused_variables)]
pub(super) async fn route_internal(
&self,
request: &InternalScrapeRequest,
) -> Result<InternalScrapeResponse, EngineError> {
match self.prepare_route(request).await {
PreparedRoute::Immediate(outcome) => outcome,
PreparedRoute::Ready(plan) => self.run_sequential(plan).await,
}
}
async fn prepare_route(&self, request: &InternalScrapeRequest) -> PreparedRoute {
let start_time = Instant::now();
let (host, request) = self.apply_host_hints(request).await;
let mut candidates = self.select_optimal_engines(&request);
self.metrics.record_candidates(candidates.len());
if candidates.is_empty() {
if let Some(outcome) = self.handle_no_candidates(&request).await {
return PreparedRoute::Immediate(outcome);
}
warn!("No suitable engines available for request");
self.metrics.failed_requests.fetch_add(1, Ordering::Relaxed);
return PreparedRoute::Immediate(Err(EngineError::AllEnginesFailed(
"No suitable engines available".to_string(),
)));
}
if self.strategy == LoadBalancingStrategy::RoundRobin {
let start_index = self.get_next_round_robin_index(candidates.len());
candidates.rotate_left(start_index);
}
debug!(
"Selected {} candidate engines using {:?} strategy",
candidates.len(),
self.strategy
);
if self.race_mode_enabled && candidates.len() > 1 {
return PreparedRoute::Immediate(
self.route_race_mode(&request, candidates, start_time).await,
);
}
PreparedRoute::Ready(RoutePlan {
host,
request,
candidates,
start_time,
})
}
async fn run_sequential(&self, plan: RoutePlan) -> Result<InternalScrapeResponse, EngineError> {
let RoutePlan {
host,
request,
candidates,
start_time,
} = plan;
let max_attempts = self.max_engine_attempts.max(1).min(candidates.len());
let max_retries = self.max_retries.max(1);
let mut total_attempts = 0;
let mut last_error = None;
#[cfg_attr(not(feature = "content"), allow(unused_mut))]
let mut force_needs_js = false;
let mut tracker = RetryTracker::new_default();
let mut last_reason: Option<RetryReason> = None;
let context = AttemptContext {
request: &request,
host: host.as_deref(),
start_time,
max_retries,
};
let mut state = AttemptState {
tracker: &mut tracker,
last_error: &mut last_error,
last_reason: &mut last_reason,
force_needs_js: &mut force_needs_js,
};
for (score, engine) in candidates.into_iter().take(max_attempts) {
total_attempts += 1;
match self
.attempt_engine(engine, score, total_attempts, &context, &mut state)
.await
{
AttemptOutcome::Done(result) => return result,
AttemptOutcome::Retry => {}
}
}
warn!("All engines failed for request to {}", request.url);
self.metrics.total_requests.fetch_add(1, Ordering::Relaxed);
self.metrics.failed_requests.fetch_add(1, Ordering::Relaxed);
Err(last_error
.unwrap_or_else(|| EngineError::AllEnginesFailed("All engines failed".to_string())))
}
async fn attempt_engine(
&self,
engine: Arc<dyn ScraperEngine>,
score: f64,
total_attempts: usize,
context: &AttemptContext<'_>,
state: &mut AttemptState<'_>,
) -> AttemptOutcome {
let request = context.request;
let host = context.host;
let start_time = context.start_time;
let engine_name = engine.name();
self.metrics.record_engine_selection(engine_name);
self.metrics.record_attempt();
debug!(
"Trying engine {} with score {:.2} for request to {}",
engine_name, score, request.url
);
let remaining = request
.timeout
.checked_sub(start_time.elapsed())
.unwrap_or(Duration::from_millis(0));
if remaining.is_zero() {
return AttemptOutcome::Done(Err(EngineError::Timeout(request.timeout)));
}
let directive = self.compute_attempt_directive(total_attempts, *state.last_reason, host);
let (attempt_request, session_identity) = self.build_attempt_request(
request,
host,
directive,
total_attempts,
*state.force_needs_js,
remaining,
);
let engine_start = Instant::now();
let engine_mrt = engine.max_response_time();
let effective_timeout = std::cmp::min(remaining, engine_mrt);
match tokio::time::timeout(effective_timeout, engine.scrape(&attempt_request)).await {
Ok(Ok(response)) => {
let report = AttemptReport {
engine_name,
response_time: engine_start.elapsed(),
total_attempts,
};
self.finish_attempt_response(
&report,
response,
attempt_request.needs_js,
session_identity.as_ref(),
context,
state,
)
.await
}
Ok(Err(e)) => {
let report = AttemptReport {
engine_name,
response_time: engine_start.elapsed(),
total_attempts,
};
self.handle_attempt_failure(report, e, context, state).await
}
Err(_elapsed) => {
let report = AttemptReport {
engine_name,
response_time: engine_start.elapsed(),
total_attempts,
};
self.handle_attempt_timeout(
report,
effective_timeout,
engine_mrt,
remaining,
context,
state,
)
.await
}
}
}
}