use super::super::check_js_upgrade_probe;
use super::super::EngineRouter;
use super::{AttemptContext, AttemptOutcome, AttemptState};
use crate::engines::engine_client::{EngineError, InternalScrapeRequest, InternalScrapeResponse};
use crate::engines::session::SessionIdentity;
#[cfg(feature = "content")]
use crate::utils::retry::RetryReason;
use log::{debug, warn};
use std::sync::atomic::Ordering;
use std::sync::Arc;
use std::time::Duration;
pub(super) struct AttemptReport<'a> {
pub(super) engine_name: &'a str,
pub(super) response_time: Duration,
pub(super) total_attempts: usize,
}
impl EngineRouter {
#[cfg_attr(not(feature = "content"), allow(unused_variables))]
pub(super) async fn finish_attempt_response(
&self,
report: &AttemptReport<'_>,
response: InternalScrapeResponse,
attempt_needs_js: bool,
session_identity: Option<&Arc<SessionIdentity>>,
context: &AttemptContext<'_>,
state: &mut AttemptState<'_>,
) -> AttemptOutcome {
let request = context.request;
let host = context.host;
let start_time = context.start_time;
let engine_name = report.engine_name;
let response_time = report.response_time;
#[cfg(feature = "content")]
let response = match self
.record_antibot_or_reject(response, report, session_identity, context, state)
.await
{
Ok(resp) => resp,
Err(outcome) => return outcome,
};
if let Some(outcome) = self
.probe_js_upgrade(
&response,
attempt_needs_js,
request,
engine_name,
response_time,
)
.await
{
return outcome;
}
AttemptOutcome::Done(
self.finish_success(
response,
engine_name,
response_time,
start_time,
host,
session_identity,
)
.await,
)
}
#[cfg(feature = "content")]
#[allow(clippy::result_large_err)] async fn record_antibot_or_reject(
&self,
response: InternalScrapeResponse,
report: &AttemptReport<'_>,
session_identity: Option<&Arc<SessionIdentity>>,
context: &AttemptContext<'_>,
state: &mut AttemptState<'_>,
) -> Result<InternalScrapeResponse, AttemptOutcome> {
let request = context.request;
let max_retries = context.max_retries;
let engine_name = report.engine_name;
let response_time = report.response_time;
let total_attempts = report.total_attempts;
match self
.record_antibot_failure(
response,
&request.url,
engine_name,
response_time,
session_identity,
)
.await
{
Err(challenge_reason) => {
*state.last_error = Some(EngineError::AntiBotDetected(challenge_reason));
*state.force_needs_js = true;
let reason = RetryReason::AntiBot;
state.tracker.record(reason);
*state.last_reason = Some(reason);
if self.should_stop_after_retry_check(
state.tracker,
reason,
total_attempts,
max_retries,
) {
return Err(AttemptOutcome::Done(Err(state
.last_error
.take()
.unwrap_or_else(|| {
EngineError::AllEnginesFailed("Max retries reached".to_string())
}))));
}
Err(AttemptOutcome::Retry)
}
Ok(resp) => Ok(resp),
}
}
async fn probe_js_upgrade(
&self,
response: &InternalScrapeResponse,
attempt_needs_js: bool,
request: &InternalScrapeRequest,
engine_name: &str,
response_time: Duration,
) -> Option<AttemptOutcome> {
if attempt_needs_js {
return None;
}
let verdict = check_js_upgrade_probe(response);
if !verdict.upgrade {
return None;
}
let mut js_request = request.clone();
js_request.needs_js = true;
if self.select_optimal_engines(&js_request).is_empty() {
warn!(
"JS upgrade probe hit (score={}, reason={}) but no \
JS-capable engine is registered for the needs_js \
re-route; degrading to the original HTTP response \
for {}",
verdict.score, verdict.reason, request.url
);
None
} else {
self.update_engine_stats(engine_name, false, response_time);
self.metrics.record_engine_failure(
engine_name,
&format!("js-upgrade-probe: {}", verdict.reason),
);
debug!(
"Engine {} returned SPA shell (probe score={}, reason={}); \
re-routing with needs_js=true to dispatch browser engine",
engine_name, verdict.score, verdict.reason
);
Some(AttemptOutcome::Done(
Box::pin(self.route_internal(&js_request)).await,
))
}
}
pub(super) async fn handle_attempt_failure(
&self,
report: AttemptReport<'_>,
e: EngineError,
context: &AttemptContext<'_>,
state: &mut AttemptState<'_>,
) -> AttemptOutcome {
let engine_name = report.engine_name;
let response_time = report.response_time;
let total_attempts = report.total_attempts;
let max_retries = context.max_retries;
self.record_engine_error(engine_name, response_time, &e);
if e.is_retryable() {
self.circuit_breaker.record_failure(engine_name).await;
let reason = e.retry_reason();
state.tracker.record(reason);
*state.last_reason = Some(reason);
if self.should_stop_after_retry_check(
state.tracker,
reason,
total_attempts,
max_retries,
) {
return AttemptOutcome::Done(Err(e));
}
warn!(
"Engine {} failed with retryable error: {}, trying next engine \
(reason={:?}, tracker total={})",
engine_name,
e,
reason,
state.tracker.total()
);
*state.last_error = Some(e);
return AttemptOutcome::Retry;
}
warn!(
"Engine {} failed with non-retryable error: {}",
engine_name, e
);
self.metrics.total_requests.fetch_add(1, Ordering::Relaxed);
self.metrics.failed_requests.fetch_add(1, Ordering::Relaxed);
AttemptOutcome::Done(Err(e))
}
pub(super) async fn handle_attempt_timeout(
&self,
report: AttemptReport<'_>,
effective_timeout: Duration,
engine_mrt: Duration,
remaining: Duration,
context: &AttemptContext<'_>,
state: &mut AttemptState<'_>,
) -> AttemptOutcome {
let engine_name = report.engine_name;
let response_time = report.response_time;
let total_attempts = report.total_attempts;
let max_retries = context.max_retries;
self.update_engine_stats(engine_name, false, response_time);
let timeout_err =
Self::mrt_timeout_error(engine_name, effective_timeout, engine_mrt, remaining);
self.metrics
.record_engine_failure(engine_name, &timeout_err.to_string());
self.circuit_breaker.record_failure(engine_name).await;
let reason = timeout_err.retry_reason();
state.tracker.record(reason);
*state.last_reason = Some(reason);
if self.should_stop_after_retry_check(state.tracker, reason, total_attempts, max_retries) {
return AttemptOutcome::Done(Err(timeout_err));
}
*state.last_error = Some(timeout_err);
AttemptOutcome::Retry
}
}