use super::super::EngineRouter;
use crate::common::metrics_shim::{counter, histogram};
use crate::engines::engine_client::{EngineError, InternalScrapeResponse};
use crate::engines::session::SessionIdentity;
use crate::utils::host_discipline::ObserveOutcome;
use log::{info, warn};
use std::sync::atomic::Ordering;
use std::sync::Arc;
use std::time::{Duration, Instant};
fn parse_retry_after_header(
headers: &std::collections::HashMap<String, String>,
) -> Option<Duration> {
headers
.iter()
.find(|(k, _)| k.eq_ignore_ascii_case("retry-after"))
.and_then(|(_, v)| crate::utils::retry_after::parse_retry_after_value(v))
.map(Duration::from_secs)
}
impl EngineRouter {
pub(super) async fn finish_success(
&self,
response: InternalScrapeResponse,
engine_name: &str,
response_time: Duration,
start_time: Instant,
host: Option<&str>,
session_identity: Option<&Arc<SessionIdentity>>,
) -> Result<InternalScrapeResponse, EngineError> {
self.update_engine_stats(engine_name, true, response_time);
self.circuit_breaker.record_success(engine_name).await;
counter!(
"crawlrs_engine_success_total",
"engine" => engine_name.to_string(),
"result" => "success"
)
.increment(1);
histogram!(
"crawlrs_engine_duration_seconds",
"engine" => engine_name.to_string()
)
.record(response_time.as_secs_f64());
self.metrics.total_requests.fetch_add(1, Ordering::Relaxed);
self.metrics
.successful_requests
.fetch_add(1, Ordering::Relaxed);
self.metrics
.record_engine_latency(engine_name, response_time);
self.metrics.record_engine_success(engine_name);
info!(
"Engine {} succeeded in {:?}, total time: {:?}",
engine_name,
response_time,
start_time.elapsed()
);
if let Some(host) = host {
if (200..300).contains(&response.status_code) {
self.host_discipline
.observe_success_with_latency(host, response_time);
} else if response.status_code == 429 {
self.host_discipline.observe(
host,
ObserveOutcome::RateLimited {
retry_after: parse_retry_after_header(&response.headers),
},
);
}
}
if let Some(identity) = session_identity {
if (200..300).contains(&response.status_code) {
self.session.mark_good(identity);
} else if matches!(response.status_code, 401 | 403 | 429) {
self.session.mark_bad(identity);
}
self.session
.capture_cookies(identity, host.unwrap_or(""), &response.headers);
}
Ok(response)
}
pub(super) fn record_engine_error(
&self,
engine_name: &str,
response_time: Duration,
error: &EngineError,
) {
self.update_engine_stats(engine_name, false, response_time);
counter!(
"crawlrs_engine_success_total",
"engine" => engine_name.to_string(),
"result" => "failure"
)
.increment(1);
histogram!(
"crawlrs_engine_duration_seconds",
"engine" => engine_name.to_string()
)
.record(response_time.as_secs_f64());
self.metrics
.record_engine_failure(engine_name, &error.to_string());
}
pub(super) fn mrt_timeout_error(
engine_name: &str,
effective_timeout: Duration,
engine_mrt: Duration,
remaining: Duration,
) -> EngineError {
let is_overall_timeout = effective_timeout <= remaining;
let timeout_err = if is_overall_timeout {
warn!(
"Request overall timeout (remaining={:?} <= mrt={:?}); \
engine={} cancelled at effective_timeout={:?}",
remaining, engine_mrt, engine_name, effective_timeout
);
EngineError::Timeout(effective_timeout)
} else {
let mrt_err = EngineError::EngineMrtExceeded {
engine: engine_name.to_string(),
mrt: effective_timeout,
};
warn!(
"Engine {} exceeded MRT (effective_timeout={:?}, mrt={:?}, remaining={:?}); \
waterfall fallback to next engine",
engine_name, effective_timeout, engine_mrt, remaining
);
mrt_err
};
timeout_err
}
}