use super::super::EngineRouter;
use crate::engines::engine_client::InternalScrapeResponse;
use crate::engines::router::check_antibot_response;
use crate::engines::session::SessionIdentity;
use crate::utils::host_discipline::ObserveOutcome;
use log::warn;
use std::sync::Arc;
use std::time::Duration;
fn is_known_waf(tech: crate::engines::antibot::AntiBotTech) -> bool {
use crate::engines::antibot::AntiBotTech;
matches!(
tech,
AntiBotTech::Cloudflare
| AntiBotTech::Akamai
| AntiBotTech::PerimeterX
| AntiBotTech::DataDome
| AntiBotTech::Imperva
| AntiBotTech::Sucuri
| AntiBotTech::Kasada
| AntiBotTech::AwsWaf
)
}
impl EngineRouter {
pub(super) async fn record_antibot_failure(
&self,
response: InternalScrapeResponse,
request_url: &str,
engine_name: &str,
response_time: Duration,
session_identity: Option<&Arc<SessionIdentity>>,
) -> Result<InternalScrapeResponse, String> {
if let Some(detection) = check_antibot_response(&response, request_url) {
if detection.needs_browser {
self.observe_antibot_host(request_url, &detection, response.status_code);
self.update_engine_stats(engine_name, false, response_time);
if let Some(identity) = session_identity {
self.session.mark_bad(identity);
}
self.metrics
.record_engine_failure(engine_name, &detection.reason);
self.snapshot_failure(
request_url,
response.status_code,
&detection.reason,
response,
)
.await;
warn!(
"Engine {} returned anti-bot challenge ({:?}): {}, \
forcing needs_js for subsequent attempts",
engine_name, detection.tech, detection.reason
);
return Err(detection.reason);
}
}
Ok(response)
}
fn observe_antibot_host(
&self,
request_url: &str,
detection: &crate::engines::antibot::Detection,
status_code: u16,
) {
if let Ok(u) = url::Url::parse(request_url) {
if let Some(host) = u.host_str() {
self.record_host_browser_hint(host);
if detection.tech == crate::engines::antibot::AntiBotTech::RateLimited {
self.host_discipline.observe(
host,
ObserveOutcome::RateLimited {
retry_after: detection.retry_after_secs.map(Duration::from_secs),
},
);
} else if status_code == 403 && !is_known_waf(detection.tech) {
self.host_discipline
.observe(host, ObserveOutcome::Forbidden { challenge: false });
}
}
}
}
}
impl EngineRouter {
pub(super) async fn snapshot_failure(
&self,
url: &str,
status_code: u16,
reason: &str,
response: InternalScrapeResponse,
) {
if let Some(snapshotter) = &self.failure_snapshot {
let InternalScrapeResponse {
content,
screenshot,
headers,
..
} = response;
snapshotter
.snapshot(
crate::infrastructure::observability::failure_snapshot::SnapshotInput {
url,
status_code,
reason,
content,
headers: &headers,
screenshot,
},
)
.await;
}
}
}
#[cfg(test)]
mod snapshot_failure_tests {
use super::*;
use std::collections::HashMap;
use std::fs;
fn sample_response() -> InternalScrapeResponse {
InternalScrapeResponse {
js_degraded: false,
captured_responses: Vec::new(),
status_code: 403,
content: "<html>challenge</html>".to_string(),
screenshot: None,
content_type: "text/html".to_string(),
headers: HashMap::new(),
response_time_ms: 42,
final_url: None,
}
}
fn dir_file_count(dir: &std::path::Path) -> usize {
fs::read_dir(dir).expect("read_dir").count()
}
#[tokio::test]
async fn snapshot_failure_none_short_circuits_without_panic() {
let router = EngineRouter::new(Vec::new());
assert!(router.failure_snapshot.is_none(), "default must be None");
router
.snapshot_failure("https://x.com/a", 403, "waf", sample_response())
.await;
}
#[tokio::test]
async fn snapshot_failure_with_snapshotter_writes_archive() {
let tmp = tempfile::tempdir().expect("tempdir");
let dir = tmp.path().join("snapshots");
let mut router = EngineRouter::new(Vec::new());
router.set_failure_snapshotter(Some(Arc::new(
crate::infrastructure::observability::failure_snapshot::FailureSnapshotter::new(
dir.clone(),
200,
),
)));
router
.snapshot_failure("https://x.com/a", 403, "waf", sample_response())
.await;
assert_eq!(dir_file_count(&dir), 2, "html + meta.json");
}
}