use super::super::EngineRouter;
use crate::engines::engine_client::{EngineError, InternalScrapeRequest, InternalScrapeResponse};
use log::debug;
#[cfg(feature = "engine-moli")]
use log::warn;
#[cfg(feature = "engine-moli")]
use std::sync::atomic::Ordering;
impl EngineRouter {
pub(super) async fn apply_host_hints(
&self,
request: &InternalScrapeRequest,
) -> (Option<String>, InternalScrapeRequest) {
let host = url::Url::parse(&request.url)
.ok()
.and_then(|u| u.host_str().map(str::to_string));
let request = if let Some(host) = &host {
let needs_js_hint = self.host_needs_browser(host);
let tls_suspect = self.host_discipline.tls_suspect(host);
if needs_js_hint {
debug!(
"host {} has anti-bot hint, routing with needs_js=true",
host
);
}
if needs_js_hint || tls_suspect {
let mut hinted = request.clone();
hinted.needs_js |= needs_js_hint;
hinted.needs_tls_fingerprint |= tls_suspect;
hinted
} else {
request.clone()
}
} else {
request.clone()
};
if let Some(host) = &host {
self.host_discipline.gate(host).await;
}
(host, request)
}
#[cfg_attr(
not(feature = "engine-moli"),
allow(clippy::unused_async, unused_variables)
)]
pub(super) async fn handle_no_candidates(
&self,
request: &InternalScrapeRequest,
) -> Option<Result<InternalScrapeResponse, EngineError>> {
#[cfg(feature = "engine-moli")]
if let Some(outcome) = self.try_static_memory_fallback(request).await {
return Some(outcome);
}
None
}
#[cfg(feature = "engine-moli")]
async fn try_static_memory_fallback(
&self,
request: &InternalScrapeRequest,
) -> Option<Result<InternalScrapeResponse, EngineError>> {
if !request.needs_js {
return None;
}
let watermark = self.memory.as_ref()?;
if watermark.level() != crate::engines::memory_watermark::MemoryLevel::Critical {
return None;
}
self.metrics.failed_requests.fetch_add(1, Ordering::Relaxed);
if self.memory_fallback_to_static {
if let Some(static_engine) = self.engines.iter().find(|e| e.name() == "reqwest") {
warn!(
"memory Critical without JS engines, degrading request to static engine (js_degraded): {}",
request.url
);
let mut static_request = request.clone();
static_request.needs_js = false;
let fallback_started = std::time::Instant::now();
let mut response = match static_engine.scrape(&static_request).await {
Ok(resp) => resp,
Err(e) => {
return Some(Err(EngineError::AllEnginesFailed(format!(
"static fallback after memory degradation failed: {e}"
))))
}
};
response.js_degraded = true;
let engine_name = static_engine.name();
let response_time = fallback_started.elapsed();
self.update_engine_stats(engine_name, true, response_time);
self.circuit_breaker.record_success(engine_name).await;
self.metrics
.record_engine_latency(engine_name, response_time);
self.metrics.record_engine_success(engine_name);
self.metrics.total_requests.fetch_add(1, Ordering::Relaxed);
self.metrics
.successful_requests
.fetch_add(1, Ordering::Relaxed);
return Some(Ok(response));
}
}
Some(Err(EngineError::MemoryConstrained(
"no JS engine available under Critical memory watermark".to_string(),
)))
}
}