use super::super::EngineRouter;
use crate::engines::engine_client::InternalScrapeRequest;
use crate::engines::session::SessionIdentity;
use crate::utils::retry::{RetryDirective, RetryReason};
use crate::utils::ua_pool::UaProfile;
use log::{debug, warn};
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
impl EngineRouter {
pub(super) fn compute_attempt_directive(
&self,
total_attempts: usize,
last_reason: Option<RetryReason>,
host: Option<&str>,
) -> RetryDirective {
let mut directive = if total_attempts == 1 {
RetryDirective::default()
} else {
let directive_attempt = (total_attempts - 2) as u32;
RetryDirective::for_attempt(
last_reason.unwrap_or(RetryReason::Transient),
directive_attempt,
)
};
if let Some(host) = host {
if self.host_discipline.is_stalled(host) && !directive.rotate_proxy {
warn!(
"host {} stalled by consecutive rate limits, forcing proxy rotation \
(attempt {})",
host, total_attempts
);
directive.rotate_proxy = true;
}
}
directive
}
pub(super) fn build_attempt_request(
&self,
request: &InternalScrapeRequest,
host: Option<&str>,
directive: RetryDirective,
total_attempts: usize,
force_needs_js: bool,
remaining: Duration,
) -> (InternalScrapeRequest, Option<Arc<SessionIdentity>>) {
let mut headers = request.headers.clone();
let mut attempt_proxy = request.proxy.clone();
let session_identity = self.resolve_attempt_identity(
request,
host,
directive,
total_attempts,
&mut headers,
&mut attempt_proxy,
);
let attempt_request = InternalScrapeRequest {
url: request.url.clone(),
method: request.method,
headers,
timeout: remaining,
needs_js: request.needs_js || force_needs_js || directive.force_browser,
needs_screenshot: request.needs_screenshot,
screenshot_config: request.screenshot_config.clone(),
mobile: request.mobile,
proxy: attempt_proxy,
skip_tls_verification: request.skip_tls_verification,
needs_tls_fingerprint: request.needs_tls_fingerprint,
use_fire_engine: request.use_fire_engine,
actions: request.actions.clone(),
body: request.body.clone(),
sync_wait_ms: request.sync_wait_ms,
block_ads: request.block_ads,
block_media: request.block_media,
session_id: request.session_id.clone(),
wait_for: request.wait_for.clone(),
needs_mllm: request.needs_mllm,
};
(attempt_request, session_identity)
}
fn resolve_attempt_identity(
&self,
request: &InternalScrapeRequest,
host: Option<&str>,
directive: RetryDirective,
total_attempts: usize,
headers: &mut HashMap<String, String>,
attempt_proxy: &mut Option<String>,
) -> Option<Arc<SessionIdentity>> {
if self.session.is_enabled() {
let host_key = host.unwrap_or("");
if directive.rotate_ua || self.host_discipline.is_stalled(host_key) {
self.session.retire(host_key, request.session_id.as_deref());
}
let identity =
self.session
.acquire(host_key, request.session_id.as_deref(), request.mobile);
let profile = identity.ua_profile(self.session.ua_pool());
apply_ua_profile_headers(headers, profile);
if let Some(cookie) = self.session.cookie_header(&identity, host_key) {
headers.insert("Cookie".to_string(), cookie);
}
*attempt_proxy = identity.proxy_url.clone().or(request.proxy.clone());
debug!(
"Attempt {}: session identity #{} (proxy={}, ua={}, AL={}, sec-ch-ua={})",
total_attempts,
identity.id,
identity
.proxy_url
.as_deref()
.map(crate::utils::proxy::redact_proxy_url)
.unwrap_or_else(|| "<none>".to_string()),
profile.ua,
profile.accept_language,
if profile.sec_ch_ua.is_empty() {
"<none>"
} else {
profile.sec_ch_ua
}
);
Some(identity)
} else if directive.rotate_ua {
let profile = self
.ua_pool
.pick_seeded((total_attempts - 1) as u64, request.mobile);
apply_ua_profile_headers(headers, profile);
debug!(
"Attempt {}: rotated UA via pick_seeded(seed={}) -> {} ({}, AL={}, sec-ch-ua={})",
total_attempts,
total_attempts - 1,
profile.ua,
if profile.mobile { "mobile" } else { "desktop" },
profile.accept_language,
if profile.sec_ch_ua.is_empty() {
"<none>"
} else {
profile.sec_ch_ua
}
);
None
} else {
None
}
}
}
fn apply_ua_profile_headers(headers: &mut HashMap<String, String>, profile: &UaProfile) {
headers.insert("User-Agent".to_string(), profile.ua.to_string());
headers.insert(
"Accept-Language".to_string(),
profile.accept_language.to_string(),
);
if profile.sec_ch_ua.is_empty() {
headers.remove("sec-ch-ua");
} else {
headers.insert("sec-ch-ua".to_string(), profile.sec_ch_ua.to_string());
}
}