use futures::StreamExt;
use parking_lot::Mutex;
use serde::{Deserialize, Serialize};
use sha2::Digest as _;
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::Semaphore;
use tracing::{debug, warn};
use url::Url;
use crate::render::chrome::browser::{Browser, BrowserConfig};
use crate::render::chrome::handler::HandlerConfig;
use crate::render::chrome::page::Page;
use crate::render::chrome_protocol::cdp::browser_protocol::browser::BrowserContextId;
use crate::render::chrome_protocol::cdp::browser_protocol::emulation::{
UserAgentBrandVersion, UserAgentMetadata,
};
use crate::render::chrome_protocol::cdp::browser_protocol::network::{
BlockPattern, ClearBrowserCacheParams, Cookie, CookieParam, CookiePartitionKey, CookiePriority,
CookieSameSite, CookieSourceScheme, EventLoadingFailed, EventLoadingFinished,
EventRequestWillBeSent, EventResponseReceived, ResourceType, SetBlockedUrLsParams,
SetBypassServiceWorkerParams, SetUserAgentOverrideParams as NetworkSetUserAgentOverrideParams,
TimeSinceEpoch,
};
use crate::render::chrome_protocol::cdp::browser_protocol::page::{
AddScriptToEvaluateOnNewDocumentParams, CaptureScreenshotParams, NavigateParams,
};
use crate::render::chrome_protocol::cdp::browser_protocol::security::SetIgnoreCertificateErrorsParams;
use crate::render::chrome_protocol::cdp::browser_protocol::service_worker::UnregisterParams as ServiceWorkerUnregisterParams;
use crate::render::chrome_protocol::cdp::browser_protocol::storage::ClearDataForOriginParams;
use crate::render::chrome_protocol::cdp::browser_protocol::target::{
CreateBrowserContextParams, CreateTargetParams, SetAutoAttachParams,
};
use crate::render::chrome_protocol::cdp::js_protocol::runtime::EvaluateParams;
use std::collections::HashMap;
#[derive(Debug, Clone)]
pub enum ScreenshotCaptureMode {
Viewport,
FullPage,
Element { selector: String },
}
pub fn parse_screenshot_mode(s: &str) -> std::result::Result<ScreenshotCaptureMode, String> {
let t = s.trim();
let lower = t.to_ascii_lowercase();
if lower == "viewport" {
return Ok(ScreenshotCaptureMode::Viewport);
}
if lower == "fullpage" || lower == "full" || lower == "full_page" {
return Ok(ScreenshotCaptureMode::FullPage);
}
if let Some(rest) = t
.strip_prefix("element:")
.or_else(|| t.strip_prefix("Element:"))
{
let sel = rest.trim();
if sel.is_empty() {
return Err(format!(
"screenshot-mode: `element:` requires a selector (got `{s}`)"
));
}
return Ok(ScreenshotCaptureMode::Element {
selector: sel.to_string(),
});
}
Err(format!(
"screenshot-mode: expected `viewport|fullpage|element:<selector>`, got `{s}`"
))
}
pub fn parse_screenshot_mode_or_default(opt: Option<&str>) -> ScreenshotCaptureMode {
match opt {
Some(s) => parse_screenshot_mode(s).unwrap_or(ScreenshotCaptureMode::FullPage),
None => ScreenshotCaptureMode::FullPage,
}
}
fn collect_block_resource_patterns(kinds: &[String]) -> Vec<String> {
let mut out: Vec<String> = Vec::new();
for kind in kinds {
let patterns: &[&str] = match kind.as_str() {
"image" => &[
"*.png", "*.jpg", "*.jpeg", "*.gif", "*.webp", "*.avif", "*.svg", "*.ico",
],
"font" => &["*.woff", "*.woff2", "*.ttf", "*.otf", "*.eot"],
"media" => &["*.mp3", "*.mp4", "*.webm", "*.ogg", "*.m3u8", "*.mpd"],
"stylesheet" => &["*.css"],
"script" => &["*.js", "*.mjs"],
"analytics" => &[
"*://*.google-analytics.com/*",
"*://*.googletagmanager.com/*",
"*://*.segment.com/*",
"*://*.hotjar.com/*",
"*://*.mixpanel.com/*",
],
other => {
tracing::debug!(kind = other, "unknown block-resource kind — ignoring");
continue;
}
};
out.extend(patterns.iter().map(|s| s.to_string()));
}
out
}
fn diff(start: f64, end: f64) -> Option<f64> {
if start >= 0.0 && end >= start {
Some(end - start)
} else {
None
}
}
const SHADOW_ROOT_CAPTURE_JS: &str = r#"
;(() => {
if (Element.prototype.__crawlexAttachShadowPatched__) return;
const original = Element.prototype.attachShadow;
Object.defineProperty(Element.prototype, "__crawlexAttachShadowPatched__", { value: true });
Element.prototype.attachShadow = function(init) {
const root = original.call(this, init);
try { Object.defineProperty(this, "__crawlexShadowRoot", { value: root, configurable: true }); } catch (_) {}
return root;
};
})();
"#;
const DOM_CLEANUP_JS: &str = r#"
;(() => {
const removeConsent = __REMOVE_CONSENT__;
const removeOverlays = __REMOVE_OVERLAYS__;
const consentRe = /(cookie|cookies|consent|privacy|gdpr|ccpa|didomi|onetrust|trustarc|quantcast|iubenda|cookiebot)/i;
const shouldDrop = (el) => {
const idClass = `${el.id || ""} ${el.className || ""} ${el.getAttribute("aria-label") || ""}`;
if (removeConsent && consentRe.test(idClass)) return true;
if (!removeOverlays) return false;
const style = window.getComputedStyle(el);
if (!style) return false;
const rect = el.getBoundingClientRect();
const coversViewport = rect.width >= innerWidth * 0.35 && rect.height >= innerHeight * 0.18;
const highZ = Number.parseInt(style.zIndex || "0", 10) >= 1000;
return coversViewport && highZ && (style.position === "fixed" || style.position === "sticky");
};
for (const el of Array.from(document.querySelectorAll("body *"))) {
try {
if (shouldDrop(el)) el.remove();
} catch (_) {}
}
})();
"#;
const FLATTEN_SHADOW_DOM_JS: &str = r#"
;(() => {
const walk = (node) => {
if (!node || node.nodeType !== Node.ELEMENT_NODE) return;
const root = node.shadowRoot || node.__crawlexShadowRoot;
if (root && !node.querySelector(":scope > template[data-crawlex-shadow-root]")) {
const tpl = document.createElement("template");
tpl.setAttribute("data-crawlex-shadow-root", root.mode || "closed");
tpl.innerHTML = root.innerHTML;
node.appendChild(tpl);
}
for (const child of Array.from(node.children || [])) walk(child);
};
walk(document.documentElement);
return document.documentElement.outerHTML;
})();
"#;
pub fn build_launch_args(
bundle: &crate::identity::IdentityBundle,
proxy: Option<&Url>,
user_data_dir: &std::path::Path,
languages: &str,
extra: &[String],
) -> Vec<String> {
build_launch_args_with_gpu(
bundle,
proxy,
user_data_dir,
languages,
crate::config::GpuPolicy::Compat,
extra,
)
}
pub fn build_launch_args_with_gpu(
bundle: &crate::identity::IdentityBundle,
proxy: Option<&Url>,
user_data_dir: &std::path::Path,
languages: &str,
gpu_policy: crate::config::GpuPolicy,
extra: &[String],
) -> Vec<String> {
let win_w = if bundle.viewport_w == 0 {
1920
} else {
bundle.viewport_w
};
let win_h = if bundle.viewport_h == 0 {
1080
} else {
bundle.viewport_h
};
let dpr = if bundle.device_pixel_ratio > 0.0 {
bundle.device_pixel_ratio
} else {
1.0
};
let mut flags: Vec<String> = vec![
"--disable-dev-shm-usage".into(),
"--disable-blink-features=AutomationControlled".into(),
"--disable-features=IsolateOrigins,site-per-process,Translate,MediaRouter,WebRtcHideLocalIpsWithMdns".into(),
"--enable-features=VaapiVideoDecoder,AcceptCHFrame,ZstdContentEncoding,EnableTLS13KyberPQ".into(),
"--force-webrtc-ip-handling-policy=disable_non_proxied_udp".into(),
"--no-first-run".into(),
"--no-default-browser-check".into(),
format!("--user-agent={}", bundle.ua),
format!("--lang={languages}"),
format!("--window-size={win_w},{win_h}"),
format!("--force-device-scale-factor={dpr}"),
"--js-flags=--noexpose-wasm".into(),
];
match gpu_policy {
crate::config::GpuPolicy::Compat => {
flags.push("--disable-gpu".into());
}
crate::config::GpuPolicy::Stealth => {
flags.push("--enable-gpu-rasterization".into());
flags.push("--use-gl=angle".into());
flags.push("--use-angle=swiftshader".into());
}
}
if let Some(p) = proxy {
flags.push(format!("--proxy-server={}", p));
flags.push("--proxy-bypass-list=<-loopback>".into());
}
flags.push(format!("--user-data-dir={}", user_data_dir.display()));
if !extra.is_empty() {
flags.extend(extra.iter().cloned());
}
flags
}
fn ua_ch_brands(brands_json: &str, major: &str) -> Vec<UserAgentBrandVersion> {
if let Ok(val) = serde_json::from_str::<serde_json::Value>(brands_json) {
if let Some(arr) = val.as_array() {
let parsed: Vec<UserAgentBrandVersion> = arr
.iter()
.filter_map(|e| {
let brand = e.get("brand")?.as_str()?.to_string();
let version = e.get("version")?.as_str()?.to_string();
Some(UserAgentBrandVersion { brand, version })
})
.collect();
if !parsed.is_empty() {
return parsed;
}
}
}
vec![
UserAgentBrandVersion {
brand: "Google Chrome".into(),
version: major.to_string(),
},
UserAgentBrandVersion {
brand: "Chromium".into(),
version: major.to_string(),
},
UserAgentBrandVersion {
brand: "Not_A Brand".into(),
version: "24".into(),
},
]
}
#[allow(dead_code)]
fn build_accept_language(locale: Option<&str>) -> String {
match locale {
None => "en-US,en;q=0.9".to_string(),
Some(v) => {
let trimmed = v.trim();
if trimmed.contains(',') || trimmed.contains(';') {
trimmed.to_string()
} else if trimmed.starts_with("en") {
"en-US,en;q=0.9".to_string()
} else {
format!("{trimmed},en;q=0.9")
}
}
}
}
use crate::config::Config;
use crate::identity::IdentityBundle;
use crate::impersonate::Profile;
use crate::render::stealth::{
render_shim_from_bundle, render_shim_from_bundle_with_calibration,
render_worker_shim_from_bundle, CalibrationOverrides,
};
use crate::render::{RenderedPage, Renderer, WaitStrategy};
use crate::storage::Storage;
use crate::{Error, Result};
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
struct PersistedSessionState {
#[serde(default)]
cookies: Vec<PersistedCookie>,
#[serde(default)]
origins: Vec<PersistedOriginState>,
}
impl PersistedSessionState {
fn merge_cookies(&mut self, cookies: Vec<PersistedCookie>) {
let mut merged: HashMap<(String, String, String), PersistedCookie> = self
.cookies
.drain(..)
.map(|cookie| {
(
(
cookie.name.clone(),
cookie.domain.clone(),
cookie.path.clone(),
),
cookie,
)
})
.collect();
for cookie in cookies {
merged.insert(
(
cookie.name.clone(),
cookie.domain.clone(),
cookie.path.clone(),
),
cookie,
);
}
self.cookies = merged.into_values().collect();
self.cookies
.sort_by(|a, b| (&a.domain, &a.path, &a.name).cmp(&(&b.domain, &b.path, &b.name)));
}
fn upsert_origin(&mut self, state: PersistedOriginState) {
self.origins.retain(|entry| entry.origin != state.origin);
self.origins.push(state);
self.origins.sort_by(|a, b| a.origin.cmp(&b.origin));
}
fn origin_state(&self, origin: &str) -> Option<&PersistedOriginState> {
self.origins.iter().find(|entry| entry.origin == origin)
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
struct PersistedCookie {
name: String,
value: String,
domain: String,
path: String,
secure: bool,
http_only: bool,
#[serde(default)]
same_site: Option<CookieSameSite>,
#[serde(default)]
expires: Option<f64>,
#[serde(default)]
priority: Option<CookiePriority>,
#[serde(default)]
source_scheme: Option<CookieSourceScheme>,
#[serde(default)]
source_port: Option<i64>,
#[serde(default)]
same_party: Option<bool>,
#[serde(default)]
partition_key: Option<CookiePartitionKey>,
}
impl From<Cookie> for PersistedCookie {
fn from(cookie: Cookie) -> Self {
Self {
name: cookie.name,
value: cookie.value,
domain: cookie.domain,
path: cookie.path,
secure: cookie.secure,
http_only: cookie.http_only,
same_site: cookie.same_site,
expires: if cookie.session {
None
} else {
Some(cookie.expires)
},
priority: Some(cookie.priority),
source_scheme: Some(cookie.source_scheme),
source_port: Some(cookie.source_port),
same_party: None,
partition_key: cookie.partition_key,
}
}
}
impl PersistedCookie {
fn to_cookie_param(&self, fallback_url: &Url) -> Result<CookieParam> {
let cookie_url = self
.cookie_url()
.unwrap_or_else(|| fallback_url.as_str().to_string());
let mut builder = CookieParam::builder()
.name(self.name.clone())
.value(self.value.clone())
.url(cookie_url)
.domain(self.domain.clone())
.path(self.path.clone())
.secure(self.secure)
.http_only(self.http_only);
if let Some(same_site) = self.same_site.clone() {
builder = builder.same_site(same_site);
}
if let Some(expires) = self.expires {
builder = builder.expires(TimeSinceEpoch::new(expires));
}
if let Some(priority) = self.priority.clone() {
builder = builder.priority(priority);
}
if let Some(source_scheme) = self.source_scheme.clone() {
builder = builder.source_scheme(source_scheme);
}
if let Some(source_port) = self.source_port {
builder = builder.source_port(source_port);
}
if let Some(same_party) = self.same_party {
builder = builder.same_party(same_party);
}
if let Some(partition_key) = self.partition_key.clone() {
builder = builder.partition_key(partition_key);
}
builder
.build()
.map_err(|e| Error::Render(format!("cookie param build: {e}")))
}
fn cookie_url(&self) -> Option<String> {
let host = self.domain.trim_start_matches('.');
if host.is_empty() {
return None;
}
let scheme = if self.secure { "https" } else { "http" };
let path = if self.path.is_empty() {
"/"
} else {
&self.path
};
Some(format!("{scheme}://{host}{path}"))
}
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
struct PersistedOriginState {
#[serde(default)]
origin: String,
#[serde(default)]
local_storage: Vec<StorageEntry>,
#[serde(default)]
session_storage: Vec<StorageEntry>,
#[serde(default)]
manifest_url: Option<String>,
#[serde(default)]
service_workers: Vec<PersistedServiceWorker>,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
struct StorageEntry {
key: String,
value: String,
}
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
struct PersistedServiceWorker {
#[serde(default)]
scope: String,
#[serde(default)]
active_script_url: Option<String>,
#[serde(default)]
waiting_script_url: Option<String>,
#[serde(default)]
installing_script_url: Option<String>,
}
impl PersistedServiceWorker {
fn script_urls(&self) -> impl Iterator<Item = &str> {
[
self.active_script_url.as_deref(),
self.waiting_script_url.as_deref(),
self.installing_script_url.as_deref(),
]
.into_iter()
.flatten()
}
}
fn collect_post_body(
req: &crate::render::chrome_protocol::cdp::browser_protocol::network::Request,
) -> Vec<u8> {
let Some(entries) = req.post_data_entries.as_ref() else {
return Vec::new();
};
let mut out = Vec::new();
for entry in entries {
if let Some(b) = entry.bytes.as_ref() {
out.extend_from_slice(<crate::render::chrome_wire::Binary as AsRef<[u8]>>::as_ref(
b,
));
}
}
out
}
pub struct RenderPool {
config: Arc<Config>,
storage: Arc<dyn Storage>,
sem: Arc<Semaphore>,
browsers: dashmap::DashMap<String, Arc<Browser>>,
browser_locks: dashmap::DashMap<String, Arc<tokio::sync::Mutex<()>>>,
contexts: dashmap::DashMap<String, BrowserContextId>,
context_locks: dashmap::DashMap<String, Arc<tokio::sync::Mutex<()>>>,
session_states: dashmap::DashMap<String, PersistedSessionState>,
mru: Mutex<std::collections::VecDeque<String>>,
user_data_root: std::path::PathBuf,
effective_profile: Mutex<Option<Profile>>,
shim_cache: Mutex<Option<String>>,
worker_shim_cache: Mutex<Option<Arc<String>>>,
bundle_cache: Mutex<Option<Arc<IdentityBundle>>>,
resolved_chrome_path: tokio::sync::Mutex<Option<String>>,
#[cfg(feature = "lua-hooks")]
lua_host: parking_lot::RwLock<Option<Arc<crate::hooks::lua::LuaHookHost>>>,
counters: parking_lot::RwLock<Option<Arc<crate::metrics::Counters>>>,
page_pool: Arc<crate::render::page_pool::PagePool>,
telemetry_tracker: Arc<Mutex<crate::antibot::telemetry::TelemetryTracker>>,
render_scope: Arc<parking_lot::RwLock<crate::config::RenderSessionScope>>,
cdp_capabilities:
parking_lot::RwLock<Option<crate::render::cdp_capabilities::EndpointCapabilities>>,
calibration_cache: Arc<crate::render::calibration::CalibrationCache>,
calibration_origin: tokio::sync::OnceCell<crate::render::calibration::CalibrationOrigin>,
}
impl RenderPool {
pub fn new(config: Arc<Config>, storage: Arc<dyn Storage>) -> Self {
Self::new_with_scope(
config.clone(),
storage,
Arc::new(parking_lot::RwLock::new(config.render_session_scope)),
)
}
pub fn new_with_scope(
config: Arc<Config>,
storage: Arc<dyn Storage>,
render_scope: Arc<parking_lot::RwLock<crate::config::RenderSessionScope>>,
) -> Self {
let cap = config.max_concurrent_render.max(1);
let user_data_root =
std::env::temp_dir().join(format!("crawlex-chrome-{}", std::process::id()));
let _ = std::fs::create_dir_all(&user_data_root);
let page_pool_limits = crate::render::page_pool::PagePoolLimits {
max_pages_per_context: config.max_pages_per_context.max(1),
..Default::default()
};
Self {
config,
storage,
sem: Arc::new(Semaphore::new(cap)),
browsers: dashmap::DashMap::new(),
browser_locks: dashmap::DashMap::new(),
contexts: dashmap::DashMap::new(),
context_locks: dashmap::DashMap::new(),
session_states: dashmap::DashMap::new(),
mru: Mutex::new(std::collections::VecDeque::new()),
user_data_root,
effective_profile: Mutex::new(None),
shim_cache: Mutex::new(None),
worker_shim_cache: Mutex::new(None),
bundle_cache: Mutex::new(None),
resolved_chrome_path: tokio::sync::Mutex::new(None),
#[cfg(feature = "lua-hooks")]
lua_host: parking_lot::RwLock::new(None),
counters: parking_lot::RwLock::new(None),
page_pool: Arc::new(crate::render::page_pool::PagePool::new(page_pool_limits)),
telemetry_tracker: Arc::new(Mutex::new(
crate::antibot::telemetry::TelemetryTracker::new(),
)),
render_scope,
cdp_capabilities: parking_lot::RwLock::new(None),
calibration_cache: Arc::new(crate::render::calibration::CalibrationCache::new()),
calibration_origin: tokio::sync::OnceCell::new(),
}
}
pub fn calibration_cache(&self) -> &Arc<crate::render::calibration::CalibrationCache> {
&self.calibration_cache
}
pub async fn ensure_calibrated(
&self,
page: &Page,
key: crate::render::calibration::CalibrationKey,
) -> Result<Arc<crate::render::calibration::EffectiveFingerprint>> {
if let Some(hit) = self.calibration_cache.get(&key) {
return Ok(hit);
}
let origin = self
.calibration_origin
.get_or_try_init(|| async {
crate::render::calibration::serve_calibration_origin()
.await
.map_err(|e| Error::Render(format!("calibration origin: {e}")))
})
.await?;
let calibrate_url = origin.calibrate_url();
let nav = NavigateParams::builder()
.url(calibrate_url.clone())
.build()
.map_err(|e| Error::Render(format!("calibration NavigateParams: {e}")))?;
page.execute(nav)
.await
.map_err(|e| Error::Render(format!("calibration navigate: {e}")))?;
let params = EvaluateParams::builder()
.expression(crate::render::calibration::CALIBRATION_PROBE_JS.to_string())
.return_by_value(true)
.await_promise(true)
.build()
.map_err(|e| Error::Render(format!("calibration EvaluateParams: {e}")))?;
let res = page
.evaluate_expression(params)
.await
.map_err(|e| Error::Render(format!("calibration evaluate: {e}")))?;
let json = match res.value() {
Some(v) => v
.as_str()
.map(|s| s.to_string())
.unwrap_or_else(|| v.to_string()),
None => return Err(Error::Render("calibration probe returned no value".into())),
};
let mut fp = crate::render::calibration::parse_probe(&json).map_err(Error::Render)?;
fp.mismatch_count = crate::render::calibration::count_mismatches(
&fp,
self.config.locale.as_deref(),
self.config.timezone.as_deref(),
);
if fp.policy.is_empty() {
fp.policy = "report-only".to_string();
}
let summary = crate::render::calibration::CalibrationSummary {
browser_product: &fp.browser_product,
platform: &fp.platform,
locale: &fp.locale,
timezone: &fp.timezone,
webgl_renderer: if fp.webgl.unmasked_renderer.is_empty() {
fp.webgl.renderer.as_str()
} else {
fp.webgl.unmasked_renderer.as_str()
},
mismatch_count: fp.mismatch_count,
policy: &fp.policy,
};
let fingerprint_id = key.fingerprint_id();
tracing::info!(
event = "calibration.summary",
fingerprint_id = fingerprint_id.as_str(),
browser_product = summary.browser_product,
platform = summary.platform,
locale = summary.locale,
timezone = summary.timezone,
webgl_renderer = summary.webgl_renderer,
mismatch_count = summary.mismatch_count,
policy = summary.policy,
session_mode = self.config.external_cdp_session_mode.as_str(),
"external CDP session calibration"
);
if std::env::var("CRAWLEX_CALIBRATION_REPORT").as_deref() == Ok("full") {
let report = crate::render::calibration::CalibrationCache::format_full_report(&fp);
tracing::info!(
event = "calibration.report",
fingerprint_id = fingerprint_id.as_str(),
report = report.as_str(),
"external CDP session calibration (full report)"
);
}
Ok(self.calibration_cache.insert(key, fp))
}
fn expected_identity_for_session(&self) -> crate::render::calibration::ExpectedIdentity {
let bundle = self.bundle();
let platform = match bundle.ua_platform.trim_matches('"') {
"" => None,
other => Some(other.to_string()),
};
crate::render::calibration::ExpectedIdentity {
browser_family: Some("Chromium".to_string()),
browser_major: Some(bundle.ua_major.to_string()),
platform,
locale: self.config.locale.clone().filter(|s| !s.is_empty()),
timezone: self.config.timezone.clone().filter(|s| !s.is_empty()),
proxy_egress_ipv4: None,
profile_id: Some(bundle.id.clone()),
min_storage_quota: None,
}
}
fn calibration_key_for(
&self,
proxy: Option<&Url>,
session_id: &str,
ctx_id: Option<&BrowserContextId>,
) -> crate::render::calibration::CalibrationKey {
let bundle = self.bundle();
let ctx_part = match ctx_id {
Some(c) => format!("{session_id}|{}", c.inner()),
None => format!("{session_id}|<default>"),
};
crate::render::calibration::CalibrationKey {
endpoint: self.config.external_cdp_url.clone().unwrap_or_default(),
seed: bundle.canvas_audio_seed.to_string(),
proxy: proxy.map(|u| u.to_string()).unwrap_or_default(),
locale: self.config.locale.clone().unwrap_or_default(),
timezone: self.config.timezone.clone().unwrap_or_default(),
profile: bundle.id.clone(),
context: ctx_part,
session_mode: self.config.external_cdp_session_mode.as_str().to_string(),
}
}
pub fn set_counters(&self, counters: Arc<crate::metrics::Counters>) {
*self.counters.write() = Some(counters);
}
pub fn page_pool(&self) -> &crate::render::page_pool::PagePool {
&self.page_pool
}
fn counters_opt(&self) -> Option<Arc<crate::metrics::Counters>> {
self.counters.read().as_ref().cloned()
}
const CHROME_CANDIDATES: &'static [&'static str] = &[
"google-chrome",
"google-chrome-stable",
"google-chrome-beta",
"google-chrome-unstable",
"chromium",
"chromium-browser",
"chrome",
];
fn detect_chrome(chrome_path: Option<&str>) -> Option<(String, u32)> {
let candidates: Vec<String> = match chrome_path {
Some(p) => vec![p.to_string()],
None => Self::CHROME_CANDIDATES
.iter()
.map(|s| s.to_string())
.collect(),
};
for bin in candidates {
let out = match std::process::Command::new(&bin).arg("--version").output() {
Ok(o) => o,
Err(_) => continue,
};
let text = String::from_utf8_lossy(&out.stdout);
for tok in text.split_whitespace() {
if let Some(major) = tok.split('.').next() {
if let Ok(v) = major.parse::<u32>() {
if v >= 90 {
return Some((bin, v));
}
}
}
}
}
None
}
pub(crate) fn detect_chrome_major(chrome_path: Option<&str>) -> Option<u32> {
Self::detect_chrome(chrome_path).map(|(_, v)| v)
}
async fn resolve_chrome_path_async(&self) -> Result<String> {
{
let guard = self.resolved_chrome_path.lock().await;
if let Some(p) = guard.as_ref() {
return Ok(p.clone());
}
}
let resolved = self.resolve_uncached().await?;
*self.resolved_chrome_path.lock().await = Some(resolved.clone());
Ok(resolved)
}
async fn resolve_uncached(&self) -> Result<String> {
if let Some(p) = self.config.chrome_path.as_deref() {
if std::process::Command::new(p)
.arg("--version")
.output()
.is_ok()
{
return Ok(p.to_string());
}
return Err(Error::Render(format!(
"chrome_path `{p}` did not respond to --version"
)));
}
#[cfg(feature = "chromium-fetcher")]
{
if !self.config.auto_fetch_chromium {
return Err(Error::Render(
"auto-fetch disabled and no --chrome-path set; re-enable \
(remove --no-fetch-chromium) or pass an explicit path"
.into(),
));
}
return self.fetch_chromium().await;
}
#[cfg(not(feature = "chromium-fetcher"))]
{
Err(Error::Render(
"`chromium-fetcher` feature disabled — rebuild with it on \
or pass --chrome-path explicitly"
.into(),
))
}
}
#[cfg(feature = "chromium-fetcher")]
async fn fetch_chromium(&self) -> Result<String> {
use crate::render::chrome_fetcher::{BrowserFetcher, BrowserFetcherOptions};
let cache_dir = directories::ProjectDirs::from("", "", "crawlex")
.map(|p| p.cache_dir().join("chromium"))
.ok_or_else(|| {
Error::Render("could not determine user cache dir for Chromium fetch".into())
})?;
let _ = std::fs::create_dir_all(&cache_dir);
tracing::info!(
cache = %cache_dir.display(),
"no system Chrome found; fetching pinned Chromium-for-Testing (first run only)"
);
let opts = BrowserFetcherOptions::builder()
.with_path(&cache_dir)
.build()
.map_err(|e| Error::Render(format!("chromium fetcher options: {e}")))?;
let fetcher = BrowserFetcher::new(opts);
let install = fetcher
.fetch()
.await
.map_err(|e| Error::Render(format!("chromium fetch failed: {e}")))?;
let path = install.executable_path.to_string_lossy().to_string();
tracing::info!(path = %path, "Chromium ready");
Ok(path)
}
fn resolve_profile(&self) -> Profile {
if let Some(p) = *self.effective_profile.lock() {
return p;
}
let cfg_profile = self.config.user_agent_profile;
let resolved = if self.config.profile_autodetect {
match Self::detect_chrome_major(self.config.chrome_path.as_deref()) {
Some(major) if major != cfg_profile.major_version() => {
let p = Profile::from_detected_major(major);
tracing::info!(
detected = major,
configured = cfg_profile.major_version(),
chosen = p.major_version(),
"render pool auto-aligned profile to installed Chrome"
);
p
}
_ => cfg_profile,
}
} else {
cfg_profile
};
*self.effective_profile.lock() = Some(resolved);
resolved
}
fn bundle(&self) -> Arc<IdentityBundle> {
if let Some(b) = self.bundle_cache.lock().as_ref() {
return b.clone();
}
let profile = self.resolve_profile();
let seed = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos() as u64)
.unwrap_or(0xc0ffee);
let b = IdentityBundle::from_profile_with_overrides(
profile,
self.config.identity_preset,
self.config.locale.as_deref(),
self.config.timezone.as_deref(),
self.config.user_agent_override.as_deref(),
seed,
)
.expect("invalid render identity config");
let arc = Arc::new(b);
*self.bundle_cache.lock() = Some(arc.clone());
arc
}
fn shim_js(&self) -> String {
if let Some(s) = self.shim_cache.lock().as_ref() {
return s.clone();
}
let bundle = self.bundle();
let mut js = render_shim_from_bundle(&bundle);
if self.config.dom_capture.flatten_shadow_dom {
js.push_str(SHADOW_ROOT_CAPTURE_JS);
}
*self.shim_cache.lock() = Some(js.clone());
js
}
fn worker_shim_js(&self) -> Arc<String> {
if let Some(s) = self.worker_shim_cache.lock().as_ref() {
return s.clone();
}
let bundle = self.bundle();
let js = Arc::new(render_worker_shim_from_bundle(&bundle));
*self.worker_shim_cache.lock() = Some(js.clone());
js
}
#[cfg(feature = "lua-hooks")]
pub fn set_lua_host(&self, host: Arc<crate::hooks::lua::LuaHookHost>) {
*self.lua_host.write() = Some(host);
}
pub async fn preflight(&self) -> Result<String> {
let primary = self.effective_primary_provider();
let primary_err = match self.preflight_for_provider(primary).await {
Ok(v) => return Ok(v),
Err(e) => e,
};
let chain = self.config.provider_fallback.chain_after(primary);
if chain.is_empty() {
return Err(primary_err);
}
let original = primary_err.to_string();
let mut attempts: Vec<(crate::config::BrowserProvider, String)> = Vec::new();
for next in chain {
if !self.provider_usable_for_fallback(next) {
let reason = self.unusable_reason(next);
tracing::warn!(
event = "provider.fallback.skipped",
source = primary.as_str(),
destination = next.as_str(),
reason = %reason,
"render pool: provider fallback skipped — candidate not usable"
);
attempts.push((next, reason.to_string()));
continue;
}
tracing::warn!(
event = "provider.fallback",
source = primary.as_str(),
destination = next.as_str(),
reason = %original,
mismatch_policy = self.config.mismatch_policy.as_str(),
session_mode = self.config.external_cdp_session_mode.as_str(),
"render pool: provider fallback attempt"
);
match self.preflight_for_provider(next).await {
Ok(v) => return Ok(v),
Err(e) => attempts.push((next, e.to_string())),
}
}
let path_str = attempts
.iter()
.map(|(p, e)| format!("{}=> {}", p.as_str(), e))
.collect::<Vec<_>>()
.join("; ");
Err(Error::Render(format!(
"preflight failed for primary `{}`: {}; fallback attempted: [{}]",
primary.as_str(),
original,
path_str
)))
}
fn effective_primary_provider(&self) -> crate::config::BrowserProvider {
use crate::config::BrowserProvider;
match self.config.browser_provider {
BrowserProvider::Stock => BrowserProvider::Stock,
BrowserProvider::Cdp => BrowserProvider::Cdp,
BrowserProvider::Auto => {
if self.config.external_cdp_url.is_some() {
BrowserProvider::Cdp
} else {
BrowserProvider::Stock
}
}
}
}
fn provider_usable_for_fallback(&self, p: crate::config::BrowserProvider) -> bool {
use crate::config::BrowserProvider;
match p {
BrowserProvider::Cdp => self.config.external_cdp_url.is_some(),
BrowserProvider::Stock => true,
BrowserProvider::Auto => false,
}
}
fn unusable_reason(&self, p: crate::config::BrowserProvider) -> &'static str {
use crate::config::BrowserProvider;
match p {
BrowserProvider::Cdp => "no external_cdp_url configured",
BrowserProvider::Auto => "auto is a meta-selector, not a concrete fallback target",
BrowserProvider::Stock => "",
}
}
async fn preflight_for_provider(&self, p: crate::config::BrowserProvider) -> Result<String> {
use crate::config::BrowserProvider;
match p {
BrowserProvider::Cdp => {
let endpoint = self.config.external_cdp_url.as_deref().ok_or_else(|| {
Error::Render(
"external CDP preflight failed: external_cdp_url not configured".into(),
)
})?;
let probe = crate::render::cdp_probe::probe(endpoint)
.await
.map_err(|msg| {
Error::Render(format!("external CDP preflight failed: {msg}"))
})?;
let caps = crate::render::cdp_capabilities::EndpointCapabilities::detect(&probe);
tracing::info!(
event = "provider.selected",
provider = BrowserProvider::Cdp.as_str(),
endpoint = endpoint,
external_cdp_url = endpoint,
ws_debugger_url = probe.web_socket_debugger_url.as_str(),
browser = probe.browser.as_str(),
endpoint_kind = caps.kind.as_str(),
identity_params = caps.identity_params,
vendor = caps.vendor.as_deref().unwrap_or(""),
session_mode = self.config.external_cdp_session_mode.as_str(),
"render pool: using external CDP endpoint"
);
*self.cdp_capabilities.write() = Some(caps);
Ok(endpoint.to_string())
}
BrowserProvider::Stock => {
let path = self.resolve_chrome_path_async().await?;
tracing::info!(
event = "provider.selected",
provider = BrowserProvider::Stock.as_str(),
chrome = %path,
"render pool: using Chrome binary"
);
Ok(path)
}
BrowserProvider::Auto => Err(Error::Render(
"internal: Auto is not a concrete preflight target".into(),
)),
}
}
fn browser_key_for(&self, proxy: Option<&Url>) -> String {
self.config
.external_cdp_url
.as_ref()
.map(|u| format!("external:{u}"))
.unwrap_or_else(|| proxy.map(|u| u.to_string()).unwrap_or_default())
}
fn spawn_child_shim_watcher(&self, browser: Arc<Browser>) {
let shim = self.shim_js();
tokio::spawn(async move {
let mut seen: std::collections::HashSet<
crate::render::chrome_protocol::cdp::browser_protocol::target::TargetId,
> = std::collections::HashSet::new();
let install = AddScriptToEvaluateOnNewDocumentParams {
source: shim,
world_name: None,
include_command_line_api: Some(false),
run_immediately: Some(true),
};
loop {
tokio::time::sleep(Duration::from_millis(250)).await;
let pages = match browser.pages().await {
Ok(p) => p,
Err(_) => break,
};
for p in pages {
let id = p.target_id().clone();
if seen.insert(id.clone()) {
if let Err(e) = p.execute(install.clone()).await {
tracing::debug!(target = ?id, ?e, "shim install on child target failed (likely worker — handled via Runtime.evaluate)");
} else {
tracing::debug!(target = ?id, "stealth shim installed on child target");
}
}
}
}
});
}
fn register_browser(&self, key: String, browser: Arc<Browser>) {
self.browsers.insert(key.clone(), browser);
if let Some(c) = self.counters_opt() {
c.browsers_active
.store(self.browsers.len(), std::sync::atomic::Ordering::Relaxed);
}
let max_browsers = self.config.max_browsers.max(1);
{
let mut mru = self.mru.lock();
mru.retain(|k| k != &key);
mru.push_front(key.clone());
while mru.len() > max_browsers {
if let Some(old) = mru.pop_back() {
if let Some((_, b)) = self.browsers.remove(&old) {
drop(b);
self.browser_locks.remove(&old);
let prefix = if old.is_empty() {
String::new()
} else {
format!("{old}|")
};
let ctx_keys: Vec<String> = self
.contexts
.iter()
.filter(|kv| {
if prefix.is_empty() {
!kv.key().contains('|')
} else {
kv.key().starts_with(&prefix)
}
})
.map(|kv| kv.key().clone())
.collect();
for ck in &ctx_keys {
self.page_pool.drop_context(ck);
}
if prefix.is_empty() {
self.contexts.retain(|k, _| k.contains('|'));
self.context_locks.retain(|k, _| k.contains('|'));
} else {
self.contexts.retain(|k, _| !k.starts_with(&prefix));
self.context_locks.retain(|k, _| !k.starts_with(&prefix));
}
tracing::debug!(proxy = %old, "render pool evicted browser");
}
}
}
}
if let Some(c) = self.counters_opt() {
c.browsers_active
.store(self.browsers.len(), std::sync::atomic::Ordering::Relaxed);
c.contexts_active
.store(self.contexts.len(), std::sync::atomic::Ordering::Relaxed);
}
}
async fn ensure_browser(&self, proxy: Option<&Url>) -> Result<Arc<Browser>> {
let key = self.browser_key_for(proxy);
if let Some(b) = self.browsers.get(&key) {
return Ok(b.clone());
}
let lock = self
.browser_locks
.entry(key.clone())
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
.clone();
let _guard = lock.lock().await;
if let Some(b) = self.browsers.get(&key) {
return Ok(b.clone());
}
if let Some(endpoint) = self.config.external_cdp_url.as_deref() {
if proxy.is_some() {
tracing::debug!(
external_cdp_url = endpoint,
"external CDP is configured; per-job proxy launch flags are owned by the external browser"
);
}
let stealth = crate::render::chrome::browser::StealthConfig {
runtime_enable_skip: true,
worker_init_script: Some(self.worker_shim_js()),
};
let handler_config = HandlerConfig {
request_timeout: Duration::from_secs(60),
stealth,
..Default::default()
};
let proxy_str = proxy.map(|u| u.to_string());
let connect_url = {
let caps_guard = self.cdp_capabilities.read();
let caps = caps_guard
.clone()
.unwrap_or_else(crate::render::cdp_capabilities::EndpointCapabilities::generic);
let hints = crate::render::cdp_capabilities::IdentityHints {
seed: None,
timezone: self.config.timezone.as_deref(),
locale: self.config.locale.as_deref(),
proxy: proxy_str.as_deref(),
geoip: None,
};
caps.build_connect_url(endpoint, &hints)
.map_err(|e| Error::Render(format!("external cdp connect url: {e}")))?
};
let (browser, mut handler) =
Browser::connect_with_config(connect_url.as_str(), handler_config)
.await
.map_err(|e| Error::Render(format!("external cdp connect: {e}")))?;
tokio::spawn(async move {
while let Some(ev) = handler.next().await {
if let Err(e) = ev {
warn!(?e, "cdp handler error");
}
}
});
let auto_attach = SetAutoAttachParams::builder()
.auto_attach(true)
.wait_for_debugger_on_start(false)
.flatten(true)
.build()
.map_err(|e| Error::Render(format!("setAutoAttach build: {e}")))?;
if let Err(e) = browser.execute(auto_attach).await {
warn!(
?e,
"Target.setAutoAttach failed; child targets won't get the stealth shim"
);
}
let browser = Arc::new(browser);
self.spawn_child_shim_watcher(browser.clone());
self.register_browser(key, browser.clone());
return Ok(browser);
}
let bundle = self.bundle();
let chrome_path = self.resolve_chrome_path_async().await?;
let mut builder = BrowserConfig::builder();
builder = builder.chrome_executable(&chrome_path);
builder = builder.no_sandbox();
let languages = bundle.chrome_lang_arg();
let udd_name = {
use sha2::Digest as _;
let h = sha2::Sha256::digest(key.as_bytes());
hex::encode(&h[..8])
};
let udd = self.user_data_root.join(udd_name);
let _ = std::fs::create_dir_all(&udd);
let flags = build_launch_args_with_gpu(
&bundle,
proxy,
&udd,
&languages,
self.config.gpu_policy,
&self.config.chrome_flags,
);
builder = builder.args(flags);
builder = builder.request_timeout(Duration::from_secs(60));
builder = builder.stealth_runtime_enable_skip(true);
builder = builder.stealth_worker_shim((*self.worker_shim_js()).clone());
let cfg = builder
.build()
.map_err(|e| Error::Render(format!("BrowserConfig: {e}")))?;
let (browser, mut handler) = Browser::launch(cfg)
.await
.map_err(|e| Error::Render(format!("launch: {e}")))?;
tokio::spawn(async move {
while let Some(ev) = handler.next().await {
if let Err(e) = ev {
warn!(?e, "cdp handler error");
}
}
});
let auto_attach = SetAutoAttachParams::builder()
.auto_attach(true)
.wait_for_debugger_on_start(false)
.flatten(true)
.build()
.map_err(|e| Error::Render(format!("setAutoAttach build: {e}")))?;
if let Err(e) = browser.execute(auto_attach).await {
warn!(
?e,
"Target.setAutoAttach failed; child targets won't get the stealth shim"
);
}
let browser = Arc::new(browser);
{
let bro = browser.clone();
let shim = self.shim_js();
tokio::spawn(async move {
let mut seen: std::collections::HashSet<
crate::render::chrome_protocol::cdp::browser_protocol::target::TargetId,
> = std::collections::HashSet::new();
let install = AddScriptToEvaluateOnNewDocumentParams {
source: shim,
world_name: None,
include_command_line_api: Some(false),
run_immediately: Some(true),
};
loop {
tokio::time::sleep(Duration::from_millis(250)).await;
let pages = match bro.pages().await {
Ok(p) => p,
Err(_) => break,
};
for p in pages {
let id = p.target_id().clone();
if seen.insert(id.clone()) {
if let Err(e) = p.execute(install.clone()).await {
tracing::debug!(target = ?id, ?e, "shim install on child target failed (likely worker — handled via Runtime.evaluate)");
} else {
tracing::debug!(target = ?id, "stealth shim installed on child target");
}
}
}
}
});
}
self.browsers.insert(key.clone(), browser.clone());
if let Some(c) = self.counters_opt() {
c.browsers_active
.store(self.browsers.len(), std::sync::atomic::Ordering::Relaxed);
}
let max_browsers = self.config.max_browsers.max(1);
{
let mut mru = self.mru.lock();
mru.retain(|k| k != &key);
mru.push_front(key.clone());
while mru.len() > max_browsers {
if let Some(old) = mru.pop_back() {
if let Some((_, b)) = self.browsers.remove(&old) {
drop(b);
self.browser_locks.remove(&old);
let prefix = if old.is_empty() {
String::new()
} else {
format!("{old}|")
};
let ctx_keys: Vec<String> = self
.contexts
.iter()
.filter(|kv| {
if prefix.is_empty() {
!kv.key().contains('|')
} else {
kv.key().starts_with(&prefix)
}
})
.map(|kv| kv.key().clone())
.collect();
for ck in &ctx_keys {
self.page_pool.drop_context(ck);
}
if prefix.is_empty() {
self.contexts.retain(|k, _| k.contains('|'));
self.context_locks.retain(|k, _| k.contains('|'));
} else {
self.contexts.retain(|k, _| !k.starts_with(&prefix));
self.context_locks.retain(|k, _| !k.starts_with(&prefix));
}
tracing::debug!(proxy = %old, "render pool evicted browser");
}
}
}
}
if let Some(c) = self.counters_opt() {
c.browsers_active
.store(self.browsers.len(), std::sync::atomic::Ordering::Relaxed);
c.contexts_active
.store(self.contexts.len(), std::sync::atomic::Ordering::Relaxed);
}
Ok(browser)
}
fn session_id_for_url(&self, url: &Url) -> String {
let host = url.host_str().unwrap_or_default();
let safe = |scope: &str| {
scope.replace(
|c: char| !(c.is_ascii_alphanumeric() || c == '.' || c == '-'),
"_",
)
};
match *self.render_scope.read() {
crate::config::RenderSessionScope::RegistrableDomain => {
let scope =
crate::discovery::subdomains::registrable_domain(host).unwrap_or_else(|| {
if host.is_empty() {
let digest = sha2::Sha256::digest(url.as_str().as_bytes());
format!("url-{}", hex::encode(&digest[..8]))
} else if let Some(port) = url.port() {
format!("{host}-{port}")
} else {
host.to_string()
}
});
format!("render-{}", safe(&scope))
}
crate::config::RenderSessionScope::Host => {
let scope = if host.is_empty() {
let digest = sha2::Sha256::digest(url.as_str().as_bytes());
format!("url-{}", hex::encode(&digest[..8]))
} else if let Some(port) = url.port() {
format!("{host}-{port}")
} else {
host.to_string()
};
format!("render-host-{}", safe(&scope))
}
crate::config::RenderSessionScope::Origin => {
let origin = url.origin().ascii_serialization();
let digest = sha2::Sha256::digest(origin.as_bytes());
let label = if origin.is_empty() || origin == "null" {
"null".to_string()
} else {
safe(&origin)
};
format!("render-origin-{label}-{}", hex::encode(&digest[..6]))
}
crate::config::RenderSessionScope::Url => {
let digest = sha2::Sha256::digest(url.as_str().as_bytes());
format!("render-url-{}", hex::encode(&digest[..8]))
}
}
}
fn browser_session_key(browser_key: &str, session_id: &str) -> String {
if browser_key.is_empty() {
session_id.to_string()
} else {
format!("{browser_key}|{session_id}")
}
}
async fn ensure_session_state_loaded(&self, session_id: &str) -> Result<()> {
if self.session_states.contains_key(session_id) {
return Ok(());
}
let state = match self.storage.load_state(session_id).await? {
Some(json) => serde_json::from_str::<PersistedSessionState>(&json)
.unwrap_or_else(|_| PersistedSessionState::default()),
None => PersistedSessionState::default(),
};
self.session_states.insert(session_id.to_string(), state);
Ok(())
}
async fn ensure_session_context(
&self,
browser_key: &str,
browser: &Browser,
session_id: &str,
) -> Result<Option<BrowserContextId>> {
if self.is_persistent_external_cdp() {
return Ok(None);
}
self.ensure_session_state_loaded(session_id).await?;
let key = Self::browser_session_key(browser_key, session_id);
if let Some(ctx_id) = self.contexts.get(&key) {
return Ok(Some(ctx_id.clone()));
}
let lock = self
.context_locks
.entry(key.clone())
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
.clone();
let _guard = lock.lock().await;
if let Some(ctx_id) = self.contexts.get(&key) {
return Ok(Some(ctx_id.clone()));
}
let ctx_id = browser
.create_browser_context(CreateBrowserContextParams::default())
.await
.map_err(|e| Error::Render(format!("create_browser_context: {e}")))?;
self.contexts.insert(key, ctx_id.clone());
if let Some(c) = self.counters_opt() {
c.contexts_active
.store(self.contexts.len(), std::sync::atomic::Ordering::Relaxed);
}
Ok(Some(ctx_id))
}
fn is_persistent_external_cdp(&self) -> bool {
self.config.external_cdp_url.is_some()
&& self.config.external_cdp_session_mode.is_persistent()
}
pub async fn drop_session(&self, session_id: &str) {
if self.is_persistent_external_cdp() {
tracing::debug!(
event = "session.drop.skipped",
session_id,
session_mode = self.config.external_cdp_session_mode.as_str(),
"persistent external CDP session — backend state preserved"
);
return;
}
let mut to_dispose: Vec<(String, String, BrowserContextId)> = Vec::new();
let suffix = format!("|{session_id}");
for entry in self.contexts.iter() {
let key = entry.key();
let browser_key = if key == session_id {
String::new()
} else if let Some(prefix) = key.strip_suffix(&suffix) {
prefix.to_string()
} else {
continue;
};
to_dispose.push((browser_key, key.clone(), entry.value().clone()));
}
for (browser_key, ctx_key, ctx_id) in to_dispose {
self.page_pool.drop_context(&ctx_key);
if let Some(browser) = self.browsers.get(&browser_key).map(|b| b.clone()) {
Self::clear_chrome_supercookies(&browser, session_id).await;
if let Err(e) = browser.dispose_browser_context(ctx_id).await {
tracing::debug!(session_id, ?e, "dispose_browser_context failed");
}
}
self.contexts.remove(&ctx_key);
self.context_locks.remove(&ctx_key);
}
self.session_states.remove(session_id);
if let Some(c) = self.counters_opt() {
c.contexts_active
.store(self.contexts.len(), std::sync::atomic::Ordering::Relaxed);
}
}
async fn clear_chrome_supercookies(browser: &Browser, session_id: &str) {
if let Err(e) = browser
.execute(SetBypassServiceWorkerParams::new(true))
.await
{
warn!(
session_id,
?e,
"Network.setBypassServiceWorker unsupported — continuing rotation"
);
}
if let Err(e) = browser
.execute(SetIgnoreCertificateErrorsParams::new(false))
.await
{
warn!(
session_id,
?e,
"Security.setIgnoreCertificateErrors unsupported — continuing rotation"
);
}
if let Err(e) = browser
.execute(ClearDataForOriginParams::new("*", "all"))
.await
{
warn!(
session_id,
?e,
"Storage.clearDataForOrigin unsupported — supercookies may persist"
);
}
if let Err(e) = browser
.execute(ServiceWorkerUnregisterParams::new("*"))
.await
{
tracing::debug!(
session_id,
?e,
"ServiceWorker.unregister(\"*\") returned — expected when no SW matched"
);
}
if let Err(e) = browser.execute(ClearBrowserCacheParams::default()).await {
warn!(
session_id,
?e,
"Network.clearBrowserCache unsupported — continuing rotation"
);
}
}
fn session_state(&self, session_id: &str) -> PersistedSessionState {
self.session_states
.get(session_id)
.map(|entry| entry.clone())
.unwrap_or_default()
}
async fn restore_session_state(
&self,
page: &Page,
target_url: &Url,
session_id: &str,
) -> Result<()> {
let state = self.session_state(session_id);
if state.cookies.is_empty() && state.origins.is_empty() {
return Ok(());
}
let cookie_params: Vec<CookieParam> = state
.cookies
.iter()
.filter_map(|cookie| cookie.to_cookie_param(target_url).ok())
.collect();
if !cookie_params.is_empty() {
if let Err(e) = page.set_cookies(cookie_params).await {
debug!(session_id, ?e, "cookie restore failed");
}
}
let origin = target_url.origin().ascii_serialization();
let Some(origin_state) = state.origin_state(&origin) else {
return Ok(());
};
let payload = serde_json::to_string(origin_state)
.map_err(|e| Error::Render(format!("state payload encode: {e}")))?;
let script = format!(
r#"(function() {{
const state = {payload};
const restore = (storage, items) => {{
try {{
storage.clear();
for (const item of items || []) {{
storage.setItem(item.key, item.value);
}}
}} catch (_) {{}}
}};
try {{
if (location.origin === state.origin) {{
restore(window.localStorage, state.local_storage);
restore(window.sessionStorage, state.session_storage);
}}
}} catch (_) {{}}
}})();"#,
);
page.execute(AddScriptToEvaluateOnNewDocumentParams {
source: script,
world_name: None,
include_command_line_api: Some(false),
run_immediately: Some(true),
})
.await
.map_err(|e| Error::Render(format!("state restore inject: {e}")))?;
Ok(())
}
async fn capture_session_state(
&self,
page: &Page,
final_url: &Url,
session_id: &str,
) -> Result<PersistedOriginState> {
let cookies = match page.get_cookies().await {
Ok(cookies) => cookies.into_iter().map(PersistedCookie::from).collect(),
Err(e) => {
debug!(session_id, ?e, "cookie capture failed");
Vec::new()
}
};
let origin = final_url.origin().ascii_serialization();
let origin_state = if !origin.is_empty() && origin != "null" {
Self::capture_origin_state(page, &origin)
.await
.unwrap_or_else(|e| {
debug!(session_id, ?e, "origin state capture failed");
PersistedOriginState::default()
})
} else {
PersistedOriginState::default()
};
let mut state = self.session_state(session_id);
state.merge_cookies(cookies);
if !origin.is_empty() && origin != "null" {
state.upsert_origin(origin_state.clone());
}
let json = serde_json::to_string(&state)
.map_err(|e| Error::Render(format!("state encode: {e}")))?;
self.storage.save_state(session_id, &json).await?;
self.session_states.insert(session_id.to_string(), state);
Ok(origin_state)
}
async fn capture_origin_state(page: &Page, origin: &str) -> Result<PersistedOriginState> {
let params = EvaluateParams::builder()
.expression(
r#"(() => ({
manifest_url: (() => {
try {
for (const link of document.querySelectorAll('link[rel]')) {
const rel = (link.getAttribute('rel') || '').toLowerCase();
if (rel.split(/\s+/).includes('manifest') && typeof link.href === 'string') {
return link.href;
}
}
} catch (_) {}
return null;
})(),
local_storage: (() => {
try {
const out = [];
for (let i = 0; i < window.localStorage.length; i++) {
const key = window.localStorage.key(i);
if (key !== null) out.push({ key, value: window.localStorage.getItem(key) ?? '' });
}
return out;
} catch (_) {
return [];
}
})(),
session_storage: (() => {
try {
const out = [];
for (let i = 0; i < window.sessionStorage.length; i++) {
const key = window.sessionStorage.key(i);
if (key !== null) out.push({ key, value: window.sessionStorage.getItem(key) ?? '' });
}
return out;
} catch (_) {
return [];
}
})(),
service_workers: await (async () => {
try {
const api = navigator.serviceWorker;
if (!api || typeof api.getRegistrations !== 'function') return [];
const regs = await api.getRegistrations();
return regs.map((reg) => ({
scope: reg.scope || '',
active_script_url: reg.active?.scriptURL ?? null,
waiting_script_url: reg.waiting?.scriptURL ?? null,
installing_script_url: reg.installing?.scriptURL ?? null,
}));
} catch (_) {
return [];
}
})(),
}))()"#,
)
.await_promise(true)
.return_by_value(true)
.build()
.map_err(|e| Error::Render(format!("origin state params: {e}")))?;
let res = page
.evaluate_expression(params)
.await
.map_err(|e| Error::Render(format!("origin state eval: {e}")))?;
let mut state = res
.value()
.cloned()
.map(serde_json::from_value::<PersistedOriginState>)
.transpose()
.map_err(|e| Error::Render(format!("origin state decode: {e}")))?
.unwrap_or_default();
state.origin = origin.to_string();
Ok(state)
}
async fn install_stealth(&self, page: &Page) -> Result<()> {
self.install_stealth_with_overrides(page, None).await
}
async fn install_stealth_with_overrides(
&self,
page: &Page,
overrides: Option<&CalibrationOverrides>,
) -> Result<()> {
let source = match overrides {
Some(o) if !o.is_empty() => {
let bundle = self.bundle();
let mut js = render_shim_from_bundle_with_calibration(&bundle, Some(o));
if self.config.dom_capture.flatten_shadow_dom {
js.push_str(SHADOW_ROOT_CAPTURE_JS);
}
js
}
_ => self.shim_js(),
};
page.execute(AddScriptToEvaluateOnNewDocumentParams {
source,
world_name: None,
include_command_line_api: Some(false),
run_immediately: Some(true),
})
.await
.map_err(|e| Error::Render(format!("stealth inject: {e}")))?;
Ok(())
}
fn calibration_overrides_from_fingerprint(
fp: &crate::render::calibration::EffectiveFingerprint,
) -> CalibrationOverrides {
fn s(v: &str) -> Option<String> {
let t = v.trim();
if t.is_empty() {
None
} else {
Some(t.to_string())
}
}
let mut o = CalibrationOverrides::default();
o.user_agent = s(&fp.user_agent);
o.platform = s(&fp.platform);
o.locale = s(&fp.locale);
o.timezone = s(&fp.timezone);
o.webgl_unmasked_vendor = s(&fp.webgl.unmasked_vendor);
o.webgl_unmasked_renderer = s(&fp.webgl.unmasked_renderer);
o.webgpu_adapter_description = fp.webgpu_adapter.as_deref().and_then(s);
if let Some(pm) = fp.performance_memory.as_ref() {
if pm.js_heap_size_limit > 0 {
o.heap_size_limit = Some(pm.js_heap_size_limit);
}
}
let mut mics = 0u8;
let mut cams = 0u8;
let mut spk = 0u8;
for d in &fp.media_devices {
let l = d.to_ascii_lowercase();
if l.contains("audioinput") {
mics = mics.saturating_add(1);
} else if l.contains("videoinput") {
cams = cams.saturating_add(1);
} else if l.contains("audiooutput") {
spk = spk.saturating_add(1);
}
}
if mics > 0 {
o.media_mic_count = Some(mics);
}
if cams > 0 {
o.media_cam_count = Some(cams);
}
if spk > 0 {
o.media_speaker_count = Some(spk);
}
o
}
async fn install_spa_observer(&self, page: &Page) -> Result<()> {
if !self.config.collect_runtime_routes && !self.config.collect_network_endpoints {
return Ok(());
}
page.execute(AddScriptToEvaluateOnNewDocumentParams {
source: crate::render::spa_observer::observer_js(),
world_name: None,
include_command_line_api: Some(false),
run_immediately: Some(true),
})
.await
.map_err(|e| Error::Render(format!("spa observer inject: {e}")))?;
Ok(())
}
async fn reinject_spa_observer(&self, page: &Page) {
if !self.config.collect_runtime_routes && !self.config.collect_network_endpoints {
return;
}
let Ok(params) = EvaluateParams::builder()
.expression(crate::render::spa_observer::observer_js())
.return_by_value(true)
.build()
else {
return;
};
if let Err(e) = page.evaluate_expression(params).await {
tracing::debug!(?e, "spa observer post-nav reinject failed");
}
}
async fn prepare_dom_capture(&self, page: &Page) {
if !self.config.dom_capture.remove_overlays
&& !self.config.dom_capture.remove_consent_popups
{
return;
}
let js = DOM_CLEANUP_JS
.replace(
"__REMOVE_CONSENT__",
if self.config.dom_capture.remove_consent_popups {
"true"
} else {
"false"
},
)
.replace(
"__REMOVE_OVERLAYS__",
if self.config.dom_capture.remove_overlays {
"true"
} else {
"false"
},
);
if let Err(e) = crate::render::interact::eval_js(page, &js).await {
tracing::debug!(?e, "dom capture cleanup failed");
}
}
async fn capture_dom_html(&self, page: &Page) -> Result<String> {
if self.config.dom_capture.flatten_shadow_dom {
match crate::render::interact::eval_js(page, FLATTEN_SHADOW_DOM_JS).await {
Ok(v) => {
if let Some(html) = v.as_str() {
return Ok(html.to_string());
}
}
Err(e) => tracing::debug!(?e, "shadow DOM flatten capture failed"),
}
}
page.content()
.await
.map_err(|e| Error::Render(format!("content: {e}")))
}
pub(crate) async fn collect_spa_observations(
page: &Page,
) -> crate::render::spa_observer::CollectedObservations {
let Ok(params) = EvaluateParams::builder()
.expression(crate::render::spa_observer::collect_expression())
.await_promise(true)
.return_by_value(true)
.build()
else {
return Default::default();
};
let res = match page.evaluate_expression(params).await {
Ok(r) => r,
Err(e) => {
tracing::debug!(?e, "spa observer collect eval failed");
return Default::default();
}
};
match res.value().cloned() {
Some(v) => match serde_json::from_value::<
crate::render::spa_observer::CollectedObservations,
>(v.clone())
{
Ok(o) => o,
Err(e) => {
tracing::debug!(?e, value=%v, "spa observer decode failed");
Default::default()
}
},
None => {
tracing::debug!("spa observer collect returned no value");
Default::default()
}
}
}
async fn install_vitals_observers(page: &Page) -> Result<()> {
const OBSERVERS_JS: &str = r#"
(() => {
try {
window.__mb_cls = 0;
window.__mb_lcp = 0;
window.__mb_tbt = 0;
window.__mb_fcp = null;
window.__mb_longest = 0;
new PerformanceObserver((list) => {
for (const e of list.getEntries()) {
if (!e.hadRecentInput) window.__mb_cls += e.value;
}
}).observe({ type: 'layout-shift', buffered: true });
new PerformanceObserver((list) => {
for (const e of list.getEntries()) {
if (e.startTime > window.__mb_lcp) window.__mb_lcp = e.startTime;
}
}).observe({ type: 'largest-contentful-paint', buffered: true });
new PerformanceObserver((list) => {
for (const e of list.getEntries()) {
const blocking = e.duration - 50;
if (blocking > 0) window.__mb_tbt += blocking;
if (e.duration > window.__mb_longest) window.__mb_longest = e.duration;
}
}).observe({ type: 'longtask', buffered: true });
new PerformanceObserver((list) => {
for (const e of list.getEntries()) {
if (e.name === 'first-contentful-paint') window.__mb_fcp = e.startTime;
}
}).observe({ type: 'paint', buffered: true });
} catch (_) {}
})();
"#;
page.execute(AddScriptToEvaluateOnNewDocumentParams {
source: OBSERVERS_JS.to_string(),
world_name: None,
include_command_line_api: Some(false),
run_immediately: Some(true),
})
.await
.map_err(|e| Error::Render(format!("observer inject: {e}")))?;
Ok(())
}
async fn probe_inp(page: &Page) -> Result<Option<f64>> {
let params = EvaluateParams::builder()
.expression(crate::metrics::INP_PROBE_JS)
.await_promise(true)
.return_by_value(true)
.build()
.map_err(|e| Error::Render(format!("inp params: {e}")))?;
let res = page
.evaluate_expression(params)
.await
.map_err(|e| Error::Render(format!("inp eval: {e}")))?;
let Some(v) = res.value() else {
return Ok(None);
};
Ok(v.as_f64())
}
async fn collect_vitals(page: &Page) -> Result<crate::metrics::WebVitals> {
let params = EvaluateParams::builder()
.expression(crate::metrics::WEB_VITALS_JS)
.await_promise(true)
.return_by_value(true)
.build()
.map_err(|e| Error::Render(format!("vitals params: {e}")))?;
let res = page
.evaluate_expression(params)
.await
.map_err(|e| Error::Render(format!("vitals eval: {e}")))?;
let Some(v) = res.value() else {
return Ok(crate::metrics::WebVitals::default());
};
serde_json::from_value::<crate::metrics::WebVitals>(v.clone())
.map_err(|e| Error::Render(format!("vitals decode: {e}")))
}
async fn wait_network_idle(page: &Page, idle_ms: u64) -> Result<()> {
let mut starts = page
.event_listener::<EventRequestWillBeSent>()
.await
.map_err(|e| Error::Render(format!("listener start: {e}")))?;
let mut finishes = page
.event_listener::<EventLoadingFinished>()
.await
.map_err(|e| Error::Render(format!("listener finish: {e}")))?;
let mut fails = page
.event_listener::<EventLoadingFailed>()
.await
.map_err(|e| Error::Render(format!("listener fail: {e}")))?;
let ceiling = tokio::time::sleep(Duration::from_secs(30));
tokio::pin!(ceiling);
let mut in_flight: i64 = 0;
let idle_window = Duration::from_millis(idle_ms);
loop {
let quiet_timer = tokio::time::sleep(idle_window);
tokio::pin!(quiet_timer);
tokio::select! {
_ = &mut ceiling => {
return Ok(());
}
_ = starts.next() => {
in_flight = in_flight.saturating_add(1);
}
_ = finishes.next() => {
in_flight = in_flight.saturating_sub(1).max(0);
}
_ = fails.next() => {
in_flight = in_flight.saturating_sub(1).max(0);
}
_ = &mut quiet_timer, if in_flight == 0 => {
return Ok(());
}
}
}
}
pub(crate) async fn wait_for(page: &Page, wait: &WaitStrategy) -> Result<()> {
match wait {
WaitStrategy::Load | WaitStrategy::DomContentLoaded => {
page.wait_for_navigation()
.await
.map_err(|e| Error::Render(format!("wait_for_navigation: {e}")))?;
}
WaitStrategy::NetworkIdle { idle_ms } => {
page.wait_for_navigation()
.await
.map_err(|e| Error::Render(format!("wait_for_navigation: {e}")))?;
Self::wait_network_idle(page, *idle_ms).await?;
}
WaitStrategy::Selector { css, timeout_ms } => {
page.wait_for_navigation()
.await
.map_err(|e| Error::Render(format!("wait_for_navigation: {e}")))?;
let deadline = std::time::Instant::now() + Duration::from_millis(*timeout_ms);
loop {
if page.find_element(css.clone()).await.is_ok() {
break;
}
if std::time::Instant::now() >= deadline {
return Err(Error::Render(format!("selector timeout: {css}")));
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
}
WaitStrategy::Fixed { ms } => {
page.wait_for_navigation()
.await
.map_err(|e| Error::Render(format!("wait_for_navigation: {e}")))?;
tokio::time::sleep(Duration::from_millis(*ms)).await;
}
WaitStrategy::ReadingDwell { .. } => {
page.wait_for_navigation()
.await
.map_err(|e| Error::Render(format!("wait_for_navigation: {e}")))?;
}
}
Ok(())
}
async fn wait_for_ready_state(page: &Page, states: &[&str], timeout: Duration) -> Result<()> {
let deadline = std::time::Instant::now() + timeout;
let params = EvaluateParams::builder()
.expression("document.readyState")
.return_by_value(true)
.build()
.map_err(|e| Error::Render(format!("readyState params: {e}")))?;
loop {
match page.evaluate_expression(params.clone()).await {
Ok(res) => {
if let Some(state) = res.value().and_then(|v| v.as_str()) {
if states.contains(&state) {
return Ok(());
}
}
}
Err(e) => {
debug!(?e, "readyState probe failed while settling page");
}
}
if std::time::Instant::now() >= deadline {
return Ok(());
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
}
pub(crate) async fn settle_after_actions(page: &Page, wait: &WaitStrategy) -> Result<()> {
Self::settle_after_actions_with_dwell(page, wait, None).await
}
pub(crate) async fn settle_after_actions_with_dwell(
page: &Page,
wait: &WaitStrategy,
reading_dwell: Option<&crate::config::ReadingDwellConfig>,
) -> Result<()> {
match wait {
WaitStrategy::Load => {
Self::wait_for_ready_state(page, &["complete"], Duration::from_secs(10)).await?;
tokio::time::sleep(Duration::from_millis(150)).await;
}
WaitStrategy::DomContentLoaded => {
Self::wait_for_ready_state(
page,
&["interactive", "complete"],
Duration::from_secs(10),
)
.await?;
tokio::time::sleep(Duration::from_millis(150)).await;
}
WaitStrategy::NetworkIdle { idle_ms } => {
Self::wait_for_ready_state(
page,
&["interactive", "complete"],
Duration::from_secs(10),
)
.await?;
Self::wait_network_idle(page, *idle_ms).await?;
}
WaitStrategy::Selector { css, timeout_ms } => {
Self::wait_for_ready_state(
page,
&["interactive", "complete"],
Duration::from_secs(10),
)
.await?;
let deadline = std::time::Instant::now() + Duration::from_millis(*timeout_ms);
loop {
if page.find_element(css.clone()).await.is_ok() {
break;
}
if std::time::Instant::now() >= deadline {
return Err(Error::Render(format!(
"selector timeout after actions: {css}"
)));
}
tokio::time::sleep(Duration::from_millis(50)).await;
}
}
WaitStrategy::Fixed { ms } => {
tokio::time::sleep(Duration::from_millis(*ms)).await;
}
WaitStrategy::ReadingDwell { wpm, jitter_ms } => {
Self::wait_for_ready_state(
page,
&["interactive", "complete"],
Duration::from_secs(10),
)
.await?;
Self::apply_reading_dwell(page, *wpm, *jitter_ms, 500, 10_000).await;
}
}
if !matches!(wait, WaitStrategy::ReadingDwell { .. }) {
if let Some(cfg) = reading_dwell.filter(|c| c.enabled) {
Self::apply_reading_dwell(page, cfg.wpm, cfg.jitter_ms, cfg.min_ms, cfg.max_ms)
.await;
}
}
Ok(())
}
async fn apply_reading_dwell(page: &Page, wpm: u32, jitter_ms: u64, min_ms: u64, max_ms: u64) {
use rand::rngs::SmallRng;
use rand::SeedableRng;
let words = match crate::render::interact::eval_js(
page,
"(document.body && document.body.innerText) ? document.body.innerText.split(/\\s+/).filter(Boolean).length : 0",
)
.await
{
Ok(v) => v.as_u64().unwrap_or(0),
Err(_) => 0,
};
let seed = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos() as u64)
.unwrap_or(0);
let mut rng = SmallRng::seed_from_u64(seed);
let ms =
crate::wait_strategy::compute_dwell_ms(words, wpm, jitter_ms, min_ms, max_ms, &mut rng);
tracing::debug!(words, ms, "reading dwell");
tokio::time::sleep(Duration::from_millis(ms)).await;
}
pub(crate) async fn capture_screenshot_mode(
page: &Page,
mode: ScreenshotCaptureMode,
) -> Option<Vec<u8>> {
use base64::Engine;
let params = match mode {
ScreenshotCaptureMode::Viewport => CaptureScreenshotParams::builder()
.from_surface(true)
.build(),
ScreenshotCaptureMode::FullPage => CaptureScreenshotParams::builder()
.capture_beyond_viewport(true)
.from_surface(true)
.build(),
ScreenshotCaptureMode::Element { ref selector } => {
use crate::render::chrome_protocol::cdp::browser_protocol::dom::{
GetBoxModelParams, GetDocumentParams, QuerySelectorParams,
};
use crate::render::chrome_protocol::cdp::browser_protocol::page::Viewport;
let doc = match page.execute(GetDocumentParams::default()).await {
Ok(r) => r.root.node_id,
Err(e) => {
debug!(?e, "GetDocument for element screenshot failed");
return None;
}
};
let qs = QuerySelectorParams::builder()
.node_id(doc)
.selector(selector.clone())
.build()
.ok()?;
let found = match page.execute(qs).await {
Ok(r) => r.node_id,
Err(e) => {
debug!(?e, "QuerySelector for element screenshot failed");
return None;
}
};
let box_model = match page
.execute(GetBoxModelParams::builder().node_id(found).build())
.await
{
Ok(r) => r.model.clone(),
Err(e) => {
debug!(?e, "GetBoxModel for element screenshot failed");
return None;
}
};
let q = &box_model.content;
if q.inner().len() < 8 {
return None;
}
let q = q.inner();
let xs = [q[0], q[2], q[4], q[6]];
let ys = [q[1], q[3], q[5], q[7]];
let x = xs.iter().cloned().fold(f64::INFINITY, f64::min);
let y = ys.iter().cloned().fold(f64::INFINITY, f64::min);
let x_max = xs.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
let y_max = ys.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
let width = (x_max - x).max(1.0);
let height = (y_max - y).max(1.0);
let clip = Viewport::builder()
.x(x)
.y(y)
.width(width)
.height(height)
.scale(1.0)
.build()
.ok()?;
CaptureScreenshotParams::builder()
.from_surface(true)
.clip(clip)
.build()
}
};
match page.execute(params).await {
Ok(r) => base64::engine::general_purpose::STANDARD
.decode(&r.data)
.ok(),
Err(e) => {
debug!(?e, "screenshot failed");
None
}
}
}
fn push_unique_url(urls: &mut Vec<Url>, candidate: Url) {
if !urls.iter().any(|existing| existing == &candidate) {
urls.push(candidate);
}
}
}
#[async_trait::async_trait]
impl crate::identity::SessionDropTarget for RenderPool {
async fn drop_session(&self, id: &str) {
RenderPool::drop_session(self, id).await;
}
}
impl Drop for RenderPool {
fn drop(&mut self) {
self.browsers.clear();
self.browser_locks.clear();
self.contexts.clear();
self.context_locks.clear();
self.session_states.clear();
let _ = std::fs::remove_dir_all(&self.user_data_root);
}
}
impl RenderPool {
pub(crate) async fn fetch_manifest_json(
page: &Page,
manifest: &Url,
) -> Option<serde_json::Value> {
let manifest_str = manifest.as_str().replace('`', "%60");
let expr = format!(
r#"(async () => {{
try {{
const r = await fetch(`{manifest}`, {{ credentials: 'include' }});
if (!r.ok) return null;
return await r.json();
}} catch (_) {{ return null; }}
}})()"#,
manifest = manifest_str
);
let params = EvaluateParams::builder()
.expression(expr)
.await_promise(true)
.return_by_value(true)
.build()
.ok()?;
let res = page.evaluate_expression(params).await.ok()?;
let v = res.value().cloned()?;
if v.is_null() {
None
} else {
Some(v)
}
}
pub(crate) async fn collect_indexeddb_inventory(
page: &Page,
_final_url: &Url,
) -> Vec<serde_json::Value> {
const EXPR: &str = r#"(async () => {
try {
if (!('indexedDB' in window) || typeof indexedDB.databases !== 'function') return [];
const metas = await indexedDB.databases();
const out = [];
for (const m of metas) {
if (!m || !m.name) continue;
const db = await new Promise((resolve) => {
try {
const req = indexedDB.open(m.name, m.version);
req.onsuccess = () => resolve(req.result);
req.onerror = () => resolve(null);
req.onblocked = () => resolve(null);
} catch (_) { resolve(null); }
});
if (!db) { out.push({ db_name: m.name, version: m.version, stores: [] }); continue; }
const stores = [];
try {
for (const name of Array.from(db.objectStoreNames)) {
try {
const tx = db.transaction(name, 'readonly');
const os = tx.objectStore(name);
stores.push({
name: String(name),
key_path: os.keyPath === null ? null : (Array.isArray(os.keyPath) ? os.keyPath : String(os.keyPath)),
auto_increment: !!os.autoIncrement,
indexes: Array.from(os.indexNames).map((n) => String(n)),
});
tx.abort && tx.abort();
} catch (_) {}
}
} catch (_) {}
try { db.close(); } catch (_) {}
out.push({ db_name: m.name, version: m.version, stores });
}
return out;
} catch (_) { return []; }
})()"#;
let Ok(params) = EvaluateParams::builder()
.expression(EXPR)
.await_promise(true)
.return_by_value(true)
.build()
else {
return Vec::new();
};
let Ok(res) = page.evaluate_expression(params).await else {
return Vec::new();
};
match res.value().cloned() {
Some(serde_json::Value::Array(items)) => items,
_ => Vec::new(),
}
}
pub(crate) async fn collect_cache_storage_inventory(
page: &Page,
_final_url: &Url,
) -> Vec<serde_json::Value> {
const EXPR: &str = r#"(async () => {
try {
if (!('caches' in window)) return [];
const names = await caches.keys();
const out = [];
for (const name of names) {
try {
const c = await caches.open(name);
const reqs = await c.keys();
const keys = reqs.slice(0, 500).map((r) => r.url);
out.push({ cache_name: name, keys, total: reqs.length });
} catch (_) {
out.push({ cache_name: name, keys: [], total: 0 });
}
}
return out;
} catch (_) { return []; }
})()"#;
let Ok(params) = EvaluateParams::builder()
.expression(EXPR)
.await_promise(true)
.return_by_value(true)
.build()
else {
return Vec::new();
};
let Ok(res) = page.evaluate_expression(params).await else {
return Vec::new();
};
match res.value().cloned() {
Some(serde_json::Value::Array(items)) => items,
_ => Vec::new(),
}
}
pub(crate) async fn capture_pwa_state_snapshot(
page: &Page,
final_url: &Url,
include_deep_storage: bool,
) -> serde_json::Value {
let origin = final_url.origin().ascii_serialization();
let origin_state = if !origin.is_empty() && origin != "null" {
Self::capture_origin_state(page, &origin)
.await
.unwrap_or_default()
} else {
PersistedOriginState::default()
};
let observations = Self::collect_spa_observations(page).await;
let manifest_json = match origin_state.manifest_url.as_deref() {
Some(raw) => match Url::parse(raw) {
Ok(manifest_url) => Self::fetch_manifest_json(page, &manifest_url).await,
Err(_) => None,
},
None => None,
};
let indexeddb = if include_deep_storage {
Self::collect_indexeddb_inventory(page, final_url).await
} else {
Vec::new()
};
let cache_storage = if include_deep_storage {
Self::collect_cache_storage_inventory(page, final_url).await
} else {
Vec::new()
};
let is_spa = !observations.routes.is_empty() || final_url.fragment().is_some();
let PersistedOriginState {
origin,
local_storage,
session_storage,
manifest_url,
service_workers,
} = origin_state;
let crate::render::spa_observer::CollectedObservations {
routes,
endpoints,
idb_audit,
} = observations;
if !idb_audit.is_empty() {
tracing::debug!(
count = idb_audit.len(),
"idb-audit observer captured write order"
);
}
serde_json::json!({
"final_url": final_url.as_str(),
"origin": origin,
"manifest_url": manifest_url,
"manifest": manifest_json,
"service_workers": service_workers,
"local_storage": local_storage,
"session_storage": session_storage,
"runtime_routes": routes,
"network_endpoints": endpoints,
"idb_audit": idb_audit,
"indexeddb": indexeddb,
"cache_storage": cache_storage,
"is_spa": is_spa,
})
}
#[allow(clippy::too_many_arguments)]
async fn persist_spa_artifacts(
&self,
seed_url: &Url,
final_url: &Url,
session_id: &str,
observations: &crate::render::spa_observer::CollectedObservations,
indexeddb: &[serde_json::Value],
cache_storage: &[serde_json::Value],
manifest_json: Option<&serde_json::Value>,
service_workers: &[PersistedServiceWorker],
) {
use crate::storage::{ArtifactKind, ArtifactMeta};
let save = |kind: ArtifactKind, value: serde_json::Value| {
let bytes = match serde_json::to_vec(&value) {
Ok(b) => b,
Err(e) => {
debug!(?e, kind = kind.wire_str(), "spa artifact encode failed");
return None;
}
};
Some((kind, bytes))
};
let mut queued: Vec<(ArtifactKind, Vec<u8>)> = Vec::new();
if self.config.collect_runtime_routes && !observations.routes.is_empty() {
if let Ok(v) = serde_json::to_value(&observations.routes) {
if let Some(q) = save(ArtifactKind::SnapshotRuntimeRoutes, v) {
queued.push(q);
}
}
}
if self.config.collect_network_endpoints && !observations.endpoints.is_empty() {
if let Ok(v) = serde_json::to_value(&observations.endpoints) {
if let Some(q) = save(ArtifactKind::SnapshotNetworkEndpoints, v) {
queued.push(q);
}
}
}
if !indexeddb.is_empty() {
if let Some(q) = save(
ArtifactKind::SnapshotIndexedDb,
serde_json::Value::Array(indexeddb.to_vec()),
) {
queued.push(q);
}
}
if !cache_storage.is_empty() {
if let Some(q) = save(
ArtifactKind::SnapshotCacheStorage,
serde_json::Value::Array(cache_storage.to_vec()),
) {
queued.push(q);
}
}
if let Some(m) = manifest_json {
if let Some(q) = save(ArtifactKind::SnapshotManifest, m.clone()) {
queued.push(q);
}
}
if self.config.collect_service_workers && !service_workers.is_empty() {
if let Ok(v) = serde_json::to_value(service_workers) {
if let Some(q) = save(ArtifactKind::SnapshotServiceWorkers, v) {
queued.push(q);
}
}
}
for (kind, bytes) in queued {
let meta = ArtifactMeta {
url: seed_url,
final_url: Some(final_url),
session_id,
kind,
name: None,
step_id: None,
step_kind: None,
selector: None,
mime: None,
};
if let Err(e) = self.storage.save_artifact(&meta, &bytes).await {
debug!(?e, kind = kind.wire_str(), "spa artifact save failed");
}
}
}
}
#[async_trait::async_trait]
impl Renderer for RenderPool {
async fn render(
&self,
url: &Url,
wait: &WaitStrategy,
collect_vitals: bool,
screenshot: bool,
actions: Option<&[crate::render::actions::Action]>,
proxy: Option<&Url>,
) -> Result<RenderedPage> {
let policy = self.config.action_policy.clone();
let actions_vec: Option<Vec<crate::render::actions::Action>> = actions.map(|a| a.to_vec());
self.render_core(url, wait, collect_vitals, screenshot, proxy, move |page| {
Box::pin(async move {
if let Some(list) = actions_vec {
if !list.is_empty() {
crate::render::actions::execute_with_policy(page, &list, &policy).await?;
return Ok(true);
}
}
Ok(false)
})
})
.await
}
}
impl RenderPool {
#[cfg(feature = "cdp-backend")]
pub async fn render_with_script(
&self,
url: &Url,
wait: &WaitStrategy,
script: &crate::script::ScriptSpec,
events: Option<std::sync::Arc<dyn crate::events::EventSink>>,
run_id: Option<u64>,
proxy: Option<&Url>,
) -> Result<(RenderedPage, crate::script::RunOutcome)> {
use std::sync::Arc;
let plan =
crate::script::plan(script).map_err(|e| Error::Render(format!("script-plan: {e}")))?;
let plan = Arc::new(plan);
let session_id = self.session_id_for_url(url);
let policy = self.config.action_policy.clone();
let outcome_slot: Arc<parking_lot::Mutex<Option<crate::script::RunOutcome>>> =
Arc::new(parking_lot::Mutex::new(None));
let outcome_for_closure = outcome_slot.clone();
let plan_for_closure = plan.clone();
let storage_for_closure = self.storage.clone();
let url_for_closure = url.clone();
let events_for_closure = events.clone();
let run_id_for_closure = run_id;
let wait_for_runner = wait.clone();
let collect_vitals = self.config.collect_web_vitals;
let screenshot = self.config.output.screenshot;
let rendered = self
.render_core(url, wait, collect_vitals, screenshot, proxy, move |page| {
let outcome_slot = outcome_for_closure;
let plan = plan_for_closure;
let session_id = session_id;
let policy = policy;
let storage = storage_for_closure;
let url_for_runner = url_for_closure;
let events = events_for_closure;
let run_id = run_id_for_closure;
let wait = wait_for_runner;
Box::pin(async move {
let mut runner =
crate::script::ScriptRunner::new(page, &plan, session_id, &policy)
.with_wait_strategy(wait)
.with_storage(storage)
.with_url(url_for_runner);
if let Some(sink) = events {
runner = runner.with_sink(sink);
}
if let Some(run_id) = run_id {
runner = runner.with_run_id(run_id);
}
let outcome = runner.run().await?;
let ran_any = !outcome.steps.is_empty();
*outcome_slot.lock() = Some(outcome);
Ok(ran_any)
})
})
.await?;
let outcome = outcome_slot.lock().take().unwrap_or_default();
Ok((rendered, outcome))
}
async fn render_core<F>(
&self,
url: &Url,
wait: &WaitStrategy,
collect_vitals: bool,
screenshot: bool,
proxy: Option<&Url>,
run_custom: F,
) -> Result<RenderedPage>
where
F: for<'p> FnOnce(&'p Page) -> futures::future::BoxFuture<'p, Result<bool>> + Send,
{
let _permit = self.sem.clone().acquire_owned().await.unwrap();
let render_started = std::time::Instant::now();
let browser = self.ensure_browser(proxy).await?;
let browser_key = self.browser_key_for(proxy);
let session_id = self.session_id_for_url(url);
let ctx_id = self
.ensure_session_context(&browser_key, &browser, &session_id)
.await?;
let ctx_key = Self::browser_session_key(&browser_key, &session_id);
let lease: crate::render::page_pool::PageLease = {
if let Some(pooled) = self.page_pool.try_acquire(&ctx_key) {
let blank = NavigateParams::builder()
.url("about:blank".to_string())
.build()
.map_err(|e| Error::Render(format!("blank NavigateParams: {e}")))?;
match pooled.page.execute(blank).await {
Ok(_) => {
if let Some(c) = self.counters_opt() {
c.pages_reused
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
crate::render::page_pool::PageLease::new(self.page_pool.clone(), pooled)
}
Err(e) => {
tracing::debug!(
?e,
"pool page reset failed; discarding and creating fresh"
);
self.page_pool.release_dirty_key(&ctx_key);
let mut params = CreateTargetParams::new("about:blank");
params.browser_context_id = ctx_id.clone();
let fresh = browser
.new_page(params)
.await
.map_err(|e| Error::Render(format!("new_page: {e}")))?;
self.page_pool.register_fresh(&ctx_key);
self.install_stealth(&fresh).await?;
self.install_spa_observer(&fresh).await?;
if let Some(c) = self.counters_opt() {
c.pages_created
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
let pooled_new =
crate::render::page_pool::PooledPage::new(fresh, ctx_key.clone());
crate::render::page_pool::PageLease::new(self.page_pool.clone(), pooled_new)
}
}
} else {
let mut params = CreateTargetParams::new("about:blank");
params.browser_context_id = ctx_id.clone();
let fresh = browser
.new_page(params)
.await
.map_err(|e| Error::Render(format!("new_page: {e}")))?;
self.page_pool.register_fresh(&ctx_key);
self.install_stealth(&fresh).await?;
self.install_spa_observer(&fresh).await?;
if let Some(c) = self.counters_opt() {
c.pages_created
.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
}
let pooled = crate::render::page_pool::PooledPage::new(fresh, ctx_key.clone());
crate::render::page_pool::PageLease::new(self.page_pool.clone(), pooled)
}
};
if let Some(c) = self.counters_opt() {
c.tabs_active.store(
self.page_pool.total_in_flight(),
std::sync::atomic::Ordering::Relaxed,
);
}
let page: Page = lease.page().clone();
if self.config.external_cdp_url.is_some() {
let cal_key = self.calibration_key_for(proxy, &session_id, ctx_id.as_ref());
match self.ensure_calibrated(&page, cal_key).await {
Ok(fp) => {
let expected = self.expected_identity_for_session();
let mismatches =
crate::render::calibration::classify_mismatches(&fp, &expected);
if !mismatches.is_empty() {
let policy = self.config.mismatch_policy;
let report =
crate::render::calibration::MismatchReport::new(policy, &mismatches);
let payload = serde_json::to_string(&report).unwrap_or_else(|_| {
format!(
"{{\"policy\":\"{}\",\"critical\":{},\"unreconciled_critical\":{}}}",
report.policy, report.critical, report.unreconciled_critical
)
});
tracing::warn!(
event = "calibration.mismatch",
policy = report.policy,
critical = report.critical,
non_critical = report.non_critical,
unreconciled_critical = report.unreconciled_critical,
categories = ?report.categories,
report = payload.as_str(),
"calibration mismatch detected"
);
if policy == crate::config::MismatchPolicy::Strict
&& crate::render::calibration::has_unreconciled_critical(&mismatches)
{
let cats: Vec<&str> = mismatches
.iter()
.filter(|m| {
m.severity
== crate::render::calibration::MismatchSeverity::Critical
&& !m.reconcilable
})
.map(|m| m.category.as_str())
.collect();
return Err(Error::Render(format!(
"strict calibration policy: unreconciled critical mismatch in {} — aborted before target navigation",
cats.join(",")
)));
}
}
let overrides = Self::calibration_overrides_from_fingerprint(&fp);
if !overrides.is_empty() {
if let Err(e) = self
.install_stealth_with_overrides(&page, Some(&overrides))
.await
{
tracing::debug!(
?e,
"calibration-aware stealth inject failed; falling back to bundle shim"
);
} else {
tracing::debug!(
event = "stealth.calibration.applied",
"external CDP shim updated with calibrated values"
);
}
}
}
Err(e) => tracing::warn!(
?e,
"calibration failed; continuing to target without cached fingerprint"
),
}
}
if !self.is_persistent_external_cdp() {
self.restore_session_state(&page, url, &session_id).await?;
}
let bundle = self.bundle();
let brands = ua_ch_brands(&bundle.ua_brands, &bundle.ua_major.to_string());
let full_version_list = ua_ch_brands(&bundle.ua_full_version_list, &bundle.ua_full_version);
let ch_platform = bundle.ua_platform.trim_matches('"');
let mobile = bundle.is_mobile();
let (platform_version, architecture, model, bitness) = match ch_platform {
"Windows" => ("10.0.0", "x86", "", Some("64".to_string())),
"macOS" => ("10.15.7", "arm", "", Some("64".to_string())),
"Android" => ("14.0.0", "arm", "Pixel 7", None),
_ => ("6.5.0", "x86", "", Some("64".to_string())),
};
let metadata = UserAgentMetadata {
brands: Some(brands),
full_version_list: Some(full_version_list),
platform: ch_platform.to_string(),
platform_version: platform_version.into(),
architecture: architecture.into(),
model: model.into(),
mobile,
bitness,
wow64: Some(false),
form_factors: None,
};
if let Err(e) = page
.execute(NetworkSetUserAgentOverrideParams {
user_agent: bundle.ua.clone(),
accept_language: Some(bundle.accept_language.clone()),
platform: Some(bundle.platform.clone()),
user_agent_metadata: Some(metadata),
})
.await
{
tracing::debug!(?e, "Network.setUserAgentOverride failed");
}
let mut all_block_urls: Vec<String> =
collect_block_resource_patterns(&self.config.block_resources);
if !self.config.reject_resource_types.is_empty() {
if screenshot {
tracing::warn!(
requested = ?self.config.reject_resource_types
.iter()
.map(|r| r.as_str())
.collect::<Vec<_>>(),
"screenshot requested — disabling reject_resource_types to preserve visual fidelity"
);
} else {
for rt in &self.config.reject_resource_types {
all_block_urls.extend(rt.url_patterns().iter().map(|s| s.to_string()));
}
}
}
if !all_block_urls.is_empty() {
let patterns: Vec<BlockPattern> = all_block_urls
.into_iter()
.map(|url_pattern| BlockPattern {
url_pattern,
block: true,
})
.collect();
if let Err(e) = page
.execute(SetBlockedUrLsParams {
url_patterns: Some(patterns),
})
.await
{
tracing::debug!(?e, "SetBlockedURLs failed (continuing)");
}
}
if collect_vitals {
Self::install_vitals_observers(&page).await?;
}
let captured = Arc::new(Mutex::new(Vec::<Url>::new()));
let mut events = page
.event_listener::<EventRequestWillBeSent>()
.await
.map_err(|e| Error::Render(format!("event_listener: {e}")))?;
{
let captured = captured.clone();
let storage = self.storage.clone();
let tracker = self.telemetry_tracker.clone();
let sid = session_id.clone();
tokio::spawn(async move {
while let Some(ev) = events.next().await {
let Ok(u) = Url::parse(&ev.request.url) else {
continue;
};
captured.lock().push(u.clone());
let body = collect_post_body(&ev.request);
let req = crate::antibot::telemetry::ObservedRequest {
url: &u,
method: &ev.request.method,
body: &body,
session_id: &sid,
};
if let Some(telem) = crate::antibot::telemetry::classify_request(&req) {
let threshold_hit = {
let mut t = tracker.lock();
t.observe(&sid, telem.vendor, telem.observed_at)
};
if let Err(e) = storage.record_telemetry(&telem).await {
tracing::debug!(?e, "record_telemetry failed");
}
tracing::debug!(
vendor = telem.vendor.as_str(),
endpoint = %telem.endpoint,
payload_size = telem.payload_size,
pattern_label = telem.pattern_label,
preemptive_rotate = threshold_hit,
"vendor telemetry observed"
);
if threshold_hit {
tracing::info!(
vendor = telem.vendor.as_str(),
session_id = %sid,
"vendor telemetry volume threshold hit — recommending RotateProxy"
);
}
}
}
});
}
let main_frame = page
.mainframe()
.await
.map_err(|e| Error::Render(format!("mainframe: {e}")))?;
let main_document_status = Arc::new(Mutex::new(None::<u16>));
let resource_map: Arc<Mutex<HashMap<String, crate::metrics::ResourceSample>>> =
Arc::new(Mutex::new(HashMap::new()));
let mut resp_events = page
.event_listener::<EventResponseReceived>()
.await
.map_err(|e| Error::Render(format!("resp listener: {e}")))?;
{
let map = resource_map.clone();
let status = main_document_status.clone();
let main_frame = main_frame.clone();
tokio::spawn(async move {
while let Some(ev) = resp_events.next().await {
let is_main_document = ev.r#type == ResourceType::Document
&& match main_frame.as_ref() {
Some(frame_id) => ev.frame_id.as_ref() == Some(frame_id),
None => true,
};
if is_main_document {
*status.lock() = Some(ev.response.status as u16);
}
if !collect_vitals {
continue;
}
let r = &ev.response;
let id = format!("{}", ev.request_id.inner());
let mut sample = crate::metrics::ResourceSample {
url: r.url.clone(),
mime_type: Some(r.mime_type.clone()),
resource_type: Some(format!("{:?}", ev.r#type)),
status: Some(r.status as u16),
from_cache: Some(
r.from_disk_cache.unwrap_or(false)
|| r.from_service_worker.unwrap_or(false),
),
protocol: r.protocol.clone(),
remote_ip: r.remote_ip_address.clone(),
remote_port: r.remote_port.map(|p| p as u16),
encoded_data_length: Some(r.encoded_data_length),
..Default::default()
};
if let Some(t) = r.timing.as_ref() {
sample.request_time = Some(t.request_time);
sample.dns_start = Some(t.dns_start);
sample.dns_end = Some(t.dns_end);
sample.connect_start = Some(t.connect_start);
sample.connect_end = Some(t.connect_end);
sample.ssl_start = Some(t.ssl_start);
sample.ssl_end = Some(t.ssl_end);
sample.send_start = Some(t.send_start);
sample.send_end = Some(t.send_end);
sample.receive_headers_start = Some(t.receive_headers_start);
sample.receive_headers_end = Some(t.receive_headers_end);
sample.dns_ms = diff(t.dns_start, t.dns_end);
sample.connect_ms = diff(t.connect_start, t.connect_end);
sample.ssl_ms = diff(t.ssl_start, t.ssl_end);
sample.send_ms = diff(t.send_start, t.send_end);
sample.wait_ms = diff(t.send_end, t.receive_headers_start);
}
map.lock().insert(id, sample);
}
});
}
if collect_vitals {
let mut fin_events = page
.event_listener::<EventLoadingFinished>()
.await
.map_err(|e| Error::Render(format!("fin listener: {e}")))?;
{
let map = resource_map.clone();
tokio::spawn(async move {
while let Some(ev) = fin_events.next().await {
let id = format!("{}", ev.request_id.inner());
let mut guard = map.lock();
if let Some(s) = guard.get_mut(&id) {
s.loading_finished_ms = Some(*ev.timestamp.inner());
s.transfer_size = Some(ev.encoded_data_length);
if let (Some(rhs), Some(rt)) = (s.receive_headers_start, s.request_time)
{
let fin_ms = (*ev.timestamp.inner() - rt) * 1000.0;
s.receive_ms = Some((fin_ms - rhs).max(0.0));
}
}
}
});
}
}
if self.config.warmup.enabled {
let warmup_url = self.config.warmup.render_template(url);
let target_str = url.to_string();
if !warmup_url.is_empty() && warmup_url != target_str {
tracing::info!(
warmup_url = %warmup_url,
dwell_ms = self.config.warmup.dwell_ms,
"warmup navigation"
);
match NavigateParams::builder().url(warmup_url.clone()).build() {
Ok(warm_nav) => {
if let Err(e) = page.execute(warm_nav).await {
tracing::warn!(?e, warmup_url = %warmup_url, "warmup navigate failed; proceeding to target");
} else if self.config.warmup.dwell_ms > 0 {
tokio::time::sleep(Duration::from_millis(self.config.warmup.dwell_ms))
.await;
}
}
Err(e) => {
tracing::warn!(?e, warmup_url = %warmup_url, "warmup NavigateParams build failed; skipping");
}
}
}
}
let nav = NavigateParams::builder()
.url(url.to_string())
.build()
.map_err(|e| Error::Render(format!("NavigateParams: {e}")))?;
let _ = page
.execute(nav)
.await
.map_err(|e| Error::Render(format!("navigate: {e}")))?;
self.reinject_spa_observer(&page).await;
Self::wait_for(&page, wait).await?;
let mutated_after_load = run_custom(&page).await?;
#[cfg(feature = "lua-hooks")]
let mut mutated_after_load = mutated_after_load;
#[cfg(feature = "lua-hooks")]
{
let host_opt = { self.lua_host.read().as_ref().cloned() };
if let Some(host) = host_opt {
mutated_after_load = true;
let mut ctx = crate::hooks::HookContext::new(url.clone(), 0);
let _ = host
.fire_with_page(
crate::hooks::HookEvent::AfterLoad,
&mut ctx,
Some(page.clone()),
)
.await;
let _ = host
.fire_with_page(
crate::hooks::HookEvent::AfterIdle,
&mut ctx,
Some(page.clone()),
)
.await;
}
}
if mutated_after_load {
Self::settle_after_actions_with_dwell(&page, wait, self.config.reading_dwell.as_ref())
.await?;
} else if let Some(cfg) = self.config.reading_dwell.as_ref().filter(|c| c.enabled) {
Self::apply_reading_dwell(&page, cfg.wpm, cfg.jitter_ms, cfg.min_ms, cfg.max_ms).await;
}
let mut vitals = if collect_vitals {
Self::collect_vitals(&page).await.unwrap_or_default()
} else {
crate::metrics::WebVitals::default()
};
if collect_vitals {
if let Ok(Some(inp)) = Self::probe_inp(&page).await {
vitals.interaction_to_next_paint_ms = Some(inp);
}
}
self.prepare_dom_capture(&page).await;
let html = self.capture_dom_html(&page).await?;
let screenshot_png = if screenshot {
let mode =
parse_screenshot_mode_or_default(self.config.output.screenshot_mode.as_deref());
Self::capture_screenshot_mode(&page, mode).await
} else {
None
};
let final_url: Url = {
let evaled = crate::render::interact::eval_js(&page, "window.location.href")
.await
.ok()
.and_then(|v| v.as_str().map(|s| s.to_string()))
.and_then(|s| Url::parse(&s).ok());
match evaled {
Some(u) => u,
None => match page.url().await {
Ok(Some(u)) => Url::parse(&u).unwrap_or_else(|_| url.clone()),
_ => url.clone(),
},
}
};
let origin_state = self
.capture_session_state(&page, &final_url, &session_id)
.await
.unwrap_or_default();
let observations =
if self.config.collect_runtime_routes || self.config.collect_network_endpoints {
Self::collect_spa_observations(&page).await
} else {
Default::default()
};
let indexeddb_inventory = if self.config.collect_indexeddb {
Self::collect_indexeddb_inventory(&page, &final_url).await
} else {
Vec::new()
};
let cache_storage_inventory = if self.config.collect_cache_storage {
Self::collect_cache_storage_inventory(&page, &final_url).await
} else {
Vec::new()
};
let manifest_json: Option<serde_json::Value> = if self.config.collect_manifest {
match origin_state.manifest_url.as_deref() {
Some(raw) => match Url::parse(raw) {
Ok(u) => Self::fetch_manifest_json(&page, &u).await,
Err(_) => None,
},
None => None,
}
} else {
None
};
let challenge_raw = crate::antibot::detect_from_html(&html, &final_url, None);
let status_snapshot = *main_document_status.lock();
let release_dirty = challenge_raw.is_some()
|| status_snapshot == Some(0)
|| matches!(status_snapshot, Some(s) if s >= 500);
if release_dirty {
if let Err(e) = page.close().await {
debug!(?e, "page close error (ignored)");
}
lease.release_dirty();
} else {
lease.release_clean();
}
if let Some(c) = self.counters_opt() {
c.tabs_active.store(
self.page_pool.total_in_flight(),
std::sync::atomic::Ordering::Relaxed,
);
}
let mut captured_urls = std::mem::take(&mut *captured.lock());
let manifest_url = origin_state
.manifest_url
.as_deref()
.and_then(|raw| Url::parse(raw).ok());
if let Some(url) = manifest_url.clone() {
Self::push_unique_url(&mut captured_urls, url);
}
let mut service_worker_urls = Vec::new();
for worker in &origin_state.service_workers {
for raw in worker.script_urls() {
if let Ok(url) = Url::parse(raw) {
Self::push_unique_url(&mut captured_urls, url.clone());
Self::push_unique_url(&mut service_worker_urls, url);
}
}
}
let mut runtime_routes: Vec<Url> = Vec::new();
for r in &observations.routes {
if let Ok(u) = Url::parse(&r.url).or_else(|_| final_url.join(&r.url)) {
if matches!(u.scheme(), "http" | "https") {
Self::push_unique_url(&mut runtime_routes, u.clone());
if self.config.collect_runtime_routes {
Self::push_unique_url(&mut captured_urls, u);
}
}
}
}
let mut network_endpoints: Vec<Url> = Vec::new();
for e in &observations.endpoints {
if let Ok(u) = Url::parse(&e.url).or_else(|_| final_url.join(&e.url)) {
if matches!(u.scheme(), "http" | "https") {
Self::push_unique_url(&mut network_endpoints, u.clone());
if self.config.collect_network_endpoints {
Self::push_unique_url(&mut captured_urls, u);
}
}
}
}
let is_spa = !runtime_routes.is_empty()
|| (final_url.fragment().is_some() && final_url.fragment() != url.fragment());
self.persist_spa_artifacts(
url,
&final_url,
&session_id,
&observations,
&indexeddb_inventory,
&cache_storage_inventory,
manifest_json.as_ref(),
&origin_state.service_workers,
)
.await;
let resources = if collect_vitals {
resource_map.lock().drain().map(|(_, v)| v).collect()
} else {
Vec::new()
};
let status = status_snapshot.unwrap_or(0);
let challenge = challenge_raw
.map(|raw| raw.into_signal(&final_url, session_id.clone(), proxy.cloned()));
if let Some(c) = self.counters_opt() {
let ok = matches!(status, 200..=399);
c.record_render(render_started.elapsed(), ok);
}
Ok(RenderedPage {
session_id,
final_url,
html_post_js: html,
captured_urls,
manifest_url,
service_worker_urls,
status,
vitals,
resources,
screenshot_png,
challenge,
runtime_routes,
network_endpoints,
is_spa,
})
}
}
#[cfg(test)]
mod storage_clear_audit {
use super::*;
use crate::render::chrome_wire::MethodType;
#[test]
fn clear_chrome_supercookies_wires_expected_cdp_methods() {
assert_eq!(
SetBypassServiceWorkerParams::method_id().as_ref(),
"Network.setBypassServiceWorker"
);
assert_eq!(
SetIgnoreCertificateErrorsParams::method_id().as_ref(),
"Security.setIgnoreCertificateErrors"
);
assert_eq!(
ClearDataForOriginParams::method_id().as_ref(),
"Storage.clearDataForOrigin"
);
assert_eq!(
ServiceWorkerUnregisterParams::method_id().as_ref(),
"ServiceWorker.unregister"
);
assert_eq!(
ClearBrowserCacheParams::method_id().as_ref(),
"Network.clearBrowserCache"
);
}
#[test]
fn clear_data_for_origin_uses_wildcard_and_all_storage_types() {
let params = ClearDataForOriginParams::new("*", "all");
assert_eq!(params.origin, "*");
assert_eq!(params.storage_types, "all");
}
#[test]
fn set_ignore_certificate_errors_resets_to_false() {
let params = SetIgnoreCertificateErrorsParams::new(false);
assert!(!params.ignore);
}
#[test]
fn set_bypass_service_worker_enables_bypass() {
let params = SetBypassServiceWorkerParams::new(true);
assert!(params.bypass);
}
}
#[cfg(test)]
mod provider_fallback_wiring {
use super::*;
use crate::config::{BrowserProvider, Config, ProviderFallbackConfig};
use crate::storage::memory::MemoryStorage;
use std::sync::Arc;
fn build_pool(config: Config) -> RenderPool {
RenderPool::new(Arc::new(config), Arc::new(MemoryStorage::new()))
}
#[test]
fn effective_primary_auto_with_endpoint_picks_cdp() {
let mut cfg = Config::default();
cfg.browser_provider = BrowserProvider::Auto;
cfg.external_cdp_url = Some("http://127.0.0.1:9222".into());
let p = build_pool(cfg);
assert_eq!(p.effective_primary_provider(), BrowserProvider::Cdp);
}
#[test]
fn effective_primary_auto_without_endpoint_picks_stock() {
let mut cfg = Config::default();
cfg.browser_provider = BrowserProvider::Auto;
cfg.external_cdp_url = None;
let p = build_pool(cfg);
assert_eq!(p.effective_primary_provider(), BrowserProvider::Stock);
}
#[test]
fn effective_primary_concrete_providers_map_to_themselves() {
for primary in [BrowserProvider::Stock, BrowserProvider::Cdp] {
let mut cfg = Config::default();
cfg.browser_provider = primary;
if primary == BrowserProvider::Cdp {
cfg.external_cdp_url = Some("http://127.0.0.1:9222".into());
}
let p = build_pool(cfg);
assert_eq!(p.effective_primary_provider(), primary);
}
}
#[test]
fn cdp_fallback_unusable_without_endpoint() {
let cfg = Config::default(); let p = build_pool(cfg);
assert!(!p.provider_usable_for_fallback(BrowserProvider::Cdp));
assert_eq!(
p.unusable_reason(BrowserProvider::Cdp),
"no external_cdp_url configured"
);
}
#[test]
fn auto_never_usable_as_fallback_target() {
let p = build_pool(Config::default());
assert!(!p.provider_usable_for_fallback(BrowserProvider::Auto));
}
#[test]
fn stock_always_usable_as_fallback_target() {
let p = build_pool(Config::default());
assert!(p.provider_usable_for_fallback(BrowserProvider::Stock));
}
#[tokio::test(flavor = "current_thread")]
async fn preflight_without_fallback_surfaces_primary_cdp_error() {
let mut cfg = Config::default();
cfg.browser_provider = BrowserProvider::Cdp;
cfg.external_cdp_url = Some("http://127.0.0.1:1".into());
cfg.provider_fallback = ProviderFallbackConfig::default();
let pool = build_pool(cfg);
let err = pool.preflight().await.unwrap_err();
let msg = format!("{err}");
assert!(msg.contains("external CDP preflight failed"), "{msg}");
assert!(
!msg.contains("fallback attempted"),
"primary failure must not carry a fallback path when fallback is off: {msg}"
);
}
#[tokio::test(flavor = "current_thread")]
async fn preflight_with_fallback_to_stock_reports_full_path_when_both_fail() {
let mut cfg = Config::default();
cfg.browser_provider = BrowserProvider::Cdp;
cfg.external_cdp_url = Some("http://127.0.0.1:1".into());
cfg.chrome_path = Some("/nonexistent/crawlex-slice-36-no-chrome".into());
cfg.provider_fallback = ProviderFallbackConfig {
enabled: true,
order: vec![BrowserProvider::Stock],
};
let pool = build_pool(cfg);
let err = pool.preflight().await.unwrap_err();
let msg = format!("{err}");
assert!(msg.contains("primary `cdp`"), "{msg}");
assert!(msg.contains("fallback attempted"), "{msg}");
assert!(msg.contains("stock=>"), "{msg}");
}
#[tokio::test(flavor = "current_thread")]
async fn preflight_with_fallback_skips_cdp_when_endpoint_missing() {
let mut cfg = Config::default();
cfg.browser_provider = BrowserProvider::Stock;
cfg.chrome_path = Some("/nonexistent/crawlex-slice-36-no-chrome".into());
cfg.provider_fallback = ProviderFallbackConfig {
enabled: true,
order: vec![BrowserProvider::Cdp],
};
let pool = build_pool(cfg);
let err = pool.preflight().await.unwrap_err();
let msg = format!("{err}");
assert!(msg.contains("primary `stock`"), "{msg}");
assert!(
msg.contains("no external_cdp_url configured"),
"skipped reason must be in the surfaced error: {msg}"
);
}
}
#[cfg(test)]
mod reject_resource_type_wiring {
use super::*;
use crate::config::RejectResourceType;
#[test]
fn block_resource_patterns_match_typed_variants() {
let string_form = collect_block_resource_patterns(&[
"image".into(),
"media".into(),
"font".into(),
"stylesheet".into(),
]);
let mut typed_form: Vec<String> = Vec::new();
for rt in [
RejectResourceType::Image,
RejectResourceType::Media,
RejectResourceType::Font,
RejectResourceType::Stylesheet,
] {
typed_form.extend(rt.url_patterns().iter().map(|s| s.to_string()));
}
assert_eq!(string_form, typed_form);
}
#[test]
fn typed_image_patterns_cover_canonical_extensions() {
let pats: Vec<&str> = RejectResourceType::Image.url_patterns().to_vec();
for required in [
"*.png", "*.jpg", "*.jpeg", "*.gif", "*.webp", "*.avif", "*.svg", "*.ico",
] {
assert!(pats.contains(&required), "missing pattern: {required}");
}
}
}