use std::{
collections::{HashMap, HashSet},
fs, future,
path::{Path, PathBuf},
sync::{
Arc, Mutex,
atomic::{AtomicBool, Ordering},
},
time::{Duration, Instant, SystemTime, UNIX_EPOCH},
};
use chromiumoxide::{
Page as CdpPage,
cdp::{
browser_protocol::{
accessibility::{AxNode, AxValue, GetFullAxTreeParams, QueryAxTreeParams},
browser::{
GetWindowForTargetParams, PermissionDescriptor, PermissionSetting,
SetDownloadBehaviorBehavior, SetDownloadBehaviorParams, SetPermissionParams,
},
dom::{BackendNodeId, GetBoxModelParams, GetDocumentParams, ResolveNodeParams},
emulation::{
ClearDeviceMetricsOverrideParams, SetDeviceMetricsOverrideParams,
SetGeolocationOverrideParams, SetLocaleOverrideParams, SetTimezoneOverrideParams,
SetTouchEmulationEnabledParams, SetUserAgentOverrideParams, UserAgentBrandVersion,
UserAgentMetadata,
},
input::{
DispatchKeyEventParams, DispatchKeyEventType, DispatchMouseEventParams,
DispatchMouseEventType, MouseButton,
},
network::{
Cookie, CookieParam, DeleteCookiesParams, EnableParams as NetworkEnableParams,
EventRequestWillBeSent, EventResponseReceived, Headers, ResourceType,
SetExtraHttpHeadersParams,
},
page::{
AddScriptToEvaluateOnNewDocumentParams, CaptureScreenshotFormat,
EventLifecycleEvent, FrameId, PrintToPdfParams, SetBypassCspParams,
Viewport as CdpClipViewport,
},
target::GetTargetsParams,
},
js_protocol::runtime::{CallFunctionOnParams, EvaluateParams, ExecutionContextId},
},
page::ScreenshotParams,
};
use futures::StreamExt;
use serde_json::Value;
use tokio::{sync::Mutex as AsyncMutex, time};
use crate::{
antibot::{self, AntibotVerdict},
ax::compact_outline,
error::{Result, VoidCrawlError},
input::{HumanizeOptions, Rng, humanized_path},
interrupt::InterruptRegistry,
response::{ResponseCapture, ResponseCaptureLimits},
selector::{self, RawRect, SelectorEntry, SelectorKind, SelectorResolution},
stealth::StealthConfig,
viewport::{ScrollTarget, Viewport},
};
fn runtime_seed() -> u64 {
SystemTime::now().duration_since(UNIX_EPOCH).map_or(0x1234_5678_9ABC_DEF0, |d| {
d.as_secs() ^ u64::from(d.subsec_nanos()).rotate_left(32)
})
}
#[derive(Debug, Clone)]
pub struct PageResponse {
pub html: String,
pub url: String,
pub status_code: Option<u16>,
pub redirected: bool,
pub headers: Vec<(String, String)>,
pub antibot: Option<AntibotVerdict>,
pub endpoints: Option<Vec<String>>,
pub endpoints_truncated: bool,
pub endpoint_sanitizer_version: Option<&'static str>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[allow(
clippy::struct_excessive_bools,
reason = "state snapshot intentionally exposes independent routing flags"
)]
pub struct TabInstrumentationState {
pub low_cdp: bool,
pub network_enabled: bool,
pub runtime_enabled: bool,
pub utility_world_enabled: bool,
pub pre_navigation_stealth: bool,
}
pub const ENDPOINT_SANITIZER_VERSION: &str = "ep-2026.06.06";
const MAX_ENDPOINTS: usize = 256;
pub fn safe_endpoint(raw_url: &str) -> Option<String> {
let head = raw_url.split(['?', '#']).next().unwrap_or("");
let (scheme, rest) = head.split_once("://")?;
let scheme = scheme.to_ascii_lowercase();
if scheme != "http" && scheme != "https" {
return None;
}
let (authority, path) = match rest.split_once('/') {
Some((a, p)) => (a, format!("/{p}")),
None => (rest, String::new()),
};
let host_port = authority.rsplit_once('@').map_or(authority, |(_, hp)| hp).to_ascii_lowercase();
let host = bare_host(&host_port);
if host.is_empty() || is_local_host(host) {
return None;
}
let safe_path: String = path
.split('/')
.map(|seg| if is_safe_segment(seg) { seg } else { ":redacted" })
.collect::<Vec<_>>()
.join("/");
Some(format!("{scheme}://{host_port}{safe_path}"))
}
fn bare_host(host_port: &str) -> &str {
if let Some(after) = host_port.strip_prefix('[') {
return after.split(']').next().unwrap_or("");
}
host_port.split(':').next().unwrap_or("")
}
fn is_local_host(host: &str) -> bool {
if host == "::1"
|| host == "::"
|| host.starts_with("fe80:")
|| host.starts_with("fc")
|| host.starts_with("fd")
{
return true;
}
let mdns_local = host.rsplit_once('.').is_some_and(|(_, tld)| tld == "local");
if host == "localhost" || host == "0.0.0.0" || mdns_local {
return true;
}
if host.starts_with("127.")
|| host.starts_with("10.")
|| host.starts_with("192.168.")
|| host.starts_with("169.254.")
{
return true;
}
let second_octet = |s: &str| s.split('.').nth(1).and_then(|o| o.parse::<u8>().ok());
if host.starts_with("172.") {
return second_octet(host).is_some_and(|o| (16..=31).contains(&o));
}
if host.starts_with("100.") {
return second_octet(host).is_some_and(|o| (64..=127).contains(&o));
}
false
}
fn is_safe_segment(seg: &str) -> bool {
if seg.is_empty() {
return true;
}
if seg.contains(['@', '=', ';', '%', ':']) {
return false;
}
if !seg.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '-' | '_' | '.' | '~')) {
return false;
}
if seg.len() > 15 {
return false;
}
let digits = seg.chars().filter(char::is_ascii_digit).count();
if digits >= 9 {
return false;
}
if seg.len() >= 12 && seg.chars().all(|c| c.is_ascii_hexdigit()) {
return false;
}
if seg.len() >= 12 {
let has_lower = seg.chars().any(|c| c.is_ascii_lowercase());
let has_upper = seg.chars().any(|c| c.is_ascii_uppercase());
let has_digit = seg.chars().any(|c| c.is_ascii_digit());
if has_lower && has_upper && has_digit {
return false;
}
}
true
}
fn finalize_endpoints(seen: &HashSet<String>, capture: bool) -> Option<Vec<String>> {
if !capture {
return None;
}
let mut v: Vec<String> = seen.iter().cloned().collect();
v.sort();
Some(v)
}
fn flatten_headers(value: &serde_json::Value) -> Vec<(String, String)> {
value
.as_object()
.map(|map| {
map.iter()
.filter_map(|(k, v)| v.as_str().map(|s| (k.to_lowercase(), s.to_string())))
.collect()
})
.unwrap_or_default()
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)]
pub struct Bbox {
pub x: u32,
pub y: u32,
pub width: u32,
pub height: u32,
}
#[derive(Debug, Clone)]
pub struct ScreenshotOptions {
pub path: Option<PathBuf>,
pub bbox: Option<Bbox>,
pub selector: Option<SelectorEntry>,
pub viewport: Option<Viewport>,
pub scroll: Option<ScrollTarget>,
pub full_page: bool,
}
impl Default for ScreenshotOptions {
fn default() -> Self {
Self {
path: None,
bbox: None,
selector: None,
viewport: None,
scroll: None,
full_page: true,
}
}
}
impl ScreenshotOptions {
pub fn with_path(mut self, path: impl Into<PathBuf>) -> Self {
self.path = Some(path.into());
self
}
pub fn with_bbox(mut self, bbox: Bbox) -> Self {
self.bbox = Some(bbox);
self
}
pub fn with_selector(mut self, selector: SelectorEntry) -> Self {
self.selector = Some(selector);
self
}
pub fn with_viewport(mut self, viewport: Viewport) -> Self {
self.viewport = Some(viewport);
self
}
pub fn with_scroll(mut self, scroll: ScrollTarget) -> Self {
self.scroll = Some(scroll);
self
}
pub fn viewport_only(mut self) -> Self {
self.full_page = false;
self
}
}
#[derive(Debug)]
pub enum ScreenshotOutput {
Bytes(Vec<u8>),
Path(PathBuf),
}
#[derive(Debug, Clone)]
pub struct DownloadOutcome {
pub path: PathBuf,
pub bytes: u64,
pub content_type: Option<String>,
}
#[derive(Debug)]
pub struct DownloadCapture {
dir: PathBuf,
before: HashSet<PathBuf>,
max_bytes: u64,
}
impl DownloadCapture {
pub async fn wait(self, page: &Page, timeout: Duration) -> Result<DownloadOutcome> {
let result = self.poll(timeout).await;
page.reset_download_behavior().await;
result
}
pub async fn poll(&self, timeout: Duration) -> Result<DownloadOutcome> {
wait_for_new_download(&self.dir, &self.before, self.max_bytes, timeout).await
}
}
const DOCUMENT_SNAPSHOT_JS: &str = r#"
(() => {
const MAX = {
headings: 80,
textBlocks: 240,
links: 160,
controls: 160,
forms: 60,
formControls: 30,
textChars: 700,
smallChars: 220
};
const clean = (value) => String(value || '').replace(/\s+/g, ' ').trim();
const clip = (value, limit) => {
const text = clean(value);
return text.length > limit ? text.slice(0, Math.max(0, limit - 3)) + '...' : text;
};
const visible = (el) => {
if (!el || !el.isConnected) return false;
const style = window.getComputedStyle(el);
if (!style || style.display === 'none' || style.visibility === 'hidden') return false;
const rect = el.getBoundingClientRect();
return rect.width > 0 && rect.height > 0;
};
const attr = (el, name) => {
const value = el.getAttribute(name);
return value == null || value === '' ? null : clip(value, MAX.smallChars);
};
const labelText = (el) => {
const id = el.id ? CSS.escape(el.id) : null;
const label = id ? document.querySelector(`label[for="${id}"]`) : null;
return clip(
el.getAttribute('aria-label')
|| el.getAttribute('title')
|| el.getAttribute('placeholder')
|| (label && label.textContent)
|| el.value
|| el.textContent
|| el.name
|| '',
MAX.smallChars
);
};
const control = (el) => ({
tag: el.tagName.toLowerCase(),
type: attr(el, 'type'),
role: attr(el, 'role'),
name: labelText(el) || null,
placeholder: attr(el, 'placeholder'),
disabled: Boolean(el.disabled || el.getAttribute('aria-disabled') === 'true')
});
const all = (selector) => Array.from(document.querySelectorAll(selector)).filter(visible);
const unique = (items) => Array.from(new Set(items));
const headingNodes = all('h1,h2,h3,h4,h5,h6');
const headings = headingNodes.slice(0, MAX.headings).map((el) => ({
level: Number(el.tagName.slice(1)),
text: clip(el.textContent, MAX.smallChars)
})).filter((h) => h.text);
const textNodes = unique([
...all('main p, main li, article p, article li, section p, blockquote, body > p, td, th'),
...all('[role="main"] p, [role="article"] p')
]).filter((el) => clean(el.textContent).length >= 20);
const text_blocks = textNodes.slice(0, MAX.textBlocks).map((el) => ({
tag: el.tagName.toLowerCase(),
text: clip(el.textContent, MAX.textChars)
})).filter((b) => b.text);
const linkNodes = all('a[href]');
const links = linkNodes.slice(0, MAX.links).map((el) => ({
text: clip(el.textContent || el.getAttribute('aria-label') || el.href, MAX.smallChars),
href: clip(el.href, MAX.smallChars)
})).filter((l) => l.href);
const controlNodes = all('button,input,select,textarea,[role="button"],[role="link"],[role="textbox"],[role="combobox"],[contenteditable="true"]');
const controls = controlNodes.slice(0, MAX.controls).map(control);
const formNodes = all('form');
const forms = formNodes.slice(0, MAX.forms).map((form) => {
const fields = Array.from(form.querySelectorAll('button,input,select,textarea,[role="button"],[role="textbox"],[role="combobox"]'))
.filter(visible)
.slice(0, MAX.formControls)
.map(control);
return {
action: attr(form, 'action') || (form.action ? clip(form.action, MAX.smallChars) : null),
method: clip(form.method || 'get', 20).toLowerCase(),
controls: fields
};
});
return {
url: location.href,
title: document.title || null,
headings,
text_blocks,
links,
controls,
forms,
total: {
headings: headingNodes.length,
text_blocks: textNodes.length,
links: linkNodes.length,
controls: controlNodes.length,
forms: formNodes.length
}
};
})()
"#;
#[derive(Debug)]
pub struct Page {
inner: CdpPage,
interrupts: Arc<InterruptRegistry>,
download_armed: AtomicBool,
cursor: Mutex<(f64, f64)>,
capture_lock: Arc<AsyncMutex<()>>,
viewport_override: Mutex<Option<Viewport>>,
network_enabled: AtomicBool,
runtime_enabled: AtomicBool,
pre_navigation_stealth: AtomicBool,
}
impl Page {
pub(crate) fn new(
inner: CdpPage,
capture_lock: Arc<AsyncMutex<()>>,
interrupts: Arc<InterruptRegistry>,
) -> Self {
Self {
inner,
interrupts,
download_armed: AtomicBool::new(false),
cursor: Mutex::new((0.0, 0.0)),
network_enabled: AtomicBool::new(false),
runtime_enabled: AtomicBool::new(false),
pre_navigation_stealth: AtomicBool::new(false),
capture_lock,
viewport_override: Mutex::new(None),
}
}
pub(crate) fn cdp(&self) -> &CdpPage {
&self.inner
}
pub(crate) fn clone_handle(&self) -> Self {
Self::new(self.inner.clone(), Arc::clone(&self.capture_lock), Arc::clone(&self.interrupts))
}
pub(crate) fn capture_lock(&self) -> Arc<AsyncMutex<()>> {
Arc::clone(&self.capture_lock)
}
pub async fn window_id(&self) -> Result<i64> {
let params =
GetWindowForTargetParams::builder().target_id(self.inner.target_id().clone()).build();
let result = self
.inner
.execute(params)
.await
.map_err(|e| VoidCrawlError::PageError(format!("getWindowForTarget: {e}")))?;
Ok(result.result.window_id.inner().to_owned())
}
pub async fn alone_in_window(&self) -> Result<bool> {
let mine = self.window_id().await?;
let targets = self
.inner
.execute(GetTargetsParams::default())
.await
.map_err(|e| VoidCrawlError::PageError(format!("getTargets: {e}")))?;
let own_target = self.target_id();
for info in &targets.result.target_infos {
if info.r#type != "page" || info.target_id.inner() == &own_target {
continue;
}
let params =
GetWindowForTargetParams::builder().target_id(info.target_id.clone()).build();
if let Ok(result) = self.inner.execute(params).await
&& *result.result.window_id.inner() == mine
{
return Ok(false);
}
}
Ok(true)
}
pub async fn ensure_active(&self) -> Result<()> {
self.interrupts.page_is_active(&self.target_id()).await
}
pub(crate) fn belongs_to_interrupt_registry(&self, registry: &Arc<InterruptRegistry>) -> bool {
Arc::ptr_eq(&self.interrupts, registry)
}
pub fn is_download_armed(&self) -> bool {
self.download_armed.load(Ordering::Relaxed)
}
pub fn target_id(&self) -> String {
self.inner.target_id().inner().clone()
}
pub fn instrumentation_state(&self) -> TabInstrumentationState {
let network_enabled = self.network_enabled.load(Ordering::Relaxed);
let runtime_enabled = self.runtime_enabled.load(Ordering::Relaxed);
let pre_navigation_stealth = self.pre_navigation_stealth.load(Ordering::Relaxed);
TabInstrumentationState {
low_cdp: !(network_enabled || runtime_enabled),
network_enabled,
runtime_enabled,
utility_world_enabled: false,
pre_navigation_stealth,
}
}
async fn ensure_network_enabled(&self) -> Result<()> {
if !self.network_enabled.load(Ordering::Relaxed) {
self.inner
.execute(NetworkEnableParams::default())
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
self.network_enabled.store(true, Ordering::Relaxed);
}
Ok(())
}
async fn ensure_runtime_enabled(&self) -> Result<()> {
if !self.runtime_enabled.load(Ordering::Relaxed) {
self.inner
.enable_runtime()
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
self.runtime_enabled.store(true, Ordering::Relaxed);
}
Ok(())
}
async fn frame_execution_context_with_runtime(
&self,
frame_id: FrameId,
frame_url_pattern: &str,
) -> Result<ExecutionContextId> {
self.ensure_runtime_enabled().await?;
for attempt in 0..20 {
if let Some(context_id) = self
.inner
.frame_execution_context(frame_id.clone())
.await
.map_err(|e| VoidCrawlError::JsEvalError(e.to_string()))?
{
return Ok(context_id);
}
if attempt < 19 {
time::sleep(Duration::from_millis(50)).await;
}
}
Err(VoidCrawlError::FrameNotFound(format!(
"{frame_url_pattern:?}: matched frame has no scriptable execution context \
(sandboxed without allow-scripts, cross-process, or not yet loaded)"
)))
}
pub(crate) async fn apply_stealth(&self, cfg: &StealthConfig) -> Result<()> {
self.pre_navigation_stealth.store(true, Ordering::Relaxed);
if cfg.use_builtin_stealth {
if let Some(ua) = &cfg.user_agent {
self.inner
.enable_stealth_mode_with_agent(ua)
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
} else {
self.inner
.enable_stealth_mode()
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
}
}
let override_ua = if let Some(ua) = cfg.user_agent.clone() {
Some(ua)
} else if cfg.use_builtin_stealth {
None
} else {
probe_user_agent(&self.inner).await?.map(|ua| dehead(&ua))
};
if let Some(ua) = override_ua {
let (nav_platform, metadata) = client_hints_for_ua(&ua);
let mut builder = SetUserAgentOverrideParams::builder()
.user_agent(ua)
.accept_language(&cfg.locale)
.platform(nav_platform);
if let Some(metadata) = metadata {
builder = builder.user_agent_metadata(metadata);
}
let params = builder.build().map_err(VoidCrawlError::PageError)?;
self.inner
.execute(params)
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
}
self.set_viewport(Viewport::custom(cfg.viewport_width, cfg.viewport_height)).await?;
if cfg.bypass_csp {
let csp = SetBypassCspParams::new(true);
self.inner.execute(csp).await.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
}
if let Some(js) = &cfg.inject_js {
let params = AddScriptToEvaluateOnNewDocumentParams::new(js.clone());
self.inner
.execute(params)
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
}
Ok(())
}
pub async fn add_init_script(&self, script: &str) -> Result<()> {
self.ensure_active().await?;
self.inner
.execute(AddScriptToEvaluateOnNewDocumentParams::new(script.to_string()))
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
Ok(())
}
pub async fn expect_responses(
&self,
patterns: Vec<(String, String)>,
timeout: Duration,
limits: ResponseCaptureLimits,
) -> Result<ResponseCapture> {
ResponseCapture::arm(self.inner.clone(), patterns, timeout, limits).await
}
pub async fn navigate(&self, url: &str) -> Result<()> {
self.ensure_active().await?;
self.inner.goto(url).await.map_err(|e| VoidCrawlError::NavigationFailed(e.to_string()))?;
Ok(())
}
pub async fn goto_and_wait_for_idle(
&self,
url: &str,
timeout: Duration,
) -> Result<PageResponse> {
self.goto_and_wait_for_idle_with_capture(url, timeout, false).await
}
#[allow(
clippy::cognitive_complexity,
reason = "a single navigate select-loop reads more clearly inline than split across helpers"
)]
pub async fn goto_and_wait_for_idle_with_capture(
&self,
url: &str,
timeout: Duration,
capture_endpoints: bool,
) -> Result<PageResponse> {
self.ensure_active().await?;
self.ensure_network_enabled().await?;
let started = Instant::now();
let mut lifecycle = self
.inner
.event_listener::<EventLifecycleEvent>()
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
let mut network = self
.inner
.event_listener::<EventResponseReceived>()
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
let mut requests = if capture_endpoints {
Some(
self.inner
.event_listener::<EventRequestWillBeSent>()
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?,
)
} else {
None
};
self.inner.goto(url).await.map_err(|e| VoidCrawlError::NavigationFailed(e.to_string()))?;
let deadline = time::sleep(timeout);
tokio::pin!(deadline);
let mut status_code: Option<u16> = None;
let mut redirect_count: u32 = 0;
let mut headers: Vec<(String, String)> = Vec::new();
let mut endpoints: HashSet<String> = HashSet::new();
let mut endpoints_truncated = false;
loop {
tokio::select! {
biased;
maybe_lifecycle = lifecycle.next() => {
match maybe_lifecycle {
Some(event) if event.name == "networkIdle" => break,
Some(_) => {}
None => break,
}
}
maybe_network = network.next() => {
if let Some(event) = maybe_network {
if event.r#type == ResourceType::Document {
#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)]
let code = event.response.status as u16;
if (300..400).contains(&code) {
redirect_count += 1;
} else if code != 0 {
status_code = Some(code);
headers = flatten_headers(event.response.headers.inner());
}
}
}
}
maybe_request = async {
match requests.as_mut() {
Some(s) => s.next().await,
None => future::pending().await,
}
}, if capture_endpoints => {
if let Some(event) = maybe_request {
if matches!(event.r#type, Some(ResourceType::Xhr | ResourceType::Fetch)) {
if let Some(ep) = safe_endpoint(&event.request.url) {
if !endpoints.contains(&ep) {
if endpoints.len() < MAX_ENDPOINTS {
endpoints.insert(ep);
} else {
endpoints_truncated = true;
}
}
}
}
}
}
() = &mut deadline => {
return Err(VoidCrawlError::NavigationTimeout {
url: url.to_string(),
wait_phase: "networkidle".to_string(),
timeout_secs: timeout.as_secs_f64(),
elapsed_secs: started.elapsed().as_secs_f64(),
});
}
}
}
let html = self.content().await?;
let final_url = self.url().await?.unwrap_or_default();
let antibot = status_code.map(|c| antibot::classify(c, &headers, &html));
Ok(PageResponse {
html,
url: final_url,
status_code,
redirected: redirect_count > 0,
headers,
antibot,
endpoints: finalize_endpoints(&endpoints, capture_endpoints),
endpoints_truncated,
endpoint_sanitizer_version: capture_endpoints.then_some(ENDPOINT_SANITIZER_VERSION),
})
}
pub async fn wait_for_navigation(&self) -> Result<()> {
self.inner
.wait_for_navigation()
.await
.map_err(|e| VoidCrawlError::NavigationFailed(e.to_string()))?;
Ok(())
}
pub async fn wait_for_network_idle(&self, timeout: Duration) -> Result<Option<String>> {
self.ensure_network_enabled().await?;
let mut events = self
.inner
.event_listener::<EventLifecycleEvent>()
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
let deadline = time::sleep(timeout);
tokio::pin!(deadline);
let mut got_almost_idle = false;
loop {
tokio::select! {
biased;
maybe_event = events.next() => {
match maybe_event {
Some(event) => {
match event.name.as_str() {
"networkIdle" => return Ok(Some("networkIdle".into())),
"networkAlmostIdle" => { got_almost_idle = true; }
_ => {} }
}
None => break, }
}
() = &mut deadline => break,
}
}
if got_almost_idle { Ok(Some("networkAlmostIdle".into())) } else { Ok(None) }
}
pub async fn wait_for_selector(&self, selector: &str, timeout: Duration) -> Result<()> {
let sel_lit = serde_json::to_string(selector)
.map_err(|e| VoidCrawlError::Other(format!("selector encode: {e}")))?;
let timeout_ms = u64::try_from(timeout.as_millis()).unwrap_or(u64::MAX);
let js = format!(
"new Promise((resolve, reject) => {{\
const sel = {sel_lit};\
if (document.querySelector(sel)) return resolve(true);\
const root = document.documentElement || document.body;\
const obs = new MutationObserver(() => {{\
if (document.querySelector(sel)) {{\
obs.disconnect();\
clearTimeout(t);\
resolve(true);\
}}\
}});\
obs.observe(root, {{ childList: true, subtree: true }});\
const t = setTimeout(() => {{\
obs.disconnect();\
reject(new Error('wait_for_selector timeout: ' + sel));\
}}, {timeout_ms});\
}})"
);
let params = EvaluateParams::builder()
.expression(js)
.return_by_value(true)
.await_promise(true)
.build()
.map_err(VoidCrawlError::JsEvalError)?;
match self.inner.evaluate_expression(params).await {
Ok(_) => Ok(()),
Err(e) => {
let msg = e.to_string();
if msg.contains("wait_for_selector timeout") {
Err(VoidCrawlError::Timeout(format!(
"selector {selector:?} did not appear within {timeout_ms}ms"
)))
} else {
Err(VoidCrawlError::JsEvalError(msg))
}
}
}
}
pub async fn content(&self) -> Result<String> {
self.inner.content().await.map_err(|e| VoidCrawlError::PageError(e.to_string()))
}
pub async fn title(&self) -> Result<Option<String>> {
self.inner.get_title().await.map_err(|e| VoidCrawlError::PageError(e.to_string()))
}
pub async fn url(&self) -> Result<Option<String>> {
self.inner.url().await.map_err(|e| VoidCrawlError::PageError(e.to_string()))
}
pub async fn document_snapshot(&self) -> Result<Value> {
let result = self
.inner
.evaluate(DOCUMENT_SNAPSHOT_JS)
.await
.map_err(|e| VoidCrawlError::JsEvalError(e.to_string()))?;
Ok(result.value().cloned().unwrap_or(Value::Null))
}
pub async fn evaluate_js(&self, expression: &str) -> Result<Value> {
self.ensure_active().await?;
let result = self
.inner
.evaluate(expression)
.await
.map_err(|e| VoidCrawlError::JsEvalError(e.to_string()))?;
match result.value() {
Some(v) => Ok(v.clone()),
None => Ok(Value::Null),
}
}
pub async fn evaluate_js_in_frame(
&self,
frame_url_pattern: &str,
expression: &str,
) -> Result<Value> {
self.ensure_active().await?;
let frame_id = self.resolve_frame(frame_url_pattern).await?;
let context_id =
self.frame_execution_context_with_runtime(frame_id, frame_url_pattern).await?;
let params = EvaluateParams::builder()
.expression(expression)
.context_id(context_id)
.return_by_value(true)
.await_promise(true)
.build()
.map_err(VoidCrawlError::JsEvalError)?;
let result = self
.inner
.evaluate_expression(params)
.await
.map_err(|e| VoidCrawlError::JsEvalError(e.to_string()))?;
match result.value() {
Some(v) => Ok(v.clone()),
None => Ok(Value::Null),
}
}
async fn resolve_frame(&self, pattern: &str) -> Result<FrameId> {
let frames =
self.inner.frames().await.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
let mut matched: Vec<(FrameId, String)> = Vec::new();
for frame_id in frames {
let url = self
.inner
.frame_url(frame_id.clone())
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
if let Some(url) = url {
if url.contains(pattern) {
matched.push((frame_id, url));
}
}
}
match matched.len() {
0 => Err(VoidCrawlError::FrameNotFound(pattern.to_string())),
1 => Ok(matched.swap_remove(0).0),
n => {
let urls = matched.iter().map(|(_, u)| u.as_str()).collect::<Vec<_>>().join(", ");
Err(VoidCrawlError::AmbiguousFrame(format!(
"{pattern:?} matched {n} frames ({urls}); use a more specific substring"
)))
}
}
}
pub async fn frame_urls(&self) -> Result<Vec<String>> {
let frames =
self.inner.frames().await.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
let mut urls = Vec::with_capacity(frames.len());
for frame_id in frames {
if let Some(url) = self
.inner
.frame_url(frame_id)
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?
{
urls.push(url);
}
}
Ok(urls)
}
pub async fn set_viewport(&self, viewport: Viewport) -> Result<()> {
let metrics = SetDeviceMetricsOverrideParams::new(
i64::from(viewport.width),
i64::from(viewport.height),
viewport.device_scale_factor,
viewport.mobile,
);
self.inner.execute(metrics).await.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
self.inner
.execute(SetTouchEmulationEnabledParams::new(viewport.has_touch))
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
if let Some(ua) = viewport.user_agent.clone() {
let (nav_platform, metadata) = if viewport.mobile {
mobile_ua_platform_and_metadata(&ua)
} else {
client_hints_for_ua(&ua)
};
let mut builder =
SetUserAgentOverrideParams::builder().user_agent(ua).platform(nav_platform);
if let Some(metadata) = metadata {
builder = builder.user_agent_metadata(metadata);
}
let params = builder.build().map_err(VoidCrawlError::PageError)?;
self.inner
.execute(params)
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
}
*self
.viewport_override
.lock()
.map_err(|_| VoidCrawlError::Other("viewport lock poisoned".into()))? = Some(viewport);
self.wait_for_repaint().await
}
pub async fn clear_viewport(&self) -> Result<()> {
self.inner
.execute(ClearDeviceMetricsOverrideParams {})
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
self.inner
.execute(SetTouchEmulationEnabledParams::new(false))
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
*self
.viewport_override
.lock()
.map_err(|_| VoidCrawlError::Other("viewport lock poisoned".into()))? = None;
self.wait_for_repaint().await
}
pub fn current_viewport(&self) -> Option<Viewport> {
self.viewport_override.lock().ok().and_then(|guard| guard.clone())
}
pub async fn screenshot_png(&self) -> Result<Vec<u8>> {
match self.screenshot(ScreenshotOptions::default()).await? {
ScreenshotOutput::Bytes(b) => Ok(b),
ScreenshotOutput::Path(_) => unreachable!("no path supplied"),
}
}
pub async fn screenshot(&self, opts: ScreenshotOptions) -> Result<ScreenshotOutput> {
if opts.bbox.is_some() && opts.selector.is_some() {
return Err(VoidCrawlError::Other(
"ScreenshotOptions: `bbox` and `selector` are mutually exclusive".into(),
));
}
let restore_viewport = if let Some(ref viewport) = opts.viewport {
let prev = self.current_viewport();
self.set_viewport(viewport.clone()).await?;
Some(prev)
} else {
None
};
let result = self.screenshot_inner(&opts).await;
if let Some(prev) = restore_viewport {
let restored = match prev {
Some(v) => self.set_viewport(v).await,
None => self.clear_viewport().await,
};
let _ = restored;
}
result
}
async fn screenshot_inner(&self, opts: &ScreenshotOptions) -> Result<ScreenshotOutput> {
let restore_scroll = match opts.scroll {
Some(_) => Some(self.scroll_position().await?),
None => None,
};
let bbox_shift = if let Some(target) = opts.scroll {
self.scroll_to(target).await?;
self.scroll_position().await?
} else {
(0.0, 0.0)
};
let effective_bbox: Option<(Bbox, bool)> = if let Some(bbox) = opts.bbox {
Some((bbox, true))
} else if let Some(entry) = &opts.selector {
match self.resolve_selector(entry).await? {
SelectorResolution::Resolved { bbox } => Some((bbox, false)),
SelectorResolution::Empty { reason } => {
return Err(VoidCrawlError::ElementNotVisible(reason));
}
SelectorResolution::Ambiguous { reason, .. } => {
return Err(VoidCrawlError::AmbiguousSelector(reason));
}
}
} else {
None
};
let mut builder = ScreenshotParams::builder().format(CaptureScreenshotFormat::Png);
if let Some((bbox, apply_shift)) = effective_bbox {
let (shift_x, shift_y) = if apply_shift { bbox_shift } else { (0.0, 0.0) };
builder = builder
.clip(CdpClipViewport {
x: f64::from(bbox.x) + shift_x,
y: f64::from(bbox.y) + shift_y,
width: f64::from(bbox.width),
height: f64::from(bbox.height),
scale: 1.0,
})
.capture_beyond_viewport(true);
} else if opts.full_page {
builder = builder.full_page(true);
} else if let Some(vp) = self.current_viewport() {
builder = builder.clip(CdpClipViewport {
x: 0.0,
y: 0.0,
width: f64::from(vp.width),
height: f64::from(vp.height),
scale: 1.0,
});
}
let capture_guard = self.capture_lock.lock().await;
self.inner
.bring_to_front()
.await
.map_err(|e| VoidCrawlError::ScreenshotError(e.to_string()))?;
let bytes = self
.inner
.screenshot(builder.build())
.await
.map_err(|e| VoidCrawlError::ScreenshotError(e.to_string()))?;
drop(capture_guard);
if let Some((x, y)) = restore_scroll {
let _ = self.evaluate_js(&format!("window.scrollTo({x}, {y})")).await;
}
if let Some(path) = opts.path.clone() {
fs::write(&path, &bytes).map_err(|e| {
VoidCrawlError::ScreenshotError(format!("write {}: {e}", path.display()))
})?;
Ok(ScreenshotOutput::Path(path))
} else {
Ok(ScreenshotOutput::Bytes(bytes))
}
}
pub(crate) async fn scroll_position(&self) -> Result<(f64, f64)> {
let value = self.evaluate_js("[window.scrollX, window.scrollY]").await?;
let arr = value.as_array().ok_or_else(|| {
VoidCrawlError::JsEvalError("scroll position: expected a [x, y] array".into())
})?;
let x = arr.first().and_then(serde_json::Value::as_f64).unwrap_or(0.0);
let y = arr.get(1).and_then(serde_json::Value::as_f64).unwrap_or(0.0);
Ok((x, y))
}
#[allow(
clippy::cast_precision_loss,
reason = "scroll offsets are CSS pixels, always far below f64's 2^52 exact-integer range"
)]
pub(crate) async fn scroll_to(&self, target: ScrollTarget) -> Result<()> {
let y = match target {
ScrollTarget::Pixels(y) => y as f64,
ScrollTarget::Viewports(n) => {
let height = self.evaluate_js("window.innerHeight").await?.as_f64().unwrap_or(0.0);
height * n
}
};
self.evaluate_js(&format!("window.scrollTo(0, {y})")).await?;
self.wait_for_repaint().await
}
async fn wait_for_repaint(&self) -> Result<()> {
let _ = time::timeout(
Duration::from_millis(500),
self.evaluate_js(
"new Promise(r => requestAnimationFrame(() => requestAnimationFrame(r)))",
),
)
.await;
Ok(())
}
pub async fn pdf_bytes(&self) -> Result<Vec<u8>> {
let params = PrintToPdfParams::default();
self.inner.pdf(params).await.map_err(|e| VoidCrawlError::PdfError(e.to_string()))
}
pub async fn download_to_dir(
&self,
url: &str,
dir: &Path,
timeout: Duration,
max_bytes: u64,
) -> Result<DownloadOutcome> {
self.ensure_active().await?;
let outcome = self.run_download(url, dir, timeout, max_bytes).await;
self.reset_download_behavior().await;
outcome
}
pub async fn arm_download(&self, dir: &Path, max_bytes: u64) -> Result<DownloadCapture> {
self.ensure_active().await?;
let params = SetDownloadBehaviorParams::builder()
.behavior(SetDownloadBehaviorBehavior::AllowAndName)
.download_path(dir.to_string_lossy().into_owned())
.build()
.map_err(VoidCrawlError::PageError)?;
self.inner.execute(params).await.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
self.download_armed.store(true, Ordering::Relaxed);
Ok(DownloadCapture { dir: dir.to_path_buf(), before: dir_entries(dir), max_bytes })
}
pub async fn reset_download_behavior(&self) {
if let Ok(params) = SetDownloadBehaviorParams::builder()
.behavior(SetDownloadBehaviorBehavior::Default)
.build()
{
let _ = self.inner.execute(params).await;
}
self.download_armed.store(false, Ordering::Relaxed);
}
async fn run_download(
&self,
url: &str,
dir: &Path,
timeout: Duration,
max_bytes: u64,
) -> Result<DownloadOutcome> {
let before = dir_entries(dir);
let params = SetDownloadBehaviorParams::builder()
.behavior(SetDownloadBehaviorBehavior::AllowAndName)
.download_path(dir.to_string_lossy().into_owned())
.build()
.map_err(VoidCrawlError::PageError)?;
self.inner.execute(params).await.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
self.download_armed.store(true, Ordering::Relaxed);
if let Some(origin) = origin_of(url) {
let _ = self.inner.goto(&origin).await;
}
let url_json = serde_json::to_string(url).unwrap_or_else(|_| "''".to_string());
let js =
DOWNLOAD_JS.replace("__URL__", &url_json).replace("__MAX__", &max_bytes.to_string());
self.evaluate_js(&js).await?;
const POLL: Duration = Duration::from_millis(200);
let deadline = time::Instant::now() + timeout;
let mut settle = SettleTracker::new();
let mut content_type: Option<String> = None;
let mut done = false;
loop {
if let Ok(state) = self.evaluate_js("window.__vcDl || null").await {
if let Some(ct) = state.get("ct").and_then(|v| v.as_str()) {
content_type = Some(strip_mime_params(ct));
}
if let Some(err) = state.get("err").and_then(|v| v.as_str()) {
return Err(VoidCrawlError::Other(format!("download failed: {err}")));
}
if state.get("done").and_then(Value::as_bool) == Some(true) {
done = true;
}
}
if done {
if let Some(outcome) = settle.poll(dir, &before, max_bytes)? {
return Ok(DownloadOutcome { content_type, ..outcome });
}
}
if time::Instant::now() >= deadline {
return Err(VoidCrawlError::Timeout(format!(
"download did not complete within {}s",
timeout.as_secs()
)));
}
time::sleep(POLL).await;
}
}
pub async fn get_full_ax_tree(&self, depth: Option<i64>) -> Result<Value> {
let params = GetFullAxTreeParams { depth, frame_id: None };
let resp = self
.inner
.execute(params)
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
serde_json::to_value(&resp.result.nodes)
.map_err(|e| VoidCrawlError::PageError(e.to_string()))
}
pub async fn ax_tree_outline(&self, depth: Option<i64>) -> Result<String> {
let tree = self.get_full_ax_tree(depth).await?;
let nodes = tree.as_array().map_or(&[][..], Vec::as_slice);
Ok(compact_outline(nodes))
}
pub async fn query_ax_tree(&self, role: Option<&str>, name: Option<&str>) -> Result<Value> {
let nodes = self.query_ax_nodes(role, name).await?;
serde_json::to_value(&nodes).map_err(|e| VoidCrawlError::PageError(e.to_string()))
}
async fn query_ax_nodes(&self, role: Option<&str>, name: Option<&str>) -> Result<Vec<AxNode>> {
let doc = self
.inner
.execute(GetDocumentParams::default())
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
let params = QueryAxTreeParams {
node_id: Some(doc.result.root.node_id),
accessible_name: name.map(str::to_string),
role: role.map(str::to_string),
..Default::default()
};
let resp = self
.inner
.execute(params)
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
Ok(resp.result.nodes)
}
pub async fn click_by_role(
&self,
role: &str,
name: &str,
nth: usize,
humanize: bool,
) -> Result<()> {
self.ensure_active().await?;
let nodes = self.query_ax_nodes(Some(role), Some(name)).await?;
let backends: Vec<_> =
nodes.iter().filter(|n| !n.ignored).filter_map(|n| n.backend_dom_node_id).collect();
let backend_id = backends.get(nth).copied().ok_or_else(|| {
VoidCrawlError::PageError(format!(
"no AX node with role={role:?} name={name:?} at index {nth} (found {} match(es))",
backends.len()
))
})?;
let resolved = self
.inner
.execute(ResolveNodeParams { backend_node_id: Some(backend_id), ..Default::default() })
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
let object_id = resolved.result.object.object_id.ok_or_else(|| {
VoidCrawlError::PageError("AX node could not be resolved to a DOM handle".into())
})?;
if humanize {
let scroll = CallFunctionOnParams::builder()
.object_id(object_id)
.function_declaration(
"function(){ this.scrollIntoView({block:'center',inline:'center'}); }",
)
.await_promise(false)
.build()
.map_err(VoidCrawlError::PageError)?;
self.inner
.execute(scroll)
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
let bm = self
.inner
.execute(GetBoxModelParams {
backend_node_id: Some(backend_id),
..Default::default()
})
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
let q = bm.result.model.content.inner();
if q.len() < 8 {
return Err(VoidCrawlError::PageError(
"element has no box-model content quad".into(),
));
}
let cx = (q[0] + q[2] + q[4] + q[6]) / 4.0;
let cy = (q[1] + q[3] + q[5] + q[7]) / 4.0;
return self.click_xy(cx, cy, true).await;
}
let call = CallFunctionOnParams::builder()
.object_id(object_id)
.function_declaration(
"function(){ this.scrollIntoView({block:'center',inline:'center'}); this.click(); }",
)
.await_promise(false)
.build()
.map_err(VoidCrawlError::PageError)?;
self.inner.execute(call).await.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
Ok(())
}
pub async fn resolve_selector(&self, entry: &SelectorEntry) -> Result<SelectorResolution> {
match entry.kind {
SelectorKind::Jsonld => Ok(SelectorResolution::Empty {
reason: "jsonld selectors address non-visual structured data (a <script> tag \
has no render box); not resolved to a rectangle"
.into(),
}),
SelectorKind::Regex => Ok(SelectorResolution::Empty {
reason: "regex selectors match raw HTML text, which has no canonical DOM \
element; not resolved to a rectangle"
.into(),
}),
SelectorKind::Visual => Ok(self.resolve_visual_selector(entry).await?),
SelectorKind::Role => self.resolve_role_selector(entry).await,
SelectorKind::Css
| SelectorKind::Xpath
| SelectorKind::Attr
| SelectorKind::GlobalId => self.resolve_dom_selector(entry).await,
}
}
async fn resolve_visual_selector(&self, entry: &SelectorEntry) -> Result<SelectorResolution> {
let (Some(x), Some(y)) = (entry.x, entry.y) else {
return Ok(SelectorResolution::Empty {
reason: "visual selector requires both x and y".into(),
});
};
if x < 0.0 || y < 0.0 {
return Ok(SelectorResolution::Empty {
reason: format!("visual point ({x}, {y}) has a negative coordinate"),
});
}
let dims = self
.evaluate_js("[window.innerWidth, window.innerHeight]")
.await?
.as_array()
.cloned()
.unwrap_or_default();
let (vw, vh) = (
dims.first().and_then(Value::as_f64).unwrap_or(f64::INFINITY),
dims.get(1).and_then(Value::as_f64).unwrap_or(f64::INFINITY),
);
if x > vw || y > vh {
return Ok(SelectorResolution::Empty {
reason: format!(
"visual point ({x}, {y}) is outside the current viewport ({vw}x{vh})"
),
});
}
Ok(SelectorResolution::Resolved {
bbox: RawRect { x, y, width: 1.0, height: 1.0 }.to_bbox(),
})
}
async fn resolve_role_selector(&self, entry: &SelectorEntry) -> Result<SelectorResolution> {
let name = entry.name.as_deref();
let nodes = self.query_ax_nodes(Some(&entry.value), name).await?;
let backends: Vec<_> =
nodes.iter().filter(|n| !n.ignored).filter_map(|n| n.backend_dom_node_id).collect();
let describe = || format!("role={:?} name={:?}", entry.value, name.unwrap_or(""));
if backends.is_empty() {
return Ok(SelectorResolution::Empty {
reason: format!("{} matched no AX nodes", describe()),
});
}
let mut visible = Vec::with_capacity(backends.len());
for backend_id in &backends {
let bm = self
.inner
.execute(GetBoxModelParams {
backend_node_id: Some(*backend_id),
..Default::default()
})
.await;
let Ok(bm) = bm else { continue };
let q = bm.result.model.border.inner();
if q.len() < 8 {
continue;
}
let xs = [q[0], q[2], q[4], q[6]];
let ys = [q[1], q[3], q[5], q[7]];
let left = xs.iter().copied().fold(f64::INFINITY, f64::min);
let top = ys.iter().copied().fold(f64::INFINITY, f64::min);
let right = xs.iter().copied().fold(f64::NEG_INFINITY, f64::max);
let bottom = ys.iter().copied().fold(f64::NEG_INFINITY, f64::max);
let (width, height) = (right - left, bottom - top);
if width > 0.0 && height > 0.0 {
visible.push(RawRect { x: left, y: top, width, height });
}
}
Ok(selector::pick_resolution(backends.len(), &visible, entry.nth, describe))
}
async fn resolve_dom_selector(&self, entry: &SelectorEntry) -> Result<SelectorResolution> {
let candidates = selector::candidates_js(entry).ok_or_else(|| {
VoidCrawlError::PageError(format!("{:?} has no DOM candidate step", entry.kind))
})?;
let count_js = format!("({candidates}).length");
let count = self.evaluate_js(&count_js).await?.as_u64().ok_or_else(|| {
VoidCrawlError::JsEvalError("candidate count was not a number".into())
})?;
let total_matches = usize::try_from(count).map_err(|_| {
VoidCrawlError::JsEvalError(format!("implausible candidate count: {count}"))
})?;
let rects_js = selector::visible_rects_js(&candidates);
let raw: Value = self.evaluate_js(&rects_js).await?;
let visible: Vec<RawRect> = serde_json::from_value(raw)
.map_err(|e| VoidCrawlError::JsEvalError(format!("rect decode failed: {e}")))?;
let describe = || format!("{:?} {:?}", entry.kind, entry.value);
Ok(selector::pick_resolution(total_matches, &visible, entry.nth, describe))
}
pub async fn ax_outline_in_frame(
&self,
frame_url_pattern: &str,
depth: Option<i64>,
) -> Result<String> {
let frame_id = self.resolve_frame(frame_url_pattern).await?;
let resp = self
.inner
.execute(GetFullAxTreeParams { depth, frame_id: Some(frame_id) })
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
let nodes = serde_json::to_value(&resp.result.nodes)
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
Ok(compact_outline(nodes.as_array().map_or(&[][..], Vec::as_slice)))
}
pub async fn click_ax_in_frame(
&self,
frame_url_pattern: &str,
role: &str,
name: &str,
nth: usize,
humanize: bool,
) -> Result<()> {
let q = self.ax_content_quad_in_frame(frame_url_pattern, role, name, nth).await?;
let cx = (q[0] + q[2] + q[4] + q[6]) / 4.0;
let cy = (q[1] + q[3] + q[5] + q[7]) / 4.0;
self.click_xy(cx, cy, humanize).await
}
pub async fn ax_box_in_frame(
&self,
frame_url_pattern: &str,
role: &str,
name: &str,
nth: usize,
) -> Result<Vec<f64>> {
let q = self.ax_content_quad_in_frame(frame_url_pattern, role, name, nth).await?;
let xs = [q[0], q[2], q[4], q[6]];
let ys = [q[1], q[3], q[5], q[7]];
let left = xs.iter().copied().fold(f64::INFINITY, f64::min);
let top = ys.iter().copied().fold(f64::INFINITY, f64::min);
let right = xs.iter().copied().fold(f64::NEG_INFINITY, f64::max);
let bottom = ys.iter().copied().fold(f64::NEG_INFINITY, f64::max);
Ok(vec![left, top, right - left, bottom - top])
}
async fn ax_content_quad_in_frame(
&self,
frame_url_pattern: &str,
role: &str,
name: &str,
nth: usize,
) -> Result<Vec<f64>> {
let backend_id = self.ax_backend_in_frame(frame_url_pattern, role, name, nth).await?;
let bm = self
.inner
.execute(GetBoxModelParams { backend_node_id: Some(backend_id), ..Default::default() })
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
let quad = bm.result.model.content.inner().clone();
if quad.len() < 8 {
return Err(VoidCrawlError::PageError("AX node has no box-model content quad".into()));
}
Ok(quad)
}
async fn ax_backend_in_frame(
&self,
frame_url_pattern: &str,
role: &str,
name: &str,
nth: usize,
) -> Result<BackendNodeId> {
fn ax_text(v: Option<&AxValue>) -> &str {
v.and_then(|a| a.value.as_ref()).and_then(Value::as_str).unwrap_or("")
}
let frame_id = self.resolve_frame(frame_url_pattern).await?;
let resp = self
.inner
.execute(GetFullAxTreeParams { depth: None, frame_id: Some(frame_id) })
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
let matched = resp
.result
.nodes
.iter()
.filter(|n| {
!n.ignored
&& ax_text(n.role.as_ref()) == role
&& (name.is_empty() || ax_text(n.name.as_ref()) == name)
})
.filter_map(|n| n.backend_dom_node_id)
.nth(nth);
matched.ok_or_else(|| {
VoidCrawlError::PageError(format!(
"no AX node with role={role:?} name={name:?} at index {nth} in frame {frame_url_pattern:?}"
))
})
}
pub async fn move_mouse(&self, x: f64, y: f64, humanize: bool) -> Result<()> {
self.ensure_active().await?;
if humanize {
let start = *self
.cursor
.lock()
.map_err(|_| VoidCrawlError::Other("cursor lock poisoned".into()))?;
let mut rng = Rng::seed(runtime_seed());
let path = humanized_path(start, (x, y), &HumanizeOptions::default(), &mut rng);
for step in path {
time::sleep(Duration::from_millis(step.delay_ms)).await;
self.dispatch_mouse_event(
DispatchMouseEventType::MouseMoved,
step.x,
step.y,
None,
None,
None,
None,
None,
)
.await?;
}
} else {
self.dispatch_mouse_event(
DispatchMouseEventType::MouseMoved,
x,
y,
None,
None,
None,
None,
None,
)
.await?;
}
*self.cursor.lock().map_err(|_| VoidCrawlError::Other("cursor lock poisoned".into()))? =
(x, y);
Ok(())
}
pub async fn click_xy(&self, x: f64, y: f64, humanize: bool) -> Result<()> {
self.ensure_active().await?;
self.move_mouse(x, y, humanize).await?;
self.dispatch_mouse_event(
DispatchMouseEventType::MousePressed,
x,
y,
Some(MouseButton::Left),
Some(1),
None,
None,
None,
)
.await?;
self.dispatch_mouse_event(
DispatchMouseEventType::MouseReleased,
x,
y,
Some(MouseButton::Left),
Some(1),
None,
None,
None,
)
.await?;
Ok(())
}
pub async fn set_geolocation(
&self,
latitude: f64,
longitude: f64,
accuracy: Option<f64>,
) -> Result<()> {
self.ensure_active().await?;
let grant = SetPermissionParams {
permission: PermissionDescriptor::new("geolocation"),
setting: PermissionSetting::Granted,
origin: None,
embedded_origin: None,
browser_context_id: None,
};
self.inner.execute(grant).await.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
let params = SetGeolocationOverrideParams {
latitude: Some(latitude),
longitude: Some(longitude),
accuracy: Some(accuracy.unwrap_or(50.0)),
..Default::default()
};
self.inner.execute(params).await.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
Ok(())
}
pub async fn set_locale(&self, locale: &str) -> Result<()> {
self.ensure_active().await?;
let params = SetLocaleOverrideParams { locale: Some(locale.to_string()) };
self.inner.execute(params).await.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
Ok(())
}
pub async fn set_timezone(&self, timezone_id: &str) -> Result<()> {
self.ensure_active().await?;
let params = SetTimezoneOverrideParams::new(timezone_id.to_string());
self.inner.execute(params).await.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
Ok(())
}
pub async fn query_selector(&self, selector: &str) -> Result<Option<String>> {
let js = format!(
"(function(){{ var el = document.querySelector({selector:?}); \
return el === null ? null : el.innerHTML; }})()"
);
let result = self
.inner
.evaluate_expression(js)
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
let val: Value = match result.into_value() {
Ok(v) => v,
Err(_) => return Ok(None),
};
match val {
Value::Null => Ok(None),
Value::String(s) => Ok(Some(s)),
other => Ok(Some(other.to_string())),
}
}
pub async fn query_selector_all(&self, selector: &str) -> Result<Vec<String>> {
let js = format!("[...document.querySelectorAll({selector:?})].map(e => e.innerHTML)");
let val: Value = self
.inner
.evaluate_expression(js)
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?
.into_value()
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
match val {
Value::Array(arr) => Ok(arr
.into_iter()
.map(|v| match v {
Value::String(s) => s,
other => other.to_string(),
})
.collect()),
_ => Ok(Vec::new()),
}
}
pub async fn click_element(&self, selector: &str) -> Result<()> {
self.ensure_active().await?;
let el = self
.inner
.find_element(selector)
.await
.map_err(|e| VoidCrawlError::ElementNotFound(e.to_string()))?;
el.click().await.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
Ok(())
}
pub async fn type_into(&self, selector: &str, text: &str) -> Result<()> {
self.ensure_active().await?;
let el = self
.inner
.find_element(selector)
.await
.map_err(|e| VoidCrawlError::ElementNotFound(e.to_string()))?;
el.focus().await.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
el.type_str(text).await.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
Ok(())
}
pub async fn set_headers(&self, headers: HashMap<String, String>) -> Result<()> {
self.ensure_active().await?;
self.ensure_network_enabled().await?;
let json_val =
serde_json::to_value(&headers).map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
let params = SetExtraHttpHeadersParams::new(Headers::new(json_val));
self.inner.execute(params).await.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
Ok(())
}
pub async fn get_cookies(&self) -> Result<Vec<Cookie>> {
self.inner.get_cookies().await.map_err(|e| VoidCrawlError::PageError(e.to_string()))
}
pub async fn set_cookie(&self, cookie: CookieParam) -> Result<()> {
self.ensure_active().await?;
self.inner
.set_cookie(cookie)
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
Ok(())
}
pub async fn set_cookies(&self, cookies: Vec<CookieParam>) -> Result<()> {
self.ensure_active().await?;
self.inner
.set_cookies(cookies)
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
Ok(())
}
pub async fn delete_cookies(&self, cookies: Vec<DeleteCookiesParams>) -> Result<()> {
self.ensure_active().await?;
self.inner
.delete_cookies(cookies)
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
Ok(())
}
#[allow(clippy::too_many_arguments)]
pub async fn dispatch_mouse_event(
&self,
event_type: DispatchMouseEventType,
x: f64,
y: f64,
button: Option<MouseButton>,
click_count: Option<i64>,
delta_x: Option<f64>,
delta_y: Option<f64>,
modifiers: Option<i64>,
) -> Result<()> {
self.ensure_active().await?;
let mut builder = DispatchMouseEventParams::builder().r#type(event_type).x(x).y(y);
if let Some(b) = button {
builder = builder.button(b);
}
if let Some(c) = click_count {
builder = builder.click_count(c);
}
if let Some(dx) = delta_x {
builder = builder.delta_x(dx);
}
if let Some(dy) = delta_y {
builder = builder.delta_y(dy);
}
if let Some(m) = modifiers {
builder = builder.modifiers(m);
}
let params = builder.build().map_err(VoidCrawlError::PageError)?;
self.inner.execute(params).await.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
Ok(())
}
pub async fn dispatch_key_event(
&self,
event_type: DispatchKeyEventType,
key: Option<&str>,
code: Option<&str>,
text: Option<&str>,
modifiers: Option<i64>,
) -> Result<()> {
self.ensure_active().await?;
let mut builder = DispatchKeyEventParams::builder().r#type(event_type);
if let Some(k) = key {
builder = builder.key(k);
}
if let Some(c) = code {
builder = builder.code(c);
}
if let Some(t) = text {
builder = builder.text(t);
}
if let Some(m) = modifiers {
builder = builder.modifiers(m);
}
let params = builder.build().map_err(VoidCrawlError::PageError)?;
self.inner.execute(params).await.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
Ok(())
}
pub async fn close(&self) -> Result<()> {
self.inner.clone().close().await.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
Ok(())
}
pub fn inner(&self) -> &CdpPage {
&self.inner
}
}
const DOWNLOAD_JS: &str = r"(() => {
window.__vcDl = { ct: null, err: null, done: false };
(async () => {
try {
const MAX = __MAX__;
const ctrl = new AbortController();
const resp = await fetch(__URL__, { credentials: 'include', signal: ctrl.signal });
window.__vcDl.ct = resp.headers.get('content-type');
const cl = resp.headers.get('content-length');
if (cl && Number(cl) > MAX) { ctrl.abort(); throw new Error('content-length ' + cl + ' exceeds limit ' + MAX); }
let blob;
if (resp.body && resp.body.getReader) {
const reader = resp.body.getReader();
const chunks = []; let total = 0;
for (;;) {
const { done, value } = await reader.read();
if (done) break;
total += value.byteLength;
if (total > MAX) { ctrl.abort(); throw new Error('exceeded size limit ' + MAX + ' bytes'); }
chunks.push(value);
}
blob = new Blob(chunks);
} else {
blob = await resp.blob();
if (blob.size > MAX) throw new Error('exceeded size limit ' + MAX + ' bytes');
}
const a = document.createElement('a');
a.href = URL.createObjectURL(blob);
a.download = (__URL__.split(/[?#]/)[0].split('/').pop()) || 'download';
(document.body || document.documentElement).appendChild(a);
a.click();
window.__vcDl.done = true;
} catch (e) {
window.__vcDl.err = String((e && e.message) || e);
}
})();
return true;
})()";
fn strip_mime_params(mime: &str) -> String {
mime.split(';').next().unwrap_or(mime).trim().to_ascii_lowercase()
}
fn origin_of(url: &str) -> Option<String> {
let (scheme, rest) = url.split_once("://")?;
let host = rest.split(['/', '?', '#']).next()?;
if host.is_empty() {
return None;
}
Some(format!("{scheme}://{host}"))
}
fn dir_entries(dir: &Path) -> HashSet<PathBuf> {
fs::read_dir(dir).into_iter().flatten().flatten().map(|e| e.path()).collect()
}
fn new_complete_files(dir: &Path, before: &HashSet<PathBuf>) -> Vec<(PathBuf, u64)> {
let Ok(rd) = fs::read_dir(dir) else { return Vec::new() };
rd.flatten()
.filter_map(|entry| {
let path = entry.path();
if before.contains(&path) || path.extension().is_some_and(|e| e == "crdownload") {
return None;
}
match entry.metadata() {
Ok(m) if m.is_file() && m.len() > 0 => Some((path, m.len())),
_ => None,
}
})
.collect()
}
const SETTLE_SIGHTINGS: u32 = 3;
struct SettleTracker {
prev: Option<(PathBuf, u64)>,
stable: u32,
}
impl SettleTracker {
fn new() -> Self {
Self { prev: None, stable: 0 }
}
fn poll(
&mut self,
dir: &Path,
before: &HashSet<PathBuf>,
max_bytes: u64,
) -> Result<Option<DownloadOutcome>> {
let files = new_complete_files(dir, before);
if files.len() > 1 {
let names = files
.iter()
.filter_map(|(p, _)| p.file_name().map(|n| n.to_string_lossy().into_owned()))
.collect::<Vec<_>>()
.join(", ");
return Err(VoidCrawlError::Other(format!(
"ambiguous download: {} new files appeared ({names}); expected exactly one",
files.len()
)));
}
let Some((path, size)) = files.into_iter().next() else {
self.prev = None;
self.stable = 0;
return Ok(None);
};
if self.prev.as_ref().is_some_and(|(p, s)| *p == path && *s == size) {
self.stable += 1;
} else {
self.prev = Some((path.clone(), size));
self.stable = 1;
}
if self.stable < SETTLE_SIGHTINGS {
return Ok(None);
}
if size > max_bytes {
let _ = fs::remove_file(&path);
return Err(VoidCrawlError::Other(format!(
"download is {size} bytes, over the {max_bytes}-byte limit"
)));
}
Ok(Some(DownloadOutcome { path, bytes: size, content_type: None }))
}
}
async fn wait_for_new_download(
dir: &Path,
before: &HashSet<PathBuf>,
max_bytes: u64,
timeout: Duration,
) -> Result<DownloadOutcome> {
const POLL: Duration = Duration::from_millis(250);
let deadline = time::Instant::now() + timeout;
let mut settle = SettleTracker::new();
loop {
if let Some(outcome) = settle.poll(dir, before, max_bytes)? {
return Ok(outcome);
}
if time::Instant::now() >= deadline {
return Err(VoidCrawlError::Timeout(format!(
"no download completed within {}s",
timeout.as_secs()
)));
}
time::sleep(POLL).await;
}
}
async fn probe_user_agent(page: &CdpPage) -> Result<Option<String>> {
let probe = page
.evaluate("navigator.userAgent")
.await
.map_err(|e| VoidCrawlError::PageError(e.to_string()))?;
match probe.value().cloned() {
Some(Value::String(ua)) => Ok(Some(ua)),
_ => Ok(None),
}
}
fn dehead(ua: &str) -> String {
if ua.contains("HeadlessChrome") {
ua.replace("HeadlessChrome", "Chrome")
} else if ua.contains("Headless") {
ua.replace("Headless", "")
} else {
ua.to_string()
}
}
fn client_hints_for_ua(ua: &str) -> (String, Option<UserAgentMetadata>) {
let (nav_platform, ch_platform, platform_version) = if ua.contains("Windows") {
("Win32", "Windows", "15.0.0")
} else if ua.contains("Mac OS X") || ua.contains("Macintosh") {
("MacIntel", "macOS", "14.5.0")
} else {
("Linux x86_64", "Linux", "6.8.0")
};
let chrome_ver: Option<&str> =
ua.split("Chrome/").nth(1).and_then(|s| s.split_whitespace().next());
let major: Option<&str> = chrome_ver.and_then(|v| v.split('.').next());
let mut builder = UserAgentMetadata::builder()
.platform(ch_platform)
.platform_version(platform_version)
.architecture("x86")
.model("")
.mobile(false)
.bitness("64")
.wow64(false);
if let (Some(major), Some(full)) = (major, chrome_ver) {
builder = builder
.brands([
UserAgentBrandVersion::new("Chromium", major),
UserAgentBrandVersion::new("Google Chrome", major),
UserAgentBrandVersion::new("Not_A Brand", "24"),
])
.full_version_lists([
UserAgentBrandVersion::new("Chromium", full),
UserAgentBrandVersion::new("Google Chrome", full),
UserAgentBrandVersion::new("Not_A Brand", "24.0.0.0"),
]);
}
(nav_platform.to_string(), builder.build().ok())
}
fn mobile_ua_platform_and_metadata(ua: &str) -> (String, Option<UserAgentMetadata>) {
if ua.contains("iPad") {
return ("iPad".to_string(), None);
}
if ua.contains("iPhone") {
return ("iPhone".to_string(), None);
}
let chrome_ver: Option<&str> =
ua.split("Chrome/").nth(1).and_then(|s| s.split_whitespace().next());
let major: Option<&str> = chrome_ver.and_then(|v| v.split('.').next());
let mut builder = UserAgentMetadata::builder()
.platform("Android")
.platform_version("14.0.0")
.architecture("")
.model("")
.mobile(true)
.bitness("64")
.wow64(false);
if let (Some(major), Some(full)) = (major, chrome_ver) {
builder = builder
.brands([
UserAgentBrandVersion::new("Chromium", major),
UserAgentBrandVersion::new("Google Chrome", major),
UserAgentBrandVersion::new("Not_A Brand", "24"),
])
.full_version_lists([
UserAgentBrandVersion::new("Chromium", full),
UserAgentBrandVersion::new("Google Chrome", full),
UserAgentBrandVersion::new("Not_A Brand", "24.0.0.0"),
]);
}
("Linux armv8l".to_string(), builder.build().ok())
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic, reason = "test harness")]
mod download_tests {
use std::{fs, path::Path};
use super::{SETTLE_SIGHTINGS, SettleTracker, dir_entries, new_complete_files};
fn touch(dir: &Path, name: &str, bytes: usize) {
fs::write(dir.join(name), vec![0u8; bytes]).unwrap();
}
#[test]
fn new_complete_files_excludes_before_crdownload_and_empty() {
let d = tempfile::tempdir().unwrap();
touch(d.path(), "old.bin", 10);
let before = dir_entries(d.path());
touch(d.path(), "new.bin", 10);
touch(d.path(), "partial.crdownload", 10);
touch(d.path(), "empty.bin", 0);
let files = new_complete_files(d.path(), &before);
assert_eq!(files.len(), 1);
assert_eq!(files[0].0.file_name().unwrap(), "new.bin");
}
#[test]
fn settle_requires_stable_samples_then_accepts() {
let d = tempfile::tempdir().unwrap();
let before = dir_entries(d.path());
touch(d.path(), "f.bin", 100);
let mut s = SettleTracker::new();
for _ in 0..(SETTLE_SIGHTINGS - 1) {
assert!(s.poll(d.path(), &before, 1_000).unwrap().is_none());
}
assert_eq!(s.poll(d.path(), &before, 1_000).unwrap().unwrap().bytes, 100);
}
#[test]
fn settle_resets_when_size_still_changing() {
let d = tempfile::tempdir().unwrap();
let before = dir_entries(d.path());
touch(d.path(), "f.bin", 10);
let mut s = SettleTracker::new();
s.poll(d.path(), &before, 1_000).unwrap();
touch(d.path(), "f.bin", 20); assert!(s.poll(d.path(), &before, 1_000).unwrap().is_none());
}
#[test]
fn settle_rejects_and_deletes_oversize() {
let d = tempfile::tempdir().unwrap();
let before = dir_entries(d.path());
touch(d.path(), "big.bin", 50);
let mut s = SettleTracker::new();
let mut last = Ok(None);
for _ in 0..SETTLE_SIGHTINGS {
last = s.poll(d.path(), &before, 8);
}
assert!(last.is_err());
assert!(!d.path().join("big.bin").exists(), "oversize file should be deleted");
}
#[test]
fn settle_errors_on_multiple_new_files() {
let d = tempfile::tempdir().unwrap();
let before = dir_entries(d.path());
touch(d.path(), "a.bin", 10);
touch(d.path(), "b.bin", 10);
let mut s = SettleTracker::new();
assert!(s.poll(d.path(), &before, 1_000).is_err());
}
}
#[cfg(test)]
#[allow(clippy::unwrap_used, clippy::expect_used, reason = "test harness")]
mod tests {
use std::collections::HashSet;
use super::{client_hints_for_ua, dehead, finalize_endpoints, safe_endpoint};
#[test]
fn safe_endpoint_strips_query_and_fragment() {
assert_eq!(
safe_endpoint("https://api.example.com/v2/search?token=SECRET&q=ada#frag"),
Some("https://api.example.com/v2/search".to_string())
);
assert_eq!(
safe_endpoint("HTTPS://API.Example.COM"),
Some("https://api.example.com".to_string())
);
assert_eq!(
safe_endpoint("https://api.example.com:8443/v1/quote"),
Some("https://api.example.com:8443/v1/quote".to_string())
);
}
#[test]
fn safe_endpoint_drops_userinfo_and_nonhttp_and_local() {
assert_eq!(
safe_endpoint("https://alice:hunter2@host.com/p"),
Some("https://host.com/p".to_string())
);
assert_eq!(safe_endpoint("ws://host.com/socket"), None);
assert_eq!(safe_endpoint("data:text/html,hi"), None);
assert_eq!(safe_endpoint("http://127.0.0.1:9000/api"), None);
assert_eq!(safe_endpoint("http://localhost/api"), None);
assert_eq!(safe_endpoint("http://192.168.1.5/api"), None);
assert_eq!(safe_endpoint("http://172.16.0.9/api"), None);
assert!(safe_endpoint("http://172.32.0.1/api").is_some());
}
#[test]
fn safe_endpoint_redacts_secret_path_segments() {
let jwt = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9";
assert_eq!(
safe_endpoint(&format!("https://h.com/reset/{jwt}")),
Some("https://h.com/reset/:redacted".to_string())
);
assert_eq!(
safe_endpoint("https://h.com/u/ada@example.com/profile"),
Some("https://h.com/u/:redacted/profile".to_string())
);
assert_eq!(
safe_endpoint("https://h.com/pay/4111111111111111"),
Some("https://h.com/pay/:redacted".to_string())
);
assert_eq!(
safe_endpoint("https://h.com/users/123/profile"),
Some("https://h.com/users/123/profile".to_string())
);
}
#[test]
fn safe_endpoint_redacts_by_default_holes() {
assert_eq!(
safe_endpoint("https://h.com/v1/keys/tokentokentokentoken"),
Some("https://h.com/v1/keys/:redacted".to_string())
);
assert_eq!(
safe_endpoint("https://h.com/t/a1b2c3d4e5f6a7b8"),
Some("https://h.com/t/:redacted".to_string())
);
assert_eq!(
safe_endpoint("https://h.com/store;jsessionid=ABC123/cart"),
Some("https://h.com/:redacted/cart".to_string())
);
assert_eq!(
safe_endpoint("https://h.com/s/aB3xK9mP2qR5w"),
Some("https://h.com/s/:redacted".to_string())
);
assert_eq!(
safe_endpoint("https://q1.finance.yahoo.com/v10/finance/quoteSummary/AAPL"),
Some("https://q1.finance.yahoo.com/v10/finance/quoteSummary/AAPL".to_string())
);
}
#[test]
fn safe_endpoint_handles_ipv6_and_cgnat_local_hosts() {
assert_eq!(safe_endpoint("http://[::1]:9000/api"), None);
assert_eq!(safe_endpoint("http://[fe80::1]/api"), None);
assert_eq!(safe_endpoint("http://100.64.0.7/api"), None);
assert_eq!(safe_endpoint("http://printer.local/status"), None);
assert!(safe_endpoint("http://[2606:4700::1111]/cdn-cgi").is_some());
}
#[test]
fn finalize_endpoints_none_when_not_capturing_else_sorted() {
let mut seen = HashSet::new();
seen.insert("https://b.com/2".to_string());
seen.insert("https://a.com/1".to_string());
assert_eq!(finalize_endpoints(&seen, false), None);
assert_eq!(
finalize_endpoints(&seen, true),
Some(vec!["https://a.com/1".to_string(), "https://b.com/2".to_string()])
);
assert_eq!(finalize_endpoints(&HashSet::new(), true), Some(vec![]));
}
const LINUX_UA: &str = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36";
const WIN_UA: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36";
const MAC_UA: &str = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36";
#[test]
fn dehead_strips_headless_token() {
assert_eq!(
dehead("Mozilla/5.0 HeadlessChrome/148.0.0.0 Safari"),
"Mozilla/5.0 Chrome/148.0.0.0 Safari"
);
assert_eq!(dehead(LINUX_UA), LINUX_UA);
}
#[test]
fn platform_matches_ua_os() {
assert_eq!(client_hints_for_ua(LINUX_UA).0, "Linux x86_64");
assert_eq!(client_hints_for_ua(WIN_UA).0, "Win32");
assert_eq!(client_hints_for_ua(MAC_UA).0, "MacIntel");
let md = client_hints_for_ua(LINUX_UA).1.unwrap();
assert_eq!(md.platform, "Linux");
assert!(!md.mobile);
assert_eq!(md.architecture, "x86");
}
#[test]
fn brands_carry_chrome_major_version() {
let md = client_hints_for_ua(LINUX_UA).1.unwrap();
let brands = md.brands.unwrap();
assert!(brands.iter().any(|b| b.brand == "Google Chrome" && b.version == "148"));
assert!(brands.iter().any(|b| b.brand == "Chromium" && b.version == "148"));
assert_eq!(brands.len(), 3);
let full = md.full_version_list.unwrap();
assert!(full.iter().any(|b| b.brand == "Google Chrome" && b.version == "148.0.0.0"));
}
#[test]
fn non_chrome_ua_has_no_brands() {
let firefox = "Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0";
let (nav_platform, md) = client_hints_for_ua(firefox);
assert_eq!(nav_platform, "Linux x86_64");
assert!(md.unwrap().brands.is_none());
}
}