captchaforge 0.2.39

//! Vendor-JS scraper + auto-stealth synthesis.
//!
//! Anti-bot vendors (Cloudflare, hCaptcha, reCAPTCHA, DataDome,
//! …) ship JavaScript that probes a hundred-odd `navigator.*` /
//! `window.*` / `WebGL.*` / `canvas.*` / `battery.*` / `media.*` /
//! `font.*` / `screen.*` surfaces. Each unique probe surface is a
//! point on the fingerprint manifold. When a vendor adds a new
//! probe (every 2-4 weeks for active vendors), captchaforge's
//! stealth coverage decays unless someone manually adds the new
//! override.
//!
//! [`VendorJsScraper`] tracks the adversary on autopilot:
//!
//! 1. Periodically fetch the vendor's challenge JS payload (URL
//!    list per [`VendorJsTarget`]).
//! 2. Parse each payload's AST, extract every `navigator.X`,
//!    `window.X`, `WebGL.X`, `canvas.X`, etc. probe surface.
//! 3. Diff against the previous payload's probe surface.
//! 4. Emit a [`ScrapeReport`] of new + removed probes.
//! 5. Optionally: auto-generate stealth-coverage entries for new
//!    probes and append to a candidate-overrides TOML for human
//!    review before merging.
//!
//! ## Pure-Rust by design
//!
//! Uses regex-based extraction rather than a real JS parser. JS
//! parsers (swc, oxc, rome) are heavy + bring in 50+ deps. The
//! probe-surface extraction is regex-tractable because vendor JS
//! invariably accesses surfaces via `e.X` / `n.X` / dot-chain
//! literals, patterns a regex matches reliably. Tradeoff: we miss
//! computed-property accesses (`e[X]` where `X` is a runtime var).
//! Acceptable: those are <5% of probes in current vendor JS.

#![allow(dead_code)] // module is opt-in (wired by the (future) D2 trainer).

use std::collections::HashSet;
use std::fmt;

/// One vendor target (name + URLs whose JS payload we inspect).
///
/// Vendors split their JS across multiple files (a thin
/// loader + a fat challenge bundle); list every URL whose
/// content might carry probes so the scraper sees the whole
/// surface.
#[derive(Debug, Clone)]
pub struct VendorJsTarget {
    pub name: String,
    pub urls: Vec<String>,
}

/// One probe surface, a `navigator.*` / `window.*` / etc. access
/// the vendor's JS performs at runtime to fingerprint the visitor.
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct ProbeSurface {
    /// Receiver namespace: `"navigator"`, `"window"`,
    /// `"WebGLRenderingContext"`, `"document"`, `"screen"`, …
    pub receiver: String,
    /// Property accessed on the receiver: `"webdriver"`,
    /// `"hardwareConcurrency"`, `"plugins"`, …
    pub property: String,
}

impl ProbeSurface {
    pub fn new(receiver: impl Into<String>, property: impl Into<String>) -> Self {
        Self {
            receiver: receiver.into(),
            property: property.into(),
        }
    }
}

impl fmt::Display for ProbeSurface {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{}.{}", self.receiver, self.property)
    }
}

/// Result of one scraper run against one vendor target.
///
/// Compares the freshly-extracted probe set against the previous
/// run's set (from [`VendorJsScraper::set_baseline`]) and reports
/// what's new / what's gone / what's stable.
#[derive(Debug, Clone, Default)]
pub struct ScrapeReport {
    pub vendor: String,
    pub urls_fetched: Vec<String>,
    pub bytes_total: usize,
    pub probes_total: usize,
    pub probes_new: Vec<ProbeSurface>,
    pub probes_removed: Vec<ProbeSurface>,
    pub probes_stable: Vec<ProbeSurface>,
}

impl ScrapeReport {
    /// Render a human-readable summary suitable for a daily
    /// CI report email / Slack DM.
    pub fn render_summary(&self) -> String {
        use std::fmt::Write;
        let mut out = String::with_capacity(512);
        let _ = writeln!(out, "vendor: {}", self.vendor);
        let _ = writeln!(out, "urls fetched: {}", self.urls_fetched.len());
        let _ = writeln!(out, "bytes scanned: {}", self.bytes_total);
        let _ = writeln!(out, "total probes: {}", self.probes_total);
        let _ = writeln!(
            out,
            "new probes: {}, removed: {}, stable: {}",
            self.probes_new.len(),
            self.probes_removed.len(),
            self.probes_stable.len()
        );
        if !self.probes_new.is_empty() {
            out.push_str("\nNEW probes (need stealth coverage):\n");
            for p in &self.probes_new {
                let _ = writeln!(out, "  + {p}");
            }
        }
        if !self.probes_removed.is_empty() {
            out.push_str("\nREMOVED probes (vendor stopped checking):\n");
            for p in &self.probes_removed {
                let _ = writeln!(out, "  - {p}");
            }
        }
        out
    }
}

/// Scraper instance, keeps a per-vendor baseline of probe sets
/// across runs so successive scrapes can diff.
pub struct VendorJsScraper {
    baselines: std::collections::HashMap<String, HashSet<ProbeSurface>>,
}

impl VendorJsScraper {
    pub fn new() -> Self {
        Self {
            baselines: std::collections::HashMap::new(),
        }
    }

    /// Pin a baseline probe set for `vendor`. Subsequent scrapes
    /// of the same vendor diff against this baseline. Used for
    /// "load yesterday's baseline at start of day" workflows.
    pub fn set_baseline(&mut self, vendor: impl Into<String>, probes: HashSet<ProbeSurface>) {
        self.baselines.insert(vendor.into(), probes);
    }

    /// Fetch every URL in `target`, extract probes from each, and
    /// return the diffed [`ScrapeReport`].
    ///
    /// Network errors on a single URL are recorded in the report's
    /// `urls_fetched` list (only successfully-fetched URLs land
    /// there). A run with zero successful fetches still returns a
    /// report (operators should alert on `urls_fetched.is_empty()`).
    pub async fn scrape(
        &mut self,
        target: &VendorJsTarget,
        client: &reqwest::Client,
    ) -> ScrapeReport {
        let mut probes: HashSet<ProbeSurface> = HashSet::new();
        let mut urls_fetched = Vec::new();
        let mut bytes_total = 0usize;
        for url in &target.urls {
            // Log host only, full URLs may include vendor-specific
            // CDN paths with operator-identifying query strings. Host
            // is enough for an operator to spot a vendor outage.
            let host_label = url::Url::parse(url)
                .ok()
                .and_then(|u| u.host_str().map(str::to_string))
                .unwrap_or_else(|| "(unparseable)".to_string());
            match client.get(url).send().await {
                Ok(resp) => match resp.text().await {
                    Ok(body) => {
                        bytes_total += body.len();
                        urls_fetched.push(url.clone());
                        for p in extract_probes(&body) {
                            probes.insert(p);
                        }
                    }
                    Err(e) => {
                        tracing::warn!(host = %host_label, error = %e, "vendor_scraper: body read failed");
                    }
                },
                Err(e) => {
                    tracing::warn!(host = %host_label, error = %e, "vendor_scraper: GET failed");
                }
            }
        }

        let baseline = self
            .baselines
            .get(&target.name)
            .cloned()
            .unwrap_or_default();

        let mut probes_new: Vec<ProbeSurface> = probes.difference(&baseline).cloned().collect();
        let mut probes_removed: Vec<ProbeSurface> = baseline.difference(&probes).cloned().collect();
        let mut probes_stable: Vec<ProbeSurface> =
            probes.intersection(&baseline).cloned().collect();
        probes_new.sort();
        probes_removed.sort();
        probes_stable.sort();

        // Update the baseline for next run.
        self.baselines.insert(target.name.clone(), probes.clone());

        ScrapeReport {
            vendor: target.name.clone(),
            urls_fetched,
            bytes_total,
            probes_total: probes.len(),
            probes_new,
            probes_removed,
            probes_stable,
        }
    }

    /// Borrow the current baseline for `vendor`. Useful for tests
    /// + persistence.
    pub fn baseline(&self, vendor: &str) -> Option<&HashSet<ProbeSurface>> {
        self.baselines.get(vendor)
    }
}

impl Default for VendorJsScraper {
    fn default() -> Self {
        Self::new()
    }
}

/// Extract every probe surface from a JS payload.
///
/// Pattern: `<receiver>.<property>` where receiver is one of the
/// known fingerprint namespaces and property is a JS identifier.
/// Matches both literal-dot and `?.` optional-chain forms.
///
/// Pure function (no IO); cheap enough to call on every refresh.
pub fn extract_probes(js: &str) -> Vec<ProbeSurface> {
    let mut out: HashSet<ProbeSurface> = HashSet::new();
    for receiver in FINGERPRINT_RECEIVERS {
        // Cheap substring scan, anchor on `receiver.` and walk the
        // following identifier. Faster than compiling a regex per
        // receiver across multi-MB JS payloads.
        let needle = format!("{receiver}.");
        let bytes = js.as_bytes();
        let mut cursor = 0usize;
        while let Some(rel) = js[cursor..].find(&needle) {
            let pos = cursor + rel;
            // Word-boundary check on the LEFT: `xnavigator.foo`
            // shouldn't match `navigator.foo`.
            let left_ok = pos == 0 || !is_ident_char(bytes[pos - 1] as char);
            if !left_ok {
                cursor = pos + 1;
                continue;
            }
            let prop_start = pos + needle.len();
            // Walk the identifier after the dot.
            let mut prop_end = prop_start;
            while prop_end < bytes.len() && is_ident_char(bytes[prop_end] as char) {
                prop_end += 1;
            }
            if prop_end > prop_start {
                let prop = &js[prop_start..prop_end];
                // Skip JS reserved-words that aren't real probes.
                if !is_reserved(prop) {
                    out.insert(ProbeSurface::new(*receiver, prop));
                }
            }
            cursor = prop_end.max(pos + 1);
        }
    }
    let mut v: Vec<ProbeSurface> = out.into_iter().collect();
    v.sort();
    v
}

/// Receivers we treat as fingerprint surfaces. Adding a new entry
/// here expands the scraper's coverage; removing one is a regression.
const FINGERPRINT_RECEIVERS: &[&str] = &[
    "navigator",
    "window",
    "document",
    "screen",
    "history",
    "WebGLRenderingContext",
    "WebGL2RenderingContext",
    "CanvasRenderingContext2D",
    "OffscreenCanvas",
    "AudioContext",
    "BaseAudioContext",
    "RTCPeerConnection",
    "MediaDevices",
    "Battery",
    "BatteryManager",
    "Notification",
    "PerformanceNavigation",
    "Intl",
    "Date",
    "Math",
];

fn is_ident_char(c: char) -> bool {
    c.is_ascii_alphanumeric() || c == '_' || c == '$'
}

/// JS reserved-word filter. The receivers we scan can technically
/// be followed by these (`navigator.then` is legal syntax, just
/// nonsensical) and they're always false-positives.
fn is_reserved(word: &str) -> bool {
    matches!(
        word,
        "then"
            | "catch"
            | "finally"
            | "constructor"
            | "prototype"
            | "toString"
            | "valueOf"
            | "hasOwnProperty"
            | "isPrototypeOf"
            | "propertyIsEnumerable"
            | "__proto__"
            | "length"
    )
}

/// Bundled vendor-target list. Operators can extend with their own
/// targets via `VendorJsTarget` instances passed to
/// [`VendorJsScraper::scrape`]. The bundled list covers the four
/// most-deployed anti-bot stacks; new vendors land as additional
/// entries here as they're identified.
pub fn bundled_targets() -> Vec<VendorJsTarget> {
    vec![
        VendorJsTarget {
            name: "cloudflare-turnstile".into(),
            urls: vec!["https://challenges.cloudflare.com/turnstile/v0/api.js".into()],
        },
        VendorJsTarget {
            name: "hcaptcha".into(),
            urls: vec!["https://hcaptcha.com/1/api.js".into()],
        },
        VendorJsTarget {
            name: "recaptcha-v2".into(),
            urls: vec!["https://www.google.com/recaptcha/api.js".into()],
        },
        VendorJsTarget {
            name: "datadome".into(),
            urls: vec!["https://js.datadome.co/tags.js".into()],
        },
    ]
}

#[cfg(test)]
#[path = "vendor_scraper/tests.rs"]
mod tests;