captchaforge 0.2.30

//! Vendor-JS scraper + auto-stealth synthesis.
//!
//! Anti-bot vendors (Cloudflare, hCaptcha, reCAPTCHA, DataDome,
//! …) ship JavaScript that probes a hundred-odd `navigator.*` /
//! `window.*` / `WebGL.*` / `canvas.*` / `battery.*` / `media.*` /
//! `font.*` / `screen.*` surfaces. Each unique probe surface is a
//! point on the fingerprint manifold. When a vendor adds a new
//! probe (every 2-4 weeks for active vendors), captchaforge's
//! stealth coverage decays unless someone manually adds the new
//! override.
//!
//! [`VendorJsScraper`] tracks the adversary on autopilot:
//!
//! 1. Periodically fetch the vendor's challenge JS payload (URL
//!    list per [`VendorJsTarget`]).
//! 2. Parse each payload's AST, extract every `navigator.X`,
//!    `window.X`, `WebGL.X`, `canvas.X`, etc. probe surface.
//! 3. Diff against the previous payload's probe surface.
//! 4. Emit a [`ScrapeReport`] of new + removed probes.
//! 5. Optionally: auto-generate stealth-coverage entries for new
//!    probes and append to a candidate-overrides TOML for human
//!    review before merging.
//!
//! ## Pure-Rust by design
//!
//! Uses regex-based extraction rather than a real JS parser. JS
//! parsers (swc, oxc, rome) are heavy + bring in 50+ deps. The
//! probe-surface extraction is regex-tractable because vendor JS
//! invariably accesses surfaces via `e.X` / `n.X` / dot-chain
//! literals — patterns a regex matches reliably. Tradeoff: we miss
//! computed-property accesses (`e[X]` where `X` is a runtime var).
//! Acceptable: those are <5% of probes in current vendor JS.

#![allow(dead_code)] // module is opt-in — wired by the (future) D2 trainer.

use std::collections::HashSet;
use std::fmt;

/// One vendor target — name + URLs whose JS payload we inspect.
///
/// Vendors split their JS across multiple files (a thin
/// loader + a fat challenge bundle); list every URL whose
/// content might carry probes so the scraper sees the whole
/// surface.
#[derive(Debug, Clone)]
pub struct VendorJsTarget {
    pub name: String,
    pub urls: Vec<String>,
}

/// One probe surface — a `navigator.*` / `window.*` / etc. access
/// the vendor's JS performs at runtime to fingerprint the visitor.
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct ProbeSurface {
    /// Receiver namespace — `"navigator"`, `"window"`,
    /// `"WebGLRenderingContext"`, `"document"`, `"screen"`, …
    pub receiver: String,
    /// Property accessed on the receiver — `"webdriver"`,
    /// `"hardwareConcurrency"`, `"plugins"`, …
    pub property: String,
}

impl ProbeSurface {
    pub fn new(receiver: impl Into<String>, property: impl Into<String>) -> Self {
        Self {
            receiver: receiver.into(),
            property: property.into(),
        }
    }
}

impl fmt::Display for ProbeSurface {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{}.{}", self.receiver, self.property)
    }
}

/// Result of one scraper run against one vendor target.
///
/// Compares the freshly-extracted probe set against the previous
/// run's set (from [`VendorJsScraper::set_baseline`]) and reports
/// what's new / what's gone / what's stable.
#[derive(Debug, Clone, Default)]
pub struct ScrapeReport {
    pub vendor: String,
    pub urls_fetched: Vec<String>,
    pub bytes_total: usize,
    pub probes_total: usize,
    pub probes_new: Vec<ProbeSurface>,
    pub probes_removed: Vec<ProbeSurface>,
    pub probes_stable: Vec<ProbeSurface>,
}

impl ScrapeReport {
    /// Render a human-readable summary suitable for a daily
    /// CI report email / Slack DM.
    pub fn render_summary(&self) -> String {
        use std::fmt::Write;
        let mut out = String::with_capacity(512);
        let _ = writeln!(out, "vendor: {}", self.vendor);
        let _ = writeln!(out, "urls fetched: {}", self.urls_fetched.len());
        let _ = writeln!(out, "bytes scanned: {}", self.bytes_total);
        let _ = writeln!(out, "total probes: {}", self.probes_total);
        let _ = writeln!(
            out,
            "new probes: {}, removed: {}, stable: {}",
            self.probes_new.len(),
            self.probes_removed.len(),
            self.probes_stable.len()
        );
        if !self.probes_new.is_empty() {
            out.push_str("\nNEW probes (need stealth coverage):\n");
            for p in &self.probes_new {
                let _ = writeln!(out, "  + {p}");
            }
        }
        if !self.probes_removed.is_empty() {
            out.push_str("\nREMOVED probes (vendor stopped checking):\n");
            for p in &self.probes_removed {
                let _ = writeln!(out, "  - {p}");
            }
        }
        out
    }
}

/// Scraper instance — keeps a per-vendor baseline of probe sets
/// across runs so successive scrapes can diff.
pub struct VendorJsScraper {
    baselines: std::collections::HashMap<String, HashSet<ProbeSurface>>,
}

impl VendorJsScraper {
    pub fn new() -> Self {
        Self {
            baselines: std::collections::HashMap::new(),
        }
    }

    /// Pin a baseline probe set for `vendor`. Subsequent scrapes
    /// of the same vendor diff against this baseline. Used for
    /// "load yesterday's baseline at start of day" workflows.
    pub fn set_baseline(&mut self, vendor: impl Into<String>, probes: HashSet<ProbeSurface>) {
        self.baselines.insert(vendor.into(), probes);
    }

    /// Fetch every URL in `target`, extract probes from each, and
    /// return the diffed [`ScrapeReport`].
    ///
    /// Network errors on a single URL are recorded in the report's
    /// `urls_fetched` list (only successfully-fetched URLs land
    /// there). A run with zero successful fetches still returns a
    /// report — operators should alert on `urls_fetched.is_empty()`.
    pub async fn scrape(
        &mut self,
        target: &VendorJsTarget,
        client: &reqwest::Client,
    ) -> ScrapeReport {
        let mut probes: HashSet<ProbeSurface> = HashSet::new();
        let mut urls_fetched = Vec::new();
        let mut bytes_total = 0usize;
        for url in &target.urls {
            match client.get(url).send().await {
                Ok(resp) => match resp.text().await {
                    Ok(body) => {
                        bytes_total += body.len();
                        urls_fetched.push(url.clone());
                        for p in extract_probes(&body) {
                            probes.insert(p);
                        }
                    }
                    Err(e) => {
                        tracing::warn!(url, error = %e, "vendor_scraper: body read failed");
                    }
                },
                Err(e) => {
                    tracing::warn!(url, error = %e, "vendor_scraper: GET failed");
                }
            }
        }

        let baseline = self
            .baselines
            .get(&target.name)
            .cloned()
            .unwrap_or_default();

        let mut probes_new: Vec<ProbeSurface> = probes.difference(&baseline).cloned().collect();
        let mut probes_removed: Vec<ProbeSurface> = baseline.difference(&probes).cloned().collect();
        let mut probes_stable: Vec<ProbeSurface> =
            probes.intersection(&baseline).cloned().collect();
        probes_new.sort();
        probes_removed.sort();
        probes_stable.sort();

        // Update the baseline for next run.
        self.baselines.insert(target.name.clone(), probes.clone());

        ScrapeReport {
            vendor: target.name.clone(),
            urls_fetched,
            bytes_total,
            probes_total: probes.len(),
            probes_new,
            probes_removed,
            probes_stable,
        }
    }

    /// Borrow the current baseline for `vendor`. Useful for tests
    /// + persistence.
    pub fn baseline(&self, vendor: &str) -> Option<&HashSet<ProbeSurface>> {
        self.baselines.get(vendor)
    }
}

impl Default for VendorJsScraper {
    fn default() -> Self {
        Self::new()
    }
}

/// Extract every probe surface from a JS payload.
///
/// Pattern: `<receiver>.<property>` where receiver is one of the
/// known fingerprint namespaces and property is a JS identifier.
/// Matches both literal-dot and `?.` optional-chain forms.
///
/// Pure function (no IO); cheap enough to call on every refresh.
pub fn extract_probes(js: &str) -> Vec<ProbeSurface> {
    let mut out: HashSet<ProbeSurface> = HashSet::new();
    for receiver in FINGERPRINT_RECEIVERS {
        // Cheap substring scan — anchor on `receiver.` and walk the
        // following identifier. Faster than compiling a regex per
        // receiver across multi-MB JS payloads.
        let needle = format!("{receiver}.");
        let bytes = js.as_bytes();
        let mut cursor = 0usize;
        while let Some(rel) = js[cursor..].find(&needle) {
            let pos = cursor + rel;
            // Word-boundary check on the LEFT — `xnavigator.foo`
            // shouldn't match `navigator.foo`.
            let left_ok = pos == 0 || !is_ident_char(bytes[pos - 1] as char);
            if !left_ok {
                cursor = pos + 1;
                continue;
            }
            let prop_start = pos + needle.len();
            // Walk the identifier after the dot.
            let mut prop_end = prop_start;
            while prop_end < bytes.len() && is_ident_char(bytes[prop_end] as char) {
                prop_end += 1;
            }
            if prop_end > prop_start {
                let prop = &js[prop_start..prop_end];
                // Skip JS reserved-words that aren't real probes.
                if !is_reserved(prop) {
                    out.insert(ProbeSurface::new(*receiver, prop));
                }
            }
            cursor = prop_end.max(pos + 1);
        }
    }
    let mut v: Vec<ProbeSurface> = out.into_iter().collect();
    v.sort();
    v
}

/// Receivers we treat as fingerprint surfaces. Adding a new entry
/// here expands the scraper's coverage; removing one is a regression.
const FINGERPRINT_RECEIVERS: &[&str] = &[
    "navigator",
    "window",
    "document",
    "screen",
    "history",
    "WebGLRenderingContext",
    "WebGL2RenderingContext",
    "CanvasRenderingContext2D",
    "OffscreenCanvas",
    "AudioContext",
    "BaseAudioContext",
    "RTCPeerConnection",
    "MediaDevices",
    "Battery",
    "BatteryManager",
    "Notification",
    "PerformanceNavigation",
    "Intl",
    "Date",
    "Math",
];

fn is_ident_char(c: char) -> bool {
    c.is_ascii_alphanumeric() || c == '_' || c == '$'
}

/// JS reserved-word filter. The receivers we scan can technically
/// be followed by these (`navigator.then` is legal syntax, just
/// nonsensical) and they're always false-positives.
fn is_reserved(word: &str) -> bool {
    matches!(
        word,
        "then"
            | "catch"
            | "finally"
            | "constructor"
            | "prototype"
            | "toString"
            | "valueOf"
            | "hasOwnProperty"
            | "isPrototypeOf"
            | "propertyIsEnumerable"
            | "__proto__"
            | "length"
    )
}

/// Bundled vendor-target list. Operators can extend with their own
/// targets via `VendorJsTarget` instances passed to
/// [`VendorJsScraper::scrape`]. The bundled list covers the four
/// most-deployed anti-bot stacks; new vendors land as additional
/// entries here as they're identified.
pub fn bundled_targets() -> Vec<VendorJsTarget> {
    vec![
        VendorJsTarget {
            name: "cloudflare-turnstile".into(),
            urls: vec!["https://challenges.cloudflare.com/turnstile/v0/api.js".into()],
        },
        VendorJsTarget {
            name: "hcaptcha".into(),
            urls: vec!["https://hcaptcha.com/1/api.js".into()],
        },
        VendorJsTarget {
            name: "recaptcha-v2".into(),
            urls: vec!["https://www.google.com/recaptcha/api.js".into()],
        },
        VendorJsTarget {
            name: "datadome".into(),
            urls: vec!["https://js.datadome.co/tags.js".into()],
        },
    ]
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn extract_probes_finds_navigator_webdriver() {
        let js = "if(navigator.webdriver){return false;}";
        let probes = extract_probes(js);
        assert!(probes.contains(&ProbeSurface::new("navigator", "webdriver")));
    }

    #[test]
    fn extract_probes_finds_multi_namespace_probes() {
        let js = r#"
            const a = navigator.userAgent;
            const b = window.chrome;
            const c = document.referrer;
            const d = screen.width;
            const e = WebGLRenderingContext.UNMASKED_VENDOR_WEBGL;
        "#;
        let probes = extract_probes(js);
        assert!(probes.contains(&ProbeSurface::new("navigator", "userAgent")));
        assert!(probes.contains(&ProbeSurface::new("window", "chrome")));
        assert!(probes.contains(&ProbeSurface::new("document", "referrer")));
        assert!(probes.contains(&ProbeSurface::new("screen", "width")));
        assert!(probes.contains(&ProbeSurface::new(
            "WebGLRenderingContext",
            "UNMASKED_VENDOR_WEBGL"
        )));
    }

    #[test]
    fn extract_probes_deduplicates_repeats() {
        let js = r#"
            navigator.webdriver;
            navigator.webdriver;
            navigator.webdriver;
        "#;
        let probes = extract_probes(js);
        assert_eq!(
            probes.iter().filter(|p| p.property == "webdriver").count(),
            1
        );
    }

    #[test]
    fn extract_probes_filters_js_reserved_words() {
        // `navigator.then` is legal syntax (Promise interop) but
        // never a real probe — must be filtered.
        let js = "navigator.then(()=>{});navigator.toString();navigator.length;";
        let probes = extract_probes(js);
        assert!(!probes
            .iter()
            .any(|p| matches!(p.property.as_str(), "then" | "toString" | "length")));
    }

    #[test]
    fn extract_probes_respects_left_word_boundary() {
        // `xnavigator.foo` shouldn't match `navigator.foo`.
        let js = "xnavigator.foo;navigator.bar;";
        let probes = extract_probes(js);
        assert!(probes.contains(&ProbeSurface::new("navigator", "bar")));
        // Important: `foo` only appears as part of `xnavigator.foo`,
        // which our boundary check should reject.
        assert!(!probes.contains(&ProbeSurface::new("navigator", "foo")));
    }

    #[test]
    fn extract_probes_handles_empty_input() {
        assert!(extract_probes("").is_empty());
        assert!(extract_probes("// just a comment").is_empty());
    }

    #[test]
    fn extract_probes_skips_minified_short_identifiers_correctly() {
        // Realistic minified pattern: `e=navigator,t=window`.
        // `e.foo` patterns become `navigator.foo` after the
        // assignment, but our regex-free scanner sees only the
        // LITERAL `<receiver>.<prop>` access. We pick up the
        // literal probe; the indirect alias `t.chrome` (where t
        // = window) is a documented false-negative class — wiring
        // alias resolution would require a real JS parser, which
        // we deliberately don't pull in.
        let js = "var e=navigator;var t=window;t.chrome;navigator.platform;";
        let probes = extract_probes(js);
        // Literal access — picked up.
        assert!(probes.contains(&ProbeSurface::new("navigator", "platform")));
        // Aliased access via `t.chrome` is NOT picked up because
        // the scanner doesn't follow assignments. This is the
        // contract.
        assert!(
            !probes.contains(&ProbeSurface::new("window", "chrome")),
            "aliased access via `t.chrome` should NOT resolve to window.chrome"
        );
    }

    #[test]
    fn scrape_diff_returns_new_probes_when_baseline_is_empty() {
        let mut s = VendorJsScraper::new();
        // Manually populate a baseline via set_baseline + then
        // exercise `extract_probes` against a JS body to compute
        // the diff via the same logic the scraper runs internally.
        s.set_baseline("test_vendor", HashSet::new());
        let probes_now: HashSet<ProbeSurface> =
            extract_probes("navigator.webdriver;navigator.platform;")
                .into_iter()
                .collect();
        let baseline = s.baseline("test_vendor").unwrap().clone();
        let new: HashSet<_> = probes_now.difference(&baseline).cloned().collect();
        assert_eq!(new.len(), 2);
    }

    #[test]
    fn scrape_diff_detects_added_probes_between_runs() {
        let mut s = VendorJsScraper::new();
        let v1: HashSet<ProbeSurface> =
            extract_probes("navigator.webdriver;").into_iter().collect();
        s.set_baseline("v", v1.clone());
        let v2: HashSet<ProbeSurface> = extract_probes("navigator.webdriver;navigator.platform;")
            .into_iter()
            .collect();
        let added: Vec<_> = v2.difference(&v1).cloned().collect();
        assert_eq!(added.len(), 1);
        assert_eq!(added[0].property, "platform");
    }

    #[test]
    fn scrape_diff_detects_removed_probes_between_runs() {
        let v1: HashSet<ProbeSurface> =
            extract_probes("navigator.webdriver;navigator.userAgent;navigator.platform;")
                .into_iter()
                .collect();
        let v2: HashSet<ProbeSurface> = extract_probes("navigator.webdriver;navigator.platform;")
            .into_iter()
            .collect();
        let removed: Vec<_> = v1.difference(&v2).cloned().collect();
        assert_eq!(removed.len(), 1);
        assert_eq!(removed[0].property, "userAgent");
    }

    #[test]
    fn scrape_report_summary_lists_new_probes() {
        let r = ScrapeReport {
            vendor: "demo".into(),
            urls_fetched: vec!["https://example.com/api.js".into()],
            bytes_total: 1234,
            probes_total: 1,
            probes_new: vec![ProbeSurface::new("navigator", "newProbe")],
            probes_removed: vec![],
            probes_stable: vec![],
        };
        let s = r.render_summary();
        assert!(s.contains("vendor: demo"));
        assert!(s.contains("new probes: 1"));
        assert!(s.contains("navigator.newProbe"));
    }

    #[test]
    fn scrape_report_summary_lists_removed_probes() {
        let r = ScrapeReport {
            vendor: "demo".into(),
            urls_fetched: vec![],
            bytes_total: 0,
            probes_total: 0,
            probes_new: vec![],
            probes_removed: vec![ProbeSurface::new("window", "deprecated")],
            probes_stable: vec![],
        };
        let s = r.render_summary();
        assert!(s.contains("window.deprecated"));
        assert!(s.contains("removed:"));
    }

    #[test]
    fn bundled_targets_cover_the_four_most_deployed_vendors() {
        let targets = bundled_targets();
        let names: Vec<&str> = targets.iter().map(|t| t.name.as_str()).collect();
        assert!(names.contains(&"cloudflare-turnstile"));
        assert!(names.contains(&"hcaptcha"));
        assert!(names.contains(&"recaptcha-v2"));
        assert!(names.contains(&"datadome"));
    }

    #[test]
    fn every_bundled_target_has_at_least_one_url() {
        for t in bundled_targets() {
            assert!(!t.urls.is_empty(), "{} has no URLs", t.name);
        }
    }

    #[test]
    fn probe_surface_displays_as_dotted_path() {
        let p = ProbeSurface::new("navigator", "webdriver");
        assert_eq!(p.to_string(), "navigator.webdriver");
    }
}