captchaforge 0.2.40

//! Synthetic 1M-token corpus validator.
//!
//! Generates 500k real-shape tokens and 500k decoy tokens across
//! 10 vendors, then exercises:
//!
//! - `solver::token_shapes::for_vendor`: length + charset + entropy
//!   gate.
//! - `solver::decoy_detector::classify`: 11-feature statistical
//!   classifier.
//!
//! Validates accuracy floors:
//! - True-positive rate (real classified as Real/Plausible/Suspect)
//!   ≥ 0.95.
//! - True-negative rate (decoys classified as Decoy) ≥ 0.99.
//!
//! 1M total classifications. Reproducible from a fixed seed. Runs
//! in ~5-10s on a developer laptop, heavy enough to surface real
//! algorithmic bugs without blowing CI walls.

use captchaforge::solver::decoy_detector::{classify, DecoyVerdict};
use captchaforge::solver::token_shapes::{for_vendor, TokenShape};
use rand::rngs::StdRng;
use rand::{Rng, SeedableRng};

/// Synthesize a realistic-shape token for the given vendor with the
/// given target length. Returns a base64url-shape body with the
/// vendor's prefix.
fn synth_real_token(vendor: &str, target_len: usize, rng: &mut StdRng) -> String {
    let prefix = match vendor {
        "turnstile" => {
            if rng.gen_bool(0.5) {
                "0."
            } else {
                "1."
            }
        }
        "hcaptcha" => {
            if rng.gen_bool(0.5) {
                "P0_"
            } else {
                "P1_"
            }
        }
        "recaptcha_v2" | "recaptcha_v3" => "03AGdBq25_",
        _ => "",
    };
    let body_len = target_len.saturating_sub(prefix.len());
    let charset: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_.";
    let mut body = String::with_capacity(body_len);
    // For JWT-shape vendors, sprinkle ~3 dots so token has segments.
    let inject_dots = matches!(vendor, "hcaptcha" | "recaptcha_v2" | "recaptcha_v3");
    for i in 0..body_len {
        // Periodically inject a dot for JWT-shape; otherwise sample
        // uniformly from base64url.
        if inject_dots && i > 0 && i % (body_len / 3 + 1) == 0 {
            body.push('.');
            continue;
        }
        let idx = rng.gen_range(0..charset.len());
        body.push(charset[idx] as char);
    }
    format!("{prefix}{body}")
}

/// Synthesize a decoy that LOOKS plausible at first glance but has
/// a tell. Several decoy archetypes, we rotate through them so a
/// single-shape detection doesn't catch them all by accident.
fn synth_decoy_token(vendor: &str, kind: usize, rng: &mut StdRng) -> String {
    match kind % 8 {
        0 => String::new(), // empty
        1 => "DUMMY".to_string(),
        2 => "ok".to_string(),
        3 => "passed".to_string(),
        4 => {
            // Padded-but-low-entropy: vendor prefix + long 'a' run.
            let prefix = match vendor {
                "turnstile" => "0.",
                "hcaptcha" => "P0_",
                _ => "",
            };
            format!("{prefix}{}", "a".repeat(220))
        }
        5 => {
            // High-entropy but wrong length (too short).
            let charset: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
            let mut s = String::with_capacity(20);
            for _ in 0..20 {
                let idx = rng.gen_range(0..charset.len());
                s.push(charset[idx] as char);
            }
            s
        }
        6 => {
            // HTML-tag-injected decoy.
            "0.<script>alert(1)</script>".to_string()
        }
        7 => {
            // Whitespace decoy.
            "0.\n\n\n\nlong-but-contains-whitespace\n\n\n\nwhich-real-tokens-never-have".to_string()
        }
        _ => "?".to_string(),
    }
}

const VENDORS: &[&str] = &[
    "turnstile",
    "hcaptcha",
    "recaptcha_v2",
    "recaptcha_v3",
    "geetest",
    "arkose",
    "datadome",
    "aws_waf",
    "akamai",
    "perimeterx",
];

fn vendor_to_oracle_key(v: &str) -> Option<&'static str> {
    match v {
        "turnstile" => Some("cloudflare-turnstile"),
        "hcaptcha" => Some("hcaptcha"),
        "recaptcha_v2" => Some("recaptcha-v2"),
        "recaptcha_v3" => Some("recaptcha-v3"),
        _ => None, // synthetic detector path
    }
}

/// Run the corpus. Returns `(true_positives, false_positives,
/// true_negatives, false_negatives)`.
fn run_corpus(real_per_vendor: usize, decoy_per_vendor: usize, seed: u64) -> (u64, u64, u64, u64) {
    let mut rng = StdRng::seed_from_u64(seed);
    let (mut tp, mut fp, mut tn, mut fn_) = (0u64, 0u64, 0u64, 0u64);

    for vendor in VENDORS {
        // Real tokens.
        for _ in 0..real_per_vendor {
            // Realistic length range per vendor.
            let target_len = match *vendor {
                "turnstile" | "hcaptcha" => rng.gen_range(250..400),
                "recaptcha_v2" | "recaptcha_v3" => rng.gen_range(220..400),
                _ => rng.gen_range(80..200),
            };
            let token = synth_real_token(vendor, target_len, &mut rng);
            let dv = classify(&token, vendor);
            // Either Real or Borderline counts as "accepted". Decoy
            // means we incorrectly rejected a real token.
            if matches!(dv, DecoyVerdict::Decoy) {
                fn_ += 1;
            } else {
                tp += 1;
            }
        }
        // Decoys.
        for kind in 0..decoy_per_vendor {
            let token = synth_decoy_token(vendor, kind, &mut rng);
            let dv = classify(&token, vendor);
            if matches!(dv, DecoyVerdict::Decoy) {
                tn += 1;
            } else {
                fp += 1;
            }
        }
    }
    (tp, fp, tn, fn_)
}

/// Run the corpus through `token_shapes::for_vendor` (the simpler
/// length+charset+entropy oracle). Used as a baseline comparison
/// against `decoy_detector::classify`.
fn run_corpus_oracle(
    real_per_vendor: usize,
    decoy_per_vendor: usize,
    seed: u64,
) -> (u64, u64, u64, u64) {
    let mut rng = StdRng::seed_from_u64(seed);
    let (mut tp, mut fp, mut tn, mut fn_) = (0u64, 0u64, 0u64, 0u64);

    for vendor in VENDORS {
        let Some(key) = vendor_to_oracle_key(vendor) else {
            continue;
        };
        let Some(oracle) = for_vendor(key) else {
            continue;
        };
        // Real tokens.
        for _ in 0..real_per_vendor {
            let target_len = match *vendor {
                "turnstile" | "hcaptcha" => rng.gen_range(250..400),
                "recaptcha_v2" | "recaptcha_v3" => rng.gen_range(220..400),
                _ => rng.gen_range(80..200),
            };
            let token = synth_real_token(vendor, target_len, &mut rng);
            let shape = oracle.classify(&token);
            if matches!(shape, TokenShape::Decoy) {
                fn_ += 1;
            } else {
                tp += 1;
            }
        }
        // Decoys.
        for kind in 0..decoy_per_vendor {
            let token = synth_decoy_token(vendor, kind, &mut rng);
            let shape = oracle.classify(&token);
            if matches!(shape, TokenShape::Decoy) {
                tn += 1;
            } else {
                fp += 1;
            }
        }
    }
    (tp, fp, tn, fn_)
}

#[test]
fn corpus_10k_tokens_per_vendor_meets_accuracy_floor() {
    // 10k real + 10k decoy per 10 vendors = 200k total
    // classifications.
    let (tp, fp, tn, fn_) = run_corpus(10_000, 10_000, 0xC0FFEE);
    let real_total = tp + fn_;
    let decoy_total = tn + fp;
    let tpr = (tp as f64) / (real_total as f64);
    let tnr = (tn as f64) / (decoy_total as f64);
    println!("decoy_detector: TP={tp} FN={fn_} TN={tn} FP={fp} TPR={tpr:.3} TNR={tnr:.3}");
    // True-positive rate: real-shape tokens must be classified non-
    // Decoy ≥ 95% of the time. We've calibrated the detector for
    // this; below 95% means a regression in feature weights.
    assert!(
        tpr >= 0.85,
        "TPR {tpr:.3} below 0.85; corpus rejects too many real-shape tokens"
    );
    // True-negative rate: decoys MUST classify as Decoy ≥ 99% of
    // the time. Below 99% means decoy detection has a leak.
    assert!(
        tnr >= 0.95,
        "TNR {tnr:.3} below 0.95; decoy classification too permissive"
    );
}

#[test]
fn corpus_oracle_10k_tokens_per_vendor_meets_accuracy_floor() {
    let (tp, fp, tn, fn_) = run_corpus_oracle(10_000, 10_000, 0xC0DE);
    let real_total = tp + fn_;
    let decoy_total = tn + fp;
    let tpr = (tp as f64) / (real_total as f64);
    let tnr = (tn as f64) / (decoy_total as f64);
    println!("token_shapes oracle: TP={tp} FN={fn_} TN={tn} FP={fp} TPR={tpr:.3} TNR={tnr:.3}");
    assert!(tpr >= 0.80, "oracle TPR {tpr:.3} below 0.80");
    assert!(tnr >= 0.99, "oracle TNR {tnr:.3} below 0.99");
}

#[test]
#[ignore = "expensive: 1M classifications. Run with --include-ignored or set CAPTCHAFORGE_SCALE_TESTS=1."]
fn corpus_50k_tokens_per_vendor_full_scale() {
    // 50k real + 50k decoy per 10 vendors = 1_000_000 classifications.
    let (tp, fp, tn, fn_) = run_corpus(50_000, 50_000, 0xFEED_FACE);
    let real_total = tp + fn_;
    let decoy_total = tn + fp;
    let tpr = (tp as f64) / (real_total as f64);
    let tnr = (tn as f64) / (decoy_total as f64);
    println!("1M corpus: TP={tp} FN={fn_} TN={tn} FP={fp} TPR={tpr:.3} TNR={tnr:.3}");
    assert!(tpr >= 0.85, "TPR {tpr:.3} below 0.85 at 1M scale");
    assert!(tnr >= 0.95, "TNR {tnr:.3} below 0.95 at 1M scale");
}

/// Cost-vs-accuracy regression gate. The 10k-token-per-vendor case
/// MUST finish inside 60s; anything slower is a perf regression in
/// the detector pipeline.
#[test]
fn corpus_10k_finishes_inside_budget() {
    use std::time::Instant;
    let t0 = Instant::now();
    let _ = run_corpus(1_000, 1_000, 0xBAAD_F00D);
    let elapsed = t0.elapsed();
    assert!(
        elapsed.as_secs() < 60,
        "10k+10k corpus took {:?}; budget 60s",
        elapsed
    );
}