captchaforge 0.2.39

//! Integration tests for the vision backend.
//!
//! These tests verify that:
//! 1. YOLOv8n model downloads and loads correctly
//! 2. CRNN model loads when available
//! 3. Inference runs without panic on synthetic images
//! 4. CRNN recognition meets a real accuracy + confidence floor on a labeled
//!    eval set (`crnn_recognizes_eval_set_with_real_accuracy`), the proving
//!    test for the Python→ONNX→Rust contract
//!
//! Run with: cargo test --features vision --test vision_integration

#[cfg(feature = "vision")]
use anyhow::Result;

#[tokio::test]
#[cfg(feature = "vision")]
async fn model_hub_downloads_yolov8n() -> Result<()> {
    let hub = captchaforge::vision::ModelHub::new();
    let path = hub.resolve(captchaforge::vision::ModelId::YoloV8n).await?;
    assert!(path.exists(), "YOLOv8n model should exist after download");
    assert!(path.metadata()?.len() > 1_000_000, "model should be > 1MB");
    Ok(())
}

#[test]
#[cfg(feature = "vision")]
fn yolov8n_detects_on_synthetic_image() -> Result<()> {
    let rt = tokio::runtime::Runtime::new()?;
    let hub = captchaforge::vision::ModelHub::new();
    let path = rt.block_on(async { hub.resolve(captchaforge::vision::ModelId::YoloV8n).await })?;

    let mut detector = captchaforge::vision::YoloDetector::load(&path)?;

    // Create a synthetic image with a red square (no real objects).
    let mut img = image::RgbImage::new(640, 480);
    for pixel in img.pixels_mut() {
        *pixel = image::Rgb([128, 128, 128]);
    }
    for y in 100..300 {
        for x in 100..300 {
            img.put_pixel(x, y, image::Rgb([255, 0, 0]));
        }
    }
    let dyn_img = image::DynamicImage::ImageRgb8(img);

    let detections = detector.detect(&dyn_img)?;
    // Synthetic image has no real COCO objects, but the inference must not panic.
    println!("detections: {}", detections.len());
    Ok(())
}

#[test]
#[cfg(feature = "vision")]
fn task_to_coco_mapping_coverage() {
    use captchaforge::vision::yolo::task_to_coco_classes;

    // Known reCAPTCHA tasks that YOLO can handle.
    assert!(task_to_coco_classes("Select all images with traffic lights").is_some());
    assert!(task_to_coco_classes("Select all buses").is_some());
    assert!(task_to_coco_classes("Select all cars").is_some());
    assert!(task_to_coco_classes("Select all bicycles").is_some());
    assert!(task_to_coco_classes("Select all vehicles").is_some());
    assert!(task_to_coco_classes("Select all fire hydrants").is_some());

    // Unknown classes (solver should fall back to VLM).
    assert!(task_to_coco_classes("Select all crosswalks").is_none());
    assert!(task_to_coco_classes("Select all chimneys").is_none());
    assert!(task_to_coco_classes("Select all stairs").is_none());
    assert!(task_to_coco_classes("Select all palm trees").is_none());
}

#[test]
#[cfg(feature = "vision")]
fn crnn_recognizer_loads_when_model_available() {
    let rt = tokio::runtime::Runtime::new().unwrap();
    let hub = captchaforge::vision::ModelHub::new();
    let path = rt.block_on(async { hub.resolve(captchaforge::vision::ModelId::CrnnText).await });

    if let Ok(path) = path {
        let charset = captchaforge::vision::crnn::default_charset();
        let result = captchaforge::vision::crnn::CrnnRecognizer::load(&path, charset);
        assert!(result.is_ok(), "CRNN should load when model is available");
    } else {
        println!(
            "CRNN model not available, skipping load test (run scripts/train_crnn.py to generate)"
        );
    }
}

/// End-to-end recognition accuracy gate for the CRNN text solver.
///
/// This is the proving test the load-only smoke test above is not: it runs the
/// Python-trained ONNX model through the Rust `ort` inference path on a labeled,
/// reproducible eval set and asserts a real accuracy floor and real confidence
/// values. It will FAIL, not silently degrade, if any link in the
/// Python→ONNX→Rust contract breaks: input normalization (plain `/255`, NCHW),
/// the `[T,1,C]` output layout, the CTC blank index, the digit+uppercase
/// charset, or the `log_softmax`→`exp()` confidence fix in `crnn.rs` (a raw
/// log-prob confidence would sit below the solver's `confidence < 0.3` reject
/// gate and kill every correct read).
///
/// The eval set is machine-local (alongside the model, not committed), produced
/// by `python scripts/train_crnn.py --emit-eval-count 200` under a fixed seed.
/// Skips (does not fail) when the model or eval set is absent, matching the
/// other model-gated tests here.
#[tokio::test]
#[cfg(feature = "vision")]
async fn crnn_recognizes_eval_set_with_real_accuracy() -> Result<()> {
    let hub = captchaforge::vision::ModelHub::new();
    let Ok(model_path) = hub.resolve(captchaforge::vision::ModelId::CrnnText).await else {
        println!(
            "SKIP crnn accuracy: model absent, run `python scripts/train_crnn.py` to generate it"
        );
        return Ok(());
    };
    let eval_dir = model_path
        .parent()
        .expect("model path has a parent dir")
        .join("eval");
    let labels_path = eval_dir.join("labels.json");
    if !labels_path.exists() {
        println!(
            "SKIP crnn accuracy: eval set absent at {}, run \
             `python scripts/train_crnn.py --emit-eval-count 200`",
            labels_path.display()
        );
        return Ok(());
    }

    let labels: std::collections::BTreeMap<String, String> =
        serde_json::from_slice(&std::fs::read(&labels_path)?)?;
    assert!(
        labels.len() >= 50,
        "eval set too small to be a meaningful gate: {} samples (want >= 50)",
        labels.len()
    );

    let charset = captchaforge::vision::crnn::default_charset();
    let mut recognizer = captchaforge::vision::crnn::CrnnRecognizer::load(&model_path, charset)
        .expect("CRNN model loads");

    let (mut exact, mut char_acc_sum, mut conf_sum_correct, mut correct_n) =
        (0usize, 0.0f64, 0.0f64, 0usize);
    let n = labels.len();
    for (file, truth) in &labels {
        let img = image::open(eval_dir.join(file))?;
        let (pred, conf) = recognizer.recognize(&img)?;

        // Confidence is a probability (the `.exp()` fix guarantees (0, 1]).
        assert!(
            conf > 0.0 && conf <= 1.0,
            "confidence {conf} out of (0,1] for {file}, log_softmax→exp() contract broken"
        );

        let cacc = 1.0 - levenshtein(&pred, truth) as f64 / truth.chars().count().max(1) as f64;
        char_acc_sum += cacc.max(0.0);
        if pred == *truth {
            exact += 1;
            correct_n += 1;
            conf_sum_correct += conf as f64;
        }
    }

    let exact_rate = exact as f64 / n as f64;
    let mean_char_acc = char_acc_sum / n as f64;
    let mean_conf_correct = if correct_n > 0 {
        conf_sum_correct / correct_n as f64
    } else {
        0.0
    };
    println!(
        "CRNN eval: n={n} exact={exact_rate:.3} mean_char_acc={mean_char_acc:.3} \
         mean_conf(correct)={mean_conf_correct:.3}"
    );

    // Real floors: a broken contract craters these toward zero. The trained
    // model reaches ~0.95 exact on its own held-out set; the Rust path sees a
    // minor resampling delta (bilinear vs PIL bicubic), so the gate sits well
    // below observed-good while still catching any contract regression.
    assert!(
        exact_rate >= 0.50,
        "CRNN exact-match {exact_rate:.3} < 0.50. Python→ONNX→Rust contract regressed"
    );
    assert!(
        mean_char_acc >= 0.75,
        "CRNN mean char accuracy {mean_char_acc:.3} < 0.75, recognition contract regressed"
    );
    // The confidence fix must keep correct reads clear of the solver's 0.3 gate.
    assert!(
        mean_conf_correct >= 0.40,
        "mean confidence on correct reads {mean_conf_correct:.3} < 0.40, too close to the \
         solver's `confidence < 0.3` reject gate; the log_softmax→exp() fix may have regressed"
    );
    Ok(())
}

/// Levenshtein edit distance over Unicode scalar values (test-local helper).
#[cfg(feature = "vision")]
fn levenshtein(a: &str, b: &str) -> usize {
    let a: Vec<char> = a.chars().collect();
    let b: Vec<char> = b.chars().collect();
    let mut prev: Vec<usize> = (0..=b.len()).collect();
    let mut cur = vec![0usize; b.len() + 1];
    for (i, &ca) in a.iter().enumerate() {
        cur[0] = i + 1;
        for (j, &cb) in b.iter().enumerate() {
            let cost = if ca == cb { 0 } else { 1 };
            cur[j + 1] = (prev[j + 1] + 1).min(cur[j] + 1).min(prev[j] + cost);
        }
        std::mem::swap(&mut prev, &mut cur);
    }
    prev[b.len()]
}

#[test]
#[cfg(not(feature = "vision"))]
fn vision_module_compiles_without_feature() {
    use captchaforge::solver::CaptchaSolver;
    // When vision is disabled, the module should still compile and the
    // solvers should be inert.
    let yolo = captchaforge::solver::YoloGridSolver::new();
    assert!(!yolo.supports(&captchaforge::captcha_detect::DetectedCaptcha::RecaptchaV2));

    let crnn = captchaforge::solver::CrnnTextSolver::new();
    assert!(!crnn.supports(&captchaforge::captcha_detect::DetectedCaptcha::ImageCaptcha));
}