model-selection-rs 0.1.0

//! Nested cross-validation vs. the naive "tune-and-report-on-the-same-folds"
//! shortcut — showing the optimism bias nested CV removes.
//!
//! Run with: `cargo run --example nested_cv`

use model_selection_rs::evaluate::{cross_validate, nested_cross_validate, BoxedScorer};
use model_selection_rs::scoring::Accuracy;
use model_selection_rs::splitters::KFold;
use ndarray::{Array1, Array2};

/// Deterministic pseudo-random bit from (seed, index).
fn bit(seed: u64, i: u64) -> f64 {
    let mut z = seed
        .wrapping_mul(0x9E37_79B9_7F4A_7C15)
        .wrapping_add(i.wrapping_mul(0xBF58_476D_1CE4_E5B9));
    z ^= z >> 27;
    z = z.wrapping_mul(0x94D0_49BB_1331_11EB);
    z ^= z >> 31;
    (z & 1) as f64
}

const N: usize = 160;
const N_CANDIDATES: u64 = 80;

/// Candidate model `p`: predictions are noise seeded by `p`, read off the row id.
fn candidate_fit(
    p: &u64,
    _x: &Array2<f64>,
    _y: &Array1<f64>,
) -> impl Fn(&Array2<f64>) -> Array1<f64> {
    let p = *p;
    move |xq: &Array2<f64>| xq.column(0).mapv(|id| bit(p, id as u64))
}

fn tune(x: &Array2<f64>, y: &Array1<f64>, inner: &KFold) -> u64 {
    let mut best = (f64::NEG_INFINITY, 0);
    for p in 0..N_CANDIDATES {
        let scorers: Vec<BoxedScorer> = vec![Box::new(Accuracy)];
        let res = cross_validate(
            inner,
            x,
            y,
            move |xt, yt| candidate_fit(&p, xt, yt),
            &scorers,
            false,
        )
        .unwrap();
        if res.mean_test_score(0) > best.0 {
            best = (res.mean_test_score(0), p);
        }
    }
    best.1
}

fn main() {
    // Labels are pure noise: no model can truly beat 50%.
    let x = Array2::from_shape_fn((N, 1), |(i, _)| i as f64);
    let y = Array1::from_shape_fn(N, |i| bit(999, i as u64));

    // Naive: pick the best of many candidates on the SAME 5-fold CV we report.
    let cv = KFold::new(5).unwrap().with_shuffle(7);
    let mut naive_best = f64::NEG_INFINITY;
    for p in 0..N_CANDIDATES {
        let scorers: Vec<BoxedScorer> = vec![Box::new(Accuracy)];
        let res = cross_validate(
            &cv,
            &x,
            &y,
            move |xt, yt| candidate_fit(&p, xt, yt),
            &scorers,
            false,
        )
        .unwrap();
        naive_best = naive_best.max(res.mean_test_score(0));
    }

    // Nested: tune on inner folds, report on untouched outer folds.
    let outer = KFold::new(5).unwrap().with_shuffle(7);
    let inner = KFold::new(4).unwrap().with_shuffle(13);
    let nested =
        nested_cross_validate(&outer, &inner, &x, &y, tune, candidate_fit, &Accuracy).unwrap();

    println!("Labels are pure noise — true accuracy is 0.50.\n");
    println!(
        "  naive  (tune & report on same folds): {:.3}  <- optimistically biased",
        naive_best
    );
    println!(
        "  nested (honest, no leakage):           {:.3} +/- {:.3}",
        nested.mean_score(),
        nested.std_score()
    );
    println!(
        "\n  per-outer-fold selected candidate: {:?}",
        nested.selected_params
    );
    println!("\nThe naive number looks good only because it got to peek at the");
    println!("evaluation folds while choosing among {N_CANDIDATES} candidates. Nested CV");
    println!("does not, so it reports the truth.");
}