quietset 0.16.0

use quietset::{
    CalibrationOptions, Decision, DecisionScore, GroupKey, LatentTruthSafety, MinRequirements,
    Observation, ScoreConfig, ScoreDispersion, ScoreWeights, StabilityReport, Thresholds,
    compute_calibration, compute_calibration_with_options, compute_evaluator_label_weights,
    compute_evaluator_reliability, compute_evaluator_weights, compute_fleiss_kappa,
    compute_krippendorff_alpha, compute_weighted_majority, compute_weighted_majority_by_label,
    group_leakage, leaked_group_keys, parse_jsonl, score_all, score_all_gold_weighted,
    score_all_latent_truth, score_all_weighted,
};

fn load(filename: &str) -> Vec<quietset::Observation> {
    let path = format!("../../tests/fixtures/{}", filename);
    let content = std::fs::read_to_string(&path).unwrap();
    parse_jsonl(&content).unwrap()
}

#[test]
fn test_simple_fixture_decisions() {
    let obs = load("simple.jsonl");
    let reports = score_all(obs, &ScoreConfig::default());
    assert_eq!(reports.len(), 2);
    let a = reports.iter().find(|r| r.sample_id == "a").unwrap();
    let b = reports.iter().find(|r| r.sample_id == "b").unwrap();
    assert_eq!(a.decision, Decision::Keep, "sample a should be kept");
    // b is noisy, should not be keep
    assert_ne!(b.decision, Decision::Keep, "sample b should not be kept");
}

#[test]
fn test_stable_scores_are_kept() {
    let obs = load("stable_scores.jsonl");
    let reports = score_all(obs, &ScoreConfig::default());
    for r in &reports {
        assert_eq!(r.decision, Decision::Keep, "{} should be kept", r.sample_id);
    }
}

#[test]
fn test_budget_sensitive_is_not_kept() {
    let obs = load("budget_sensitive.jsonl");
    let reports = score_all(obs, &ScoreConfig::default());
    assert_eq!(reports.len(), 1);
    assert_ne!(
        reports[0].decision,
        Decision::Keep,
        "budget-sensitive sample should not be kept"
    );
}

#[test]
fn test_single_observation_is_review() {
    let obs = vec![quietset::Observation {
        sample_id: "solo".into(),
        label: Some("yes".into()),
        score: Some(0.99),
        ..Default::default()
    }];
    let reports = score_all(obs, &ScoreConfig::default());
    assert_eq!(reports[0].decision, Decision::Review);
    assert!((reports[0].stability_score - 0.5).abs() < 1e-10);
}

#[test]
fn test_missing_optional_fields() {
    let jsonl = r#"{"sample_id":"a","label":"yes"}
{"sample_id":"a","label":"yes"}
{"sample_id":"a","label":"no"}"#;
    let obs = parse_jsonl(jsonl).unwrap();
    let reports = score_all(obs, &ScoreConfig::default());
    assert_eq!(reports.len(), 1);
    assert!(reports[0].score_mean.is_none());
    assert!(reports[0].label_agreement.is_some());
}

#[test]
fn test_label_agreement() {
    let jsonl = r#"{"sample_id":"a","label":"yes"}
{"sample_id":"a","label":"yes"}
{"sample_id":"a","label":"no"}"#;
    let obs = parse_jsonl(jsonl).unwrap();
    let reports = score_all(obs, &ScoreConfig::default());
    let agreement = reports[0].label_agreement.unwrap();
    assert!((agreement - 2.0 / 3.0).abs() < 1e-10);
}

#[test]
fn test_invalid_jsonl_returns_error() {
    let result = parse_jsonl(
        r#"{"sample_id":"a"}
not_valid_json"#,
    );
    assert!(result.is_err());
    let err = result.unwrap_err().to_string();
    assert!(
        err.contains("line 2"),
        "error should mention line number: {err}"
    );
}

#[test]
fn test_deterministic_output_order() {
    let jsonl = r#"{"sample_id":"z","label":"a","score":0.9}
{"sample_id":"a","label":"a","score":0.9}
{"sample_id":"m","label":"a","score":0.9}
{"sample_id":"z","label":"a","score":0.9}
{"sample_id":"a","label":"a","score":0.9}
{"sample_id":"m","label":"a","score":0.9}"#;
    let obs = parse_jsonl(jsonl).unwrap();
    let reports = score_all(obs, &ScoreConfig::default());
    // Should preserve first-seen insertion order
    assert_eq!(reports[0].sample_id, "z");
    assert_eq!(reports[1].sample_id, "a");
    assert_eq!(reports[2].sample_id, "m");
}

#[test]
fn test_grouping_by_sample_id() {
    let jsonl = r#"{"sample_id":"a","score":0.9}
{"sample_id":"b","score":0.8}
{"sample_id":"a","score":0.85}"#;
    let obs = parse_jsonl(jsonl).unwrap();
    let reports = score_all(obs, &ScoreConfig::default());
    let a = reports.iter().find(|r| r.sample_id == "a").unwrap();
    assert_eq!(a.n_observations, 2);
}

#[test]
fn test_score_mean_std_range() {
    let obs = vec![
        quietset::Observation {
            sample_id: "a".into(),
            score: Some(1.0),
            ..Default::default()
        },
        quietset::Observation {
            sample_id: "a".into(),
            score: Some(3.0),
            ..Default::default()
        },
    ];
    let reports = score_all(obs, &ScoreConfig::default());
    let r = &reports[0];
    assert!((r.score_mean.unwrap() - 2.0).abs() < 1e-10);
    assert!((r.score_range.unwrap() - 2.0).abs() < 1e-10);
}

#[test]
fn test_keep_review_drop_thresholds() {
    use quietset::decision::{Thresholds, decide};
    let t = Thresholds::default();
    assert_eq!(decide(0.9, &t), Decision::Keep);
    assert_eq!(decide(0.6, &t), Decision::Review);
    assert_eq!(decide(0.3, &t), Decision::Drop);
    // boundaries
    assert_eq!(decide(0.85, &t), Decision::Keep);
    assert_eq!(decide(0.40, &t), Decision::Drop);
}

#[test]
fn test_missing_sample_id_is_error() {
    let err = parse_jsonl(r#"{}"#).unwrap_err().to_string();
    assert!(
        err.contains("sample_id"),
        "error should mention sample_id: {err}"
    );
    assert!(parse_jsonl(r#"{"label":"x"}"#).is_err());
}

#[test]
fn test_invalid_score_scale() {
    let zero = ScoreConfig {
        score_scale: 0.0,
        ..ScoreConfig::default()
    };
    assert!(zero.validate().is_err());
    let neg = ScoreConfig {
        score_scale: -1.0,
        ..ScoreConfig::default()
    };
    assert!(neg.validate().is_err());
    let nan = ScoreConfig {
        score_scale: f64::NAN,
        ..ScoreConfig::default()
    };
    assert!(nan.validate().is_err());
    assert!(ScoreConfig::default().validate().is_ok());
}

#[test]
fn test_majority_label_tie_is_deterministic() {
    // 1:1 tie between "alpha" and "beta" — alphabetically first should always win
    let jsonl =
        "{\"sample_id\":\"a\",\"label\":\"beta\"}\n{\"sample_id\":\"a\",\"label\":\"alpha\"}";
    for _ in 0..20 {
        let obs = parse_jsonl(jsonl).unwrap();
        let reports = score_all(obs, &ScoreConfig::default());
        assert_eq!(
            reports[0].majority_label.as_deref(),
            Some("alpha"),
            "tie must resolve deterministically to 'alpha'"
        );
    }
}

#[test]
fn test_seed_sensitivity_affects_score() {
    // Same label, wildly different scores across seeds — should not be Keep
    let jsonl = r#"{"sample_id":"a","label":"win","score":0.95,"seed":1}
{"sample_id":"a","label":"win","score":0.05,"seed":2}"#;
    let obs = parse_jsonl(jsonl).unwrap();
    let reports = score_all(obs, &ScoreConfig::default());
    assert!(reports[0].seed_sensitivity.is_some());
    assert_ne!(
        reports[0].decision,
        Decision::Keep,
        "seed-unstable sample should not be kept (stability={})",
        reports[0].stability_score
    );
}

#[test]
fn test_score_weights_exclude_dimension() {
    // Zero-weight a dimension and verify it doesn't affect the score
    let jsonl = r#"{"sample_id":"a","label":"win","score":0.9}
{"sample_id":"a","label":"win","score":0.1}"#;
    let obs_default = parse_jsonl(jsonl).unwrap();
    let obs_no_score = parse_jsonl(jsonl).unwrap();

    let default_score = score_all(obs_default, &ScoreConfig::default())[0].stability_score;
    let no_score_weight = score_all(
        obs_no_score,
        &ScoreConfig {
            weights: ScoreWeights {
                score_stability: 0.0,
                ..ScoreWeights::default()
            },
            ..ScoreConfig::default()
        },
    )[0]
    .stability_score;

    // Excluding score dimension should raise the score (score is unstable here)
    assert!(
        no_score_weight > default_score,
        "excluding unstable score dimension should raise stability_score"
    );
}

#[test]
fn test_score_nan_is_error() {
    // serde_json rejects NaN in JSON, so we construct an Observation directly
    let mut obs = quietset::Observation {
        sample_id: "a".into(),
        score: Some(f64::NAN),
        ..Default::default()
    };
    // parse_jsonl can't produce NaN (invalid JSON), but we test the API path via
    // a JSONL-round-trip workaround — just verify NaN propagation is caught
    // by constructing the error inline to confirm the error variant exists.
    obs.score = Some(f64::INFINITY);
    let jsonl = format!(
        "{{\"sample_id\":\"a\",\"score\":{}}}",
        "9999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999"
    );
    // Very large float overflows to infinity in some parsers; test what we can from the API
    drop(jsonl); // the exact JSONL path depends on serde_json behavior
    // Direct API: validate that InvalidScore error exists and is usable
    let err = quietset::Error::InvalidScore { line: 1 };
    assert!(err.to_string().contains("score"));
    let err2 = quietset::Error::InvalidBudget { line: 2 };
    assert!(err2.to_string().contains("budget"));
    drop(obs);
}

#[test]
fn test_invalid_threshold_drop_gt_keep() {
    let config = ScoreConfig {
        thresholds: Thresholds {
            keep: 0.40,
            drop: 0.85,
        }, // drop > keep — invalid
        ..ScoreConfig::default()
    };
    let err = config.validate().unwrap_err().to_string();
    assert!(
        err.contains("drop_threshold") && err.contains("keep_threshold"),
        "error should mention both thresholds: {err}"
    );
}

#[test]
fn test_threshold_out_of_range() {
    let neg = ScoreConfig {
        thresholds: Thresholds {
            keep: -0.1,
            drop: 0.0,
        },
        ..ScoreConfig::default()
    };
    assert!(neg.validate().is_err());

    let over = ScoreConfig {
        thresholds: Thresholds {
            keep: 1.1,
            drop: 0.4,
        },
        ..ScoreConfig::default()
    };
    assert!(over.validate().is_err());
}

#[test]
fn test_components_populated() {
    let jsonl = r#"{"sample_id":"a","label":"win","score":0.9,"budget":4,"seed":1,"model_id":"m1","evaluator_id":"e1"}
{"sample_id":"a","label":"win","score":0.8,"budget":8,"seed":2,"model_id":"m2","evaluator_id":"e2"}"#;
    let obs = parse_jsonl(jsonl).unwrap();
    let reports = score_all(obs, &ScoreConfig::default());
    let c = &reports[0].components;
    assert!(c.label.is_some(), "label component should be present");
    assert!(
        c.score_consistency.is_some(),
        "score_consistency should be present"
    );
    assert!(
        c.budget_robustness.is_some(),
        "budget_robustness should be present"
    );
    assert!(
        c.seed_robustness.is_some(),
        "seed_robustness should be present"
    );
    assert!(
        c.model_agreement.is_some(),
        "model_agreement should be present"
    );
    assert!(
        c.evaluator_agreement.is_some(),
        "evaluator_agreement should be present"
    );
    // all component values in [0, 1]
    for v in [
        c.label,
        c.score_consistency,
        c.budget_robustness,
        c.seed_robustness,
        c.model_agreement,
        c.evaluator_agreement,
    ]
    .into_iter()
    .flatten()
    {
        assert!((0.0..=1.0).contains(&v), "component {v} out of [0,1]");
    }
}

#[test]
fn test_negative_weight_is_error() {
    let config = ScoreConfig {
        weights: ScoreWeights {
            label_agreement: -1.0,
            ..ScoreWeights::default()
        },
        ..ScoreConfig::default()
    };
    let err = config.validate().unwrap_err().to_string();
    assert!(
        err.contains("label_agreement"),
        "error should mention field: {err}"
    );
}

#[test]
fn test_nan_weight_is_error() {
    let config = ScoreConfig {
        weights: ScoreWeights {
            score_stability: f64::NAN,
            ..ScoreWeights::default()
        },
        ..ScoreConfig::default()
    };
    assert!(config.validate().is_err());
}

#[test]
fn test_all_zero_weights_is_error() {
    let config = ScoreConfig {
        weights: ScoreWeights {
            label_agreement: 0.0,
            score_stability: 0.0,
            budget_stability: 0.0,
            seed_stability: 0.0,
            model_agreement: 0.0,
            evaluator_agreement: 0.0,
            score_sign: 0.0,
            gradient_sign: 0.0,
            update_cosine: 0.0,
            teacher_residual: 0.0,
            shuffle_seed: 0.0,
            loss_recipe: 0.0,
        },
        ..ScoreConfig::default()
    };
    let err = config.validate().unwrap_err().to_string();
    assert!(
        err.contains("zero"),
        "error should mention zero weights: {err}"
    );
}

#[test]
fn test_validate_rejects_empty_sample_id() {
    let obs = Observation {
        sample_id: "".into(),
        ..Default::default()
    };
    let err = obs.validate(1).unwrap_err().to_string();
    assert!(
        err.contains("sample_id"),
        "error should mention sample_id: {err}"
    );

    let obs_ws = Observation {
        sample_id: "   ".into(),
        ..Default::default()
    };
    assert!(
        obs_ws.validate(1).is_err(),
        "whitespace-only sample_id should fail"
    );
}

#[test]
fn test_weakest_component_tie_is_deterministic() {
    use quietset::StabilityComponents;
    let c = StabilityComponents {
        label: Some(0.5),
        score_consistency: Some(0.5),
        score_sign_agreement: None,
        budget_robustness: Some(0.5),
        seed_robustness: Some(0.5),
        model_agreement: Some(0.5),
        evaluator_agreement: Some(0.5),
        gradient_sign_agreement: None,
        update_direction_agreement: None,
        teacher_residual_stability: None,
        shuffle_seed_robustness: None,
        loss_recipe_agreement: None,
    };
    // tie resolved by fixed declaration order — "label" always wins
    for _ in 0..20 {
        let (name, val) = c.weakest().unwrap();
        assert_eq!(name, "label");
        assert_eq!(val, 0.5);
    }
}

#[test]
fn test_confidence_single_obs() {
    let obs = vec![quietset::Observation {
        sample_id: "a".into(),
        score: Some(0.9),
        ..Default::default()
    }];
    let reports = score_all(obs, &ScoreConfig::default());
    // n=1, k=3: confidence = 1/(1+3) = 0.25
    let expected_confidence = 1.0 / (1.0 + 3.0);
    assert!((reports[0].confidence - expected_confidence).abs() < 1e-9);
}

#[test]
fn test_adjusted_score_pulls_toward_half() {
    // n=2, high stability -> adjusted should be lower than raw
    let jsonl = r#"{"sample_id":"a","label":"win","score":0.95}
{"sample_id":"a","label":"win","score":0.94}"#;
    let obs = parse_jsonl(jsonl).unwrap();
    let reports = score_all(obs, &ScoreConfig::default());
    let r = &reports[0];
    assert!(
        r.adjusted_stability_score < r.stability_score,
        "adjusted={} should be < raw={}",
        r.adjusted_stability_score,
        r.stability_score
    );
}

#[test]
fn test_min_observations_demotes_keep() {
    let jsonl = r#"{"sample_id":"a","label":"win","score":0.99}
{"sample_id":"a","label":"win","score":0.98}"#;
    let obs = parse_jsonl(jsonl).unwrap();
    // Without min requirement: likely keep
    let reports_default = score_all(obs.clone(), &ScoreConfig::default());
    assert_eq!(reports_default[0].decision, Decision::Keep);
    // With min 5 observations: demoted to review
    let config = ScoreConfig {
        min_requirements: MinRequirements {
            observations: 5,
            ..Default::default()
        },
        ..ScoreConfig::default()
    };
    let reports_min = score_all(obs, &config);
    assert_eq!(
        reports_min[0].decision,
        Decision::Review,
        "should be demoted to review when n < min_observations"
    );
}

#[test]
fn test_label_margin_unanimous() {
    let jsonl = r#"{"sample_id":"a","label":"win"}
{"sample_id":"a","label":"win"}
{"sample_id":"a","label":"win"}"#;
    let obs = parse_jsonl(jsonl).unwrap();
    let reports = score_all(obs, &ScoreConfig::default());
    assert!(
        (reports[0].label_margin.unwrap() - 1.0).abs() < 1e-9,
        "unanimous -> margin = 1.0"
    );
}

#[test]
fn test_label_margin_split() {
    // 2 win, 2 loss -> margin = 0
    let jsonl = r#"{"sample_id":"x","label":"win"}
{"sample_id":"x","label":"loss"}
{"sample_id":"x","label":"win"}
{"sample_id":"x","label":"loss"}"#;
    let obs = parse_jsonl(jsonl).unwrap();
    let reports = score_all(obs, &ScoreConfig::default());
    assert!(
        (reports[0].label_margin.unwrap() - 0.0).abs() < 1e-9,
        "50/50 -> margin = 0.0"
    );
}

#[test]
fn test_label_entropy_uniform() {
    // 3 equal labels -> normalized entropy = 1.0
    let jsonl = r#"{"sample_id":"a","label":"A"}
{"sample_id":"a","label":"B"}
{"sample_id":"a","label":"C"}"#;
    let obs = parse_jsonl(jsonl).unwrap();
    let reports = score_all(obs, &ScoreConfig::default());
    let e = reports[0].label_entropy.unwrap();
    assert!(
        (e - 1.0).abs() < 1e-6,
        "uniform 3-class -> entropy = 1.0, got {e}"
    );
}

#[test]
fn test_budget_slope_positive() {
    // higher budget -> higher score: slope should be positive
    let jsonl = r#"{"sample_id":"a","score":0.5,"budget":4}
{"sample_id":"a","score":0.7,"budget":8}
{"sample_id":"a","score":0.9,"budget":16}"#;
    let obs = parse_jsonl(jsonl).unwrap();
    let reports = score_all(obs, &ScoreConfig::default());
    let slope = reports[0].budget_slope.unwrap();
    assert!(
        slope > 0.0,
        "increasing scores with budget -> positive slope, got {slope}"
    );
}

#[test]
fn test_evaluator_reliability() {
    use quietset::compute_evaluator_reliability;
    // e1 always agrees with majority, e2 disagrees sometimes
    // sample a: 1 win (e1), 1 win (e2) -> majority = win
    // sample b: 2 win (e1), 1 loss (e2) -> majority = win (unambiguous)
    let jsonl = r#"{"sample_id":"a","label":"win","evaluator_id":"e1"}
{"sample_id":"a","label":"win","evaluator_id":"e2"}
{"sample_id":"b","label":"win","evaluator_id":"e1"}
{"sample_id":"b","label":"win","evaluator_id":"e1"}
{"sample_id":"b","label":"loss","evaluator_id":"e2"}"#;
    let obs = parse_jsonl(jsonl).unwrap();
    let reports = score_all(obs.clone(), &ScoreConfig::default());
    let rel = compute_evaluator_reliability(&obs, &reports);
    assert_eq!(
        *rel.get("e1").unwrap() as i32,
        1,
        "e1 always matches majority"
    );
    assert!(rel.get("e2").unwrap() < &1.0, "e2 disagrees sometimes");
}

#[test]
fn test_min_requirements_not_overridden_by_adjusted_score() {
    // n=2, high stability — adjusted score (with low confidence_k) stays above keep threshold
    // but n < min_requirements.observations=3, so decision must be Review, not Keep
    let jsonl = r#"{"sample_id":"a","label":"win","score":0.99}
{"sample_id":"a","label":"win","score":0.98}"#;
    let obs = parse_jsonl(jsonl).unwrap();
    let config = ScoreConfig {
        decision_score: DecisionScore::Adjusted,
        confidence_k: 0.01, // near-zero k -> confidence ≈ 1 -> adjusted ≈ raw (high score)
        min_requirements: MinRequirements {
            observations: 3,
            ..Default::default()
        },
        ..ScoreConfig::default()
    };
    let reports = score_all(obs, &config);
    // adjusted score is high enough for Keep, but n=2 < 3 -> must stay Review
    assert_eq!(
        reports[0].decision,
        Decision::Review,
        "MinRequirements must take precedence; adjusted_score={:.4}",
        reports[0].adjusted_stability_score
    );
}

#[test]
fn test_adjusted_score_pulls_toward_half_with_large_k() {
    let jsonl = r#"{"sample_id":"a","label":"win","score":0.99}
{"sample_id":"a","label":"win","score":0.98}"#;
    let obs_default = parse_jsonl(jsonl).unwrap();
    let obs_large_k = parse_jsonl(jsonl).unwrap();

    let raw = score_all(obs_default, &ScoreConfig::default())[0].stability_score;
    let adj = score_all(
        obs_large_k,
        &ScoreConfig {
            confidence_k: 100.0,
            ..ScoreConfig::default()
        },
    )[0]
    .adjusted_stability_score;

    assert!(
        adj < raw,
        "large confidence_k -> adjusted score < raw score ({adj:.4} vs {raw:.4})"
    );
    assert!(
        adj > 0.5,
        "adjusted score should still be above 0.5 for high-stability sample"
    );
}

#[test]
fn test_wilson_lcb_lower_than_raw_for_small_n() {
    // 2/2 unanimous: label_agreement = 1.0 but LCB should be < 1.0
    let jsonl = r#"{"sample_id":"a","label":"win"}
{"sample_id":"a","label":"win"}"#;
    let obs = parse_jsonl(jsonl).unwrap();
    let reports = score_all(obs, &ScoreConfig::default());
    let r = &reports[0];
    assert_eq!(r.label_agreement, Some(1.0));
    // LCB at 95% for 2/2 should be well below 1.0 (roughly 0.34-0.81 range)
    let lcb = r.label_agreement_lcb.unwrap();
    assert!(lcb < 1.0, "LCB for 2/2 should be < 1.0, got {lcb}");
    assert!(lcb > 0.0, "LCB should be > 0.0");
}

#[test]
fn test_wilson_lcb_high_for_large_n() {
    // 25/25 unanimous: LCB should be very high (> 0.85)
    // For unanimous p=1: LCB = n/(n+z^2); z≈1.96, z^2≈3.84; 25/28.84≈0.867
    let jsonl: String = (0..25)
        .map(|i| format!("{{\"sample_id\":\"a\",\"label\":\"win\",\"run_id\":\"{i}\"}}\n"))
        .collect();
    let obs = parse_jsonl(&jsonl).unwrap();
    let reports = score_all(obs, &ScoreConfig::default());
    let lcb = reports[0].label_agreement_lcb.unwrap();
    assert!(lcb > 0.85, "LCB for 25/25 should be > 0.85, got {lcb}");
}

#[test]
fn test_lcb_policy_more_conservative_than_raw() {
    // 3/3 unanimous, raw stability should be keep, LCB should be lower
    let jsonl = r#"{"sample_id":"a","label":"win"}
{"sample_id":"a","label":"win"}
{"sample_id":"a","label":"win"}"#;
    let obs_raw = parse_jsonl(jsonl).unwrap();
    let obs_lcb = parse_jsonl(jsonl).unwrap();

    let raw_reports = score_all(obs_raw, &ScoreConfig::default());
    let lcb_reports = score_all(
        obs_lcb,
        &ScoreConfig {
            decision_score: DecisionScore::LowerConfidenceBound,
            ..ScoreConfig::default()
        },
    );
    // Raw should keep (all agree), LCB stability_score should be lower
    assert_eq!(raw_reports[0].decision, Decision::Keep);
    // LCB stability score is lower than raw (LCB penalises low n)
    assert!(
        lcb_reports[0].stability_score <= raw_reports[0].stability_score,
        "LCB stability should be <= raw: {} vs {}",
        lcb_reports[0].stability_score,
        raw_reports[0].stability_score
    );
}

#[test]
fn test_score_mad_less_sensitive_to_outlier() {
    // Scores: 0.9, 0.9, 0.9, 0.9, 0.0 (one outlier)
    // MAD should be much lower than std
    let obs = vec![
        quietset::Observation {
            sample_id: "a".into(),
            score: Some(0.9),
            ..Default::default()
        },
        quietset::Observation {
            sample_id: "a".into(),
            score: Some(0.9),
            ..Default::default()
        },
        quietset::Observation {
            sample_id: "a".into(),
            score: Some(0.9),
            ..Default::default()
        },
        quietset::Observation {
            sample_id: "a".into(),
            score: Some(0.9),
            ..Default::default()
        },
        quietset::Observation {
            sample_id: "a".into(),
            score: Some(0.0),
            ..Default::default()
        },
    ];
    let reports = score_all(obs, &ScoreConfig::default());
    let r = &reports[0];
    let std = r.score_std.unwrap();
    let mad = r.score_mad.unwrap();
    assert!(
        mad < std,
        "MAD ({mad:.4}) should be < std ({std:.4}) with one outlier"
    );
    assert!(
        mad < 0.01,
        "MAD should be near 0 (4 identical scores), got {mad:.4}"
    );
}

#[test]
fn test_score_iqr_even_n_uses_interpolation() {
    // Scores: 1,2,3,4 (even n). Linear interpolation (NumPy/R default) gives
    // median=2.5, Q1=1.75, Q3=3.25, IQR=1.5. Nearest-rank rounding would give
    // median=3, Q1=2, Q3=4, IQR=2 instead.
    let obs = vec![1.0, 2.0, 3.0, 4.0]
        .into_iter()
        .map(|score| quietset::Observation {
            sample_id: "a".into(),
            score: Some(score),
            ..Default::default()
        })
        .collect();
    let config = ScoreConfig {
        score_dispersion: ScoreDispersion::Iqr,
        ..ScoreConfig::default()
    };
    let reports = score_all(obs, &config);
    let iqr = reports[0].score_iqr.unwrap();
    assert!(
        (iqr - 1.5).abs() < 1e-9,
        "expected interpolated IQR 1.5, got {iqr}"
    );
}

fn sign_agreement_of(scores: &[f64]) -> Option<f64> {
    let obs: Vec<Observation> = scores
        .iter()
        .map(|&score| Observation {
            sample_id: "a".into(),
            score: Some(score),
            ..Default::default()
        })
        .collect();
    score_all(obs, &ScoreConfig::default())[0].score_sign_agreement
}

#[test]
fn test_score_sign_agreement() {
    // All same sign.
    assert_eq!(sign_agreement_of(&[0.3, 0.5, 0.2]), Some(1.0));

    // Clear majority, mixed.
    assert_eq!(sign_agreement_of(&[0.3, -0.1, 0.2, 0.15]), Some(0.75));

    // The discriminating case: score_std reads this as near-perfectly stable, but the sign
    // flipped from a positive to a negative signal — score_sign_agreement catches it.
    let obs = vec![
        Observation {
            sample_id: "a".into(),
            score: Some(0.01),
            ..Default::default()
        },
        Observation {
            sample_id: "a".into(),
            score: Some(-0.01),
            ..Default::default()
        },
    ];
    let r = &score_all(obs, &ScoreConfig::default())[0];
    assert!(r.score_std.unwrap() < 0.02, "{:?}", r.score_std);
    assert_eq!(r.score_sign_agreement, Some(0.5));

    // Zero is its own bucket, not folded into either sign: 3-way tie.
    assert_eq!(sign_agreement_of(&[0.0, 0.3, -0.3]), Some(1.0 / 3.0));

    // Zero plus a majority.
    assert_eq!(sign_agreement_of(&[0.0, 0.0, 0.3]), Some(2.0 / 3.0));

    // Single observation.
    assert_eq!(sign_agreement_of(&[0.42]), Some(1.0));

    // No numeric scores at all.
    let obs = vec![Observation {
        sample_id: "a".into(),
        label: Some("win".into()),
        ..Default::default()
    }];
    assert_eq!(
        score_all(obs, &ScoreConfig::default())[0].score_sign_agreement,
        None
    );
}

#[test]
fn test_score_sign_weight_zero_by_default_leaves_stability_score_unchanged() {
    // A sign-flipping-but-magnitude-stable fixture: score_sign_agreement is a low 0.5, but
    // the default score_sign weight (0.0) must leave stability_score identical to what it
    // was before this field became weightable.
    let obs = vec![
        Observation {
            sample_id: "a".into(),
            score: Some(0.01),
            ..Default::default()
        },
        Observation {
            sample_id: "a".into(),
            score: Some(-0.01),
            ..Default::default()
        },
    ];
    let r = &score_all(obs, &ScoreConfig::default())[0];
    assert_eq!(r.score_sign_agreement, Some(0.5));
    // score_std is tiny here, so score_consistency alone (the only active dimension) should
    // drive stability_score close to 1.0 -- score_sign_agreement's low 0.5 must not have
    // pulled it down, since its default weight is 0.0.
    assert!(
        r.stability_score > 0.9,
        "default score_sign weight (0.0) should not affect stability_score, got {}",
        r.stability_score
    );
}

#[test]
fn test_score_sign_weight_explicit_lowers_stability_score_on_sign_flip() {
    let sign_flip = vec![
        Observation {
            sample_id: "a".into(),
            score: Some(0.01),
            ..Default::default()
        },
        Observation {
            sample_id: "a".into(),
            score: Some(-0.01),
            ..Default::default()
        },
    ];
    let same_sign = vec![
        Observation {
            sample_id: "a".into(),
            score: Some(0.01),
            ..Default::default()
        },
        Observation {
            sample_id: "a".into(),
            score: Some(0.01),
            ..Default::default()
        },
    ];
    let config = ScoreConfig {
        weights: ScoreWeights {
            score_sign: 2.0,
            ..ScoreWeights::default()
        },
        ..ScoreConfig::default()
    };
    let flip_score = score_all(sign_flip, &config)[0].stability_score;
    let same_score = score_all(same_sign, &config)[0].stability_score;
    assert!(
        flip_score < same_score,
        "explicitly weighting score_sign should lower stability_score when scores flip \
         sign, got flip={flip_score} same={same_score}"
    );
}

#[test]
fn test_gold_label_changes_reliability() {
    // e1 matches majority (win is strict majority with 2 win vs 1 loss), e2 doesn't
    // With gold_label=loss on all samples, reliability reverses
    let jsonl_no_gold = r#"{"sample_id":"a","label":"win","evaluator_id":"e1"}
{"sample_id":"a","label":"win","evaluator_id":"e1"}
{"sample_id":"a","label":"loss","evaluator_id":"e2"}"#;

    let jsonl_gold = r#"{"sample_id":"a","label":"win","evaluator_id":"e1","gold_label":"loss"}
{"sample_id":"a","label":"win","evaluator_id":"e1","gold_label":"loss"}
{"sample_id":"a","label":"loss","evaluator_id":"e2","gold_label":"loss"}"#;

    let obs_no_gold = parse_jsonl(jsonl_no_gold).unwrap();
    let obs_gold = parse_jsonl(jsonl_gold).unwrap();

    let reports = score_all(obs_no_gold.clone(), &ScoreConfig::default());
    let rel_no_gold = compute_evaluator_reliability(&obs_no_gold, &reports);

    let reports_g = score_all(obs_gold.clone(), &ScoreConfig::default());
    let rel_gold = compute_evaluator_reliability(&obs_gold, &reports_g);

    // Without gold: e1 matches majority (win) -> high reliability
    // With gold (loss): e1 doesn't match gold -> low reliability
    let e1_no_gold = *rel_no_gold.get("e1").unwrap();
    let e1_gold = *rel_gold.get("e1").unwrap();
    assert!(
        e1_no_gold > e1_gold,
        "e1 reliability should be lower when gold_label differs: {e1_no_gold:.2} vs {e1_gold:.2}"
    );
}

#[test]
fn test_lcb_keep_demotions_excludes_already_unstable() {
    // The key semantic: lcb_keep_demotions counts samples where
    //   stability_score >= keep_threshold  (raw would keep)
    //   AND label_agreement_lcb < keep_threshold  (LCB would not keep)
    //
    // Critically, a sample already below keep_threshold in raw mode (e.g. split labels)
    // must NOT be counted even if its LCB is also below the threshold.
    //
    // Note: Wilson LCB at 95% confidence requires ~22+ observations of all-match to exceed 0.85,
    // so small samples (even fully-agreeing ones) still have LCB < 0.85 and are valid demotions.
    let config = ScoreConfig {
        thresholds: Thresholds {
            keep: 0.85,
            drop: 0.40,
        },
        confidence_level: 0.95,
        ..ScoreConfig::default()
    };

    let make_obs = |id: &str, label: &str| Observation {
        sample_id: id.into(),
        label: Some(label.into()),
        ..Default::default()
    };

    let mut obs = Vec::new();
    // sample_a, sample_b: fully-agreeing → stability_score = 1.0, LCB < 0.85 → demotion candidates
    for _ in 0..5 {
        obs.push(make_obs("a", "win"));
    }
    for _ in 0..2 {
        obs.push(make_obs("b", "win"));
    }
    // sample_c: 50/50 split → stability_score < 0.85 → already unstable, must NOT be counted
    obs.push(make_obs("c", "win"));
    obs.push(make_obs("c", "loss"));

    let reports = score_all(obs, &config);
    let keep_threshold = config.thresholds.keep;

    let is_demotion = |r: &&quietset::StabilityReport| {
        r.stability_score >= keep_threshold
            && r.label_agreement_lcb
                .map(|v| v < keep_threshold)
                .unwrap_or(false)
    };

    // sample_c must not appear in demotions — it's already review in raw mode
    let report_c = reports.iter().find(|r| r.sample_id == "c").unwrap();
    assert!(
        report_c.stability_score < keep_threshold,
        "sample_c stability_score should be below keep threshold: {:.4}",
        report_c.stability_score
    );
    assert!(
        !is_demotion(&report_c),
        "sample_c must not be counted as a demotion (already unstable)"
    );

    // sample_a and sample_b should be demotions (raw-keep but LCB < threshold)
    let report_a = reports.iter().find(|r| r.sample_id == "a").unwrap();
    let report_b = reports.iter().find(|r| r.sample_id == "b").unwrap();
    assert!(
        is_demotion(&report_a),
        "sample_a should be a demotion candidate"
    );
    assert!(
        is_demotion(&report_b),
        "sample_b should be a demotion candidate"
    );
}

#[test]
fn test_fleiss_kappa_perfect_agreement() {
    // 3 raters, 3 subjects, all agree → kappa = 1.0
    let jsonl = r#"{"sample_id":"a","label":"yes","evaluator_id":"e1"}
{"sample_id":"a","label":"yes","evaluator_id":"e2"}
{"sample_id":"a","label":"yes","evaluator_id":"e3"}
{"sample_id":"b","label":"no","evaluator_id":"e1"}
{"sample_id":"b","label":"no","evaluator_id":"e2"}
{"sample_id":"b","label":"no","evaluator_id":"e3"}
{"sample_id":"c","label":"yes","evaluator_id":"e1"}
{"sample_id":"c","label":"yes","evaluator_id":"e2"}
{"sample_id":"c","label":"yes","evaluator_id":"e3"}"#;
    let obs = parse_jsonl(jsonl).unwrap();
    let k = compute_fleiss_kappa(&obs).expect("should compute kappa");
    assert!(
        (k - 1.0).abs() < 1e-9,
        "perfect agreement → kappa=1.0, got {k:.6}"
    );
}

#[test]
fn test_fleiss_kappa_perfect_disagreement() {
    // 2 raters always disagree → P_obs=0, P_e=0.5 → kappa = (0-0.5)/(1-0.5) = -1.0
    let jsonl = r#"{"sample_id":"a","label":"yes","evaluator_id":"e1"}
{"sample_id":"a","label":"no","evaluator_id":"e2"}
{"sample_id":"b","label":"yes","evaluator_id":"e1"}
{"sample_id":"b","label":"no","evaluator_id":"e2"}"#;
    let obs = parse_jsonl(jsonl).unwrap();
    let k = compute_fleiss_kappa(&obs).expect("should compute kappa");
    assert!(
        (k + 1.0).abs() < 1e-9,
        "perfect disagreement → kappa=-1.0, got {k:.6}"
    );
}

#[test]
fn test_krippendorff_alpha_perfect_agreement() {
    let jsonl = r#"{"sample_id":"a","label":"win","evaluator_id":"e1"}
{"sample_id":"a","label":"win","evaluator_id":"e2"}
{"sample_id":"b","label":"loss","evaluator_id":"e1"}
{"sample_id":"b","label":"loss","evaluator_id":"e2"}"#;
    let obs = parse_jsonl(jsonl).unwrap();
    let a = compute_krippendorff_alpha(&obs).expect("should compute alpha");
    assert!(
        (a - 1.0).abs() < 1e-9,
        "perfect agreement → alpha=1.0, got {a:.6}"
    );
}

#[test]
fn test_krippendorff_alpha_perfect_disagreement() {
    // 2 raters always disagree → Do=1, De=2/3 → alpha = 1 - 1/(2/3) = -0.5
    let jsonl = r#"{"sample_id":"a","label":"yes","evaluator_id":"e1"}
{"sample_id":"a","label":"no","evaluator_id":"e2"}
{"sample_id":"b","label":"yes","evaluator_id":"e1"}
{"sample_id":"b","label":"no","evaluator_id":"e2"}"#;
    let obs = parse_jsonl(jsonl).unwrap();
    let a = compute_krippendorff_alpha(&obs).expect("should compute alpha");
    assert!(
        (a + 0.5).abs() < 1e-9,
        "perfect disagreement → alpha=-0.5, got {a:.6}"
    );
}

#[test]
fn test_kappa_and_alpha_undefined_for_single_rater() {
    // Only one rating per subject — undefined
    let jsonl = r#"{"sample_id":"a","label":"yes","evaluator_id":"e1"}
{"sample_id":"b","label":"no","evaluator_id":"e1"}"#;
    let obs = parse_jsonl(jsonl).unwrap();
    assert!(compute_fleiss_kappa(&obs).is_none());
    assert!(compute_krippendorff_alpha(&obs).is_none());
}

#[test]
fn test_calibrate_target_precision() {
    // a, b, c: 5 observations each, all agree, gold matches → stable and correct
    // e: 5 obs, all "loss", but gold="win" → stable but WRONG
    // d: 2 obs, split → unstable
    let make = |id: &str, label: &str, gold: &str, n: usize| -> Vec<Observation> {
        (0..n)
            .map(|i| Observation {
                sample_id: id.into(),
                label: Some(label.into()),
                gold_label: Some(gold.into()),
                evaluator_id: Some(format!("e{i}")),
                ..Default::default()
            })
            .collect()
    };
    let mut obs = Vec::new();
    obs.extend(make("a", "win", "win", 5));
    obs.extend(make("b", "win", "win", 5));
    obs.extend(make("c", "win", "win", 5));
    obs.push(Observation {
        sample_id: "d".into(),
        label: Some("win".into()),
        gold_label: Some("win".into()),
        ..Default::default()
    });
    obs.push(Observation {
        sample_id: "d".into(),
        label: Some("loss".into()),
        gold_label: Some("win".into()),
        ..Default::default()
    });
    obs.extend(make("e", "loss", "win", 5)); // stable but wrong

    // e is stable (all "loss") and scores 1.0 just like a/b/c; no threshold can separate it.
    // Best achievable precision: 3 correct / 4 kept (a,b,c,e all score 1.0) = 0.75
    let result = compute_calibration(&obs, &DecisionScore::Raw, 0.95, 0.75, None, 0.40, None);
    assert!(
        result.is_some(),
        "should find a threshold meeting 0.75 precision"
    );
    let r = result.unwrap();
    assert!(
        r.achieved_precision >= 0.75,
        "precision {:.3} should be >= 0.75",
        r.achieved_precision
    );
    assert!(r.keep_threshold >= 0.50 && r.keep_threshold <= 0.99);
}

#[test]
fn test_calibrate_precision_ci_contains_achieved_precision() {
    let make = |id: &str, label: &str, gold: &str, n: usize| -> Vec<Observation> {
        (0..n)
            .map(|i| Observation {
                sample_id: id.into(),
                label: Some(label.into()),
                gold_label: Some(gold.into()),
                evaluator_id: Some(format!("e{i}")),
                ..Default::default()
            })
            .collect()
    };
    let mut obs = Vec::new();
    obs.extend(make("a", "win", "win", 5));
    obs.extend(make("b", "win", "win", 5));
    obs.extend(make("c", "win", "win", 5));
    obs.extend(make("e", "loss", "win", 5)); // stable but wrong

    let result = compute_calibration(&obs, &DecisionScore::Raw, 0.95, 0.75, None, 0.40, None);
    let r = result.expect("should find a threshold meeting 0.75 precision");
    assert!(r.precision_ci_low >= 0.0 && r.precision_ci_high <= 1.0);
    assert!(
        r.precision_ci_low <= r.achieved_precision,
        "ci_low {} should be <= achieved_precision {}",
        r.precision_ci_low,
        r.achieved_precision
    );
    assert!(
        r.achieved_precision <= r.precision_ci_high,
        "achieved_precision {} should be <= ci_high {}",
        r.achieved_precision,
        r.precision_ci_high
    );
}

#[test]
fn test_calibrate_precision_ci_narrows_with_more_gold_samples() {
    let make = |id: &str, label: &str, gold: &str, n: usize| -> Vec<Observation> {
        (0..n)
            .map(|i| Observation {
                sample_id: id.into(),
                label: Some(label.into()),
                gold_label: Some(gold.into()),
                evaluator_id: Some(format!("e{i}")),
                ..Default::default()
            })
            .collect()
    };
    // Small: 3 correct + 1 wrong, all stable (score 1.0) -> n_keep=4
    let mut small = Vec::new();
    small.extend(make("a", "win", "win", 5));
    small.extend(make("b", "win", "win", 5));
    small.extend(make("c", "win", "win", 5));
    small.extend(make("e", "loss", "win", 5));

    // Large: same 3:1 correct:wrong ratio repeated 10x with distinct sample_ids -> n_keep=40
    let mut large = Vec::new();
    for i in 0..10 {
        large.extend(make(&format!("a{i}"), "win", "win", 5));
        large.extend(make(&format!("b{i}"), "win", "win", 5));
        large.extend(make(&format!("c{i}"), "win", "win", 5));
        large.extend(make(&format!("e{i}"), "loss", "win", 5));
    }

    let r_small = compute_calibration(&small, &DecisionScore::Raw, 0.95, 0.75, None, 0.40, None)
        .expect("small should find a threshold meeting 0.75 precision");
    let r_large = compute_calibration(&large, &DecisionScore::Raw, 0.95, 0.75, None, 0.40, None)
        .expect("large should find a threshold meeting 0.75 precision");

    let small_width = r_small.precision_ci_high - r_small.precision_ci_low;
    let large_width = r_large.precision_ci_high - r_large.precision_ci_low;
    assert!(
        large_width < small_width,
        "CI width with more gold samples ({large_width}) should be narrower than with few ({small_width})"
    );
}

#[test]
fn test_calibrate_no_gold_returns_none() {
    let obs = vec![
        Observation {
            sample_id: "a".into(),
            label: Some("win".into()),
            ..Default::default()
        },
        Observation {
            sample_id: "a".into(),
            label: Some("win".into()),
            ..Default::default()
        },
    ];
    assert!(compute_calibration(&obs, &DecisionScore::Raw, 0.95, 0.95, None, 0.40, None).is_none());
}

#[test]
fn test_calibrate_heldout_precision_differs_from_training() {
    let make = |id: &str, label: &str, gold: &str, n: usize| -> Vec<Observation> {
        (0..n)
            .map(|i| Observation {
                sample_id: id.into(),
                label: Some(label.into()),
                gold_label: Some(gold.into()),
                evaluator_id: Some(format!("e{i}")),
                ..Default::default()
            })
            .collect()
    };

    // Training: 9 correct stable samples + 1 wrong stable sample -> 9/10 = 0.90 precision,
    // all fully agreeing (score 1.0), so the kept set is identical across every threshold.
    let mut train = Vec::new();
    for i in 0..9 {
        train.extend(make(&format!("t{i}"), "win", "win", 5));
    }
    train.extend(make("tw", "loss", "win", 5)); // stable but wrong

    // Held-out: same shape (all stable, score 1.0) but only 2/10 correct -> true precision 0.20.
    let mut heldout = Vec::new();
    for i in 0..2 {
        heldout.extend(make(&format!("h{i}"), "win", "win", 5));
    }
    for i in 0..8 {
        heldout.extend(make(&format!("hw{i}"), "loss", "win", 5));
    }

    let result = compute_calibration(
        &train,
        &DecisionScore::Raw,
        0.95,
        0.80,
        None,
        0.40,
        Some(&heldout),
    )
    .expect("training should find a threshold meeting 0.80 precision");

    assert!(
        (result.keep_threshold - 0.99).abs() < 1e-9,
        "threshold is selected from training data"
    );
    assert_eq!(
        result.n_total, 10,
        "n_total should reflect heldout's own size, not training's"
    );
    assert_eq!(result.n_keep, 10);
    assert!(
        (result.achieved_precision - 0.20).abs() < 1e-9,
        "achieved_precision should be heldout's true precision (2/10), got {}",
        result.achieved_precision
    );
    assert!(
        result.achieved_precision < 0.80,
        "training-selected threshold overfits training data; falling below target on \
         heldout is the expected, honest signal, not a bug"
    );
    let note = result
        .note
        .expect("large train/heldout gap should attach a note");
    assert!(
        note.contains("0.900") && note.contains("0.200"),
        "note should name both the in-sample and held-out precision: {note}"
    );
    assert!(
        (result.train_precision.expect("heldout was used") - 0.90).abs() < 1e-9,
        "train_precision should be the in-sample 9/10 ratio, got {:?}",
        result.train_precision
    );
}

#[test]
fn test_calibrate_no_heldout_note_always_present() {
    let obs: Vec<Observation> = (0..10)
        .map(|i| Observation {
            sample_id: format!("s{i}"),
            label: Some("win".into()),
            gold_label: Some("win".into()),
            evaluator_id: Some(format!("e{i}")),
            ..Default::default()
        })
        .collect();
    let result = compute_calibration(&obs, &DecisionScore::Raw, 0.95, 0.75, None, 0.40, None)
        .expect("all-agreeing gold-correct samples should calibrate");
    let note = result
        .note
        .expect("achieved_precision without --heldout is always in-sample optimistic");
    assert!(note.contains("--heldout"));
    assert!(
        result.train_precision.is_none(),
        "train_precision is only populated when --heldout is used"
    );
}

#[test]
fn test_calibrate_heldout_small_gap_no_note() {
    let make = |id: &str, label: &str, gold: &str, n: usize| -> Vec<Observation> {
        (0..n)
            .map(|i| Observation {
                sample_id: id.into(),
                label: Some(label.into()),
                gold_label: Some(gold.into()),
                evaluator_id: Some(format!("e{i}")),
                ..Default::default()
            })
            .collect()
    };

    // Training: 9/10 correct (0.90 precision). Held-out: same 9/10 ratio -> zero gap.
    let mut train = Vec::new();
    for i in 0..9 {
        train.extend(make(&format!("t{i}"), "win", "win", 5));
    }
    train.extend(make("tw", "loss", "win", 5));

    let mut heldout = Vec::new();
    for i in 0..9 {
        heldout.extend(make(&format!("h{i}"), "win", "win", 5));
    }
    heldout.extend(make("hw", "loss", "win", 5));

    let result = compute_calibration(
        &train,
        &DecisionScore::Raw,
        0.95,
        0.80,
        None,
        0.40,
        Some(&heldout),
    )
    .expect("training should find a threshold meeting 0.80 precision");

    assert!(
        (result.achieved_precision - 0.90).abs() < 1e-9,
        "heldout precision should match training's 9/10 ratio"
    );
    assert!(
        result.note.is_none(),
        "a zero-gap heldout result should not attach an overfitting note, got {:?}",
        result.note
    );
}

#[test]
fn test_calibrate_heldout_no_gold_returns_none() {
    let make = |id: &str, label: &str, gold: &str, n: usize| -> Vec<Observation> {
        (0..n)
            .map(|i| Observation {
                sample_id: id.into(),
                label: Some(label.into()),
                gold_label: Some(gold.into()),
                evaluator_id: Some(format!("e{i}")),
                ..Default::default()
            })
            .collect()
    };
    let mut train = Vec::new();
    train.extend(make("a", "win", "win", 5));
    train.extend(make("b", "win", "win", 5));

    let heldout = vec![
        Observation {
            sample_id: "h".into(),
            label: Some("win".into()),
            ..Default::default()
        },
        Observation {
            sample_id: "h".into(),
            label: Some("win".into()),
            ..Default::default()
        },
    ];

    assert!(
        compute_calibration(
            &train,
            &DecisionScore::Raw,
            0.95,
            0.75,
            None,
            0.40,
            Some(&heldout)
        )
        .is_none(),
        "heldout with no gold_label should return None, mirroring the \
         training-gold-missing convention"
    );
}

#[test]
fn test_compute_calibration_matches_compute_calibration_with_options_default() {
    // Backward-compatibility guard: the plain compute_calibration must be indistinguishable
    // from compute_calibration_with_options called with CalibrationOptions::default() -- not
    // just "compiles", but bit-for-bit the same result on a fixture that exercises heldout,
    // note, and group_leakage.
    let make = |id: &str, label: &str, gold: &str, n: usize| -> Vec<Observation> {
        (0..n)
            .map(|i| Observation {
                sample_id: id.into(),
                label: Some(label.into()),
                gold_label: Some(gold.into()),
                evaluator_id: Some(format!("e{i}")),
                ..Default::default()
            })
            .collect()
    };
    let mut train = Vec::new();
    for i in 0..9 {
        train.extend(make(&format!("t{i}"), "win", "win", 5));
    }
    train.extend(make("tw", "loss", "win", 5));
    let mut heldout = Vec::new();
    for i in 0..2 {
        heldout.extend(make(&format!("h{i}"), "win", "win", 5));
    }
    for i in 0..8 {
        heldout.extend(make(&format!("hw{i}"), "loss", "win", 5));
    }

    let via_plain = compute_calibration(
        &train,
        &DecisionScore::Raw,
        0.95,
        0.80,
        None,
        0.40,
        Some(&heldout),
    )
    .expect("plain API should calibrate");
    let via_options = compute_calibration_with_options(
        &train,
        &DecisionScore::Raw,
        0.95,
        0.80,
        None,
        0.40,
        Some(&heldout),
        &CalibrationOptions::default(),
    )
    .expect("options API with defaults should calibrate identically");

    assert_eq!(via_plain.keep_threshold, via_options.keep_threshold);
    assert_eq!(via_plain.achieved_precision, via_options.achieved_precision);
    assert_eq!(via_plain.coverage, via_options.coverage);
    assert_eq!(via_plain.n_keep, via_options.n_keep);
    assert_eq!(via_plain.n_total, via_options.n_total);
    assert_eq!(via_plain.train_precision, via_options.train_precision);
    assert_eq!(via_plain.note, via_options.note);
    assert!(via_plain.group_leakage.is_none());
    assert!(via_options.group_leakage.is_none());
}

fn obs_with_group(sample_id: &str, source_root_id: &str) -> Observation {
    Observation {
        sample_id: sample_id.into(),
        label: Some("win".into()),
        gold_label: Some("win".into()),
        source_root_id: Some(source_root_id.into()),
        ..Default::default()
    }
}

fn obs_with_opening_family(sample_id: &str, opening_family: &str) -> Observation {
    Observation {
        sample_id: sample_id.into(),
        label: Some("win".into()),
        gold_label: Some("win".into()),
        opening_family: Some(opening_family.into()),
        ..Default::default()
    }
}

#[test]
fn test_group_leakage_detects_shared_source_root_id() {
    let calibration = vec![obs_with_group("a", "g1"), obs_with_group("b", "g2")];
    let heldout = vec![obs_with_group("c", "g2"), obs_with_group("d", "g3")];
    let leakage = group_leakage(&calibration, &heldout, GroupKey::SourceRootId);
    assert_eq!(leakage.train_unique_groups, 2);
    assert_eq!(leakage.heldout_unique_groups, 2);
    assert_eq!(leakage.leaked_group_count, 1);
    assert_eq!(leakage.leaked_keys_sample, vec!["g2".to_string()]);
    assert!(!leakage.leaked_keys_truncated);
    assert_eq!(leakage.train_rows, 2);
    assert_eq!(leakage.heldout_rows, 2);
    assert_eq!(leakage.rows_with_group, 4);
    assert_eq!(leakage.rows_missing_group, 0);
    assert_eq!(leakage.coverage, 1.0);
}

#[test]
fn test_group_leakage_works_for_opening_family() {
    let calibration = vec![obs_with_opening_family("a", "sicilian")];
    let heldout = vec![obs_with_opening_family("c", "sicilian")];
    let leakage = group_leakage(&calibration, &heldout, GroupKey::OpeningFamily);
    assert_eq!(leakage.group_field, "opening_family");
    assert_eq!(leakage.leaked_group_count, 1);
    assert_eq!(leakage.leaked_keys_sample, vec!["sicilian".to_string()]);
    assert_eq!(leakage.coverage, 1.0);
}

#[test]
fn test_group_leakage_reports_zero_shared_on_a_clean_split() {
    let calibration = vec![obs_with_group("a", "g1"), obs_with_group("b", "g2")];
    let heldout = vec![obs_with_group("c", "g3"), obs_with_group("d", "g4")];
    let leakage = group_leakage(&calibration, &heldout, GroupKey::SourceRootId);
    assert_eq!(leakage.leaked_group_count, 0);
    assert!(leakage.leaked_keys_sample.is_empty());
    assert_eq!(leakage.rows_missing_group, 0);
    assert_eq!(leakage.coverage, 1.0);
}

#[test]
fn test_group_leakage_counts_rows_missing_the_key_when_all_are_missing() {
    let calibration = vec![
        obs_with_group("a", "g1"),
        Observation {
            sample_id: "b".into(),
            gold_label: Some("win".into()),
            ..Default::default()
        },
    ];
    let heldout = vec![Observation {
        sample_id: "c".into(),
        gold_label: Some("win".into()),
        ..Default::default()
    }];
    let leakage = group_leakage(&calibration, &heldout, GroupKey::SourceRootId);
    assert_eq!(leakage.train_rows, 2);
    assert_eq!(leakage.heldout_rows, 1);
    assert_eq!(leakage.rows_with_group, 1, "only 'a' carries a group key");
    assert_eq!(leakage.rows_missing_group, 2, "'b' and 'c' carry none");
    // heldout carries zero group keys at all, so no leak could ever be detected regardless of
    // the training side's coverage -- leaked_group_count must not read as "verified clean".
    assert_eq!(leakage.heldout_unique_groups, 0);
    assert_eq!(leakage.leaked_group_count, 0);
}

#[test]
fn test_group_leakage_reports_partial_coverage_without_hiding_a_leak() {
    // Only some rows on each side carry the group key -- the check must still find the leak
    // among the rows that do, and report the coverage gap rather than silently excluding the
    // missing rows and calling the result "clean" or "fully checked".
    let calibration = vec![
        obs_with_group("a", "shared"),
        Observation {
            sample_id: "b".into(),
            gold_label: Some("win".into()),
            ..Default::default()
        },
    ];
    let heldout = vec![
        obs_with_group("c", "shared"),
        Observation {
            sample_id: "d".into(),
            gold_label: Some("win".into()),
            ..Default::default()
        },
    ];
    let leakage = group_leakage(&calibration, &heldout, GroupKey::SourceRootId);
    assert_eq!(
        leakage.leaked_group_count, 1,
        "the 'shared' group is still found"
    );
    assert_eq!(leakage.leaked_keys_sample, vec!["shared".to_string()]);
    assert_eq!(leakage.train_rows, 2);
    assert_eq!(leakage.heldout_rows, 2);
    assert_eq!(leakage.rows_with_group, 2);
    assert_eq!(leakage.rows_missing_group, 2);
    assert!((leakage.coverage - 0.5).abs() < 1e-9);
}

#[test]
fn test_group_leakage_sample_is_sorted_when_under_the_cap() {
    let calibration: Vec<Observation> = (0..15)
        .map(|i| obs_with_group(&format!("a{i}"), &format!("g{i:02}")))
        .collect();
    let heldout: Vec<Observation> = (0..15)
        .map(|i| obs_with_group(&format!("h{i}"), &format!("g{i:02}")))
        .collect();
    let leakage = group_leakage(&calibration, &heldout, GroupKey::SourceRootId);
    assert_eq!(leakage.leaked_group_count, 15);
    assert_eq!(
        leakage.leaked_keys_sample.len(),
        15,
        "well under the 100-key cap, so the sample holds every leaked key"
    );
    assert!(!leakage.leaked_keys_truncated);
    let mut sorted = leakage.leaked_keys_sample.clone();
    sorted.sort();
    assert_eq!(
        leakage.leaked_keys_sample, sorted,
        "keys should already be sorted"
    );
}

#[test]
fn test_group_leakage_sample_is_capped_and_flagged_truncated_over_the_limit() {
    // 150 leaked groups, well past MAX_LEAKED_KEYS_SAMPLE (100) -- leaked_group_count must
    // still report the exact total while leaked_keys_sample stays bounded.
    let calibration: Vec<Observation> = (0..150)
        .map(|i| obs_with_group(&format!("a{i}"), &format!("g{i:03}")))
        .collect();
    let heldout: Vec<Observation> = (0..150)
        .map(|i| obs_with_group(&format!("h{i}"), &format!("g{i:03}")))
        .collect();
    let leakage = group_leakage(&calibration, &heldout, GroupKey::SourceRootId);
    assert_eq!(
        leakage.leaked_group_count, 150,
        "exact count, unaffected by the cap"
    );
    assert_eq!(leakage.leaked_keys_sample.len(), 100);
    assert!(leakage.leaked_keys_truncated);
    let mut sorted = leakage.leaked_keys_sample.clone();
    sorted.sort();
    assert_eq!(
        leakage.leaked_keys_sample, sorted,
        "sample should be a sorted prefix"
    );
}

#[test]
fn test_leaked_group_keys_returns_the_complete_uncapped_list() {
    let calibration: Vec<Observation> = (0..150)
        .map(|i| obs_with_group(&format!("a{i}"), &format!("g{i:03}")))
        .collect();
    let heldout: Vec<Observation> = (0..150)
        .map(|i| obs_with_group(&format!("h{i}"), &format!("g{i:03}")))
        .collect();
    let all_keys = leaked_group_keys(&calibration, &heldout, GroupKey::SourceRootId);
    assert_eq!(
        all_keys.len(),
        150,
        "the low-level function has no cap, unlike group_leakage's embedded sample"
    );
    let mut sorted = all_keys.clone();
    sorted.sort();
    assert_eq!(all_keys, sorted);
}

#[test]
fn test_compute_calibration_attaches_leakage_only_with_both_heldout_and_key() {
    let train = vec![obs_with_group("a", "g1"), obs_with_group("b", "g2")];
    let heldout = vec![obs_with_group("c", "g2")];

    // No group_by at all: the plain, options-free API is exactly this case.
    let no_key = compute_calibration(
        &train,
        &DecisionScore::Raw,
        0.95,
        0.5,
        None,
        0.40,
        Some(&heldout),
    )
    .unwrap();
    assert!(no_key.group_leakage.is_none());

    let no_heldout = compute_calibration_with_options(
        &train,
        &DecisionScore::Raw,
        0.95,
        0.5,
        None,
        0.40,
        None,
        &CalibrationOptions {
            group_by: Some(GroupKey::SourceRootId),
        },
    )
    .unwrap();
    assert!(no_heldout.group_leakage.is_none());

    let both = compute_calibration_with_options(
        &train,
        &DecisionScore::Raw,
        0.95,
        0.5,
        None,
        0.40,
        Some(&heldout),
        &CalibrationOptions {
            group_by: Some(GroupKey::SourceRootId),
        },
    )
    .unwrap();
    assert!(both.group_leakage.is_some());
}

#[test]
fn test_compute_calibration_note_mentions_leakage_when_groups_are_shared() {
    let train = vec![obs_with_group("a", "g1"), obs_with_group("b", "g2")];
    let heldout = vec![obs_with_group("c", "g2")];
    let result = compute_calibration_with_options(
        &train,
        &DecisionScore::Raw,
        0.95,
        0.5,
        None,
        0.40,
        Some(&heldout),
        &CalibrationOptions {
            group_by: Some(GroupKey::SourceRootId),
        },
    )
    .unwrap();
    let note = result.note.expect("leaked split should attach a note");
    assert!(
        note.contains("source_root_id") && note.contains("optimistic"),
        "note should describe the leakage: {note}"
    );
}

#[test]
fn test_compute_calibration_note_mentions_incomplete_coverage_when_rows_are_missing() {
    // No group is shared, but neither side has full coverage -- the note must not read as a
    // clean bill of health, since the leakage check itself only covered part of the data.
    let train = vec![
        obs_with_group("a", "g1"),
        Observation {
            sample_id: "b".into(),
            label: Some("win".into()),
            gold_label: Some("win".into()),
            ..Default::default()
        },
    ];
    let heldout = vec![obs_with_group("c", "g3")];
    let result = compute_calibration_with_options(
        &train,
        &DecisionScore::Raw,
        0.95,
        0.5,
        None,
        0.40,
        Some(&heldout),
        &CalibrationOptions {
            group_by: Some(GroupKey::SourceRootId),
        },
    )
    .unwrap();
    assert_eq!(result.group_leakage.as_ref().unwrap().leaked_group_count, 0);
    let note = result
        .note
        .expect("incomplete coverage should attach a note even with no leak found");
    assert!(
        note.contains("coverage"),
        "note should flag the incomplete coverage: {note}"
    );
}

#[test]
fn test_compute_calibration_note_keeps_overfit_sentence_alongside_leakage_sentence() {
    let make = |id: &str, label: &str, gold: &str, group: &str, n: usize| -> Vec<Observation> {
        (0..n)
            .map(|i| Observation {
                sample_id: id.into(),
                label: Some(label.into()),
                gold_label: Some(gold.into()),
                evaluator_id: Some(format!("e{i}")),
                source_root_id: Some(group.into()),
                ..Default::default()
            })
            .collect()
    };

    // Same shape as test_calibrate_heldout_precision_differs_from_training (9/10 correct
    // stable training vs. 2/10 correct stable heldout -> an overfit gap), plus one group
    // ("g_shared") shared between a training and a held-out sample -> a leakage overlap too.
    let mut train = Vec::new();
    for i in 0..9 {
        train.extend(make(&format!("t{i}"), "win", "win", &format!("g_t{i}"), 5));
    }
    train.extend(make("tw", "loss", "win", "g_shared", 5));

    let mut heldout = Vec::new();
    heldout.extend(make("h0", "win", "win", "g_shared", 5));
    heldout.extend(make("h1", "win", "win", "g_h1", 5));
    for i in 0..8 {
        heldout.extend(make(
            &format!("hw{i}"),
            "loss",
            "win",
            &format!("g_hw{i}"),
            5,
        ));
    }

    let result = compute_calibration_with_options(
        &train,
        &DecisionScore::Raw,
        0.95,
        0.80,
        None,
        0.40,
        Some(&heldout),
        &CalibrationOptions {
            group_by: Some(GroupKey::SourceRootId),
        },
    )
    .expect("training should find a threshold meeting 0.80 precision");
    let note = result
        .note
        .expect("both leakage and overfit gap should attach a note");
    assert!(
        note.contains("source_root_id") && note.contains("overfit"),
        "note should combine both sentences: {note}"
    );
}

#[test]
fn test_profile_llm_judge_base_weights() {
    let base = ScoreWeights {
        evaluator_agreement: 2.0,
        model_agreement: 2.0,
        ..ScoreWeights::default()
    };
    assert!((base.evaluator_agreement - 2.0).abs() < 1e-9);
    assert!((base.model_agreement - 2.0).abs() < 1e-9);
    assert!((base.label_agreement - 1.0).abs() < 1e-9);
    assert!((base.budget_stability - 1.0).abs() < 1e-9);
}

#[test]
fn test_filter_confidence_threshold() {
    // Sample with 2 obs has lower confidence than sample with 10 obs
    let obs2 = vec![
        Observation {
            sample_id: "s2".into(),
            label: Some("w".into()),
            ..Default::default()
        };
        2
    ];
    let obs10 = vec![
        Observation {
            sample_id: "s10".into(),
            label: Some("w".into()),
            ..Default::default()
        };
        10
    ];
    let mut all = obs2;
    all.extend(obs10);
    let reports = score_all(all, &ScoreConfig::default());
    let r2 = reports.iter().find(|r| r.sample_id == "s2").unwrap();
    let r10 = reports.iter().find(|r| r.sample_id == "s10").unwrap();
    // confidence = n/(n+3): 2→0.40, 10→0.77
    assert!(r2.confidence < r10.confidence);
    // A filter with min_confidence=0.6 should drop s2 but keep s10
    assert!(r2.confidence < 0.6);
    assert!(r10.confidence >= 0.6);
}

#[test]
fn test_label_distribution_basic() {
    // 2 "win" + 1 "loss" → distribution {"win": ~0.667, "loss": ~0.333}
    let jsonl = r#"{"sample_id":"a","label":"win"}
{"sample_id":"a","label":"win"}
{"sample_id":"a","label":"loss"}"#;
    let obs = parse_jsonl(jsonl).unwrap();
    let reports = score_all(obs, &ScoreConfig::default());
    let r = &reports[0];
    let dist = r
        .label_distribution
        .as_ref()
        .expect("label_distribution should be Some");
    let win_frac = *dist.get("win").unwrap();
    let loss_frac = *dist.get("loss").unwrap();
    assert!(
        (win_frac - 2.0 / 3.0).abs() < 1e-9,
        "win fraction: {win_frac}"
    );
    assert!(
        (loss_frac - 1.0 / 3.0).abs() < 1e-9,
        "loss fraction: {loss_frac}"
    );
    // Most common label comes first
    let first = dist.keys().next().unwrap();
    assert_eq!(first, "win");
}

#[test]
fn test_label_distribution_none_when_no_labels() {
    let obs = vec![Observation {
        sample_id: "x".into(),
        score: Some(0.9),
        ..Default::default()
    }];
    let reports = score_all(obs, &ScoreConfig::default());
    assert!(reports[0].label_distribution.is_none());
}

#[test]
fn test_weighted_majority_overrides_majority() {
    // Evaluator A (reliable: gold matches) says "win".
    // Evaluators B and C (unreliable: gold disagrees) both say "loss".
    // Raw majority = "loss" (2 vs 1). Weighted majority should = "win".
    let obs = vec![
        Observation {
            sample_id: "s".into(),
            label: Some("win".into()),
            evaluator_id: Some("A".into()),
            gold_label: Some("win".into()),
            ..Default::default()
        },
        Observation {
            sample_id: "s".into(),
            label: Some("loss".into()),
            evaluator_id: Some("B".into()),
            gold_label: Some("win".into()),
            ..Default::default()
        },
        Observation {
            sample_id: "s".into(),
            label: Some("loss".into()),
            evaluator_id: Some("C".into()),
            gold_label: Some("win".into()),
            ..Default::default()
        },
    ];
    let reports = score_all_weighted(obs.clone(), &ScoreConfig::default());
    let r = &reports[0];
    // Raw majority is "loss"
    assert_eq!(r.majority_label.as_deref(), Some("loss"));
    // Evaluator weights: A matches gold (1/1 + smoothing), B&C don't (0/1 + smoothing)
    let truth: std::collections::HashMap<String, String> = obs
        .iter()
        .filter_map(|o| o.gold_label.clone().map(|g| (o.sample_id.clone(), g)))
        .collect();
    let weights = compute_evaluator_weights(&obs, &truth);
    let w_a = *weights.get("A").unwrap();
    let w_b = *weights.get("B").unwrap();
    assert!(w_a > w_b, "A should have higher weight than B/C");
    // Weighted majority overrides: "win" wins when A's weight > sum of B+C weights
    // A: (1+0.5)/(1+1)=0.75, B/C: (0+0.5)/(1+1)=0.25 each → win score=0.75, loss=0.50
    assert_eq!(r.weighted_majority_label.as_deref(), Some("win"));
    assert_eq!(r.majority_weighted_conflict, Some(true));
}

#[test]
fn test_gold_weighted_flips_review_to_keep() {
    // Same reliability pattern as test_weighted_majority_overrides_majority (A reliably
    // right, B/C reliably wrong per gold_label), repeated across 6 samples so A's smoothed
    // weight (6.5/7) dominates B+C's (0.5/7 each): gold-weighted confidence for "s" ≈
    // 0.867, clearing the default keep_threshold=0.85, while raw label_agreement stays at
    // 0.667 (Review). evaluator_agreement is zeroed out: with its default weight it also
    // reflects this sample's disagreement and would keep the raw and gold-weighted scores
    // too close together to flip the decision.
    let mut obs = Vec::new();
    for id in ["s", "s2", "s3", "s4", "s5", "s6"] {
        for (eval, label) in [("A", "win"), ("B", "loss"), ("C", "loss")] {
            obs.push(Observation {
                sample_id: id.into(),
                label: Some(label.into()),
                evaluator_id: Some(eval.into()),
                gold_label: Some("win".into()),
                ..Default::default()
            });
        }
    }
    let weights = ScoreWeights {
        evaluator_agreement: 0.0,
        ..ScoreWeights::default()
    };

    let raw_reports = score_all(
        obs.clone(),
        &ScoreConfig {
            weights: weights.clone(),
            ..ScoreConfig::default()
        },
    );
    let s_raw = raw_reports.iter().find(|r| r.sample_id == "s").unwrap();
    assert_eq!(s_raw.decision, Decision::Review);

    let gold_reports = score_all_gold_weighted(
        obs,
        &ScoreConfig {
            weights,
            decision_score: DecisionScore::GoldWeighted,
            ..ScoreConfig::default()
        },
    );
    let s_gold = gold_reports.iter().find(|r| r.sample_id == "s").unwrap();
    assert_eq!(s_gold.weighted_majority_label.as_deref(), Some("win"));
    assert_eq!(
        s_gold.decision,
        Decision::Keep,
        "gold-weighted stability {} should clear keep_threshold",
        s_gold.stability_score
    );
    assert!(s_gold.stability_score > s_raw.stability_score);
}

#[test]
fn test_gold_weighted_falls_back_to_raw_without_gold_labels() {
    let jsonl = r#"{"sample_id":"a","label":"win"}
{"sample_id":"a","label":"win"}
{"sample_id":"a","label":"loss"}"#;
    let raw_reports = score_all(parse_jsonl(jsonl).unwrap(), &ScoreConfig::default());
    let gold_reports = score_all_gold_weighted(
        parse_jsonl(jsonl).unwrap(),
        &ScoreConfig {
            decision_score: DecisionScore::GoldWeighted,
            ..ScoreConfig::default()
        },
    );
    assert_eq!(gold_reports[0].decision, raw_reports[0].decision);
    assert_eq!(
        gold_reports[0].stability_score,
        raw_reports[0].stability_score
    );
    assert_eq!(
        gold_reports[0].weighted_majority_label,
        gold_reports[0].majority_label
    );
}

#[test]
fn test_evaluator_label_weights_reveal_asymmetric_reliability() {
    // Evaluator D: always correct when saying "win" (3/3), always wrong when saying "loss"
    // (0/3, true label is "win" every time) — overall scalar accuracy is a middling 0.5,
    // hiding this asymmetry.
    let mut obs = Vec::new();
    for id in ["c1", "c2", "c3"] {
        obs.push(Observation {
            sample_id: id.into(),
            label: Some("win".into()),
            evaluator_id: Some("D".into()),
            gold_label: Some("win".into()),
            ..Default::default()
        });
    }
    for id in ["c4", "c5", "c6"] {
        obs.push(Observation {
            sample_id: id.into(),
            label: Some("loss".into()),
            evaluator_id: Some("D".into()),
            gold_label: Some("win".into()),
            ..Default::default()
        });
    }
    let truth: std::collections::HashMap<String, String> = obs
        .iter()
        .filter_map(|o| o.gold_label.clone().map(|g| (o.sample_id.clone(), g)))
        .collect();

    let scalar = compute_evaluator_weights(&obs, &truth);
    let w_scalar = *scalar.get("D").unwrap();
    assert!((w_scalar - 0.5).abs() < 1e-9); // (3+0.5)/(6+1)

    let by_label = compute_evaluator_label_weights(&obs, &truth);
    let w_win = *by_label.get(&("D".to_string(), "win".to_string())).unwrap();
    let w_loss = *by_label
        .get(&("D".to_string(), "loss".to_string()))
        .unwrap();
    assert!((w_win - 0.875).abs() < 1e-9); // (3+0.5)/(3+1)
    assert!((w_loss - 0.125).abs() < 1e-9); // (0+0.5)/(3+1)

    // Target sample: D says "win" (their reliable label); a neutral evaluator (no
    // evaluator_id, defaults to weight 1.0 in both schemes) says "loss". No gold_label on
    // this sample — it must not leak into its own reliability calibration above. Compare
    // scalar vs. per-label confidence in "loss": per-label should be materially lower,
    // since it discounts D's overall mediocrity for this specific, reliable "win" call.
    let target = vec![
        Observation {
            sample_id: "t".into(),
            label: Some("win".into()),
            evaluator_id: Some("D".into()),
            ..Default::default()
        },
        Observation {
            sample_id: "t".into(),
            label: Some("loss".into()),
            ..Default::default()
        },
    ];
    let (_, scalar_conf, _, _) = compute_weighted_majority(&target, Some("loss"), &scalar);
    let (_, label_conf, _, _) =
        compute_weighted_majority_by_label(&target, Some("loss"), &by_label);
    // scalar: win=0.5, loss=1.0, total=1.5 -> loss confidence = 1.0/1.5
    assert!((scalar_conf.unwrap() - (1.0 / 1.5)).abs() < 1e-9);
    // per-label: win=0.875, loss=1.0, total=1.875 -> loss confidence = 1.0/1.875
    assert!((label_conf.unwrap() - (1.0 / 1.875)).abs() < 1e-9);
    assert!(label_conf.unwrap() < scalar_conf.unwrap());
}

#[test]
fn test_latent_truth_flips_review_to_keep_via_disagreement() {
    // 3 reliable evaluators (A, B, C) + 2 systematically-biased ones (X, Y) who report the
    // *opposite* of the true label on every training sample. Training samples alternate
    // win/loss (not unanimous) so the correct label still wins the naive 3-vs-2 majority,
    // letting EM learn A/B/C's reliability from data that already looks right. No
    // `gold_label` appears anywhere — this is fully unsupervised.
    let mut obs = Vec::new();
    let training = [
        ("t1", "win"),
        ("t2", "loss"),
        ("t3", "win"),
        ("t4", "loss"),
        ("t5", "win"),
        ("t6", "loss"),
    ];
    for (id, truth) in training {
        let opposite = if truth == "win" { "loss" } else { "win" };
        for eval in ["A", "B", "C"] {
            obs.push(Observation {
                sample_id: id.into(),
                label: Some(truth.into()),
                evaluator_id: Some(eval.into()),
                ..Default::default()
            });
        }
        for eval in ["X", "Y"] {
            obs.push(Observation {
                sample_id: id.into(),
                label: Some(opposite.into()),
                evaluator_id: Some(eval.into()),
                ..Default::default()
            });
        }
    }
    // Target sample: only A (reliable, says "win") and X, Y (biased, both say "loss") weigh
    // in. Raw majority is "loss" (2 of 3).
    for (eval, label) in [("A", "win"), ("X", "loss"), ("Y", "loss")] {
        obs.push(Observation {
            sample_id: "s".into(),
            label: Some(label.into()),
            evaluator_id: Some(eval.into()),
            ..Default::default()
        });
    }
    assert!(
        obs.iter().all(|o| o.gold_label.is_none()),
        "this test must be fully unsupervised"
    );

    let weights = ScoreWeights {
        evaluator_agreement: 0.0,
        ..ScoreWeights::default()
    };

    let raw_reports = score_all(
        obs.clone(),
        &ScoreConfig {
            weights: weights.clone(),
            ..ScoreConfig::default()
        },
    );
    let s_raw = raw_reports.iter().find(|r| r.sample_id == "s").unwrap();
    assert_eq!(s_raw.majority_label.as_deref(), Some("loss"));
    assert_eq!(s_raw.decision, Decision::Review);

    let lt_reports = score_all_latent_truth(
        obs,
        &ScoreConfig {
            weights,
            decision_score: DecisionScore::LatentTruth,
            ..ScoreConfig::default()
        },
    );
    let s_lt = lt_reports.iter().find(|r| r.sample_id == "s").unwrap();
    assert_eq!(
        s_lt.latent_truth_label.as_deref(),
        Some("win"),
        "EM should recover the correct label from A's reliability, distribution: {:?}",
        s_lt.latent_truth_label_distribution
    );
    assert_eq!(s_lt.majority_latent_conflict, Some(true));
    assert_eq!(
        s_lt.decision,
        Decision::Keep,
        "latent-truth stability {} should clear keep_threshold",
        s_lt.stability_score
    );
}

#[test]
fn test_latent_truth_falls_back_to_raw_with_single_evaluator() {
    let jsonl = r#"{"sample_id":"a","label":"win","evaluator_id":"only_one"}
{"sample_id":"a","label":"win","evaluator_id":"only_one"}
{"sample_id":"a","label":"loss","evaluator_id":"only_one"}"#;
    let raw_reports = score_all(parse_jsonl(jsonl).unwrap(), &ScoreConfig::default());
    let lt_reports = score_all_latent_truth(
        parse_jsonl(jsonl).unwrap(),
        &ScoreConfig {
            decision_score: DecisionScore::LatentTruth,
            ..ScoreConfig::default()
        },
    );
    assert_eq!(lt_reports[0].decision, raw_reports[0].decision);
    assert_eq!(
        lt_reports[0].stability_score,
        raw_reports[0].stability_score
    );
    assert_eq!(lt_reports[0].latent_truth_label, None);
}

#[test]
fn test_latent_truth_falls_back_to_raw_with_single_label() {
    let jsonl = r#"{"sample_id":"a","label":"win","evaluator_id":"e1"}
{"sample_id":"a","label":"win","evaluator_id":"e2"}
{"sample_id":"b","label":"win","evaluator_id":"e1"}"#;
    let raw_reports = score_all(parse_jsonl(jsonl).unwrap(), &ScoreConfig::default());
    let lt_reports = score_all_latent_truth(
        parse_jsonl(jsonl).unwrap(),
        &ScoreConfig {
            decision_score: DecisionScore::LatentTruth,
            ..ScoreConfig::default()
        },
    );
    for (raw, lt) in raw_reports.iter().zip(lt_reports.iter()) {
        assert_eq!(lt.decision, raw.decision);
        assert_eq!(lt.stability_score, raw.stability_score);
        assert_eq!(lt.latent_truth_label, None);
    }
}

#[test]
fn test_latent_truth_ignores_gold_label() {
    // Same batch scored with and without a (wrong) gold_label on every observation — output
    // must be identical either way, since LatentTruth never consults gold_label at all
    // (locks in the "no semi-supervised init" design decision so it can't silently regress).
    let mut obs_no_gold = Vec::new();
    let mut obs_with_wrong_gold = Vec::new();
    let training = [("t1", "win"), ("t2", "loss"), ("t3", "win"), ("t4", "loss")];
    for (id, truth) in training {
        let opposite = if truth == "win" { "loss" } else { "win" };
        for eval in ["A", "B"] {
            obs_no_gold.push(Observation {
                sample_id: id.into(),
                label: Some(truth.into()),
                evaluator_id: Some(eval.into()),
                ..Default::default()
            });
            obs_with_wrong_gold.push(Observation {
                sample_id: id.into(),
                label: Some(truth.into()),
                evaluator_id: Some(eval.into()),
                gold_label: Some(opposite.into()),
                ..Default::default()
            });
        }
        obs_no_gold.push(Observation {
            sample_id: id.into(),
            label: Some(opposite.into()),
            evaluator_id: Some("X".into()),
            ..Default::default()
        });
        obs_with_wrong_gold.push(Observation {
            sample_id: id.into(),
            label: Some(opposite.into()),
            evaluator_id: Some("X".into()),
            gold_label: Some(opposite.into()),
            ..Default::default()
        });
    }

    let config = ScoreConfig {
        decision_score: DecisionScore::LatentTruth,
        ..ScoreConfig::default()
    };
    let no_gold_reports = score_all_latent_truth(obs_no_gold, &config);
    let wrong_gold_reports = score_all_latent_truth(obs_with_wrong_gold, &config);

    for (a, b) in no_gold_reports.iter().zip(wrong_gold_reports.iter()) {
        assert_eq!(a.latent_truth_label, b.latent_truth_label);
        assert_eq!(a.latent_truth_confidence, b.latent_truth_confidence);
        assert_eq!(a.decision, b.decision);
    }
}

#[test]
fn test_game_ai_profile_uses_lcb_and_stricter_mins() {
    // Verify game-ai profile config values (test the library-level config, not CLI)
    // In the CLI, game-ai sets: budget_stability×2, seed_stability×2, score_sign×2, min_obs=4,
    // min_budgets=2, min_seeds=2, min_evaluators=2, lcb. We replicate the budget/seed/min part
    // of that config here directly (score_sign is omitted since this fixture has no negative
    // scores to flip sign on, so it wouldn't change this test's outcome either way).
    let config = ScoreConfig {
        weights: ScoreWeights {
            budget_stability: 2.0,
            seed_stability: 2.0,
            ..ScoreWeights::default()
        },
        min_requirements: MinRequirements {
            observations: 4,
            budgets: 2,
            seeds: 2,
            evaluators: 2,
            ..MinRequirements::default()
        },
        decision_score: DecisionScore::LowerConfidenceBound,
        ..ScoreConfig::default()
    };
    // Sample with enough budget/seed variation to pass score but only 1 budget level → demoted
    let obs: Vec<Observation> = (0..5)
        .map(|i| Observation {
            sample_id: "p".into(),
            label: Some("win".into()),
            score: Some(0.9),
            budget: Some(4.0), // all same budget → min_budgets=2 not met
            seed: Some(i),
            ..Default::default()
        })
        .collect();
    let reports = score_all(obs, &config);
    // Would be keep by stability score, but demoted because only 1 distinct budget
    assert_ne!(
        reports[0].decision,
        Decision::Keep,
        "should be demoted by min_budgets=2"
    );
}

#[test]
fn test_policy_precision_decreases_with_lower_threshold() {
    // With gold_label: higher threshold → higher precision (fewer, better samples)
    let obs: Vec<Observation> = vec![
        // 4 stable-correct (agreement 1.0 for "win", gold="win")
        Observation {
            sample_id: "a".into(),
            label: Some("win".into()),
            gold_label: Some("win".into()),
            ..Default::default()
        },
        Observation {
            sample_id: "a".into(),
            label: Some("win".into()),
            gold_label: Some("win".into()),
            ..Default::default()
        },
        Observation {
            sample_id: "b".into(),
            label: Some("win".into()),
            gold_label: Some("win".into()),
            ..Default::default()
        },
        Observation {
            sample_id: "b".into(),
            label: Some("win".into()),
            gold_label: Some("win".into()),
            ..Default::default()
        },
        // 1 wrong (agreement 1.0 for "loss", gold="win") — stable but wrong
        Observation {
            sample_id: "c".into(),
            label: Some("loss".into()),
            gold_label: Some("win".into()),
            ..Default::default()
        },
        Observation {
            sample_id: "c".into(),
            label: Some("loss".into()),
            gold_label: Some("win".into()),
            ..Default::default()
        },
    ];
    let gold: std::collections::HashMap<String, String> = obs
        .iter()
        .filter_map(|o| o.gold_label.clone().map(|g| (o.sample_id.clone(), g)))
        .collect();
    // Score all
    let config = ScoreConfig {
        thresholds: Thresholds {
            keep: 0.0,
            drop: -1.0,
        },
        ..ScoreConfig::default()
    };
    let reports = score_all(obs, &config);
    // At threshold=0.0 all 3 are "kept"; precision = 2/3
    let all_kept: Vec<_> = reports
        .iter()
        .filter(|r| r.stability_score >= 0.0)
        .collect();
    let matches = all_kept
        .iter()
        .filter(|r| {
            gold.get(&r.sample_id)
                .and_then(|g| r.majority_label.as_deref().map(|m| m == g.as_str()))
                .unwrap_or(false)
        })
        .count();
    assert_eq!(all_kept.len(), 3);
    assert_eq!(matches, 2, "2 of 3 samples are correct");
    // At high threshold only a and b are kept (stable correct ones, same stability)
    // c has same stability but majority_label="loss"≠"win"
    // All have same stability_score since they're all unanimous 2-obs
    // This just validates our precision math is right
    let prec_all = matches as f64 / all_kept.len() as f64;
    assert!((prec_all - 2.0 / 3.0).abs() < 1e-9);
}

#[test]
fn test_active_review_high_entropy_has_higher_urgency() {
    // Sample with high entropy should rank higher urgency than sample with low entropy
    let jsonl_high = r#"{"sample_id":"noisy","label":"win"}
{"sample_id":"noisy","label":"loss"}"#;
    let jsonl_low = r#"{"sample_id":"stable","label":"win"}
{"sample_id":"stable","label":"win"}"#;
    let mut obs: Vec<Observation> = parse_jsonl(jsonl_high).unwrap();
    obs.extend(parse_jsonl(jsonl_low).unwrap());
    let reports = score_all(obs, &ScoreConfig::default());
    let noisy = reports.iter().find(|r| r.sample_id == "noisy").unwrap();
    let stable = reports.iter().find(|r| r.sample_id == "stable").unwrap();
    // noisy has higher entropy than stable
    let noisy_ent = noisy.label_entropy.unwrap_or(0.0);
    let stable_ent = stable.label_entropy.unwrap_or(0.0);
    assert!(
        noisy_ent > stable_ent,
        "noisy entropy {noisy_ent} should exceed stable {stable_ent}"
    );
    // urgency ∝ entropy: noisy > stable (actual urgency computation is in CLI)
    assert_eq!(noisy_ent, 1.0, "uniform 50/50 split = entropy 1.0");
    assert_eq!(stable_ent, 0.0, "unanimous = entropy 0.0");
}

#[test]
fn test_active_review_low_lcb_has_higher_urgency_signal() {
    // A sample with low label_agreement_lcb should have a higher urgency signal than one with high lcb
    let many_win: Vec<Observation> = (0..20)
        .map(|i| Observation {
            sample_id: "high_lcb".into(),
            label: Some("win".into()),
            evaluator_id: Some(format!("e{i}")),
            ..Default::default()
        })
        .collect();
    let two_win: Vec<Observation> = (0..2)
        .map(|i| Observation {
            sample_id: "low_lcb".into(),
            label: Some("win".into()),
            evaluator_id: Some(format!("f{i}")),
            ..Default::default()
        })
        .collect();
    let mut obs = many_win;
    obs.extend(two_win);
    let reports = score_all(obs, &ScoreConfig::default());
    let high = reports.iter().find(|r| r.sample_id == "high_lcb").unwrap();
    let low = reports.iter().find(|r| r.sample_id == "low_lcb").unwrap();
    // high_lcb sample has much higher LCB (22 obs at 100% → LCB ≈ 0.85)
    // low_lcb sample has low LCB (2 obs at 100% → LCB ≈ 0.34)
    let high_lcb_val = high.label_agreement_lcb.unwrap_or(0.0);
    let low_lcb_val = low.label_agreement_lcb.unwrap_or(0.0);
    assert!(
        high_lcb_val > low_lcb_val,
        "20-obs sample should have higher LCB than 2-obs: {high_lcb_val} vs {low_lcb_val}"
    );
    // Urgency signal for low_lcb is higher (1 - lcb is larger)
    let urgency_high = 1.0 - high_lcb_val;
    let urgency_low = 1.0 - low_lcb_val;
    assert!(
        urgency_low > urgency_high,
        "low-lcb sample urgency {urgency_low:.3} > high-lcb {urgency_high:.3}"
    );
}

#[test]
fn test_score_dispersion_mad_less_sensitive_to_outlier() {
    // 3 scores: 0.90, 0.88, 0.02 — the 0.02 is an outlier.
    // STD will be dragged down; MAD (based on median) won't be.
    let obs: Vec<Observation> = [0.90f64, 0.88, 0.02]
        .iter()
        .map(|&s| Observation {
            sample_id: "a".into(),
            score: Some(s),
            ..Default::default()
        })
        .collect();

    let std_config = ScoreConfig::default(); // ScoreDispersion::Std
    let mad_config = ScoreConfig {
        score_dispersion: ScoreDispersion::Mad,
        ..ScoreConfig::default()
    };

    let r_std = &score_all(obs.clone(), &std_config)[0];
    let r_mad = &score_all(obs, &mad_config)[0];

    let sc_std = r_std.components.score_consistency.unwrap();
    let sc_mad = r_mad.components.score_consistency.unwrap();

    assert!(
        sc_mad > sc_std,
        "MAD score_consistency ({sc_mad:.4}) should exceed STD ({sc_std:.4}) when outlier present"
    );
    // Verify both are in [0, 1]
    assert!((0.0..=1.0).contains(&sc_std));
    assert!((0.0..=1.0).contains(&sc_mad));
}

#[test]
fn test_score_dispersion_iqr_zero_gives_perfect_consistency() {
    // All scores identical → IQR=0 → score_consistency=1.0
    let obs: Vec<Observation> = [0.85f64, 0.85, 0.85, 0.85]
        .iter()
        .map(|&s| Observation {
            sample_id: "b".into(),
            score: Some(s),
            ..Default::default()
        })
        .collect();

    let iqr_config = ScoreConfig {
        score_dispersion: ScoreDispersion::Iqr,
        ..ScoreConfig::default()
    };
    let r = &score_all(obs, &iqr_config)[0];
    let sc = r.components.score_consistency.unwrap();
    assert!(
        (sc - 1.0).abs() < 1e-9,
        "identical scores → IQR=0 → score_consistency=1.0, got {sc}"
    );
}

#[test]
fn test_score_dispersion_mad_single_obs_skips_score_consistency() {
    // MAD requires >= 2 obs; with 1 obs, score_consistency should be None
    let obs = vec![Observation {
        sample_id: "c".into(),
        score: Some(0.9),
        ..Default::default()
    }];
    let mad_config = ScoreConfig {
        score_dispersion: ScoreDispersion::Mad,
        ..ScoreConfig::default()
    };
    let r = &score_all(obs, &mad_config)[0];
    // Single obs → stability_score=0.5 (review), score_consistency None
    assert!(
        r.components.score_consistency.is_none(),
        "single obs with MAD should have no score_consistency"
    );
    assert_eq!(r.decision, Decision::Review);
}

// `debug_assert!` compiles out under `--release`, so this test only panics in a debug
// build (see the trust-boundary note on `score_all`); skip it under release to avoid a
// false failure when `cargo test --release` is used.
#[test]
#[cfg(debug_assertions)]
#[should_panic(expected = "call Observation::validate() before scoring")]
fn test_score_all_panics_on_non_finite_score_bypassing_parser() {
    let obs = vec![Observation {
        sample_id: "a".into(),
        score: Some(f64::NAN),
        ..Default::default()
    }];
    let _ = score_all(obs, &ScoreConfig::default());
}

// Same trust-boundary guard as above, extended to the trajectory fields added alongside
// gradient_sign_agreement etc. — a caller bypassing parse_jsonl/validate() with a NaN/inf
// trajectory field should hit a loud debug-build panic, not a silently wrong sign_agreement
// bucket (NaN compares false to both `> 0.0` and `< 0.0`, so it would otherwise land in the
// "zero" bucket unnoticed).
#[test]
#[cfg(debug_assertions)]
#[should_panic(expected = "call Observation::validate() before scoring")]
fn test_score_all_panics_on_non_finite_trajectory_effect_bypassing_parser() {
    let obs = vec![Observation {
        sample_id: "a".into(),
        trajectory_effect: Some(f64::INFINITY),
        ..Default::default()
    }];
    let _ = score_all(obs, &ScoreConfig::default());
}

// Fixture shared by the correlated-warning and min-effective-n demotion tests below: 3
// reliable evaluators (R1-R3) + 2 evaluators (TW1, TW2) identical to each other (correct on
// the first half of training samples, both wrong together on the second half). Target sample
// "s" has one reliable evaluator + both twins -- baseline (all safety flags off) is
// decision=Keep, evaluator_effective_n≈1.8, correlated_evaluator_warning=true, converged=true.
// See latent_truth.rs's test_correlated_evaluators_lower_effective_n_and_warn for the same
// fixture at the compute_latent_truth level.
fn correlated_twins_fixture() -> Vec<Observation> {
    let mut obs = Vec::new();
    let training = [
        ("t1", "win"),
        ("t2", "loss"),
        ("t3", "win"),
        ("t4", "loss"),
        ("t5", "win"),
        ("t6", "loss"),
    ];
    for (i, (id, truth)) in training.iter().enumerate() {
        let opposite = if *truth == "win" { "loss" } else { "win" };
        for eval in ["R1", "R2", "R3"] {
            obs.push(Observation {
                sample_id: (*id).into(),
                label: Some((*truth).to_string()),
                evaluator_id: Some(eval.into()),
                ..Default::default()
            });
        }
        let twin_label = if i < 3 { *truth } else { opposite };
        for eval in ["TW1", "TW2"] {
            obs.push(Observation {
                sample_id: (*id).into(),
                label: Some(twin_label.to_string()),
                evaluator_id: Some(eval.into()),
                ..Default::default()
            });
        }
    }
    for (eval, label) in [("R1", "win"), ("TW1", "loss"), ("TW2", "loss")] {
        obs.push(Observation {
            sample_id: "s".into(),
            label: Some(label.into()),
            evaluator_id: Some(eval.into()),
            ..Default::default()
        });
    }
    obs
}

fn score_correlated_twins(safety: LatentTruthSafety) -> StabilityReport {
    let weights = ScoreWeights {
        evaluator_agreement: 0.0,
        ..ScoreWeights::default()
    };
    let reports = score_all_latent_truth(
        correlated_twins_fixture(),
        &ScoreConfig {
            weights,
            decision_score: DecisionScore::LatentTruth,
            latent_truth_safety: safety,
            ..ScoreConfig::default()
        },
    );
    reports.into_iter().find(|r| r.sample_id == "s").unwrap()
}

#[test]
fn test_latent_truth_demotes_keep_on_correlated_warning() {
    let s = score_correlated_twins(LatentTruthSafety {
        demote_on_correlated_warning: true,
        ..Default::default()
    });
    assert_eq!(s.decision, Decision::Review);
    assert_eq!(
        s.latent_truth_demotion_reason.as_deref(),
        Some("correlated_evaluator_warning")
    );
}

#[test]
fn test_latent_truth_correlated_warning_off_by_default_locks_backward_compat() {
    let s = score_correlated_twins(LatentTruthSafety::default());
    assert_eq!(
        s.decision,
        Decision::Keep,
        "with all safety flags off (the default), decision must be unaffected"
    );
    assert!(s.latent_truth_demotion_reason.is_none());
}

#[test]
fn test_latent_truth_demotes_keep_on_min_effective_n() {
    // s.evaluator_effective_n is ~1.8 (see test_correlated_evaluators_lower_effective_n_and_warn);
    // a minimum of 2.0 is comfortably above that.
    let s = score_correlated_twins(LatentTruthSafety {
        min_evaluator_effective_n: Some(2.0),
        ..Default::default()
    });
    assert_eq!(s.decision, Decision::Review);
    assert_eq!(
        s.latent_truth_demotion_reason.as_deref(),
        Some("evaluator_effective_n_below_minimum")
    );
}

#[test]
fn test_latent_truth_min_effective_n_off_by_default_locks_backward_compat() {
    let s = score_correlated_twins(LatentTruthSafety {
        min_evaluator_effective_n: None,
        ..Default::default()
    });
    assert_eq!(s.decision, Decision::Keep);
    assert!(s.latent_truth_demotion_reason.is_none());
}

#[test]
fn test_latent_truth_demotion_priority_order() {
    // Both demote_on_correlated_warning and min_evaluator_effective_n fire for "s" ->
    // correlated_evaluator_warning wins (higher priority).
    let s = score_correlated_twins(LatentTruthSafety {
        demote_on_correlated_warning: true,
        min_evaluator_effective_n: Some(2.0),
        demote_on_non_convergence: false,
    });
    assert_eq!(s.decision, Decision::Review);
    assert_eq!(
        s.latent_truth_demotion_reason.as_deref(),
        Some("correlated_evaluator_warning")
    );
}

// Fixture for non-convergence demotion tests: an adversarial-tie batch (guaranteed
// converged=false at the iteration cap, same shape as
// latent_truth.rs's test_compute_latent_truth_adversarial_tie_hits_iteration_cap) plus a
// disjoint-evaluator "clean" sample with several unanimous supporting samples reinforcing its
// evaluators' reliability. Because `converged` is one shared value per EM run, "clean" inherits
// converged=false even though its own posterior is confidently decided (baseline: Keep). The
// adversarial samples themselves stay Review/Drop already (never Keep), so the Keep->Review-only
// demotion doesn't touch them.
fn non_convergence_fixture() -> Vec<Observation> {
    let mut obs = Vec::new();
    for s in 0..6 {
        let sample_id = format!("adv{s}");
        for e in 0..4 {
            let label = if (s + e) % 2 == 0 { "a" } else { "b" };
            obs.push(Observation {
                sample_id: sample_id.clone(),
                label: Some(label.to_string()),
                evaluator_id: Some(format!("eval{e}")),
                ..Default::default()
            });
        }
    }
    for i in 0..6 {
        let sample_id = if i == 0 {
            "clean".to_string()
        } else {
            format!("clean_support{i}")
        };
        for eval in ["clean0", "clean1", "clean2"] {
            obs.push(Observation {
                sample_id: sample_id.clone(),
                label: Some("a".into()),
                evaluator_id: Some(eval.into()),
                ..Default::default()
            });
        }
    }
    obs
}

fn score_non_convergence(safety: LatentTruthSafety) -> StabilityReport {
    let weights = ScoreWeights {
        evaluator_agreement: 0.0,
        ..ScoreWeights::default()
    };
    let reports = score_all_latent_truth(
        non_convergence_fixture(),
        &ScoreConfig {
            weights,
            decision_score: DecisionScore::LatentTruth,
            latent_truth_safety: safety,
            ..ScoreConfig::default()
        },
    );
    reports
        .into_iter()
        .find(|r| r.sample_id == "clean")
        .unwrap()
}

#[test]
fn test_latent_truth_demotes_keep_on_non_convergence() {
    let clean = score_non_convergence(LatentTruthSafety {
        demote_on_non_convergence: true,
        ..Default::default()
    });
    assert_eq!(clean.latent_truth_converged, Some(false));
    assert_eq!(clean.decision, Decision::Review);
    assert_eq!(
        clean.latent_truth_demotion_reason.as_deref(),
        Some("latent_truth_not_converged")
    );
}

#[test]
fn test_latent_truth_non_convergence_off_by_default_locks_backward_compat() {
    let clean = score_non_convergence(LatentTruthSafety::default());
    assert_eq!(clean.decision, Decision::Keep);
    assert!(clean.latent_truth_demotion_reason.is_none());
}

// --- trajectory stability (Phase 1) ---

#[test]
fn test_gradient_sign_agreement() {
    let obs = vec![
        Observation {
            sample_id: "a".into(),
            gradient_sign: Some(0.4),
            ..Default::default()
        },
        Observation {
            sample_id: "a".into(),
            gradient_sign: Some(0.2),
            ..Default::default()
        },
        Observation {
            sample_id: "a".into(),
            gradient_sign: Some(-0.1),
            ..Default::default()
        },
    ];
    let r = &score_all(obs, &ScoreConfig::default())[0];
    assert_eq!(r.gradient_sign_agreement, Some(2.0 / 3.0));
}

#[test]
fn test_update_cosine_mean_and_direction_agreement() {
    let obs = vec![
        Observation {
            sample_id: "a".into(),
            update_cosine: Some(0.8),
            ..Default::default()
        },
        Observation {
            sample_id: "a".into(),
            update_cosine: Some(0.6),
            ..Default::default()
        },
        Observation {
            sample_id: "a".into(),
            update_cosine: Some(-0.2),
            ..Default::default()
        },
    ];
    let r = &score_all(obs, &ScoreConfig::default())[0];
    assert!((r.update_cosine_mean.unwrap() - 0.4).abs() < 1e-10);
    assert_eq!(r.update_direction_agreement, Some(2.0 / 3.0));
}

#[test]
fn test_teacher_residual_stability() {
    // Tight cluster of residuals -> low std -> high stability.
    let tight = vec![
        Observation {
            sample_id: "a".into(),
            teacher_residual: Some(0.01),
            ..Default::default()
        },
        Observation {
            sample_id: "a".into(),
            teacher_residual: Some(-0.01),
            ..Default::default()
        },
    ];
    let r_tight = &score_all(tight, &ScoreConfig::default())[0];
    assert!(r_tight.teacher_residual_stability.unwrap() > 0.95);

    // Wide spread -> low stability.
    let wide = vec![
        Observation {
            sample_id: "a".into(),
            teacher_residual: Some(0.9),
            ..Default::default()
        },
        Observation {
            sample_id: "a".into(),
            teacher_residual: Some(-0.9),
            ..Default::default()
        },
    ];
    let r_wide = &score_all(wide, &ScoreConfig::default())[0];
    assert!(r_wide.teacher_residual_stability.unwrap() < 0.5);

    // Absent entirely -> None.
    let none = vec![Observation {
        sample_id: "a".into(),
        ..Default::default()
    }];
    assert!(
        score_all(none, &ScoreConfig::default())[0]
            .teacher_residual_stability
            .is_none()
    );
}

#[test]
fn test_shuffle_seed_sensitivity_mirrors_seed_sensitivity() {
    let jsonl = r#"{"sample_id":"a","label":"win","score":0.95,"shuffle_seed":1}
{"sample_id":"a","label":"win","score":0.05,"shuffle_seed":2}"#;
    let obs = parse_jsonl(jsonl).unwrap();
    let reports = score_all(obs, &ScoreConfig::default());
    assert!(reports[0].shuffle_seed_sensitivity.is_some());
    assert!(reports[0].shuffle_seed_sensitivity.unwrap() > 0.5);
}

#[test]
fn test_loss_recipe_agreement_mirrors_model_agreement() {
    let obs = vec![
        Observation {
            sample_id: "a".into(),
            label: Some("win".into()),
            loss_recipe: Some("recipe_a".into()),
            ..Default::default()
        },
        Observation {
            sample_id: "a".into(),
            label: Some("win".into()),
            loss_recipe: Some("recipe_b".into()),
            ..Default::default()
        },
        Observation {
            sample_id: "a".into(),
            label: Some("loss".into()),
            loss_recipe: Some("recipe_c".into()),
            ..Default::default()
        },
    ];
    let r = &score_all(obs, &ScoreConfig::default())[0];
    assert!((r.loss_recipe_agreement.unwrap() - 2.0 / 3.0).abs() < 1e-10);
}

#[test]
fn test_dead_and_saturated_unit_rate() {
    let obs = vec![
        Observation {
            sample_id: "a".into(),
            layer_id: Some("ft".into()),
            dead_unit_count: Some(3.0),
            saturated_unit_count: Some(0.0),
            ..Default::default()
        },
        Observation {
            sample_id: "a".into(),
            layer_id: Some("ft".into()),
            dead_unit_count: Some(0.0),
            saturated_unit_count: Some(0.0),
            ..Default::default()
        },
    ];
    let r = &score_all(obs, &ScoreConfig::default())[0];
    assert_eq!(r.dead_unit_rate, Some(0.5));
    assert_eq!(r.saturated_unit_rate, Some(0.0));

    let none = vec![Observation {
        sample_id: "a".into(),
        ..Default::default()
    }];
    let r_none = &score_all(none, &ScoreConfig::default())[0];
    assert!(r_none.dead_unit_rate.is_none());
    assert!(r_none.saturated_unit_rate.is_none());
}

#[test]
fn test_trajectory_effect_mean_and_sign_agreement() {
    let obs = vec![
        Observation {
            sample_id: "a".into(),
            trajectory_effect: Some(-0.3),
            ..Default::default()
        },
        Observation {
            sample_id: "a".into(),
            trajectory_effect: Some(-0.1),
            ..Default::default()
        },
    ];
    let r = &score_all(obs, &ScoreConfig::default())[0];
    assert!((r.trajectory_effect_mean.unwrap() - (-0.2)).abs() < 1e-10);
    assert_eq!(r.trajectory_effect_sign_agreement, Some(1.0));
}

#[test]
fn test_trajectory_weights_zero_by_default_leave_stability_score_unchanged() {
    // Every new dimension present, all at their default 0.0 weight: stability_score should
    // depend only on label_agreement here, exactly as if the new fields didn't exist.
    let obs = vec![
        Observation {
            sample_id: "a".into(),
            label: Some("win".into()),
            gradient_sign: Some(0.1),
            update_cosine: Some(-0.9),
            teacher_residual: Some(5.0),
            shuffle_seed: Some(1),
            loss_recipe: Some("x".into()),
            ..Default::default()
        },
        Observation {
            sample_id: "a".into(),
            label: Some("win".into()),
            gradient_sign: Some(-0.1),
            update_cosine: Some(0.9),
            teacher_residual: Some(-5.0),
            shuffle_seed: Some(2),
            loss_recipe: Some("y".into()),
            ..Default::default()
        },
    ];
    let r = &score_all(obs, &ScoreConfig::default())[0];
    assert_eq!(
        r.stability_score, 1.0,
        "unweighted trajectory fields must not affect stability_score by default"
    );
}

#[test]
fn test_trajectory_weight_explicit_pulls_stability_score_down() {
    let obs = vec![
        Observation {
            sample_id: "a".into(),
            label: Some("win".into()),
            gradient_sign: Some(0.1),
            ..Default::default()
        },
        Observation {
            sample_id: "a".into(),
            label: Some("win".into()),
            gradient_sign: Some(-0.1),
            ..Default::default()
        },
    ];
    let config = ScoreConfig {
        weights: ScoreWeights {
            gradient_sign: 5.0,
            ..ScoreWeights::default()
        },
        ..ScoreConfig::default()
    };
    let r = &score_all(obs, &config)[0];
    assert!(
        r.stability_score < 1.0,
        "explicit gradient_sign weight should pull a sign-split sample below perfect stability, got {}",
        r.stability_score
    );
}

#[test]
fn test_trajectory_fields_absent_leaves_json_output_byte_for_byte_unchanged() {
    // No trajectory field appears anywhere in this fixture, and every new ScoreWeights entry
    // defaults to 0.0 -- so every new StabilityReport/StabilityComponents field must be
    // skip_serializing_if=None (fully absent from the JSON), not present-as-null, and every
    // pre-existing field's value must be untouched. This is the literal "old output identical"
    // check, not just "decision is the same".
    let obs = load("simple.jsonl");
    let reports = score_all(obs, &ScoreConfig::default());
    let a = reports.iter().find(|r| r.sample_id == "a").unwrap();

    let new_report_fields = [
        "gradient_sign_agreement",
        "update_cosine_mean",
        "update_direction_agreement",
        "teacher_residual_stability",
        "shuffle_seed_sensitivity",
        "loss_recipe_agreement",
        "dead_unit_rate",
        "saturated_unit_rate",
        "trajectory_effect_mean",
        "trajectory_effect_sign_agreement",
    ];
    let new_component_fields = [
        "gradient_sign_agreement",
        "update_direction_agreement",
        "teacher_residual_stability",
        "shuffle_seed_robustness",
        "loss_recipe_agreement",
    ];

    let json = serde_json::to_string(a).unwrap();
    for field in new_report_fields {
        assert!(
            !json.contains(field),
            "new field {field} must be fully absent (not even as null) when its underlying \
             Observation field is absent, got: {json}"
        );
    }
    let value: serde_json::Value = serde_json::from_str(&json).unwrap();
    let components = value["components"].as_object().unwrap();
    for field in new_component_fields {
        assert!(
            !components.contains_key(field),
            "new component {field} must be absent when its weight is 0.0 (default) and its \
             underlying field is absent"
        );
    }

    // Exact pre-existing field values are unchanged: sample "a" is 3 unanimous "win"
    // observations with tight scores (0.91/0.88/0.90) -> the same stability_score this
    // fixture has always produced.
    assert_eq!(a.n_observations, 3);
    assert_eq!(a.label_agreement, Some(1.0));
    assert!((a.score_mean.unwrap() - 0.8966666666666666).abs() < 1e-9);
    assert_eq!(a.decision, Decision::Keep);
}

#[test]
fn test_validate_rejects_non_finite_trajectory_fields() {
    let bad = Observation {
        sample_id: "a".into(),
        gradient_sign: Some(f64::NAN),
        ..Default::default()
    };
    let err = bad.validate(7).unwrap_err().to_string();
    assert!(err.contains("gradient_sign"), "{err}");
    assert!(err.contains('7'), "{err}");

    let bad_effect = Observation {
        sample_id: "a".into(),
        trajectory_effect: Some(f64::INFINITY),
        ..Default::default()
    };
    assert!(bad_effect.validate(1).is_err());
}