eval-flake-rs 0.1.0

Detect flaky LLM eval cases by tracking pass/fail across repeated runs. Returns per-case flip-rate and an overall flakiness score. Zero deps.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98

//! # eval-flake-rs
//!
//! Detect flaky LLM eval cases by tracking pass/fail outcomes across
//! repeated runs of the same suite.
//!
//! - A case is **stable** when every run agrees.
//! - A case is **flaky** when runs disagree (at least one pass and at
//!   least one fail).
//! - A case is **always failing** when every run failed.
//!
//! Flip-rate = fraction of run-to-run transitions where the outcome
//! changed. 0.0 = stable, 0.5 = maximally flaky.
//!
//! ## Example
//!
//! ```
//! use eval_flake_rs::{detect, CaseResult};
//!
//! let runs: Vec<Vec<(&str, bool)>> = vec![
//!     vec![("case-1", true),  ("case-2", true),  ("case-3", false)],
//!     vec![("case-1", true),  ("case-2", false), ("case-3", false)],
//!     vec![("case-1", true),  ("case-2", true),  ("case-3", false)],
//! ];
//! let results = detect(&runs);
//! assert!(matches!(results.iter().find(|r| r.case == "case-2").unwrap().status, CaseResult::Flaky));
//! ```

#![deny(missing_docs)]

use std::collections::BTreeMap;

/// Per-case status.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CaseResult {
    /// Every run passed.
    AlwaysPass,
    /// Every run failed.
    AlwaysFail,
    /// At least one pass and one fail.
    Flaky,
}

/// Per-case report.
#[derive(Debug, Clone)]
pub struct Report {
    /// Case name.
    pub case: String,
    /// Classification.
    pub status: CaseResult,
    /// Number of pass observations.
    pub passes: u32,
    /// Number of fail observations.
    pub fails: u32,
    /// Flip-rate over run-to-run transitions (0.0–1.0).
    pub flip_rate: f64,
}

/// Aggregate per-case status from a list of run outcomes.
///
/// `runs[r] = [(case_name, pass)]`. Missing cases in a run are simply
/// not counted for that case; cases must use the same name across runs
/// to be aligned.
pub fn detect(runs: &[Vec<(&str, bool)>]) -> Vec<Report> {
    let mut grouped: BTreeMap<String, Vec<bool>> = BTreeMap::new();
    for run in runs {
        for (case, pass) in run {
            grouped.entry((*case).to_string()).or_default().push(*pass);
        }
    }
    grouped
        .into_iter()
        .map(|(case, outcomes)| {
            let passes = outcomes.iter().filter(|p| **p).count() as u32;
            let fails = outcomes.iter().filter(|p| !**p).count() as u32;
            let status = match (passes, fails) {
                (0, _) => CaseResult::AlwaysFail,
                (_, 0) => CaseResult::AlwaysPass,
                _ => CaseResult::Flaky,
            };
            let flip_rate = if outcomes.len() < 2 {
                0.0
            } else {
                let flips = outcomes
                    .windows(2)
                    .filter(|w| w[0] != w[1])
                    .count() as f64;
                flips / (outcomes.len() - 1) as f64
            };
            Report {
                case,
                status,
                passes,
                fails,
                flip_rate,
            }
        })
        .collect()
}