eval_flake_rs/
lib.rs

1//! # eval-flake-rs
2//!
3//! Detect flaky LLM eval cases by tracking pass/fail outcomes across
4//! repeated runs of the same suite.
5//!
6//! - A case is **stable** when every run agrees.
7//! - A case is **flaky** when runs disagree (at least one pass and at
8//!   least one fail).
9//! - A case is **always failing** when every run failed.
10//!
11//! Flip-rate = fraction of run-to-run transitions where the outcome
12//! changed. 0.0 = stable, 0.5 = maximally flaky.
13//!
14//! ## Example
15//!
16//! ```
17//! use eval_flake_rs::{detect, CaseResult};
18//!
19//! let runs: Vec<Vec<(&str, bool)>> = vec![
20//!     vec![("case-1", true),  ("case-2", true),  ("case-3", false)],
21//!     vec![("case-1", true),  ("case-2", false), ("case-3", false)],
22//!     vec![("case-1", true),  ("case-2", true),  ("case-3", false)],
23//! ];
24//! let results = detect(&runs);
25//! assert!(matches!(results.iter().find(|r| r.case == "case-2").unwrap().status, CaseResult::Flaky));
26//! ```
27
28#![deny(missing_docs)]
29
30use std::collections::BTreeMap;
31
32/// Per-case status.
33#[derive(Debug, Clone, Copy, PartialEq, Eq)]
34pub enum CaseResult {
35    /// Every run passed.
36    AlwaysPass,
37    /// Every run failed.
38    AlwaysFail,
39    /// At least one pass and one fail.
40    Flaky,
41}
42
43/// Per-case report.
44#[derive(Debug, Clone)]
45pub struct Report {
46    /// Case name.
47    pub case: String,
48    /// Classification.
49    pub status: CaseResult,
50    /// Number of pass observations.
51    pub passes: u32,
52    /// Number of fail observations.
53    pub fails: u32,
54    /// Flip-rate over run-to-run transitions (0.0–1.0).
55    pub flip_rate: f64,
56}
57
58/// Aggregate per-case status from a list of run outcomes.
59///
60/// `runs[r] = [(case_name, pass)]`. Missing cases in a run are simply
61/// not counted for that case; cases must use the same name across runs
62/// to be aligned.
63pub fn detect(runs: &[Vec<(&str, bool)>]) -> Vec<Report> {
64    let mut grouped: BTreeMap<String, Vec<bool>> = BTreeMap::new();
65    for run in runs {
66        for (case, pass) in run {
67            grouped.entry((*case).to_string()).or_default().push(*pass);
68        }
69    }
70    grouped
71        .into_iter()
72        .map(|(case, outcomes)| {
73            let passes = outcomes.iter().filter(|p| **p).count() as u32;
74            let fails = outcomes.iter().filter(|p| !**p).count() as u32;
75            let status = match (passes, fails) {
76                (0, _) => CaseResult::AlwaysFail,
77                (_, 0) => CaseResult::AlwaysPass,
78                _ => CaseResult::Flaky,
79            };
80            let flip_rate = if outcomes.len() < 2 {
81                0.0
82            } else {
83                let flips = outcomes
84                    .windows(2)
85                    .filter(|w| w[0] != w[1])
86                    .count() as f64;
87                flips / (outcomes.len() - 1) as f64
88            };
89            Report {
90                case,
91                status,
92                passes,
93                fails,
94                flip_rate,
95            }
96        })
97        .collect()
98}
eval_flake_rs/lib.rs

eval_flake_rs/
lib.rs
