eval-flake-rs 0.1.0

Detect flaky LLM eval cases by tracking pass/fail across repeated runs. Returns per-case flip-rate and an overall flakiness score. Zero deps.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58

use eval_flake_rs::{detect, CaseResult};

#[test]
fn always_pass() {
    let runs = vec![
        vec![("a", true)],
        vec![("a", true)],
        vec![("a", true)],
    ];
    let r = detect(&runs);
    assert_eq!(r[0].status, CaseResult::AlwaysPass);
    assert_eq!(r[0].passes, 3);
    assert_eq!(r[0].fails, 0);
    assert_eq!(r[0].flip_rate, 0.0);
}

#[test]
fn always_fail() {
    let runs = vec![vec![("b", false)], vec![("b", false)]];
    let r = detect(&runs);
    assert_eq!(r[0].status, CaseResult::AlwaysFail);
}

#[test]
fn flaky_with_high_flip_rate() {
    let runs = vec![
        vec![("c", true)],
        vec![("c", false)],
        vec![("c", true)],
        vec![("c", false)],
    ];
    let r = detect(&runs);
    assert_eq!(r[0].status, CaseResult::Flaky);
    assert!((r[0].flip_rate - 1.0).abs() < 1e-9);
}

#[test]
fn flaky_with_low_flip_rate() {
    let runs = vec![
        vec![("d", true)],
        vec![("d", true)],
        vec![("d", false)],
        vec![("d", false)],
    ];
    let r = detect(&runs);
    assert_eq!(r[0].status, CaseResult::Flaky);
    // 1 flip out of 3 transitions ≈ 0.333
    assert!((r[0].flip_rate - 1.0 / 3.0).abs() < 1e-9);
}

#[test]
fn multiple_cases_sorted() {
    let runs = vec![vec![("z", true), ("a", false)]];
    let r = detect(&runs);
    // BTreeMap → alphabetical
    assert_eq!(r[0].case, "a");
    assert_eq!(r[1].case, "z");
}