runifold-eval-cli 0.3.0

CLI and CI quality gates for Runifold evaluations
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120

use std::path::Path;

use anyhow::Result;

use super::{ExperimentReport, resources::ResourceBudget};
use crate::dataset;

pub(super) async fn write_report(
    report: &ExperimentReport,
    output: &Path,
    junit: Option<&Path>,
    markdown: Option<&Path>,
    minimum_lower_bound: Option<f64>,
    max_flaky_case_rate: f64,
    budget: &ResourceBudget,
) -> Result<()> {
    dataset::write(output, serde_json::to_string_pretty(report)?.as_bytes()).await?;
    if let Some(path) = junit {
        dataset::write(
            path,
            junit_report(report, minimum_lower_bound, max_flaky_case_rate, budget).as_bytes(),
        )
        .await?;
    }
    if let Some(path) = markdown {
        dataset::write(
            path,
            markdown_report(report, minimum_lower_bound, max_flaky_case_rate, budget).as_bytes(),
        )
        .await?;
    }
    Ok(())
}

fn junit_report(
    report: &ExperimentReport,
    minimum_lower_bound: Option<f64>,
    max_flaky_case_rate: f64,
    budget: &ResourceBudget,
) -> String {
    let passed = report.passes(minimum_lower_bound, max_flaky_case_rate, budget);
    format!(
        "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<testsuite name=\"runifold-experiment\" tests=\"1\" failures=\"{}\">\n  <testcase classname=\"{}\" name=\"{}\">{}</testcase>\n</testsuite>\n",
        usize::from(!passed),
        xml(&report.dataset_name),
        xml(&report.candidate_version),
        if passed {
            String::new()
        } else {
            "<failure message=\"experiment quality gate failed\"/>".into()
        }
    )
}

fn markdown_report(
    report: &ExperimentReport,
    minimum_lower_bound: Option<f64>,
    max_flaky_case_rate: f64,
    budget: &ResourceBudget,
) -> String {
    let latency = report
        .resources
        .latency_ms
        .as_ref()
        .map_or_else(|| "n/a".into(), |value| format!("{:.2} ms", value.p95));
    let total_tokens = report
        .resources
        .input_tokens
        .checked_add(report.resources.output_tokens)
        .map_or_else(|| "overflow".into(), |value| value.to_string());
    let mut output = format!(
        "# Runifold Experiment\n\n- Dataset: `{}` @ `{}`\n- Candidate: `{}`\n- Samples: {}\n- Cases: {}\n- Flaky case rate: {:.2}%\n- p95 latency: {}\n- Total tokens: {}\n- Total cost: ${:.6}\n- Gate: **{}**\n\n| Score | Mean | Pass rate | Std dev | 95% CI |\n|---|---:|---:|---:|---:|\n",
        report.dataset_name,
        report.dataset_version,
        report.candidate_version,
        report.requested_samples,
        report.case_ids.len(),
        report.flaky_case_rate * 100.0,
        latency,
        total_tokens,
        report.resources.cost_usd,
        if report.passes(minimum_lower_bound, max_flaky_case_rate, budget) {
            "PASS"
        } else {
            "FAIL"
        }
    );
    for statistics in &report.statistics {
        use std::fmt::Write as _;
        let interval = match (
            statistics.confidence_lower_bound,
            statistics.confidence_upper_bound,
        ) {
            (Some(lower), Some(upper)) => format!("[{lower:.4}, {upper:.4}]"),
            _ => "n/a".into(),
        };
        let deviation = statistics
            .standard_deviation
            .map_or_else(|| "n/a".into(), |value| format!("{value:.4}"));
        let _ = writeln!(
            output,
            "| {} | {:.4} | {:.2}% | {} | {} |",
            statistics.name,
            statistics.mean,
            statistics.pass_rate * 100.0,
            deviation,
            interval
        );
    }
    output
}

fn xml(value: &str) -> String {
    value
        .replace('&', "&amp;")
        .replace('<', "&lt;")
        .replace('>', "&gt;")
        .replace('"', "&quot;")
        .replace('\'', "&apos;")
}