assay-core 2.10.0

High-performance evaluation framework for LLM agents (Core)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119

use crate::model::{TestResultRow, TestStatus};
use std::path::Path;

pub fn write_junit(suite: &str, results: &[TestResultRow], out: &Path) -> anyhow::Result<()> {
    let mut xml = String::new();
    xml.push_str(r#"<?xml version="1.0" encoding="UTF-8"?>"#);
    xml.push('\n');
    xml.push_str(&format!(r#"<testsuite name="{}">"#, escape(suite)));
    xml.push('\n');

    for r in results {
        xml.push_str(&format!(r#"  <testcase name="{}">"#, escape(&r.test_id)));
        match r.status {
            TestStatus::Pass | TestStatus::AllowedOnError => {}
            TestStatus::Skipped => {
                xml.push_str(&format!(r#"<skipped message="{}"/>"#, escape(&r.message)))
            }
            TestStatus::Warn | TestStatus::Flaky | TestStatus::Unstable => {
                // Use clear warning label in system-out
                let label = match r.status {
                    TestStatus::Flaky => "FLAKY",
                    TestStatus::Unstable => "UNSTABLE",
                    _ => "WARNING",
                };
                xml.push_str(&format!(
                    r#"<system-out>{}: {}</system-out>"#,
                    label,
                    escape(&r.message)
                ));
            }
            TestStatus::Fail => {
                xml.push_str(&format!(r#"<failure message="{}"/>"#, escape(&r.message)))
            }
            TestStatus::Error => {
                xml.push_str(&format!(r#"<error message="{}"/>"#, escape(&r.message)))
            }
        }
        xml.push_str("</testcase>\n");
    }

    xml.push_str("</testsuite>\n");
    std::fs::write(out, xml)?;
    Ok(())
}

fn escape(s: &str) -> String {
    s.replace('&', "&amp;")
        .replace('<', "&lt;")
        .replace('>', "&gt;")
        .replace('"', "&quot;")
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::model::TestResultRow;
    use crate::model::TestStatus;

    #[test]
    fn test_junit_output_structure() {
        let temp_dir = tempfile::tempdir().unwrap();
        let path = temp_dir.path().join("junit.xml");

        let results = vec![
            TestResultRow {
                test_id: "test_pass".into(),
                status: TestStatus::Pass,
                message: "ok".into(),
                score: Some(1.0),
                cached: false,
                details: serde_json::Value::Null,
                duration_ms: Some(10),
                fingerprint: None,
                skip_reason: None,
                attempts: None,
                error_policy_applied: None,
            },
            TestResultRow {
                test_id: "test_warn".into(),
                status: TestStatus::Warn,
                message: "almost".into(),
                score: Some(0.5),
                cached: false,
                details: serde_json::Value::Null,
                duration_ms: Some(10),
                fingerprint: None,
                skip_reason: None,
                attempts: None,
                error_policy_applied: None,
            },
            TestResultRow {
                test_id: "test_fail".into(),
                status: TestStatus::Fail,
                message: "bad".into(),
                score: Some(0.0),
                cached: false,
                details: serde_json::Value::Null,
                duration_ms: Some(10),
                fingerprint: None,
                skip_reason: None,
                attempts: None,
                error_policy_applied: None,
            },
        ];

        write_junit("demo", &results, &path).unwrap();

        let content = std::fs::read_to_string(path).unwrap();
        assert!(content.contains(r#"<testsuite name="demo">"#));
        // Pass
        assert!(content.contains(r#"<testcase name="test_pass">"#));
        // Warn (system-out)
        assert!(content.contains(r#"<testcase name="test_warn">"#));
        assert!(content.contains(r#"<system-out>WARNING: almost</system-out>"#));
        // Fail
        assert!(content.contains(r#"<testcase name="test_fail">"#));
        assert!(content.contains(r#"<failure message="bad"/>"#));
    }
}