assay-core 3.9.1

High-performance evaluation framework for LLM agents (Core)
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115

use assay_core::config::load_config;
use std::io::Write;
use tempfile::NamedTempFile;

#[test]
fn test_config_version_defaults() -> anyhow::Result<()> {
    let mut tmp = NamedTempFile::new()?;
    writeln!(
        tmp,
        r#"
suite: test
model: dummy
tests:
  - id: t1
    input: {{ prompt: "hi" }}
    expected: {{ type: must_contain, must_contain: ["hi"] }}
"#
    )?;

    let cfg = load_config(tmp.path(), false, false)?;
    assert_eq!(cfg.version, 0, "Default version should be 0 (legacy)");
    assert!(cfg.is_legacy());
    Ok(())
}

#[test]
fn test_config_version_explicit_v1() -> anyhow::Result<()> {
    let mut tmp = NamedTempFile::new()?;
    writeln!(
        tmp,
        r#"
configVersion: 1
suite: test
model: dummy
tests: []
"#
    )?;
    // Note: Empty tests usually error, but let's see if load_config enforces it.
    // Yes, check source: if cfg.tests.is_empty() { return Err(...) }
    // So we need a dummy test.
    writeln!(
        tmp,
        r#"
  - id: t1
    input: {{ prompt: "hi" }}
    expected: {{ type: must_contain, must_contain: ["hi"] }}
"#
    )?;

    // Re-write coherently
    let mut tmp = NamedTempFile::new()?;
    writeln!(
        tmp,
        r#"
configVersion: 1
suite: test
model: dummy
tests:
  - id: t1
    input: {{ prompt: "hi" }}
    expected: {{ type: must_contain, must_contain: ["hi"] }}
"#
    )?;

    let cfg = load_config(tmp.path(), false, false)?;
    assert_eq!(cfg.version, 1, "Explicit version 1 should be respected");
    assert!(!cfg.is_legacy());
    Ok(())
}

#[test]
fn test_legacy_mode_override() -> anyhow::Result<()> {
    let mut tmp = NamedTempFile::new()?;
    writeln!(
        tmp,
        r#"
configVersion: 1
suite: test
model: dummy
tests:
  - id: t1
    input: {{ prompt: "hi" }}
    expected: {{ type: must_contain, must_contain: ["hi"] }}
"#
    )?;

    // Legacy mode = true should force version to 0
    let cfg = load_config(tmp.path(), true, false)?;
    assert_eq!(cfg.version, 0, "Legacy mode should force version 0");
    assert!(cfg.is_legacy());
    Ok(())
}

#[test]
fn test_unsupported_version() -> anyhow::Result<()> {
    let mut tmp = NamedTempFile::new()?;
    writeln!(
        tmp,
        r#"
configVersion: 999
suite: test
model: dummy
tests:
  - id: t1
    input: {{ prompt: "hi" }}
    expected: {{ type: must_contain, must_contain: ["hi"] }}
"#
    )?;

    let res = load_config(tmp.path(), false, false);
    assert!(res.is_err());
    let err = res.err().unwrap().to_string();
    assert!(err.contains("unsupported config version 999"));
    Ok(())
}