assay-cli 6.6.2

Policy-as-code gate for MCP agent tool calls, with verifiable evidence and Linux kernel enforcement.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118

use super::*;
#[test]
fn schema_cli_validates_jsonl_inputs() {
    let dir = tempdir().unwrap();
    let input = dir.path().join("results.jsonl");
    fs::write(
        &input,
        concat!(
            r#"{"gradingResult":{"componentResults":[{"pass":true,"score":1,"reason":"Assertion passed","assertion":{"type":"equals","value":"Hello world"}}]}}"#,
            "\n"
        ),
    )
    .unwrap();

    let output = assay_schema_command()
        .arg("validate")
        .arg("--schema")
        .arg("promptfoo-cli-jsonl-component-result.v1")
        .arg("--input")
        .arg(&input)
        .arg("--jsonl")
        .arg("--format")
        .arg("json")
        .assert()
        .success()
        .get_output()
        .stdout
        .clone();
    let report: Value = serde_json::from_slice(&output).unwrap();
    assert_eq!(report["schema"], "assay.evidence.schema.validation.v1");
    assert_eq!(report["valid"], true);
    assert_eq!(report["documents"], 1);
}

#[test]
fn schema_cli_returns_exit_one_for_schema_mismatch() {
    let dir = tempdir().unwrap();
    let input = dir.path().join("receipt.json");
    fs::write(
        &input,
        json!({
            "schema": "assay.receipt.promptfoo.assertion-component.v1",
            "source_system": "promptfoo"
        })
        .to_string(),
    )
    .unwrap();

    let output = assay_schema_command()
        .arg("validate")
        .arg("--schema")
        .arg("promptfoo.assertion-component.v1")
        .arg("--input")
        .arg(&input)
        .arg("--format")
        .arg("json")
        .assert()
        .code(1)
        .get_output()
        .stdout
        .clone();
    let report: Value = serde_json::from_slice(&output).unwrap();
    assert_eq!(report["valid"], false);
    assert!(!report["errors"].as_array().unwrap().is_empty());
}

#[test]
fn schema_cli_returns_exit_two_for_invalid_json_input() {
    let dir = tempdir().unwrap();
    let input = dir.path().join("receipt.json");
    fs::write(
        &input,
        r#"{"schema":"assay.receipt.promptfoo.assertion-component.v1""#,
    )
    .unwrap();

    let output = assay_schema_command()
        .arg("validate")
        .arg("--schema")
        .arg("promptfoo.assertion-component.v1")
        .arg("--input")
        .arg(&input)
        .arg("--format")
        .arg("json")
        .assert()
        .code(2)
        .get_output()
        .stdout
        .clone();
    let report: Value = serde_json::from_slice(&output).unwrap();
    assert_eq!(report["valid"], false);
    assert_eq!(report["errors"][0]["kind"], "parse");
}

#[test]
fn schema_cli_returns_exit_two_for_empty_jsonl_input() {
    let dir = tempdir().unwrap();
    let input = dir.path().join("results.jsonl");
    fs::write(&input, "\n\n").unwrap();

    let output = assay_schema_command()
        .arg("validate")
        .arg("--schema")
        .arg("promptfoo-cli-jsonl-component-result.v1")
        .arg("--input")
        .arg(&input)
        .arg("--jsonl")
        .arg("--format")
        .arg("json")
        .assert()
        .code(2)
        .get_output()
        .stdout
        .clone();
    let report: Value = serde_json::from_slice(&output).unwrap();
    assert_eq!(report["valid"], false);
    assert_eq!(report["errors"][0]["kind"], "empty_input");
}