tandem-eval 0.7.0

Evaluation harness and regression tooling for Tandem
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68

use super::*;

fn corpus_path() -> std::path::PathBuf {
    Path::new(env!("CARGO_MANIFEST_DIR")).join("../../eval_datasets/agentic_product_authoring.yaml")
}

#[test]
fn corpus_is_versioned_and_has_all_required_scenario_kinds() {
    let document = load_agentic_dataset(&corpus_path()).expect("load authoring corpus");
    assert_eq!(document.dataset.name, "agentic_product_authoring");
    assert_eq!(document.acceptance.required_pass_rate, 1.0);
    assert!(!document.acceptance.live_provider_calls);
    let kinds = document
        .dataset
        .test_cases
        .iter()
        .map(|case| scenario_for_case(case).expect("scenario").name())
        .collect::<BTreeSet<_>>();
    for required in [
        "route",
        "chat_model_execution",
        "tool_contract",
        "identity_boundary",
        "draft_lifecycle",
        "active_artifact",
        "confirmation_boundary",
        "failure_taxonomy",
    ] {
        assert!(kinds.contains(required), "missing scenario kind {required}");
    }
}

#[test]
fn authoritative_success_claim_requires_a_persisted_artifact() {
    let outcome = json!({
        "ok": true,
        "resource": { "id": "automation-1" }
    });
    let error = validate_authoritative_claim("Created the automation draft.", &outcome, None)
        .expect_err("claim without persistence must fail");
    assert!(error.to_string().contains("persisted artifact"));
}

#[test]
fn missing_required_coverage_fails_the_gate() {
    let dataset = EvalDataset::new("coverage", "1.0");
    let thresholds = AgenticAuthoringAcceptanceThresholds {
        required_pass_rate: 1.0,
        required_coverage: vec!["tenant_isolation".to_string()],
        execution_profile: "deterministic".to_string(),
        live_provider_calls: false,
    };
    let report = build_report(
        &dataset,
        &thresholds,
        vec![AgenticAuthoringCaseResult {
            test_id: "case".to_string(),
            description: "case".to_string(),
            scenario: "route".to_string(),
            passed: true,
            tags: vec!["routing".to_string()],
            evidence: Value::Null,
            error: None,
        }],
    );
    assert!(!report.gate_passed);
    assert_eq!(report.missing_coverage, vec!["tenant_isolation"]);
}