swink-agent-eval 0.9.0

Evaluation framework for swink-agent: trajectory tracing, golden path verification, and cost governance
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125

use std::path::PathBuf;

use swink_agent_eval::{
    Assertion, AssertionKind, Attachment, EvalCase, EvalSet, FewShotExample,
    InteractionExpectation, validate_eval_case, validate_eval_set,
};

fn base_case(id: &str) -> EvalCase {
    EvalCase {
        id: id.to_string(),
        name: id.to_string(),
        description: None,
        system_prompt: "system".to_string(),
        user_messages: vec!["hello".to_string()],
        expected_trajectory: None,
        expected_response: None,
        expected_assertion: None,
        expected_interactions: None,
        few_shot_examples: vec![],
        budget: None,
        evaluators: vec![],
        metadata: serde_json::Value::Null,
        attachments: vec![],
        session_id: None,
        expected_environment_state: None,
        expected_tool_intent: None,
        semantic_tool_selection: false,
        state_capture: None,
    }
}

#[test]
fn validate_accepts_extended_v043_fields() {
    let mut case = base_case("valid");
    case.expected_assertion = Some(Assertion {
        description: "user issue is resolved".into(),
        kind: AssertionKind::ToolInvoked("lookup_order".into()),
    });
    case.expected_interactions = Some(vec![InteractionExpectation {
        from: "planner".into(),
        to: "tool".into(),
        description: "delegates lookup".into(),
    }]);
    case.few_shot_examples = vec![FewShotExample {
        input: "Where is my order?".into(),
        expected: "I can help track that.".into(),
        reasoning: Some("Establishes the expected tone.".into()),
    }];
    case.attachments = vec![
        Attachment::Path(PathBuf::from("fixtures/order.png")),
        Attachment::Base64 {
            mime: "image/png".into(),
            bytes: vec![1, 2, 3],
        },
        Attachment::Url("https://example.com/order.png".into()),
    ];

    validate_eval_case(&case).expect("extended fields should validate");
}

#[test]
fn validate_rejects_blank_assertion_tool_name() {
    let mut case = base_case("blank-tool");
    case.expected_assertion = Some(Assertion {
        description: "must invoke a tool".into(),
        kind: AssertionKind::ToolInvoked("   ".into()),
    });

    let err = validate_eval_case(&case).expect_err("blank tool name must be rejected");
    assert!(
        err.to_string()
            .contains("expected_assertion.kind.tool_name")
    );
}

#[test]
fn validate_rejects_malformed_attachment_path_and_url() {
    let mut path_case = base_case("bad-path");
    path_case.attachments = vec![Attachment::Path(PathBuf::from("../escape.png"))];
    let err = validate_eval_case(&path_case).expect_err("parent traversal path must be rejected");
    assert!(err.to_string().contains("attachments[0] path"));

    let mut url_case = base_case("bad-url");
    url_case.attachments = vec![Attachment::Url("http://example.com/image.png".into())];
    let err = validate_eval_case(&url_case).expect_err("http URL must be rejected");
    assert!(err.to_string().contains("URL must use https"));
}

#[test]
fn validate_rejects_duplicate_case_ids_within_eval_set() {
    let set = EvalSet {
        id: "set".into(),
        name: "Set".into(),
        description: None,
        cases: vec![base_case("dup"), base_case("dup")],
    };

    let err = validate_eval_set(&set).expect_err("duplicate case IDs must be rejected");
    assert!(err.to_string().contains("duplicate case id `dup`"));
}

#[test]
fn deterministic_session_id_tracks_new_case_fields() {
    let mut left = base_case("session");
    left.expected_assertion = Some(Assertion {
        description: "complete the task".into(),
        kind: AssertionKind::GoalCompleted,
    });
    left.expected_interactions = Some(vec![InteractionExpectation {
        from: "agent".into(),
        to: "reviewer".into(),
        description: "hands off the result".into(),
    }]);
    left.few_shot_examples = vec![FewShotExample {
        input: "draft".into(),
        expected: "reviewed".into(),
        reasoning: None,
    }];

    let mut right = left.clone();
    assert_eq!(left.default_session_id(), right.default_session_id());

    right.few_shot_examples[0].expected = "approved".into();
    assert_ne!(left.default_session_id(), right.default_session_id());
}