1use serde::{Deserialize, Serialize};
11use serde_json::{json, Value};
12
13use crate::dataset::ExpectedToolCall;
14use crate::evidence::RunEvidence;
15
16#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
21#[serde(tag = "assertion", rename_all = "snake_case")]
22pub enum Assertion {
23 ToolCallOrder {
27 expected: Vec<ExpectedToolCall>,
29 },
30
31 ToolCallCount {
33 name: String,
35 expected: usize,
37 },
38
39 StatePredicate {
42 pointer: String,
44 expected: Value,
46 },
47
48 NoToolCall {
50 names: Vec<String>,
52 },
53
54 MaxCost {
56 usd: f64,
58 },
59
60 MaxLatency {
62 ms: u64,
64 },
65}
66
67impl Assertion {
68 pub fn name(&self) -> String {
72 match self {
73 Assertion::ToolCallOrder { .. } => "tool_call_order".to_owned(),
74 Assertion::ToolCallCount { name, .. } => format!("tool_call_count[{name}]"),
75 Assertion::StatePredicate { pointer, .. } => format!("state[{pointer}]"),
76 Assertion::NoToolCall { .. } => "no_tool_call".to_owned(),
77 Assertion::MaxCost { .. } => "max_cost".to_owned(),
78 Assertion::MaxLatency { .. } => "max_latency".to_owned(),
79 }
80 }
81
82 pub fn evaluate(&self, evidence: &RunEvidence) -> AssertionResult {
84 match self {
85 Assertion::ToolCallOrder { expected } => eval_tool_call_order(expected, evidence),
86 Assertion::ToolCallCount { name, expected } => {
87 let observed = evidence
88 .tool_calls
89 .iter()
90 .filter(|call| &call.name == name)
91 .count();
92 AssertionResult {
93 assertion: self.name(),
94 passed: observed == *expected,
95 expected: json!({ "tool": name, "count": expected }),
96 observed: json!({ "count": observed }),
97 detail: (observed != *expected).then(|| {
98 format!("`{name}` was called {observed} time(s), expected {expected}")
99 }),
100 }
101 }
102 Assertion::StatePredicate { pointer, expected } => {
103 let observed = evidence.final_state.pointer(pointer);
104 let passed = observed == Some(expected);
105 AssertionResult {
106 assertion: self.name(),
107 passed,
108 expected: expected.clone(),
109 observed: observed.cloned().unwrap_or(Value::Null),
110 detail: (!passed).then(|| match observed {
111 None => format!("pointer `{pointer}` is not present in the final state"),
112 Some(value) => {
113 format!("value at `{pointer}` is {value}, expected {expected}")
114 }
115 }),
116 }
117 }
118 Assertion::NoToolCall { names } => {
119 let offending: Vec<&str> = evidence
120 .tool_calls
121 .iter()
122 .filter(|call| names.contains(&call.name))
123 .map(|call| call.name.as_str())
124 .collect();
125 AssertionResult {
126 assertion: self.name(),
127 passed: offending.is_empty(),
128 expected: json!({ "forbidden": names }),
129 observed: json!({ "called": offending }),
130 detail: (!offending.is_empty())
131 .then(|| format!("blacklisted tool(s) called: {}", offending.join(", "))),
132 }
133 }
134 Assertion::MaxCost { usd } => AssertionResult {
135 assertion: self.name(),
136 passed: evidence.cost_usd <= *usd,
137 expected: json!({ "max_usd": usd }),
138 observed: json!({ "usd": evidence.cost_usd }),
139 detail: (evidence.cost_usd > *usd).then(|| {
140 format!(
141 "run cost ${:.6} exceeds the ${:.6} bound",
142 evidence.cost_usd, usd
143 )
144 }),
145 },
146 Assertion::MaxLatency { ms } => AssertionResult {
147 assertion: self.name(),
148 passed: evidence.latency_ms <= *ms,
149 expected: json!({ "max_ms": ms }),
150 observed: json!({ "ms": evidence.latency_ms }),
151 detail: (evidence.latency_ms > *ms).then(|| {
152 format!(
153 "run took {} ms, over the {ms} ms bound",
154 evidence.latency_ms
155 )
156 }),
157 },
158 }
159 }
160}
161
162#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
164pub struct AssertionResult {
165 pub assertion: String,
167
168 pub passed: bool,
170
171 pub expected: Value,
173
174 pub observed: Value,
176
177 #[serde(default, skip_serializing_if = "Option::is_none")]
179 pub detail: Option<String>,
180}
181
182fn eval_tool_call_order(expected: &[ExpectedToolCall], evidence: &RunEvidence) -> AssertionResult {
187 let observed_names = evidence.tool_names();
188 let mut cursor = 0_usize;
189
190 for (position, want) in expected.iter().enumerate() {
191 let mut matched = false;
192 let mut arg_mismatch: Option<String> = None;
193 while cursor < evidence.tool_calls.len() {
194 let call = &evidence.tool_calls[cursor];
195 cursor += 1;
196 if call.name == want.name {
197 if want.matches_arguments(&call.arguments) {
198 matched = true;
199 break;
200 }
201 arg_mismatch.get_or_insert_with(|| {
204 format!(
205 "`{}` was called at trajectory position {} but its arguments {} did not \
206 satisfy the matchers",
207 want.name,
208 cursor - 1,
209 call.arguments,
210 )
211 });
212 }
213 }
214 if !matched {
215 let detail = arg_mismatch.unwrap_or_else(|| {
216 format!(
217 "expected call {} (`{}`) never appears at or after trajectory position {}",
218 position + 1,
219 want.name,
220 cursor,
221 )
222 });
223 return AssertionResult {
224 assertion: Assertion::ToolCallOrder {
225 expected: expected.to_vec(),
226 }
227 .name(),
228 passed: false,
229 expected: json!(expected),
230 observed: json!(observed_names),
231 detail: Some(detail),
232 };
233 }
234 }
235
236 AssertionResult {
237 assertion: Assertion::ToolCallOrder {
238 expected: expected.to_vec(),
239 }
240 .name(),
241 passed: true,
242 expected: json!(expected),
243 observed: json!(observed_names),
244 detail: None,
245 }
246}