Skip to main content

rusty_eval/
assertion.rs

1//! Deterministic assertions over run evidence.
2//!
3//! An [`Assertion`] is a closed set of checks — no heuristics, no model in
4//! the loop — evaluated against a recorded run's [`RunEvidence`]. Every
5//! evaluation returns an [`AssertionResult`] carrying the evidence of the
6//! verdict: what was expected, what was observed, and a human-readable
7//! detail on failure. Results are serializable, so reports preserve exactly
8//! why a run failed, not just that it did.
9
10use serde::{Deserialize, Serialize};
11use serde_json::{json, Value};
12
13use crate::dataset::ExpectedToolCall;
14use crate::evidence::RunEvidence;
15
16/// One deterministic check over a run's evidence.
17///
18/// Serialized with an `assertion` tag so assertions can live in config
19/// files and reports: `{"assertion":"tool_call_count","name":"calculator","expected":2}`.
20#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
21#[serde(tag = "assertion", rename_all = "snake_case")]
22pub enum Assertion {
23    /// The expected tool calls appear in the observed trajectory as an
24    /// ordered subsequence (extra calls in between are allowed), each
25    /// satisfying its argument matchers.
26    ToolCallOrder {
27        /// The expected subsequence, in order.
28        expected: Vec<ExpectedToolCall>,
29    },
30
31    /// The tool `name` was called exactly `expected` times.
32    ToolCallCount {
33        /// Tool name to count.
34        name: String,
35        /// Exact expected call count.
36        expected: usize,
37    },
38
39    /// The run's final state has `expected` at `pointer` (JSON pointer,
40    /// RFC 6901 — e.g. `/messages/3/content`).
41    StatePredicate {
42        /// JSON pointer into the final state object.
43        pointer: String,
44        /// Expected value, compared with `==`.
45        expected: Value,
46    },
47
48    /// None of the blacklisted tools were called.
49    NoToolCall {
50        /// Forbidden tool names.
51        names: Vec<String>,
52    },
53
54    /// The run's total journaled cost is at most `usd`.
55    MaxCost {
56        /// Cost ceiling in USD.
57        usd: f64,
58    },
59
60    /// The run's wall latency is at most `ms`.
61    MaxLatency {
62        /// Latency ceiling in milliseconds.
63        ms: u64,
64    },
65}
66
67impl Assertion {
68    /// The stable report key for this assertion. Aggregation groups results
69    /// by it, so it includes the parameters that distinguish two assertions
70    /// of the same kind (`state[/answer]` vs `state[/score]`).
71    pub fn name(&self) -> String {
72        match self {
73            Assertion::ToolCallOrder { .. } => "tool_call_order".to_owned(),
74            Assertion::ToolCallCount { name, .. } => format!("tool_call_count[{name}]"),
75            Assertion::StatePredicate { pointer, .. } => format!("state[{pointer}]"),
76            Assertion::NoToolCall { .. } => "no_tool_call".to_owned(),
77            Assertion::MaxCost { .. } => "max_cost".to_owned(),
78            Assertion::MaxLatency { .. } => "max_latency".to_owned(),
79        }
80    }
81
82    /// Evaluate against one run's evidence. Pure: same evidence, same verdict.
83    pub fn evaluate(&self, evidence: &RunEvidence) -> AssertionResult {
84        match self {
85            Assertion::ToolCallOrder { expected } => eval_tool_call_order(expected, evidence),
86            Assertion::ToolCallCount { name, expected } => {
87                let observed = evidence
88                    .tool_calls
89                    .iter()
90                    .filter(|call| &call.name == name)
91                    .count();
92                AssertionResult {
93                    assertion: self.name(),
94                    passed: observed == *expected,
95                    expected: json!({ "tool": name, "count": expected }),
96                    observed: json!({ "count": observed }),
97                    detail: (observed != *expected).then(|| {
98                        format!("`{name}` was called {observed} time(s), expected {expected}")
99                    }),
100                }
101            }
102            Assertion::StatePredicate { pointer, expected } => {
103                let observed = evidence.final_state.pointer(pointer);
104                let passed = observed == Some(expected);
105                AssertionResult {
106                    assertion: self.name(),
107                    passed,
108                    expected: expected.clone(),
109                    observed: observed.cloned().unwrap_or(Value::Null),
110                    detail: (!passed).then(|| match observed {
111                        None => format!("pointer `{pointer}` is not present in the final state"),
112                        Some(value) => {
113                            format!("value at `{pointer}` is {value}, expected {expected}")
114                        }
115                    }),
116                }
117            }
118            Assertion::NoToolCall { names } => {
119                let offending: Vec<&str> = evidence
120                    .tool_calls
121                    .iter()
122                    .filter(|call| names.contains(&call.name))
123                    .map(|call| call.name.as_str())
124                    .collect();
125                AssertionResult {
126                    assertion: self.name(),
127                    passed: offending.is_empty(),
128                    expected: json!({ "forbidden": names }),
129                    observed: json!({ "called": offending }),
130                    detail: (!offending.is_empty())
131                        .then(|| format!("blacklisted tool(s) called: {}", offending.join(", "))),
132                }
133            }
134            Assertion::MaxCost { usd } => AssertionResult {
135                assertion: self.name(),
136                passed: evidence.cost_usd <= *usd,
137                expected: json!({ "max_usd": usd }),
138                observed: json!({ "usd": evidence.cost_usd }),
139                detail: (evidence.cost_usd > *usd).then(|| {
140                    format!(
141                        "run cost ${:.6} exceeds the ${:.6} bound",
142                        evidence.cost_usd, usd
143                    )
144                }),
145            },
146            Assertion::MaxLatency { ms } => AssertionResult {
147                assertion: self.name(),
148                passed: evidence.latency_ms <= *ms,
149                expected: json!({ "max_ms": ms }),
150                observed: json!({ "ms": evidence.latency_ms }),
151                detail: (evidence.latency_ms > *ms).then(|| {
152                    format!(
153                        "run took {} ms, over the {ms} ms bound",
154                        evidence.latency_ms
155                    )
156                }),
157            },
158        }
159    }
160}
161
162/// The verdict of one [`Assertion`] on one run, with its evidence.
163#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
164pub struct AssertionResult {
165    /// The assertion's report key ([`Assertion::name`]).
166    pub assertion: String,
167
168    /// The verdict.
169    pub passed: bool,
170
171    /// What was expected (assertion parameters as JSON).
172    pub expected: Value,
173
174    /// What the run actually did (the observed quantity as JSON).
175    pub observed: Value,
176
177    /// Why it failed, in one sentence. Absent on passes.
178    #[serde(default, skip_serializing_if = "Option::is_none")]
179    pub detail: Option<String>,
180}
181
182/// Subsequence matching for [`Assertion::ToolCallOrder`]: greedy
183/// earliest-match, which is optimal for subsequence existence — taking the
184/// earliest matching candidate leaves the longest possible suffix for the
185/// rest of the pattern.
186fn eval_tool_call_order(expected: &[ExpectedToolCall], evidence: &RunEvidence) -> AssertionResult {
187    let observed_names = evidence.tool_names();
188    let mut cursor = 0_usize;
189
190    for (position, want) in expected.iter().enumerate() {
191        let mut matched = false;
192        let mut arg_mismatch: Option<String> = None;
193        while cursor < evidence.tool_calls.len() {
194            let call = &evidence.tool_calls[cursor];
195            cursor += 1;
196            if call.name == want.name {
197                if want.matches_arguments(&call.arguments) {
198                    matched = true;
199                    break;
200                }
201                // Remember the nearest near-miss for the failure detail, but
202                // keep scanning: a later same-name call may satisfy the args.
203                arg_mismatch.get_or_insert_with(|| {
204                    format!(
205                        "`{}` was called at trajectory position {} but its arguments {} did not \
206                         satisfy the matchers",
207                        want.name,
208                        cursor - 1,
209                        call.arguments,
210                    )
211                });
212            }
213        }
214        if !matched {
215            let detail = arg_mismatch.unwrap_or_else(|| {
216                format!(
217                    "expected call {} (`{}`) never appears at or after trajectory position {}",
218                    position + 1,
219                    want.name,
220                    cursor,
221                )
222            });
223            return AssertionResult {
224                assertion: Assertion::ToolCallOrder {
225                    expected: expected.to_vec(),
226                }
227                .name(),
228                passed: false,
229                expected: json!(expected),
230                observed: json!(observed_names),
231                detail: Some(detail),
232            };
233        }
234    }
235
236    AssertionResult {
237        assertion: Assertion::ToolCallOrder {
238            expected: expected.to_vec(),
239        }
240        .name(),
241        passed: true,
242        expected: json!(expected),
243        observed: json!(observed_names),
244        detail: None,
245    }
246}