use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use crate::dataset::ExpectedToolCall;
use crate::evidence::RunEvidence;
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(tag = "assertion", rename_all = "snake_case")]
pub enum Assertion {
ToolCallOrder {
expected: Vec<ExpectedToolCall>,
},
ToolCallCount {
name: String,
expected: usize,
},
StatePredicate {
pointer: String,
expected: Value,
},
NoToolCall {
names: Vec<String>,
},
MaxCost {
usd: f64,
},
MaxLatency {
ms: u64,
},
}
impl Assertion {
pub fn name(&self) -> String {
match self {
Assertion::ToolCallOrder { .. } => "tool_call_order".to_owned(),
Assertion::ToolCallCount { name, .. } => format!("tool_call_count[{name}]"),
Assertion::StatePredicate { pointer, .. } => format!("state[{pointer}]"),
Assertion::NoToolCall { .. } => "no_tool_call".to_owned(),
Assertion::MaxCost { .. } => "max_cost".to_owned(),
Assertion::MaxLatency { .. } => "max_latency".to_owned(),
}
}
pub fn evaluate(&self, evidence: &RunEvidence) -> AssertionResult {
match self {
Assertion::ToolCallOrder { expected } => eval_tool_call_order(expected, evidence),
Assertion::ToolCallCount { name, expected } => {
let observed = evidence
.tool_calls
.iter()
.filter(|call| &call.name == name)
.count();
AssertionResult {
assertion: self.name(),
passed: observed == *expected,
expected: json!({ "tool": name, "count": expected }),
observed: json!({ "count": observed }),
detail: (observed != *expected).then(|| {
format!("`{name}` was called {observed} time(s), expected {expected}")
}),
}
}
Assertion::StatePredicate { pointer, expected } => {
let observed = evidence.final_state.pointer(pointer);
let passed = observed == Some(expected);
AssertionResult {
assertion: self.name(),
passed,
expected: expected.clone(),
observed: observed.cloned().unwrap_or(Value::Null),
detail: (!passed).then(|| match observed {
None => format!("pointer `{pointer}` is not present in the final state"),
Some(value) => {
format!("value at `{pointer}` is {value}, expected {expected}")
}
}),
}
}
Assertion::NoToolCall { names } => {
let offending: Vec<&str> = evidence
.tool_calls
.iter()
.filter(|call| names.contains(&call.name))
.map(|call| call.name.as_str())
.collect();
AssertionResult {
assertion: self.name(),
passed: offending.is_empty(),
expected: json!({ "forbidden": names }),
observed: json!({ "called": offending }),
detail: (!offending.is_empty())
.then(|| format!("blacklisted tool(s) called: {}", offending.join(", "))),
}
}
Assertion::MaxCost { usd } => AssertionResult {
assertion: self.name(),
passed: evidence.cost_usd <= *usd,
expected: json!({ "max_usd": usd }),
observed: json!({ "usd": evidence.cost_usd }),
detail: (evidence.cost_usd > *usd).then(|| {
format!(
"run cost ${:.6} exceeds the ${:.6} bound",
evidence.cost_usd, usd
)
}),
},
Assertion::MaxLatency { ms } => AssertionResult {
assertion: self.name(),
passed: evidence.latency_ms <= *ms,
expected: json!({ "max_ms": ms }),
observed: json!({ "ms": evidence.latency_ms }),
detail: (evidence.latency_ms > *ms).then(|| {
format!(
"run took {} ms, over the {ms} ms bound",
evidence.latency_ms
)
}),
},
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct AssertionResult {
pub assertion: String,
pub passed: bool,
pub expected: Value,
pub observed: Value,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub detail: Option<String>,
}
fn eval_tool_call_order(expected: &[ExpectedToolCall], evidence: &RunEvidence) -> AssertionResult {
let observed_names = evidence.tool_names();
let mut cursor = 0_usize;
for (position, want) in expected.iter().enumerate() {
let mut matched = false;
let mut arg_mismatch: Option<String> = None;
while cursor < evidence.tool_calls.len() {
let call = &evidence.tool_calls[cursor];
cursor += 1;
if call.name == want.name {
if want.matches_arguments(&call.arguments) {
matched = true;
break;
}
arg_mismatch.get_or_insert_with(|| {
format!(
"`{}` was called at trajectory position {} but its arguments {} did not \
satisfy the matchers",
want.name,
cursor - 1,
call.arguments,
)
});
}
}
if !matched {
let detail = arg_mismatch.unwrap_or_else(|| {
format!(
"expected call {} (`{}`) never appears at or after trajectory position {}",
position + 1,
want.name,
cursor,
)
});
return AssertionResult {
assertion: Assertion::ToolCallOrder {
expected: expected.to_vec(),
}
.name(),
passed: false,
expected: json!(expected),
observed: json!(observed_names),
detail: Some(detail),
};
}
}
AssertionResult {
assertion: Assertion::ToolCallOrder {
expected: expected.to_vec(),
}
.name(),
passed: true,
expected: json!(expected),
observed: json!(observed_names),
detail: None,
}
}