rusty_eval/evidence.rs
1//! What a run left behind: the evidence assertions evaluate against.
2//!
3//! [`RunEvidence`] is a distilled, self-contained view of one run — the
4//! ordered tool-call trajectory, the final state, wall latency, total cost,
5//! and token usage — extracted from the run's Flight Recorder journal
6//! ([`rusty_agent_runtime::journal::Journal`]). It is deliberately plain
7//! data: serializable, clonable, and free of runtime handles, so it can
8//! travel into reports, judges, and tests fabricated by hand.
9
10use serde::{Deserialize, Serialize};
11use serde_json::Value;
12
13use rusty_agent_runtime::journal::Journal;
14use rusty_agent_runtime::record::{EventStatus, RunEventKind};
15
16/// How a run terminated, from the evaluator's point of view.
17#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
18#[serde(tag = "status", rename_all = "snake_case")]
19pub enum RunStatus {
20 /// Routing reached the end; `final_state` is the terminal state.
21 Done,
22 /// A node interrupted (human-in-the-loop). `final_state` is the
23 /// suspension-point state. An interrupted run has not finished its work;
24 /// case runs count it as not passed.
25 Interrupted,
26 /// The executor returned an error. `final_state` is null — whatever
27 /// partial state existed is not trustworthy evidence.
28 Failed {
29 /// The executor's error message.
30 error: String,
31 },
32}
33
34impl RunStatus {
35 /// `true` only for [`RunStatus::Done`].
36 pub fn is_done(&self) -> bool {
37 matches!(self, RunStatus::Done)
38 }
39}
40
41impl std::fmt::Display for RunStatus {
42 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
43 match self {
44 RunStatus::Done => f.write_str("done"),
45 RunStatus::Interrupted => f.write_str("interrupted"),
46 RunStatus::Failed { error } => write!(f, "failed: {error}"),
47 }
48 }
49}
50
51/// One journaled tool call, in trajectory order.
52#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
53pub struct ToolCallRecord {
54 /// Journal sequence number — the run's total order.
55 pub seq: u64,
56
57 /// Tool name (the `tool` field of the canonical tool-call request
58 /// payload; falls back to the recording node's id).
59 pub name: String,
60
61 /// The call's arguments (`Null` when the journaled payload carried none).
62 pub arguments: Value,
63
64 /// Measured latency of the call in milliseconds, when journaled.
65 #[serde(default, skip_serializing_if = "Option::is_none")]
66 pub latency_ms: Option<u64>,
67
68 /// Journaled cost of the call in USD, when known.
69 #[serde(default, skip_serializing_if = "Option::is_none")]
70 pub cost_usd: Option<f64>,
71
72 /// How the call ended.
73 pub status: EventStatus,
74}
75
76/// The distilled evidence of one run.
77#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
78pub struct RunEvidence {
79 /// How the run terminated.
80 pub status: RunStatus,
81
82 /// The ordered tool-call trajectory (journal `tool_call` events in
83 /// sequence order).
84 pub tool_calls: Vec<ToolCallRecord>,
85
86 /// The run's final (or suspension-point) state as JSON. `Null` for
87 /// failed runs.
88 pub final_state: Value,
89
90 /// Wall latency of the whole run in milliseconds, measured by the
91 /// experiment runner around `Executor::run`.
92 pub latency_ms: u64,
93
94 /// Total journaled cost in USD: the sum of `cost_usd` over every event
95 /// in the journal (model calls carry cost too, not only tool calls).
96 /// Events without cost contribute zero.
97 pub cost_usd: f64,
98
99 /// Total tokens reported by model calls in the run.
100 pub total_tokens: u64,
101}
102
103impl RunEvidence {
104 /// Distill a journal plus run outcome into evidence.
105 ///
106 /// Tool calls are read from `tool_call` journal events in `seq` order.
107 /// The canonical payload shape is `{"tool": name, "arguments": {...}}`
108 /// (what the runtime's recording wrappers journal); events whose payload
109 /// lacks a `tool` field fall back to the recording node's id, then to
110 /// `"unknown"` — every journaled call stays visible in the trajectory.
111 pub fn from_journal(
112 journal: &Journal,
113 status: RunStatus,
114 final_state: Value,
115 latency_ms: u64,
116 ) -> Self {
117 let events = journal.events();
118 let mut tool_calls = Vec::new();
119 let mut cost_usd = 0.0_f64;
120 let mut total_tokens = 0_u64;
121
122 for event in &events {
123 if let Some(cost) = event.cost_usd {
124 cost_usd += cost;
125 }
126 if let Some(usage) = &event.tokens {
127 total_tokens = total_tokens.saturating_add(usage.total_tokens);
128 }
129 if event.kind != RunEventKind::ToolCall {
130 continue;
131 }
132 let payload = event
133 .input
134 .as_ref()
135 .and_then(|reference| journal.resolve(reference));
136 let (name, arguments) = match payload {
137 Some(value) => {
138 let name = value
139 .get("tool")
140 .and_then(Value::as_str)
141 .map(str::to_owned)
142 .or_else(|| event.node_id.clone())
143 .unwrap_or_else(|| "unknown".to_owned());
144 let arguments = value.get("arguments").cloned().unwrap_or(Value::Null);
145 (name, arguments)
146 }
147 None => (
148 event
149 .node_id
150 .clone()
151 .unwrap_or_else(|| "unknown".to_owned()),
152 Value::Null,
153 ),
154 };
155 tool_calls.push(ToolCallRecord {
156 seq: event.seq,
157 name,
158 arguments,
159 latency_ms: event.latency_ms,
160 cost_usd: event.cost_usd,
161 status: event.status,
162 });
163 }
164
165 Self {
166 status,
167 tool_calls,
168 final_state,
169 latency_ms,
170 cost_usd,
171 total_tokens,
172 }
173 }
174
175 /// The trajectory as bare tool names, in order.
176 pub fn tool_names(&self) -> Vec<&str> {
177 self.tool_calls
178 .iter()
179 .map(|call| call.name.as_str())
180 .collect()
181 }
182}