Skip to main content

rusty_eval/
evidence.rs

1//! What a run left behind: the evidence assertions evaluate against.
2//!
3//! [`RunEvidence`] is a distilled, self-contained view of one run — the
4//! ordered tool-call trajectory, the final state, wall latency, total cost,
5//! and token usage — extracted from the run's Flight Recorder journal
6//! ([`rusty_agent_runtime::journal::Journal`]). It is deliberately plain
7//! data: serializable, clonable, and free of runtime handles, so it can
8//! travel into reports, judges, and tests fabricated by hand.
9
10use serde::{Deserialize, Serialize};
11use serde_json::Value;
12
13use rusty_agent_runtime::journal::Journal;
14use rusty_agent_runtime::record::{EventStatus, RunEventKind};
15
16/// How a run terminated, from the evaluator's point of view.
17#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
18#[serde(tag = "status", rename_all = "snake_case")]
19pub enum RunStatus {
20    /// Routing reached the end; `final_state` is the terminal state.
21    Done,
22    /// A node interrupted (human-in-the-loop). `final_state` is the
23    /// suspension-point state. An interrupted run has not finished its work;
24    /// case runs count it as not passed.
25    Interrupted,
26    /// The executor returned an error. `final_state` is null — whatever
27    /// partial state existed is not trustworthy evidence.
28    Failed {
29        /// The executor's error message.
30        error: String,
31    },
32}
33
34impl RunStatus {
35    /// `true` only for [`RunStatus::Done`].
36    pub fn is_done(&self) -> bool {
37        matches!(self, RunStatus::Done)
38    }
39}
40
41impl std::fmt::Display for RunStatus {
42    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
43        match self {
44            RunStatus::Done => f.write_str("done"),
45            RunStatus::Interrupted => f.write_str("interrupted"),
46            RunStatus::Failed { error } => write!(f, "failed: {error}"),
47        }
48    }
49}
50
51/// One journaled tool call, in trajectory order.
52#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
53pub struct ToolCallRecord {
54    /// Journal sequence number — the run's total order.
55    pub seq: u64,
56
57    /// Tool name (the `tool` field of the canonical tool-call request
58    /// payload; falls back to the recording node's id).
59    pub name: String,
60
61    /// The call's arguments (`Null` when the journaled payload carried none).
62    pub arguments: Value,
63
64    /// Measured latency of the call in milliseconds, when journaled.
65    #[serde(default, skip_serializing_if = "Option::is_none")]
66    pub latency_ms: Option<u64>,
67
68    /// Journaled cost of the call in USD, when known.
69    #[serde(default, skip_serializing_if = "Option::is_none")]
70    pub cost_usd: Option<f64>,
71
72    /// How the call ended.
73    pub status: EventStatus,
74}
75
76/// The distilled evidence of one run.
77#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
78pub struct RunEvidence {
79    /// How the run terminated.
80    pub status: RunStatus,
81
82    /// The ordered tool-call trajectory (journal `tool_call` events in
83    /// sequence order).
84    pub tool_calls: Vec<ToolCallRecord>,
85
86    /// The run's final (or suspension-point) state as JSON. `Null` for
87    /// failed runs.
88    pub final_state: Value,
89
90    /// Wall latency of the whole run in milliseconds, measured by the
91    /// experiment runner around `Executor::run`.
92    pub latency_ms: u64,
93
94    /// Total journaled cost in USD: the sum of `cost_usd` over every event
95    /// in the journal (model calls carry cost too, not only tool calls).
96    /// Events without cost contribute zero.
97    pub cost_usd: f64,
98
99    /// Total tokens reported by model calls in the run.
100    pub total_tokens: u64,
101}
102
103impl RunEvidence {
104    /// Distill a journal plus run outcome into evidence.
105    ///
106    /// Tool calls are read from `tool_call` journal events in `seq` order.
107    /// The canonical payload shape is `{"tool": name, "arguments": {...}}`
108    /// (what the runtime's recording wrappers journal); events whose payload
109    /// lacks a `tool` field fall back to the recording node's id, then to
110    /// `"unknown"` — every journaled call stays visible in the trajectory.
111    pub fn from_journal(
112        journal: &Journal,
113        status: RunStatus,
114        final_state: Value,
115        latency_ms: u64,
116    ) -> Self {
117        let events = journal.events();
118        let mut tool_calls = Vec::new();
119        let mut cost_usd = 0.0_f64;
120        let mut total_tokens = 0_u64;
121
122        for event in &events {
123            if let Some(cost) = event.cost_usd {
124                cost_usd += cost;
125            }
126            if let Some(usage) = &event.tokens {
127                total_tokens = total_tokens.saturating_add(usage.total_tokens);
128            }
129            if event.kind != RunEventKind::ToolCall {
130                continue;
131            }
132            let payload = event
133                .input
134                .as_ref()
135                .and_then(|reference| journal.resolve(reference));
136            let (name, arguments) = match payload {
137                Some(value) => {
138                    let name = value
139                        .get("tool")
140                        .and_then(Value::as_str)
141                        .map(str::to_owned)
142                        .or_else(|| event.node_id.clone())
143                        .unwrap_or_else(|| "unknown".to_owned());
144                    let arguments = value.get("arguments").cloned().unwrap_or(Value::Null);
145                    (name, arguments)
146                }
147                None => (
148                    event
149                        .node_id
150                        .clone()
151                        .unwrap_or_else(|| "unknown".to_owned()),
152                    Value::Null,
153                ),
154            };
155            tool_calls.push(ToolCallRecord {
156                seq: event.seq,
157                name,
158                arguments,
159                latency_ms: event.latency_ms,
160                cost_usd: event.cost_usd,
161                status: event.status,
162            });
163        }
164
165        Self {
166            status,
167            tool_calls,
168            final_state,
169            latency_ms,
170            cost_usd,
171            total_tokens,
172        }
173    }
174
175    /// The trajectory as bare tool names, in order.
176    pub fn tool_names(&self) -> Vec<&str> {
177        self.tool_calls
178            .iter()
179            .map(|call| call.name.as_str())
180            .collect()
181    }
182}