Skip to main content

salvor_runtime/
progress.rs

1//! Live progress: turning each persisted event into a one-line `tracing`
2//! record, emitted the instant the event becomes durable.
3//!
4//! Live emission replaces walking the log *after* a drive returned: the
5//! runtime is the only layer that knows a step
6//! happened as it happens, so it is the layer that emits the step. Every
7//! [`crate::RunCtx`] persist calls [`emit_step`] right after the store append
8//! succeeds; a subscriber installed by the binary (the CLI writes it to
9//! stderr) renders the record. Replayed events are answered from the log and
10//! never re-persisted, so they never re-emit: a resumed or recovered run
11//! streams only its genuinely new activity.
12//!
13//! # What is emitted, and what is withheld
14//!
15//! Each record carries the correlation fields an operator needs (`run_id` and
16//! `seq`) and a scannable message of the form `"<kind> <detail>"`. The detail
17//! is the same one-line summary the `history` command prints, produced by the
18//! shared [`event_kind`] / [`event_detail`] functions so a step reads the
19//! same whether you watch it live or inspect it later.
20//!
21//! The detail deliberately does **not** carry full payloads. Inputs, outputs,
22//! resume values, and error messages are truncated (see [`truncate_str`]) so
23//! a model's raw output or a tool's raw arguments never land in the progress
24//! stream in full. The untruncated form lives only in the event log, reachable
25//! through `salvor history --json`.
26
27use salvor_core::{BudgetKind, Event, RunId, SequenceNumber};
28use time::OffsetDateTime;
29
30use crate::wire::{decode_failure, decode_suspension};
31
32/// Emits one info-level progress record for a freshly persisted event.
33///
34/// Called from the [`crate::RunCtx`] IO edge the moment an append returns
35/// `Ok`, so the record reaches the subscriber while the run is still driving.
36/// The `run_id` and `seq` fields correlate the line to its place in the log;
37/// the message is `"<kind> <detail>"` with the payload truncated.
38pub(crate) fn emit_step(run_id: RunId, seq: SequenceNumber, event: &Event) {
39    tracing::info!(
40        run_id = %run_id.as_uuid(),
41        seq = seq.get(),
42        "{} {}",
43        event_kind(event),
44        event_detail(event),
45    );
46}
47
48/// The stable `kind` label for one event, matching the enum variant name so
49/// it reads the same as the wire form's `kind` tag.
50#[must_use]
51pub fn event_kind(event: &Event) -> &'static str {
52    match event {
53        Event::RunStarted { .. } => "RunStarted",
54        Event::ModelCallRequested { .. } => "ModelCallRequested",
55        Event::ModelCallCompleted { .. } => "ModelCallCompleted",
56        Event::ToolCallRequested { .. } => "ToolCallRequested",
57        Event::ToolCallCompleted { .. } => "ToolCallCompleted",
58        Event::NowObserved { .. } => "NowObserved",
59        Event::RandomObserved { .. } => "RandomObserved",
60        Event::Suspended { .. } => "Suspended",
61        Event::Resumed { .. } => "Resumed",
62        Event::BudgetExceeded { .. } => "BudgetExceeded",
63        Event::RunCompleted { .. } => "RunCompleted",
64        Event::RunFailed { .. } => "RunFailed",
65        Event::RunAbandoned { .. } => "RunAbandoned",
66        Event::GraphRunStarted { .. } => "GraphRunStarted",
67        Event::NodeEntered { .. } => "NodeEntered",
68        Event::NodeExited { .. } => "NodeExited",
69        Event::NodeSkipped { .. } => "NodeSkipped",
70        Event::BranchTaken { .. } => "BranchTaken",
71        Event::MapFannedOut { .. } => "MapFannedOut",
72        Event::MapIterationStarted { .. } => "MapIterationStarted",
73        Event::MapIterationJoined { .. } => "MapIterationJoined",
74        Event::FoldIterationStarted { .. } => "FoldIterationStarted",
75        Event::FoldIterationJoined { .. } => "FoldIterationJoined",
76        Event::FoldConverged { .. } => "FoldConverged",
77    }
78}
79
80/// The informative payload of one event, rendered as a single line. Picks the
81/// fields that matter per kind: a tool call shows its name and effect, a model
82/// completion its token usage, a suspension its reason. Hashes are shortened
83/// and every payload is truncated, so no full input, output, or error text
84/// reaches the progress stream; `salvor history --json` is the escape hatch for
85/// the untruncated envelope.
86#[must_use]
87pub fn event_detail(event: &Event) -> String {
88    match event {
89        Event::RunStarted {
90            agent_def_hash,
91            input,
92            ..
93        } => format!(
94            "agent {} input {}",
95            short_hash(agent_def_hash),
96            truncate_json(input)
97        ),
98        Event::ModelCallRequested { request_hash, .. } => {
99            format!("request {}", short_hash(request_hash))
100        }
101        Event::ModelCallCompleted { usage, .. } => format!(
102            "usage in {} out {}",
103            usage.input_tokens, usage.output_tokens
104        ),
105        Event::ToolCallRequested {
106            tool,
107            input,
108            effect,
109            idempotency_key,
110            ..
111        } => {
112            let key = idempotency_key
113                .as_deref()
114                .map_or_else(String::new, |k| format!(" key {k}"));
115            format!("{tool} [{effect:?}]{key} input {}", truncate_json(input))
116        }
117        Event::ToolCallCompleted { output, .. } => {
118            if let Some(suspension) = decode_suspension(output) {
119                format!("suspends: {}", suspension.reason)
120            } else if let Some(failure) = decode_failure(output) {
121                format!(
122                    "error ({}, {} attempt(s)): {}",
123                    failure.kind.as_str(),
124                    failure.attempts,
125                    truncate_str(&failure.message)
126                )
127            } else {
128                format!("output {}", truncate_json(output))
129            }
130        }
131        Event::NowObserved { now } => format_ts(*now),
132        Event::RandomObserved { value } => format!("value {value}"),
133        Event::Suspended { reason, .. } => format!("reason: {reason}"),
134        Event::Resumed { input } => format!("input {}", truncate_json(input)),
135        Event::BudgetExceeded { budget, observed } => {
136            format!(
137                "{} limit {}, observed {}",
138                budget_label(budget.kind),
139                fmt_num(budget.limit),
140                fmt_num(*observed)
141            )
142        }
143        Event::RunCompleted { output } => format!("output {}", truncate_json(output)),
144        Event::RunFailed { error } => format!("error: {}", truncate_str(error)),
145        Event::RunAbandoned {
146            reason,
147            unresolved_write,
148        } => {
149            let why = reason
150                .as_deref()
151                .map_or_else(|| "no reason given".to_owned(), truncate_str);
152            match unresolved_write {
153                Some(write) => format!(
154                    "abandoned: {why} (unresolved write at seq {}, tool {})",
155                    write.seq.get(),
156                    write.tool
157                ),
158                None => format!("abandoned: {why}"),
159            }
160        }
161        Event::GraphRunStarted {
162            graph_hash, input, ..
163        } => format!(
164            "graph {} input {}",
165            short_hash(graph_hash),
166            truncate_json(input)
167        ),
168        Event::NodeEntered { node } => format!("enter {node}"),
169        Event::NodeExited { node } => format!("exit {node}"),
170        Event::NodeSkipped { node, reason } => format!("skip {node}: {}", truncate_str(reason)),
171        Event::BranchTaken { node, case } => format!("branch {node} -> {case}"),
172        Event::MapFannedOut { node, items } => {
173            format!("map {node} fan-out {}", truncate_json(items))
174        }
175        Event::MapIterationStarted {
176            node,
177            index,
178            child_run,
179        } => format!("map {node}[{index}] child {}", short_hash(child_run)),
180        Event::MapIterationJoined { node, index } => format!("map {node}[{index}] joined"),
181        Event::FoldIterationStarted { node, index } => format!("fold {node}[{index}] started"),
182        Event::FoldIterationJoined { node, index } => format!("fold {node}[{index}] joined"),
183        Event::FoldConverged {
184            node,
185            winner_index,
186            reason,
187        } => format!(
188            "fold {node} converged on [{winner_index}]: {}",
189            truncate_str(reason)
190        ),
191    }
192}
193
194/// Shortens a `sha256:...` hash to its prefix and the first seven hex digits,
195/// so a line names a request without a 64-character wall of hex.
196fn short_hash(hash: &str) -> String {
197    match hash.split_once(':') {
198        Some((scheme, hex)) => {
199            let head: String = hex.chars().take(7).collect();
200            if hex.len() > 7 {
201                format!("{scheme}:{head}\u{2026}")
202            } else {
203                format!("{scheme}:{hex}")
204            }
205        }
206        None => hash.chars().take(12).collect(),
207    }
208}
209
210/// A human word for a budget dimension.
211fn budget_label(kind: BudgetKind) -> &'static str {
212    match kind {
213        BudgetKind::Steps => "steps",
214        BudgetKind::Tokens => "tokens",
215        BudgetKind::CostUsd => "cost_usd",
216        BudgetKind::WallTime => "wall_time",
217    }
218}
219
220/// Formats an `f64` budget figure without a needless `.0` when it is integral.
221/// Steps and tokens are whole numbers even though every budget dimension rides
222/// the wire as a float; the integral cutoff stays inside the range where an
223/// `f64` holds integers exactly (see the `Budget` docs).
224fn fmt_num(value: f64) -> String {
225    if value.fract() == 0.0 && value.abs() < 1e15 {
226        format!("{}", value as i64)
227    } else {
228        format!("{value}")
229    }
230}
231
232/// Formats a timestamp as `YYYY-MM-DD HH:MM:SSZ` from its components, avoiding
233/// a dependency on the `time` crate's optional `formatting` feature.
234fn format_ts(ts: OffsetDateTime) -> String {
235    let utc = ts.to_offset(time::UtcOffset::UTC);
236    format!(
237        "{:04}-{:02}-{:02} {:02}:{:02}:{:02}Z",
238        utc.year(),
239        u8::from(utc.month()),
240        utc.day(),
241        utc.hour(),
242        utc.minute(),
243        utc.second(),
244    )
245}
246
247/// Compact one-line JSON, truncated so a payload never blows out a line and
248/// never streams in full.
249fn truncate_json(value: &serde_json::Value) -> String {
250    truncate_str(&value.to_string())
251}
252
253/// Truncates a string to a scannable length with an ellipsis, so no full
254/// payload or error message reaches the progress stream.
255fn truncate_str(text: &str) -> String {
256    const CAP: usize = 80;
257    if text.chars().count() > CAP {
258        let head: String = text.chars().take(CAP).collect();
259        format!("{head}\u{2026}")
260    } else {
261        text.to_owned()
262    }
263}
264
265#[cfg(test)]
266mod tests {
267    use super::*;
268    use serde_json::json;
269
270    /// A long input is truncated, so a raw payload never reaches the stream in
271    /// full: the detail line stays capped even for a large value.
272    #[test]
273    fn detail_truncates_long_payloads() {
274        let big = "x".repeat(500);
275        let detail = event_detail(&Event::RunStarted {
276            agent_def_hash: "sha256:abcdef0123456789".into(),
277            input: json!({ "prompt": big }),
278            labels: None,
279        });
280        assert!(detail.contains('\u{2026}'), "detail should be truncated");
281        assert!(
282            detail.chars().count() < 200,
283            "truncated detail stays short: {} chars",
284            detail.chars().count()
285        );
286        // The short hash appears; the full 64-hex form does not.
287        assert!(detail.contains("sha256:abcdef0"));
288    }
289
290    /// Every kind maps to its variant name, matching the wire tag.
291    #[test]
292    fn kind_matches_variant_name() {
293        assert_eq!(
294            event_kind(&Event::RunCompleted { output: json!(1) }),
295            "RunCompleted"
296        );
297        assert_eq!(
298            event_kind(&Event::RandomObserved { value: 7 }),
299            "RandomObserved"
300        );
301    }
302}