Skip to main content

session_core/
request_trace.rs

1//! Provider request traces for offline evals (Rust-only feature; the idea is reflection-oss/mirror's
2//! `*.model-calls.jsonl`, `mirror-agent/src/http/traced_model.rs`): with `ULTRON_REQUEST_TRACE=<file>` every
3//! conversation request (the root lane's and each subagent conversation's) appends one JSON line to `<file>` once
4//! its response arrived, holding only the DELTA from the previous request of the same lane:
5//!
6//! - `kind: "start"`: the lane's first request, all its messages;
7//! - `kind: "append"`: the previous request's messages are a prefix of this one's; `messages` are the ones added
8//!   after `base` of them;
9//! - `kind: "reset"`: the prefix changed (a prompt-cache miss): `reason` (`model`, `system`, `tools`,
10//!   `compaction`, `branch_summary`, `context_edit` for `ctx.forget` and other context edits, `truncated`,
11//!   `prefix_changed`), `at` (the first message that differs) and all the messages.
12//!
13//! Plus `session`, `lane`, `request` (per lane, from 1), `model` (`provider/id`), `usage` and `stopReason` of the
14//! response, and `timestamp`. [`check`] (`ultron trace check <file>`) reports every reset with its reason. The file
15//! is opened for appending (several sessions may share one) with mode 0600; a failed write disables the trace
16//! without touching the request.
17
18use ai::{AssistantMessage, Model, TranscriptContext};
19use serde_json::{Map, Value, json};
20use session_journal::context::{BRANCH_SUMMARY_PREFIX, COMPACTION_SUMMARY_PREFIX, CONTEXT_OMITTED_TOOL_RESULT_TEXT};
21use std::collections::HashMap;
22use std::io::Write;
23use std::path::Path;
24use std::sync::{Arc, Mutex, PoisonError};
25
26pub const TRACE_ENV: &str = "ULTRON_REQUEST_TRACE";
27
28/// How a request relates to the previous request of its lane.
29#[derive(Clone, Debug, PartialEq, Eq)]
30pub enum Delta {
31    Start,
32    Append { base: usize },
33    Reset { reason: &'static str, at: usize },
34}
35
36/// One message's user text (string content or text blocks), for the summary markers.
37fn user_text(message: &Value) -> Option<String> {
38    if message["role"] != "user" {
39        return None;
40    }
41    match &message["content"] {
42        Value::String(text) => Some(text.clone()),
43        Value::Array(blocks) => Some(
44            blocks
45                .iter()
46                .filter_map(|b| b["text"].as_str())
47                .collect::<Vec<_>>()
48                .join(""),
49        ),
50        _ => None,
51    }
52}
53
54fn without_tools(message: &Value) -> Value {
55    let mut out = message.clone();
56    if let Some(map) = out.as_object_mut() {
57        map.remove("toolsAdded");
58        map.remove("toolsRemoved");
59        map.remove("tools_added");
60        map.remove("tools_removed");
61    }
62    out
63}
64
65fn mentions_omission(message: &Value) -> bool {
66    message.to_string().contains(CONTEXT_OMITTED_TOOL_RESULT_TEXT)
67}
68
69/// The delta of `current` (on `model`) from `previous` (the lane's last request and its model).
70pub fn delta(previous: Option<(&str, &[Value])>, model: &str, current: &[Value]) -> Delta {
71    let Some((previous_model, previous)) = previous else {
72        return Delta::Start;
73    };
74    let at = previous
75        .iter()
76        .zip(current)
77        .position(|(p, c)| p != c)
78        .unwrap_or(previous.len().min(current.len()));
79    if previous_model != model {
80        return Delta::Reset { reason: "model", at };
81    }
82    if at == previous.len() {
83        return Delta::Append { base: at };
84    }
85    if at == current.len() {
86        return Delta::Reset {
87            reason: "truncated",
88            at,
89        };
90    }
91    let (p, c) = (&previous[at], &current[at]);
92    let reason = if p["role"] == "system" && c["role"] == "system" {
93        if without_tools(p) == without_tools(c) {
94            "tools"
95        } else {
96            "system"
97        }
98    } else if new_marker(previous, &current[at..], COMPACTION_SUMMARY_PREFIX) {
99        "compaction"
100    } else if new_marker(previous, &current[at..], BRANCH_SUMMARY_PREFIX) {
101        "branch_summary"
102    } else if (p["role"] == c["role"] && p["timestamp"] == c["timestamp"])
103        || previous[at + 1..].contains(c)
104        || mentions_omission(c)
105    {
106        // The same message with other content (`ctx.forget` stubs, context edits), or messages dropped.
107        "context_edit"
108    } else {
109        "prefix_changed"
110    };
111    Delta::Reset { reason, at }
112}
113
114/// Whether `tail` holds a summary message (`prefix`) that `previous` does not.
115fn new_marker(previous: &[Value], tail: &[Value], prefix: &str) -> bool {
116    tail.iter()
117        .any(|m| user_text(m).is_some_and(|t| t.starts_with(prefix)) && !previous.contains(m))
118}
119
120#[derive(Default)]
121struct LaneState {
122    requests: u64,
123    model: String,
124    messages: Vec<Value>,
125}
126
127struct TraceState {
128    file: Option<std::fs::File>,
129    lanes: HashMap<(String, String), LaneState>,
130}
131
132pub struct RequestTrace {
133    state: Mutex<TraceState>,
134}
135
136impl RequestTrace {
137    /// The trace `ULTRON_REQUEST_TRACE` names (`None` when unset or empty, or when the file cannot be opened).
138    pub fn from_env(var: &dyn Fn(&str) -> Option<String>) -> Option<Arc<RequestTrace>> {
139        let path = var(TRACE_ENV).filter(|p| !p.trim().is_empty())?;
140        RequestTrace::open(Path::new(&path)).ok().map(Arc::new)
141    }
142
143    pub fn open(path: &Path) -> std::io::Result<RequestTrace> {
144        use std::os::unix::fs::OpenOptionsExt;
145        let file = std::fs::OpenOptions::new()
146            .create(true)
147            .append(true)
148            .mode(0o600)
149            .open(path)?;
150        Ok(RequestTrace {
151            state: Mutex::new(TraceState {
152                file: Some(file),
153                lanes: HashMap::new(),
154            }),
155        })
156    }
157
158    /// Record one answered request of `lane` in `session`.
159    pub fn record(
160        &self,
161        session: &str,
162        lane: &str,
163        model: &Model,
164        context: &TranscriptContext,
165        response: &AssistantMessage,
166    ) {
167        let messages: Vec<Value> = context
168            .messages
169            .iter()
170            .map(|m| serde_json::to_value(m).unwrap_or(Value::Null))
171            .collect();
172        let model_key = format!("{}/{}", model.provider, model.id);
173        let mut state = self.state.lock().unwrap_or_else(PoisonError::into_inner);
174        let lane_state = state.lanes.entry((session.to_string(), lane.to_string())).or_default();
175        let previous = (lane_state.requests > 0).then_some((lane_state.model.as_str(), lane_state.messages.as_slice()));
176        let delta = delta(previous, &model_key, &messages);
177        lane_state.requests += 1;
178        let mut line = Map::new();
179        line.insert("session".into(), json!(session));
180        line.insert("lane".into(), json!(lane));
181        line.insert("request".into(), json!(lane_state.requests));
182        let shown = match &delta {
183            Delta::Start => {
184                line.insert("kind".into(), json!("start"));
185                line.insert("base".into(), json!(0));
186                &messages[..]
187            }
188            Delta::Append { base } => {
189                line.insert("kind".into(), json!("append"));
190                line.insert("base".into(), json!(base));
191                &messages[*base..]
192            }
193            Delta::Reset { reason, at } => {
194                line.insert("kind".into(), json!("reset"));
195                line.insert("reason".into(), json!(reason));
196                line.insert("at".into(), json!(at));
197                line.insert("base".into(), json!(0));
198                &messages[..]
199            }
200        };
201        line.insert("model".into(), json!(model_key));
202        line.insert("messages".into(), json!(shown));
203        line.insert(
204            "usage".into(),
205            serde_json::to_value(&response.usage).unwrap_or(Value::Null),
206        );
207        line.insert(
208            "stopReason".into(),
209            serde_json::to_value(response.stop_reason).unwrap_or(Value::Null),
210        );
211        line.insert("timestamp".into(), json!(jsrt::date::now_ms()));
212        lane_state.model = model_key;
213        lane_state.messages = messages;
214        let mut text = Value::Object(line).to_string();
215        text.push('\n');
216        // Never append after a possibly partial line: a failed write ends the trace.
217        let failed = state
218            .file
219            .as_mut()
220            .is_some_and(|file| file.write_all(text.as_bytes()).and_then(|()| file.flush()).is_err());
221        if failed {
222            state.file = None;
223        }
224    }
225}
226
227/// One reset [`check`] found.
228#[derive(Clone, Debug, PartialEq, Eq)]
229pub struct Reset {
230    /// 1-based line number.
231    pub line: usize,
232    pub session: String,
233    pub lane: String,
234    pub request: u64,
235    pub reason: String,
236    pub at: u64,
237}
238
239/// What [`check`] found in a trace.
240#[derive(Clone, Debug, PartialEq, Eq, Default)]
241pub struct Report {
242    pub requests: usize,
243    pub lanes: usize,
244    pub resets: Vec<Reset>,
245}
246
247/// Read a trace: every request line counted, every reset listed. Malformed lines are an error naming the line.
248pub fn check(text: &str) -> Result<Report, String> {
249    let mut report = Report::default();
250    let mut lanes = std::collections::HashSet::new();
251    for (index, line) in text.lines().enumerate() {
252        if line.trim().is_empty() {
253            continue;
254        }
255        let value: Value =
256            serde_json::from_str(line).map_err(|e| format!("line {}: not a trace record ({e})", index + 1))?;
257        let (Some(kind), Some(lane)) = (value["kind"].as_str(), value["lane"].as_str()) else {
258            return Err(format!("line {}: not a trace record (no kind or lane)", index + 1));
259        };
260        let session = value["session"].as_str().unwrap_or("").to_string();
261        lanes.insert((session.clone(), lane.to_string()));
262        report.requests += 1;
263        if kind == "reset" {
264            report.resets.push(Reset {
265                line: index + 1,
266                session,
267                lane: lane.to_string(),
268                request: value["request"].as_u64().unwrap_or(0),
269                reason: value["reason"].as_str().unwrap_or("unknown").to_string(),
270                at: value["at"].as_u64().unwrap_or(0),
271            });
272        }
273    }
274    report.lanes = lanes.len();
275    Ok(report)
276}
277
278#[cfg(test)]
279mod tests {
280    use super::*;
281
282    fn system(prompt: &str, tools: &[&str]) -> Value {
283        json!({"role": "system", "content": prompt, "toolsAdded": tools, "timestamp": 1})
284    }
285    fn user(text: &str, at: i64) -> Value {
286        json!({"role": "user", "content": text, "timestamp": at})
287    }
288    fn result(text: &str, at: i64) -> Value {
289        json!({"role": "toolResult", "toolCallId": "c1", "content": [{"type": "text", "text": text}], "timestamp": at})
290    }
291    const M: &str = "anthropic/claude-sonnet-4-5";
292
293    #[test]
294    fn session_core_request_trace_delta_start_and_append() {
295        let first = [system("S", &["rlm"]), user("a", 2)];
296        assert_eq!(delta(None, M, &first), Delta::Start);
297        let second = [system("S", &["rlm"]), user("a", 2), user("b", 3)];
298        assert_eq!(delta(Some((M, &first)), M, &second), Delta::Append { base: 2 });
299        // A repeated request (a retry) appends nothing.
300        assert_eq!(delta(Some((M, &second)), M, &second), Delta::Append { base: 3 });
301    }
302
303    #[test]
304    fn session_core_request_trace_delta_reset_reasons() {
305        let base = [
306            system("S", &["rlm"]),
307            user("a", 2),
308            result("long output", 3),
309            user("b", 4),
310        ];
311        let reset = |current: &[Value], model: &str| delta(Some((M, &base)), model, current);
312        assert_eq!(
313            reset(
314                &[base[0].clone(), base[1].clone(), base[2].clone(), base[3].clone()],
315                "openai/gpt"
316            ),
317            Delta::Reset { reason: "model", at: 4 }
318        );
319        assert_eq!(
320            reset(&[system("S2", &["rlm"]), base[1].clone()], M),
321            Delta::Reset {
322                reason: "system",
323                at: 0
324            }
325        );
326        assert_eq!(
327            reset(&[system("S", &["rlm", "x"]), base[1].clone()], M),
328            Delta::Reset { reason: "tools", at: 0 }
329        );
330        let summary = user(&format!("{COMPACTION_SUMMARY_PREFIX}gist</summary>"), 5);
331        assert_eq!(
332            reset(&[base[0].clone(), summary, base[3].clone()], M),
333            Delta::Reset {
334                reason: "compaction",
335                at: 1
336            }
337        );
338        let branch = user(&format!("{BRANCH_SUMMARY_PREFIX}gist</summary>"), 5);
339        assert_eq!(
340            reset(&[base[0].clone(), base[1].clone(), branch], M),
341            Delta::Reset {
342                reason: "branch_summary",
343                at: 2
344            }
345        );
346        // `ctx.forget`: the tool result's content replaced by the omission stub (same message, same timestamp).
347        assert_eq!(
348            reset(
349                &[
350                    base[0].clone(),
351                    base[1].clone(),
352                    result(CONTEXT_OMITTED_TOOL_RESULT_TEXT, 3),
353                    base[3].clone()
354                ],
355                M
356            ),
357            Delta::Reset {
358                reason: "context_edit",
359                at: 2
360            }
361        );
362        // A message dropped from the middle.
363        assert_eq!(
364            reset(&[base[0].clone(), base[1].clone(), base[3].clone()], M),
365            Delta::Reset {
366                reason: "context_edit",
367                at: 2
368            }
369        );
370        assert_eq!(
371            reset(&[base[0].clone(), base[1].clone()], M),
372            Delta::Reset {
373                reason: "truncated",
374                at: 2
375            }
376        );
377        assert_eq!(
378            reset(&[base[0].clone(), user("other", 9)], M),
379            Delta::Reset {
380                reason: "prefix_changed",
381                at: 1
382            }
383        );
384    }
385
386    #[test]
387    fn session_core_request_trace_check_lists_resets() {
388        let text = [
389            r#"{"session":"s","lane":"main","request":1,"kind":"start","base":0}"#,
390            r#"{"session":"s","lane":"main","request":2,"kind":"append","base":2}"#,
391            "",
392            r#"{"session":"s","lane":"conversation:2","request":1,"kind":"start","base":0}"#,
393            r#"{"session":"s","lane":"main","request":3,"kind":"reset","reason":"compaction","at":1}"#,
394        ]
395        .join("\n");
396        let report = check(&text).unwrap();
397        assert_eq!(report.requests, 4);
398        assert_eq!(report.lanes, 2);
399        assert_eq!(
400            report.resets,
401            vec![Reset {
402                line: 5,
403                session: "s".into(),
404                lane: "main".into(),
405                request: 3,
406                reason: "compaction".into(),
407                at: 1,
408            }]
409        );
410        assert_eq!(
411            check("{\"kind\":1}").unwrap_err(),
412            "line 1: not a trace record (no kind or lane)"
413        );
414        assert!(check("nope").unwrap_err().starts_with("line 1: not a trace record ("));
415    }
416
417    #[test]
418    fn session_core_request_trace_opens_only_when_named() {
419        assert!(RequestTrace::from_env(&|_| None).is_none());
420        assert!(RequestTrace::from_env(&|_| Some("  ".into())).is_none());
421        let dir = tempfile::tempdir().unwrap();
422        let path = dir.path().join("t.jsonl");
423        let wanted = path.display().to_string();
424        assert!(RequestTrace::from_env(&|k| (k == TRACE_ENV).then(|| wanted.clone())).is_some());
425        assert!(path.exists());
426    }
427}