eval_magic/adapters/
codex_transcript.rs

1//! Codex transcript (event-stream) parsing.
2//!
3//! Codex emits a JSONL event stream;
4//! `item.completed` events whose item type is not an agent message / reasoning /
5//! plan update become ordered [`ToolInvocation`]s. Produces the same
6//! [`TranscriptSummary`] shape as the Claude adapter, but with Codex's token
7//! accounting (excludes cached input tokens).
8
9use crate::adapters::claude_code_transcript::TranscriptSummary;
10use crate::core::ToolInvocation;
11use serde_json::{Map, Value};
12use std::fs;
13use std::io;
14use std::path::Path;
15
16const NON_TOOL_ITEMS: [&str; 3] = ["agent_message", "reasoning", "plan_update"];
17const ARG_OMIT_KEYS: [&str; 6] = ["id", "type", "status", "output", "result", "error"];
18
19fn read_events(jsonl_path: &Path) -> io::Result<Vec<Value>> {
20    let raw = fs::read_to_string(jsonl_path)?;
21    let mut out = Vec::new();
22    for line in raw.split('\n') {
23        if line.is_empty() {
24            continue;
25        }
26        // Skip malformed lines rather than failing the whole parse.
27        if let Ok(v) = serde_json::from_str::<Value>(line) {
28            out.push(v);
29        }
30    }
31    Ok(out)
32}
33
34fn stringify_value(v: &Value) -> String {
35    match v {
36        Value::String(s) => s.clone(),
37        other => serde_json::to_string(other).unwrap_or_default(),
38    }
39}
40
41/// First present of `output`, `result`, `error` (a present-but-null value still
42/// counts), stringified.
43fn maybe_result(item: &Map<String, Value>) -> Option<String> {
44    ["output", "result", "error"]
45        .into_iter()
46        .find_map(|k| item.get(k).map(stringify_value))
47}
48
49/// All item keys except the structural ones, preserving JSON key order; `None`
50/// when nothing remains.
51fn item_args(item: &Map<String, Value>) -> Option<Value> {
52    let mut args = Map::new();
53    for (key, value) in item {
54        if ARG_OMIT_KEYS.contains(&key.as_str()) {
55            continue;
56        }
57        args.insert(key.clone(), value.clone());
58    }
59    if args.is_empty() {
60        None
61    } else {
62        Some(Value::Object(args))
63    }
64}
65
66fn extract_invocations(records: &[Value]) -> Vec<ToolInvocation> {
67    let mut invocations = Vec::new();
68    for record in records {
69        if record.get("type").and_then(Value::as_str) != Some("item.completed") {
70            continue;
71        }
72        let Some(item) = record.get("item").and_then(Value::as_object) else {
73            continue;
74        };
75        let Some(item_type) = item.get("type").and_then(Value::as_str) else {
76            continue;
77        };
78        if NON_TOOL_ITEMS.contains(&item_type) {
79            continue;
80        }
81        let ordinal = invocations.len() as u32;
82        invocations.push(ToolInvocation {
83            name: item_type.to_string(),
84            args: item_args(item),
85            result: maybe_result(item).map(Value::String),
86            ordinal,
87        });
88    }
89    invocations
90}
91
92/// Parse a Codex event stream into ordered tool invocations.
93pub fn parse_codex_events(jsonl_path: &Path) -> io::Result<Vec<ToolInvocation>> {
94    Ok(extract_invocations(&read_events(jsonl_path)?))
95}
96
97fn parse_millis(s: &str) -> Option<i64> {
98    chrono::DateTime::parse_from_rfc3339(s)
99        .ok()
100        .map(|dt| dt.timestamp_millis())
101}
102
103/// Parse a Codex event stream into a full [`TranscriptSummary`].
104pub fn parse_codex_events_full(jsonl_path: &Path) -> io::Result<TranscriptSummary> {
105    let records = read_events(jsonl_path)?;
106
107    let mut first_ts: Option<i64> = None;
108    let mut last_ts: Option<i64> = None;
109    let mut timestamp_count = 0usize;
110    let mut final_text: Option<String> = None;
111    let mut total_tokens: Option<i64> = None;
112
113    for record in &records {
114        if let Some(ts_str) = record.get("timestamp").and_then(Value::as_str)
115            && let Some(ts) = parse_millis(ts_str)
116        {
117            if first_ts.is_none() {
118                first_ts = Some(ts);
119            }
120            last_ts = Some(ts);
121            timestamp_count += 1;
122        }
123
124        let rtype = record.get("type").and_then(Value::as_str);
125
126        if rtype == Some("item.completed")
127            && let Some(item) = record.get("item").and_then(Value::as_object)
128            && item.get("type").and_then(Value::as_str) == Some("agent_message")
129            && let Some(text) = item.get("text").and_then(Value::as_str)
130        {
131            final_text = Some(text.to_string());
132        }
133
134        if rtype == Some("turn.completed")
135            && let Some(usage) = record.get("usage").and_then(Value::as_object)
136        {
137            let get = |k: &str| usage.get(k).and_then(Value::as_i64).unwrap_or(0);
138            let sum = get("input_tokens") + get("output_tokens") + get("reasoning_output_tokens");
139            total_tokens = Some(total_tokens.unwrap_or(0) + sum);
140        }
141    }
142
143    let duration_ms = match (first_ts, last_ts) {
144        (Some(f), Some(l)) if timestamp_count >= 2 => Some(l - f),
145        _ => None,
146    };
147
148    Ok(TranscriptSummary {
149        tool_invocations: extract_invocations(&records),
150        total_tokens,
151        duration_ms,
152        final_text,
153    })
154}
155
156#[cfg(test)]
157mod tests {
158    use super::*;
159    use crate::core::ToolInvocation;
160    use serde_json::{Value, json};
161    use std::fs;
162    use std::path::Path;
163    use tempfile::TempDir;
164
165    fn write_jsonl(path: &Path, lines: &[Value]) {
166        let body = lines
167            .iter()
168            .map(|l| l.to_string())
169            .collect::<Vec<_>>()
170            .join("\n");
171        fs::write(path, format!("{body}\n")).unwrap();
172    }
173
174    #[test]
175    fn extracts_completed_tool_items_with_ordinals_args_and_results() {
176        let dir = TempDir::new().unwrap();
177        let path = dir.path().join("items.jsonl");
178        write_jsonl(
179            &path,
180            &[
181                json!({"type": "item.started", "timestamp": "2026-06-07T10:00:00.000Z", "item": {"id": "item_1", "type": "command_execution", "command": "bash -lc 'bun test'", "status": "in_progress"}}),
182                json!({"type": "item.completed", "timestamp": "2026-06-07T10:00:02.000Z", "item": {"id": "item_1", "type": "command_execution", "command": "bash -lc 'bun test'", "output": "2 pass\n0 fail", "status": "completed"}}),
183                json!({"type": "item.completed", "item": {"id": "item_2", "type": "file_change", "path": "src/app.ts", "status": "completed"}}),
184                json!({"type": "item.completed", "item": {"id": "item_3", "type": "agent_message", "text": "Done."}}),
185            ],
186        );
187
188        let result = parse_codex_events(&path).unwrap();
189        assert_eq!(result.len(), 2);
190        assert_eq!(
191            result[0],
192            ToolInvocation {
193                name: "command_execution".into(),
194                args: Some(json!({"command": "bash -lc 'bun test'"})),
195                result: Some(Value::String("2 pass\n0 fail".into())),
196                ordinal: 0,
197            }
198        );
199        assert_eq!(
200            result[1],
201            ToolInvocation {
202                name: "file_change".into(),
203                args: Some(json!({"path": "src/app.ts"})),
204                result: None,
205                ordinal: 1,
206            }
207        );
208    }
209
210    #[test]
211    fn skips_malformed_jsonl_lines() {
212        let dir = TempDir::new().unwrap();
213        let path = dir.path().join("malformed.jsonl");
214        let good = json!({"type": "item.completed", "item": {"id": "item_1", "type": "web_search", "query": "codex exec json"}});
215        fs::write(&path, format!("{good}\nnot valid json\n")).unwrap();
216        assert_eq!(
217            parse_codex_events(&path).unwrap(),
218            vec![ToolInvocation {
219                name: "web_search".into(),
220                args: Some(json!({"query": "codex exec json"})),
221                result: None,
222                ordinal: 0,
223            }]
224        );
225    }
226
227    #[test]
228    fn preserves_text_fields_on_non_message_tool_items() {
229        let dir = TempDir::new().unwrap();
230        let path = dir.path().join("tool-text.jsonl");
231        write_jsonl(
232            &path,
233            &[
234                json!({"type": "item.completed", "item": {"id": "item_1", "type": "web_search", "query": "codex events", "text": "search summary"}}),
235            ],
236        );
237        assert_eq!(
238            parse_codex_events(&path).unwrap(),
239            vec![ToolInvocation {
240                name: "web_search".into(),
241                args: Some(json!({"query": "codex events", "text": "search summary"})),
242                result: None,
243                ordinal: 0,
244            }]
245        );
246    }
247
248    #[test]
249    fn does_not_treat_agent_messages_reasoning_or_plan_updates_as_tools() {
250        let dir = TempDir::new().unwrap();
251        let path = dir.path().join("non-tools.jsonl");
252        write_jsonl(
253            &path,
254            &[
255                json!({"type": "item.completed", "item": {"id": "a", "type": "agent_message"}}),
256                json!({"type": "item.completed", "item": {"id": "b", "type": "reasoning"}}),
257                json!({"type": "item.completed", "item": {"id": "c", "type": "plan_update"}}),
258            ],
259        );
260        assert_eq!(parse_codex_events(&path).unwrap(), vec![]);
261    }
262
263    #[test]
264    fn extracts_invocations_last_agent_text_usage_and_duration() {
265        let dir = TempDir::new().unwrap();
266        let path = dir.path().join("full.jsonl");
267        write_jsonl(
268            &path,
269            &[
270                json!({"type": "thread.started", "timestamp": "2026-06-07T10:00:00.000Z"}),
271                json!({"type": "item.completed", "timestamp": "2026-06-07T10:00:03.000Z", "item": {"id": "item_1", "type": "command_execution", "command": "ls", "output": "README.md"}}),
272                json!({"type": "item.completed", "timestamp": "2026-06-07T10:00:04.000Z", "item": {"id": "item_2", "type": "agent_message", "text": "First."}}),
273                json!({"type": "item.completed", "timestamp": "2026-06-07T10:00:05.000Z", "item": {"id": "item_3", "type": "agent_message", "text": "Final."}}),
274                json!({"type": "turn.completed", "timestamp": "2026-06-07T10:00:10.000Z", "usage": {"input_tokens": 100, "cached_input_tokens": 75, "output_tokens": 20, "reasoning_output_tokens": 5}}),
275            ],
276        );
277
278        let full = parse_codex_events_full(&path).unwrap();
279        assert_eq!(
280            full.tool_invocations,
281            vec![ToolInvocation {
282                name: "command_execution".into(),
283                args: Some(json!({"command": "ls"})),
284                result: Some(Value::String("README.md".into())),
285                ordinal: 0,
286            }]
287        );
288        assert_eq!(full.final_text, Some("Final.".into()));
289        assert_eq!(full.total_tokens, Some(125)); // cached input excluded
290        assert_eq!(full.duration_ms, Some(10_000));
291    }
292
293    #[test]
294    fn returns_null_usage_and_duration_when_sparse() {
295        let dir = TempDir::new().unwrap();
296        let path = dir.path().join("sparse.jsonl");
297        write_jsonl(
298            &path,
299            &[
300                json!({"type": "item.completed", "timestamp": "2026-06-07T10:00:00.000Z", "item": {"id": "item_1", "type": "agent_message", "text": "Done."}}),
301            ],
302        );
303        let full = parse_codex_events_full(&path).unwrap();
304        assert_eq!(full.final_text, Some("Done.".into()));
305        assert_eq!(full.total_tokens, None);
306        assert_eq!(full.duration_ms, None);
307    }
308}
eval_magic/adapters/codex_transcript.rs

eval_magic/adapters/
codex_transcript.rs
