eval_magic/adapters/codex/
transcript.rs

1//! Codex transcript (event-stream) parsing.
2//!
3//! Codex emits a JSONL event stream;
4//! `item.completed` events whose item type is not an agent message / reasoning /
5//! plan update become ordered [`ToolInvocation`]s. Produces the same
6//! [`TranscriptSummary`] shape as the Claude adapter, but with Codex's token
7//! accounting (excludes cached input tokens).
8
9use crate::adapters::TranscriptSummary;
10use crate::adapters::transcript::TranscriptEvent;
11use crate::adapters::transcript::read_jsonl;
12use crate::core::ToolInvocation;
13use serde_json::{Map, Value};
14use std::io;
15use std::path::Path;
16
17const NON_TOOL_ITEMS: [&str; 3] = ["agent_message", "reasoning", "plan_update"];
18const ARG_OMIT_KEYS: [&str; 6] = ["id", "type", "status", "output", "result", "error"];
19
20fn stringify_value(v: &Value) -> String {
21    match v {
22        Value::String(s) => s.clone(),
23        other => serde_json::to_string(other).unwrap_or_default(),
24    }
25}
26
27/// First present of `output`, `result`, `error` (a present-but-null value still
28/// counts), stringified.
29fn maybe_result(item: &Map<String, Value>) -> Option<String> {
30    ["output", "result", "error"]
31        .into_iter()
32        .find_map(|k| item.get(k).map(stringify_value))
33}
34
35/// All item keys except the structural ones, preserving JSON key order; `None`
36/// when nothing remains.
37fn item_args(item: &Map<String, Value>) -> Option<Value> {
38    let mut args = Map::new();
39    for (key, value) in item {
40        if ARG_OMIT_KEYS.contains(&key.as_str()) {
41            continue;
42        }
43        args.insert(key.clone(), value.clone());
44    }
45    if args.is_empty() {
46        None
47    } else {
48        Some(Value::Object(args))
49    }
50}
51
52fn extract_invocations(records: &[Value]) -> Vec<ToolInvocation> {
53    let mut invocations = Vec::new();
54    for record in records {
55        if record.get("type").and_then(Value::as_str) != Some("item.completed") {
56            continue;
57        }
58        let Some(item) = record.get("item").and_then(Value::as_object) else {
59            continue;
60        };
61        let Some(item_type) = item.get("type").and_then(Value::as_str) else {
62            continue;
63        };
64        if NON_TOOL_ITEMS.contains(&item_type) {
65            continue;
66        }
67        let ordinal = invocations.len() as u32;
68        invocations.push(ToolInvocation {
69            name: item_type.to_string(),
70            args: item_args(item),
71            result: maybe_result(item).map(Value::String),
72            ordinal,
73        });
74    }
75    invocations
76}
77
78fn extract_events(records: &[Value]) -> Vec<TranscriptEvent> {
79    let mut events = Vec::new();
80    for record in records {
81        if record.get("type").and_then(Value::as_str) != Some("item.completed") {
82            continue;
83        }
84        let Some(item) = record.get("item").and_then(Value::as_object) else {
85            continue;
86        };
87        let Some(item_type) = item.get("type").and_then(Value::as_str) else {
88            continue;
89        };
90        let ordinal = events.len() as u32;
91        if item_type == "agent_message" {
92            if let Some(text) = item.get("text").and_then(Value::as_str) {
93                events.push(TranscriptEvent::AssistantMessage {
94                    ordinal,
95                    text: text.to_string(),
96                });
97            }
98        } else if !NON_TOOL_ITEMS.contains(&item_type) {
99            events.push(TranscriptEvent::ToolInvocation {
100                ordinal,
101                name: item_type.to_string(),
102                args: item_args(item),
103                result: maybe_result(item).map(Value::String),
104            });
105        }
106    }
107    events
108}
109
110/// Parse a Codex event stream into ordered tool invocations.
111pub fn parse_codex_events(jsonl_path: &Path) -> io::Result<Vec<ToolInvocation>> {
112    Ok(extract_invocations(&read_jsonl::<Value>(jsonl_path)?))
113}
114
115fn parse_millis(s: &str) -> Option<i64> {
116    chrono::DateTime::parse_from_rfc3339(s)
117        .ok()
118        .map(|dt| dt.timestamp_millis())
119}
120
121/// Parse a Codex event stream into a full [`TranscriptSummary`].
122pub fn parse_codex_events_full(jsonl_path: &Path) -> io::Result<TranscriptSummary> {
123    let records = read_jsonl::<Value>(jsonl_path)?;
124
125    let mut first_ts: Option<i64> = None;
126    let mut last_ts: Option<i64> = None;
127    let mut timestamp_count = 0usize;
128    let mut final_text: Option<String> = None;
129    let mut total_tokens: Option<i64> = None;
130    let session_id = records.iter().find_map(|record| {
131        (record.get("type").and_then(Value::as_str) == Some("thread.started"))
132            .then(|| record.get("thread_id").and_then(Value::as_str))
133            .flatten()
134            .map(str::to_string)
135    });
136
137    for record in &records {
138        if let Some(ts_str) = record.get("timestamp").and_then(Value::as_str)
139            && let Some(ts) = parse_millis(ts_str)
140        {
141            if first_ts.is_none() {
142                first_ts = Some(ts);
143            }
144            last_ts = Some(ts);
145            timestamp_count += 1;
146        }
147
148        let rtype = record.get("type").and_then(Value::as_str);
149
150        if rtype == Some("item.completed")
151            && let Some(item) = record.get("item").and_then(Value::as_object)
152            && item.get("type").and_then(Value::as_str) == Some("agent_message")
153            && let Some(text) = item.get("text").and_then(Value::as_str)
154        {
155            final_text = Some(text.to_string());
156        }
157
158        if rtype == Some("turn.completed")
159            && let Some(usage) = record.get("usage").and_then(Value::as_object)
160        {
161            let get = |k: &str| usage.get(k).and_then(Value::as_i64).unwrap_or(0);
162            let sum = get("input_tokens") + get("output_tokens") + get("reasoning_output_tokens");
163            total_tokens = Some(total_tokens.unwrap_or(0) + sum);
164        }
165    }
166
167    let duration_ms = match (first_ts, last_ts) {
168        (Some(f), Some(l)) if timestamp_count >= 2 => Some(l - f),
169        _ => None,
170    };
171
172    Ok(TranscriptSummary {
173        tool_invocations: extract_invocations(&records),
174        events: extract_events(&records),
175        session_id,
176        total_tokens,
177        duration_ms,
178        final_text,
179    })
180}
181
182#[cfg(test)]
183mod tests {
184    use super::*;
185    use crate::core::ToolInvocation;
186    use serde_json::{Value, json};
187    use std::fs;
188    use std::path::Path;
189    use tempfile::TempDir;
190
191    fn write_jsonl(path: &Path, lines: &[Value]) {
192        let body = lines
193            .iter()
194            .map(|l| l.to_string())
195            .collect::<Vec<_>>()
196            .join("\n");
197        fs::write(path, format!("{body}\n")).unwrap();
198    }
199
200    #[test]
201    fn extracts_completed_tool_items_with_ordinals_args_and_results() {
202        let dir = TempDir::new().unwrap();
203        let path = dir.path().join("items.jsonl");
204        write_jsonl(
205            &path,
206            &[
207                json!({"type": "item.started", "timestamp": "2026-06-07T10:00:00.000Z", "item": {"id": "item_1", "type": "command_execution", "command": "bash -lc 'bun test'", "status": "in_progress"}}),
208                json!({"type": "item.completed", "timestamp": "2026-06-07T10:00:02.000Z", "item": {"id": "item_1", "type": "command_execution", "command": "bash -lc 'bun test'", "output": "2 pass\n0 fail", "status": "completed"}}),
209                json!({"type": "item.completed", "item": {"id": "item_2", "type": "file_change", "path": "src/app.ts", "status": "completed"}}),
210                json!({"type": "item.completed", "item": {"id": "item_3", "type": "agent_message", "text": "Done."}}),
211            ],
212        );
213
214        let result = parse_codex_events(&path).unwrap();
215        assert_eq!(result.len(), 2);
216        assert_eq!(
217            result[0],
218            ToolInvocation {
219                name: "command_execution".into(),
220                args: Some(json!({"command": "bash -lc 'bun test'"})),
221                result: Some(Value::String("2 pass\n0 fail".into())),
222                ordinal: 0,
223            }
224        );
225        assert_eq!(
226            result[1],
227            ToolInvocation {
228                name: "file_change".into(),
229                args: Some(json!({"path": "src/app.ts"})),
230                result: None,
231                ordinal: 1,
232            }
233        );
234    }
235
236    #[test]
237    fn skips_malformed_jsonl_lines() {
238        let dir = TempDir::new().unwrap();
239        let path = dir.path().join("malformed.jsonl");
240        let good = json!({"type": "item.completed", "item": {"id": "item_1", "type": "web_search", "query": "codex exec json"}});
241        fs::write(&path, format!("{good}\nnot valid json\n")).unwrap();
242        assert_eq!(
243            parse_codex_events(&path).unwrap(),
244            vec![ToolInvocation {
245                name: "web_search".into(),
246                args: Some(json!({"query": "codex exec json"})),
247                result: None,
248                ordinal: 0,
249            }]
250        );
251    }
252
253    #[test]
254    fn preserves_text_fields_on_non_message_tool_items() {
255        let dir = TempDir::new().unwrap();
256        let path = dir.path().join("tool-text.jsonl");
257        write_jsonl(
258            &path,
259            &[
260                json!({"type": "item.completed", "item": {"id": "item_1", "type": "web_search", "query": "codex events", "text": "search summary"}}),
261            ],
262        );
263        assert_eq!(
264            parse_codex_events(&path).unwrap(),
265            vec![ToolInvocation {
266                name: "web_search".into(),
267                args: Some(json!({"query": "codex events", "text": "search summary"})),
268                result: None,
269                ordinal: 0,
270            }]
271        );
272    }
273
274    #[test]
275    fn does_not_treat_agent_messages_reasoning_or_plan_updates_as_tools() {
276        let dir = TempDir::new().unwrap();
277        let path = dir.path().join("non-tools.jsonl");
278        write_jsonl(
279            &path,
280            &[
281                json!({"type": "item.completed", "item": {"id": "a", "type": "agent_message"}}),
282                json!({"type": "item.completed", "item": {"id": "b", "type": "reasoning"}}),
283                json!({"type": "item.completed", "item": {"id": "c", "type": "plan_update"}}),
284            ],
285        );
286        assert_eq!(parse_codex_events(&path).unwrap(), vec![]);
287    }
288
289    #[test]
290    fn extracts_invocations_last_agent_text_usage_and_duration() {
291        let dir = TempDir::new().unwrap();
292        let path = dir.path().join("full.jsonl");
293        write_jsonl(
294            &path,
295            &[
296                json!({"type": "thread.started", "thread_id": "thread-codex-1", "timestamp": "2026-06-07T10:00:00.000Z"}),
297                json!({"type": "item.completed", "timestamp": "2026-06-07T10:00:03.000Z", "item": {"id": "item_1", "type": "command_execution", "command": "ls", "output": "README.md"}}),
298                json!({"type": "item.completed", "timestamp": "2026-06-07T10:00:04.000Z", "item": {"id": "item_2", "type": "agent_message", "text": "First."}}),
299                json!({"type": "item.completed", "timestamp": "2026-06-07T10:00:05.000Z", "item": {"id": "item_3", "type": "agent_message", "text": "Final."}}),
300                json!({"type": "turn.completed", "timestamp": "2026-06-07T10:00:10.000Z", "usage": {"input_tokens": 100, "cached_input_tokens": 75, "output_tokens": 20, "reasoning_output_tokens": 5}}),
301            ],
302        );
303
304        let full = parse_codex_events_full(&path).unwrap();
305        assert_eq!(
306            full.tool_invocations,
307            vec![ToolInvocation {
308                name: "command_execution".into(),
309                args: Some(json!({"command": "ls"})),
310                result: Some(Value::String("README.md".into())),
311                ordinal: 0,
312            }]
313        );
314        assert_eq!(full.session_id.as_deref(), Some("thread-codex-1"));
315        assert_eq!(
316            serde_json::to_value(&full.events).unwrap(),
317            json!([
318                {
319                    "type": "tool_invocation",
320                    "ordinal": 0,
321                    "name": "command_execution",
322                    "args": {"command": "ls"},
323                    "result": "README.md"
324                },
325                {
326                    "type": "assistant_message",
327                    "ordinal": 1,
328                    "text": "First."
329                },
330                {
331                    "type": "assistant_message",
332                    "ordinal": 2,
333                    "text": "Final."
334                }
335            ])
336        );
337        assert_eq!(full.final_text, Some("Final.".into()));
338        assert_eq!(full.total_tokens, Some(125)); // cached input excluded
339        assert_eq!(full.duration_ms, Some(10_000));
340    }
341
342    #[test]
343    fn returns_null_usage_and_duration_when_sparse() {
344        let dir = TempDir::new().unwrap();
345        let path = dir.path().join("sparse.jsonl");
346        write_jsonl(
347            &path,
348            &[
349                json!({"type": "item.completed", "timestamp": "2026-06-07T10:00:00.000Z", "item": {"id": "item_1", "type": "agent_message", "text": "Done."}}),
350            ],
351        );
352        let full = parse_codex_events_full(&path).unwrap();
353        assert_eq!(full.final_text, Some("Done.".into()));
354        assert_eq!(full.total_tokens, None);
355        assert_eq!(full.duration_ms, None);
356    }
357}
eval_magic/adapters/codex/transcript.rs

eval_magic/adapters/codex/
transcript.rs
