eval_magic/adapters/codex/
transcript.rs

1//! Codex transcript (event-stream) parsing.
2//!
3//! Codex emits a JSONL event stream;
4//! `item.completed` events whose item type is not an agent message / reasoning /
5//! plan update become ordered [`ToolInvocation`]s. Produces the same
6//! [`TranscriptSummary`] shape as the Claude adapter, but with Codex's blended
7//! token accounting: non-cached input plus output.
8
9use crate::adapters::TranscriptSummary;
10use crate::adapters::transcript::TranscriptEvent;
11use crate::adapters::transcript::read_jsonl;
12use crate::core::ToolInvocation;
13use serde_json::{Map, Value};
14use std::io;
15use std::path::Path;
16
17const NON_TOOL_ITEMS: [&str; 3] = ["agent_message", "reasoning", "plan_update"];
18const ARG_OMIT_KEYS: [&str; 6] = ["id", "type", "status", "output", "result", "error"];
19
20fn stringify_value(v: &Value) -> String {
21    match v {
22        Value::String(s) => s.clone(),
23        other => serde_json::to_string(other).unwrap_or_default(),
24    }
25}
26
27/// First present of `output`, `result`, `error` (a present-but-null value still
28/// counts), stringified.
29fn maybe_result(item: &Map<String, Value>) -> Option<String> {
30    ["output", "result", "error"]
31        .into_iter()
32        .find_map(|k| item.get(k).map(stringify_value))
33}
34
35/// All item keys except the structural ones, preserving JSON key order; `None`
36/// when nothing remains.
37fn item_args(item: &Map<String, Value>) -> Option<Value> {
38    let mut args = Map::new();
39    for (key, value) in item {
40        if ARG_OMIT_KEYS.contains(&key.as_str()) {
41            continue;
42        }
43        args.insert(key.clone(), value.clone());
44    }
45    if args.is_empty() {
46        None
47    } else {
48        Some(Value::Object(args))
49    }
50}
51
52fn extract_invocations(records: &[Value]) -> Vec<ToolInvocation> {
53    let mut invocations = Vec::new();
54    for record in records {
55        if record.get("type").and_then(Value::as_str) != Some("item.completed") {
56            continue;
57        }
58        let Some(item) = record.get("item").and_then(Value::as_object) else {
59            continue;
60        };
61        let Some(item_type) = item.get("type").and_then(Value::as_str) else {
62            continue;
63        };
64        if NON_TOOL_ITEMS.contains(&item_type) {
65            continue;
66        }
67        let ordinal = invocations.len() as u32;
68        invocations.push(ToolInvocation {
69            name: item_type.to_string(),
70            args: item_args(item),
71            result: maybe_result(item).map(Value::String),
72            ordinal,
73        });
74    }
75    invocations
76}
77
78fn extract_events(records: &[Value]) -> Vec<TranscriptEvent> {
79    let mut events = Vec::new();
80    for record in records {
81        if record.get("type").and_then(Value::as_str) != Some("item.completed") {
82            continue;
83        }
84        let Some(item) = record.get("item").and_then(Value::as_object) else {
85            continue;
86        };
87        let Some(item_type) = item.get("type").and_then(Value::as_str) else {
88            continue;
89        };
90        let ordinal = events.len() as u32;
91        if item_type == "agent_message" {
92            if let Some(text) = item.get("text").and_then(Value::as_str) {
93                events.push(TranscriptEvent::AssistantMessage {
94                    ordinal,
95                    text: text.to_string(),
96                });
97            }
98        } else if !NON_TOOL_ITEMS.contains(&item_type) {
99            events.push(TranscriptEvent::ToolInvocation {
100                ordinal,
101                name: item_type.to_string(),
102                args: item_args(item),
103                result: maybe_result(item).map(Value::String),
104            });
105        }
106    }
107    events
108}
109
110/// Parse a Codex event stream into ordered tool invocations.
111pub fn parse_codex_events(jsonl_path: &Path) -> io::Result<Vec<ToolInvocation>> {
112    Ok(extract_invocations(&read_jsonl::<Value>(jsonl_path)?))
113}
114
115fn parse_millis(s: &str) -> Option<i64> {
116    chrono::DateTime::parse_from_rfc3339(s)
117        .ok()
118        .map(|dt| dt.timestamp_millis())
119}
120
121/// Parse a Codex event stream into a full [`TranscriptSummary`].
122pub fn parse_codex_events_full(jsonl_path: &Path) -> io::Result<TranscriptSummary> {
123    let records = read_jsonl::<Value>(jsonl_path)?;
124
125    let mut first_ts: Option<i64> = None;
126    let mut last_ts: Option<i64> = None;
127    let mut timestamp_count = 0usize;
128    let mut final_text: Option<String> = None;
129    let mut total_tokens: Option<i64> = None;
130    let session_id = records.iter().find_map(|record| {
131        (record.get("type").and_then(Value::as_str) == Some("thread.started"))
132            .then(|| record.get("thread_id").and_then(Value::as_str))
133            .flatten()
134            .map(str::to_string)
135    });
136
137    for record in &records {
138        if let Some(ts_str) = record.get("timestamp").and_then(Value::as_str)
139            && let Some(ts) = parse_millis(ts_str)
140        {
141            if first_ts.is_none() {
142                first_ts = Some(ts);
143            }
144            last_ts = Some(ts);
145            timestamp_count += 1;
146        }
147
148        let rtype = record.get("type").and_then(Value::as_str);
149
150        if rtype == Some("item.completed")
151            && let Some(item) = record.get("item").and_then(Value::as_object)
152            && item.get("type").and_then(Value::as_str) == Some("agent_message")
153            && let Some(text) = item.get("text").and_then(Value::as_str)
154        {
155            final_text = Some(text.to_string());
156        }
157
158        if rtype == Some("turn.completed")
159            && let Some(usage) = record.get("usage").and_then(Value::as_object)
160            && (usage.contains_key("input_tokens") || usage.contains_key("output_tokens"))
161        {
162            let get = |k: &str| usage.get(k).and_then(Value::as_i64).unwrap_or(0);
163            let blended = get("input_tokens")
164                .saturating_add(get("output_tokens"))
165                .saturating_sub(get("cached_input_tokens"))
166                .max(0);
167            total_tokens = Some(total_tokens.unwrap_or(0) + blended);
168        }
169    }
170
171    let duration_ms = match (first_ts, last_ts) {
172        (Some(f), Some(l)) if timestamp_count >= 2 => Some(l - f),
173        _ => None,
174    };
175
176    Ok(TranscriptSummary {
177        tool_invocations: extract_invocations(&records),
178        events: extract_events(&records),
179        session_id,
180        total_tokens,
181        duration_ms,
182        final_text,
183    })
184}
185
186#[cfg(test)]
187mod tests {
188    use super::*;
189    use crate::core::ToolInvocation;
190    use serde_json::{Value, json};
191    use std::fs;
192    use std::path::Path;
193    use tempfile::TempDir;
194
195    fn write_jsonl(path: &Path, lines: &[Value]) {
196        let body = lines
197            .iter()
198            .map(|l| l.to_string())
199            .collect::<Vec<_>>()
200            .join("\n");
201        fs::write(path, format!("{body}\n")).unwrap();
202    }
203
204    #[test]
205    fn extracts_completed_tool_items_with_ordinals_args_and_results() {
206        let dir = TempDir::new().unwrap();
207        let path = dir.path().join("items.jsonl");
208        write_jsonl(
209            &path,
210            &[
211                json!({"type": "item.started", "timestamp": "2026-06-07T10:00:00.000Z", "item": {"id": "item_1", "type": "command_execution", "command": "bash -lc 'bun test'", "status": "in_progress"}}),
212                json!({"type": "item.completed", "timestamp": "2026-06-07T10:00:02.000Z", "item": {"id": "item_1", "type": "command_execution", "command": "bash -lc 'bun test'", "output": "2 pass\n0 fail", "status": "completed"}}),
213                json!({"type": "item.completed", "item": {"id": "item_2", "type": "file_change", "path": "src/app.ts", "status": "completed"}}),
214                json!({"type": "item.completed", "item": {"id": "item_3", "type": "agent_message", "text": "Done."}}),
215            ],
216        );
217
218        let result = parse_codex_events(&path).unwrap();
219        assert_eq!(result.len(), 2);
220        assert_eq!(
221            result[0],
222            ToolInvocation {
223                name: "command_execution".into(),
224                args: Some(json!({"command": "bash -lc 'bun test'"})),
225                result: Some(Value::String("2 pass\n0 fail".into())),
226                ordinal: 0,
227            }
228        );
229        assert_eq!(
230            result[1],
231            ToolInvocation {
232                name: "file_change".into(),
233                args: Some(json!({"path": "src/app.ts"})),
234                result: None,
235                ordinal: 1,
236            }
237        );
238    }
239
240    #[test]
241    fn skips_malformed_jsonl_lines() {
242        let dir = TempDir::new().unwrap();
243        let path = dir.path().join("malformed.jsonl");
244        let good = json!({"type": "item.completed", "item": {"id": "item_1", "type": "web_search", "query": "codex exec json"}});
245        fs::write(&path, format!("{good}\nnot valid json\n")).unwrap();
246        assert_eq!(
247            parse_codex_events(&path).unwrap(),
248            vec![ToolInvocation {
249                name: "web_search".into(),
250                args: Some(json!({"query": "codex exec json"})),
251                result: None,
252                ordinal: 0,
253            }]
254        );
255    }
256
257    #[test]
258    fn preserves_text_fields_on_non_message_tool_items() {
259        let dir = TempDir::new().unwrap();
260        let path = dir.path().join("tool-text.jsonl");
261        write_jsonl(
262            &path,
263            &[
264                json!({"type": "item.completed", "item": {"id": "item_1", "type": "web_search", "query": "codex events", "text": "search summary"}}),
265            ],
266        );
267        assert_eq!(
268            parse_codex_events(&path).unwrap(),
269            vec![ToolInvocation {
270                name: "web_search".into(),
271                args: Some(json!({"query": "codex events", "text": "search summary"})),
272                result: None,
273                ordinal: 0,
274            }]
275        );
276    }
277
278    #[test]
279    fn does_not_treat_agent_messages_reasoning_or_plan_updates_as_tools() {
280        let dir = TempDir::new().unwrap();
281        let path = dir.path().join("non-tools.jsonl");
282        write_jsonl(
283            &path,
284            &[
285                json!({"type": "item.completed", "item": {"id": "a", "type": "agent_message"}}),
286                json!({"type": "item.completed", "item": {"id": "b", "type": "reasoning"}}),
287                json!({"type": "item.completed", "item": {"id": "c", "type": "plan_update"}}),
288            ],
289        );
290        assert_eq!(parse_codex_events(&path).unwrap(), vec![]);
291    }
292
293    #[test]
294    fn extracts_invocations_last_agent_text_usage_and_duration() {
295        let dir = TempDir::new().unwrap();
296        let path = dir.path().join("full.jsonl");
297        write_jsonl(
298            &path,
299            &[
300                json!({"type": "thread.started", "thread_id": "thread-codex-1", "timestamp": "2026-06-07T10:00:00.000Z"}),
301                json!({"type": "item.completed", "timestamp": "2026-06-07T10:00:03.000Z", "item": {"id": "item_1", "type": "command_execution", "command": "ls", "output": "README.md"}}),
302                json!({"type": "item.completed", "timestamp": "2026-06-07T10:00:04.000Z", "item": {"id": "item_2", "type": "agent_message", "text": "First."}}),
303                json!({"type": "item.completed", "timestamp": "2026-06-07T10:00:05.000Z", "item": {"id": "item_3", "type": "agent_message", "text": "Final."}}),
304                json!({"type": "turn.completed", "timestamp": "2026-06-07T10:00:10.000Z", "usage": {"input_tokens": 100, "cached_input_tokens": 75, "output_tokens": 20, "reasoning_output_tokens": 5}}),
305            ],
306        );
307
308        let full = parse_codex_events_full(&path).unwrap();
309        assert_eq!(
310            full.tool_invocations,
311            vec![ToolInvocation {
312                name: "command_execution".into(),
313                args: Some(json!({"command": "ls"})),
314                result: Some(Value::String("README.md".into())),
315                ordinal: 0,
316            }]
317        );
318        assert_eq!(full.session_id.as_deref(), Some("thread-codex-1"));
319        assert_eq!(
320            serde_json::to_value(&full.events).unwrap(),
321            json!([
322                {
323                    "type": "tool_invocation",
324                    "ordinal": 0,
325                    "name": "command_execution",
326                    "args": {"command": "ls"},
327                    "result": "README.md"
328                },
329                {
330                    "type": "assistant_message",
331                    "ordinal": 1,
332                    "text": "First."
333                },
334                {
335                    "type": "assistant_message",
336                    "ordinal": 2,
337                    "text": "Final."
338                }
339            ])
340        );
341        assert_eq!(full.final_text, Some("Final.".into()));
342        assert_eq!(full.total_tokens, Some(45)); // non-cached input plus output
343        assert_eq!(full.duration_ms, Some(10_000));
344    }
345
346    #[test]
347    fn reports_codex_blended_usage_without_double_counting_reasoning() {
348        let dir = TempDir::new().unwrap();
349        let path = dir.path().join("codex-usage.jsonl");
350        write_jsonl(
351            &path,
352            &[json!({
353                "type": "turn.completed",
354                "usage": {
355                    "input_tokens": 886_850,
356                    "cached_input_tokens": 833_024,
357                    "output_tokens": 10_083,
358                    "reasoning_output_tokens": 6_070
359                }
360            })],
361        );
362
363        let full = parse_codex_events_full(&path).unwrap();
364        assert_eq!(full.total_tokens, Some(63_909));
365        assert_eq!(full.duration_ms, None);
366    }
367
368    #[test]
369    fn turn_without_input_or_output_leaves_tokens_null() {
370        let dir = TempDir::new().unwrap();
371        let path = dir.path().join("cached-only-usage.jsonl");
372        write_jsonl(
373            &path,
374            &[json!({
375                "type": "turn.completed",
376                "usage": {"cached_input_tokens": 100}
377            })],
378        );
379
380        let full = parse_codex_events_full(&path).unwrap();
381        assert_eq!(full.total_tokens, None);
382    }
383
384    #[test]
385    fn returns_null_usage_and_duration_when_sparse() {
386        let dir = TempDir::new().unwrap();
387        let path = dir.path().join("sparse.jsonl");
388        write_jsonl(
389            &path,
390            &[
391                json!({"type": "item.completed", "timestamp": "2026-06-07T10:00:00.000Z", "item": {"id": "item_1", "type": "agent_message", "text": "Done."}}),
392            ],
393        );
394        let full = parse_codex_events_full(&path).unwrap();
395        assert_eq!(full.final_text, Some("Done.".into()));
396        assert_eq!(full.total_tokens, None);
397        assert_eq!(full.duration_ms, None);
398    }
399}
eval_magic/adapters/codex/transcript.rs

eval_magic/adapters/codex/
transcript.rs
