eval_magic/adapters/codex/
transcript.rs

1//! Codex transcript (event-stream) parsing.
2//!
3//! Codex emits a JSONL event stream;
4//! `item.completed` events whose item type is not an agent message / reasoning /
5//! plan update become ordered [`ToolInvocation`]s. Produces the same
6//! [`TranscriptSummary`] shape as the Claude adapter, but with Codex's token
7//! accounting (excludes cached input tokens).
8
9use crate::adapters::TranscriptSummary;
10use crate::adapters::transcript::read_jsonl;
11use crate::core::ToolInvocation;
12use serde_json::{Map, Value};
13use std::io;
14use std::path::Path;
15
16const NON_TOOL_ITEMS: [&str; 3] = ["agent_message", "reasoning", "plan_update"];
17const ARG_OMIT_KEYS: [&str; 6] = ["id", "type", "status", "output", "result", "error"];
18
19fn stringify_value(v: &Value) -> String {
20    match v {
21        Value::String(s) => s.clone(),
22        other => serde_json::to_string(other).unwrap_or_default(),
23    }
24}
25
26/// First present of `output`, `result`, `error` (a present-but-null value still
27/// counts), stringified.
28fn maybe_result(item: &Map<String, Value>) -> Option<String> {
29    ["output", "result", "error"]
30        .into_iter()
31        .find_map(|k| item.get(k).map(stringify_value))
32}
33
34/// All item keys except the structural ones, preserving JSON key order; `None`
35/// when nothing remains.
36fn item_args(item: &Map<String, Value>) -> Option<Value> {
37    let mut args = Map::new();
38    for (key, value) in item {
39        if ARG_OMIT_KEYS.contains(&key.as_str()) {
40            continue;
41        }
42        args.insert(key.clone(), value.clone());
43    }
44    if args.is_empty() {
45        None
46    } else {
47        Some(Value::Object(args))
48    }
49}
50
51fn extract_invocations(records: &[Value]) -> Vec<ToolInvocation> {
52    let mut invocations = Vec::new();
53    for record in records {
54        if record.get("type").and_then(Value::as_str) != Some("item.completed") {
55            continue;
56        }
57        let Some(item) = record.get("item").and_then(Value::as_object) else {
58            continue;
59        };
60        let Some(item_type) = item.get("type").and_then(Value::as_str) else {
61            continue;
62        };
63        if NON_TOOL_ITEMS.contains(&item_type) {
64            continue;
65        }
66        let ordinal = invocations.len() as u32;
67        invocations.push(ToolInvocation {
68            name: item_type.to_string(),
69            args: item_args(item),
70            result: maybe_result(item).map(Value::String),
71            ordinal,
72        });
73    }
74    invocations
75}
76
77/// Parse a Codex event stream into ordered tool invocations.
78pub fn parse_codex_events(jsonl_path: &Path) -> io::Result<Vec<ToolInvocation>> {
79    Ok(extract_invocations(&read_jsonl::<Value>(jsonl_path)?))
80}
81
82fn parse_millis(s: &str) -> Option<i64> {
83    chrono::DateTime::parse_from_rfc3339(s)
84        .ok()
85        .map(|dt| dt.timestamp_millis())
86}
87
88/// Parse a Codex event stream into a full [`TranscriptSummary`].
89pub fn parse_codex_events_full(jsonl_path: &Path) -> io::Result<TranscriptSummary> {
90    let records = read_jsonl::<Value>(jsonl_path)?;
91
92    let mut first_ts: Option<i64> = None;
93    let mut last_ts: Option<i64> = None;
94    let mut timestamp_count = 0usize;
95    let mut final_text: Option<String> = None;
96    let mut total_tokens: Option<i64> = None;
97
98    for record in &records {
99        if let Some(ts_str) = record.get("timestamp").and_then(Value::as_str)
100            && let Some(ts) = parse_millis(ts_str)
101        {
102            if first_ts.is_none() {
103                first_ts = Some(ts);
104            }
105            last_ts = Some(ts);
106            timestamp_count += 1;
107        }
108
109        let rtype = record.get("type").and_then(Value::as_str);
110
111        if rtype == Some("item.completed")
112            && let Some(item) = record.get("item").and_then(Value::as_object)
113            && item.get("type").and_then(Value::as_str) == Some("agent_message")
114            && let Some(text) = item.get("text").and_then(Value::as_str)
115        {
116            final_text = Some(text.to_string());
117        }
118
119        if rtype == Some("turn.completed")
120            && let Some(usage) = record.get("usage").and_then(Value::as_object)
121        {
122            let get = |k: &str| usage.get(k).and_then(Value::as_i64).unwrap_or(0);
123            let sum = get("input_tokens") + get("output_tokens") + get("reasoning_output_tokens");
124            total_tokens = Some(total_tokens.unwrap_or(0) + sum);
125        }
126    }
127
128    let duration_ms = match (first_ts, last_ts) {
129        (Some(f), Some(l)) if timestamp_count >= 2 => Some(l - f),
130        _ => None,
131    };
132
133    Ok(TranscriptSummary {
134        tool_invocations: extract_invocations(&records),
135        total_tokens,
136        duration_ms,
137        final_text,
138    })
139}
140
141#[cfg(test)]
142mod tests {
143    use super::*;
144    use crate::core::ToolInvocation;
145    use serde_json::{Value, json};
146    use std::fs;
147    use std::path::Path;
148    use tempfile::TempDir;
149
150    fn write_jsonl(path: &Path, lines: &[Value]) {
151        let body = lines
152            .iter()
153            .map(|l| l.to_string())
154            .collect::<Vec<_>>()
155            .join("\n");
156        fs::write(path, format!("{body}\n")).unwrap();
157    }
158
159    #[test]
160    fn extracts_completed_tool_items_with_ordinals_args_and_results() {
161        let dir = TempDir::new().unwrap();
162        let path = dir.path().join("items.jsonl");
163        write_jsonl(
164            &path,
165            &[
166                json!({"type": "item.started", "timestamp": "2026-06-07T10:00:00.000Z", "item": {"id": "item_1", "type": "command_execution", "command": "bash -lc 'bun test'", "status": "in_progress"}}),
167                json!({"type": "item.completed", "timestamp": "2026-06-07T10:00:02.000Z", "item": {"id": "item_1", "type": "command_execution", "command": "bash -lc 'bun test'", "output": "2 pass\n0 fail", "status": "completed"}}),
168                json!({"type": "item.completed", "item": {"id": "item_2", "type": "file_change", "path": "src/app.ts", "status": "completed"}}),
169                json!({"type": "item.completed", "item": {"id": "item_3", "type": "agent_message", "text": "Done."}}),
170            ],
171        );
172
173        let result = parse_codex_events(&path).unwrap();
174        assert_eq!(result.len(), 2);
175        assert_eq!(
176            result[0],
177            ToolInvocation {
178                name: "command_execution".into(),
179                args: Some(json!({"command": "bash -lc 'bun test'"})),
180                result: Some(Value::String("2 pass\n0 fail".into())),
181                ordinal: 0,
182            }
183        );
184        assert_eq!(
185            result[1],
186            ToolInvocation {
187                name: "file_change".into(),
188                args: Some(json!({"path": "src/app.ts"})),
189                result: None,
190                ordinal: 1,
191            }
192        );
193    }
194
195    #[test]
196    fn skips_malformed_jsonl_lines() {
197        let dir = TempDir::new().unwrap();
198        let path = dir.path().join("malformed.jsonl");
199        let good = json!({"type": "item.completed", "item": {"id": "item_1", "type": "web_search", "query": "codex exec json"}});
200        fs::write(&path, format!("{good}\nnot valid json\n")).unwrap();
201        assert_eq!(
202            parse_codex_events(&path).unwrap(),
203            vec![ToolInvocation {
204                name: "web_search".into(),
205                args: Some(json!({"query": "codex exec json"})),
206                result: None,
207                ordinal: 0,
208            }]
209        );
210    }
211
212    #[test]
213    fn preserves_text_fields_on_non_message_tool_items() {
214        let dir = TempDir::new().unwrap();
215        let path = dir.path().join("tool-text.jsonl");
216        write_jsonl(
217            &path,
218            &[
219                json!({"type": "item.completed", "item": {"id": "item_1", "type": "web_search", "query": "codex events", "text": "search summary"}}),
220            ],
221        );
222        assert_eq!(
223            parse_codex_events(&path).unwrap(),
224            vec![ToolInvocation {
225                name: "web_search".into(),
226                args: Some(json!({"query": "codex events", "text": "search summary"})),
227                result: None,
228                ordinal: 0,
229            }]
230        );
231    }
232
233    #[test]
234    fn does_not_treat_agent_messages_reasoning_or_plan_updates_as_tools() {
235        let dir = TempDir::new().unwrap();
236        let path = dir.path().join("non-tools.jsonl");
237        write_jsonl(
238            &path,
239            &[
240                json!({"type": "item.completed", "item": {"id": "a", "type": "agent_message"}}),
241                json!({"type": "item.completed", "item": {"id": "b", "type": "reasoning"}}),
242                json!({"type": "item.completed", "item": {"id": "c", "type": "plan_update"}}),
243            ],
244        );
245        assert_eq!(parse_codex_events(&path).unwrap(), vec![]);
246    }
247
248    #[test]
249    fn extracts_invocations_last_agent_text_usage_and_duration() {
250        let dir = TempDir::new().unwrap();
251        let path = dir.path().join("full.jsonl");
252        write_jsonl(
253            &path,
254            &[
255                json!({"type": "thread.started", "timestamp": "2026-06-07T10:00:00.000Z"}),
256                json!({"type": "item.completed", "timestamp": "2026-06-07T10:00:03.000Z", "item": {"id": "item_1", "type": "command_execution", "command": "ls", "output": "README.md"}}),
257                json!({"type": "item.completed", "timestamp": "2026-06-07T10:00:04.000Z", "item": {"id": "item_2", "type": "agent_message", "text": "First."}}),
258                json!({"type": "item.completed", "timestamp": "2026-06-07T10:00:05.000Z", "item": {"id": "item_3", "type": "agent_message", "text": "Final."}}),
259                json!({"type": "turn.completed", "timestamp": "2026-06-07T10:00:10.000Z", "usage": {"input_tokens": 100, "cached_input_tokens": 75, "output_tokens": 20, "reasoning_output_tokens": 5}}),
260            ],
261        );
262
263        let full = parse_codex_events_full(&path).unwrap();
264        assert_eq!(
265            full.tool_invocations,
266            vec![ToolInvocation {
267                name: "command_execution".into(),
268                args: Some(json!({"command": "ls"})),
269                result: Some(Value::String("README.md".into())),
270                ordinal: 0,
271            }]
272        );
273        assert_eq!(full.final_text, Some("Final.".into()));
274        assert_eq!(full.total_tokens, Some(125)); // cached input excluded
275        assert_eq!(full.duration_ms, Some(10_000));
276    }
277
278    #[test]
279    fn returns_null_usage_and_duration_when_sparse() {
280        let dir = TempDir::new().unwrap();
281        let path = dir.path().join("sparse.jsonl");
282        write_jsonl(
283            &path,
284            &[
285                json!({"type": "item.completed", "timestamp": "2026-06-07T10:00:00.000Z", "item": {"id": "item_1", "type": "agent_message", "text": "Done."}}),
286            ],
287        );
288        let full = parse_codex_events_full(&path).unwrap();
289        assert_eq!(full.final_text, Some("Done.".into()));
290        assert_eq!(full.total_tokens, None);
291        assert_eq!(full.duration_ms, None);
292    }
293}
eval_magic/adapters/codex/transcript.rs

eval_magic/adapters/codex/
transcript.rs
