eval_magic/adapters/opencode/
transcript.rs

1//! OpenCode transcript (event-stream) parsing.
2//!
3//! `opencode run --format json` emits a JSONL event stream whose envelopes
4//! carry `{type, timestamp (epoch ms), sessionID, ...data}`. `tool_use` events
5//! hold a self-contained tool part — the name at `part.tool`, the arguments at
6//! `part.state.input`, and the outcome at `part.state.output` (completed) or
7//! `part.state.error` (error) — which become ordered [`ToolInvocation`]s.
8//! Token usage is summed from `step_finish` parts (`part.tokens`, cache
9//! reads excluded — matching the codex parser's accounting), the final
10//! message is the last `text` part's text, and the duration is the spread of
11//! the envelope timestamps.
12
13use crate::adapters::TranscriptSummary;
14use crate::adapters::transcript::read_jsonl;
15use crate::core::ToolInvocation;
16use serde_json::Value;
17use std::io;
18use std::path::Path;
19
20fn extract_invocations(records: &[Value]) -> Vec<ToolInvocation> {
21    let mut invocations = Vec::new();
22    for record in records {
23        if record.get("type").and_then(Value::as_str) != Some("tool_use") {
24            continue;
25        }
26        let Some(part) = record.get("part").and_then(Value::as_object) else {
27            continue;
28        };
29        let Some(name) = part.get("tool").and_then(Value::as_str) else {
30            continue;
31        };
32        let state = part.get("state");
33        let args = state
34            .and_then(|s| s.get("input"))
35            .filter(|input| input.is_object())
36            .cloned();
37        // Completed states carry `output`, error states carry `error` (both
38        // strings in OpenCode's SDK shapes).
39        let result = state
40            .and_then(|s| s.get("output").or_else(|| s.get("error")))
41            .and_then(Value::as_str)
42            .map(str::to_string);
43        let ordinal = invocations.len() as u32;
44        invocations.push(ToolInvocation {
45            name: name.to_string(),
46            args,
47            result: result.map(Value::String),
48            ordinal,
49        });
50    }
51    invocations
52}
53
54/// Parse an OpenCode `--format json` event stream into ordered tool
55/// invocations.
56pub fn parse_opencode_events(jsonl_path: &Path) -> io::Result<Vec<ToolInvocation>> {
57    Ok(extract_invocations(&read_jsonl::<Value>(jsonl_path)?))
58}
59
60/// Parse an OpenCode `--format json` event stream into a full
61/// [`TranscriptSummary`].
62pub fn parse_opencode_events_full(jsonl_path: &Path) -> io::Result<TranscriptSummary> {
63    let records = read_jsonl::<Value>(jsonl_path)?;
64
65    let mut first_ts: Option<i64> = None;
66    let mut last_ts: Option<i64> = None;
67    let mut timestamp_count = 0usize;
68    let mut final_text: Option<String> = None;
69    let mut total_tokens: Option<i64> = None;
70
71    for record in &records {
72        // OpenCode envelope timestamps are epoch milliseconds (numbers), not
73        // RFC 3339 strings.
74        if let Some(ts) = record.get("timestamp").and_then(Value::as_i64) {
75            if first_ts.is_none() {
76                first_ts = Some(ts);
77            }
78            last_ts = Some(ts);
79            timestamp_count += 1;
80        }
81
82        let rtype = record.get("type").and_then(Value::as_str);
83
84        if rtype == Some("text")
85            && let Some(text) = record
86                .get("part")
87                .and_then(|p| p.get("text"))
88                .and_then(Value::as_str)
89        {
90            final_text = Some(text.to_string());
91        }
92
93        if rtype == Some("step_finish")
94            && let Some(tokens) = record
95                .get("part")
96                .and_then(|p| p.get("tokens"))
97                .and_then(Value::as_object)
98        {
99            let get = |k: &str| tokens.get(k).and_then(Value::as_i64).unwrap_or(0);
100            // cache.read/write excluded, matching the codex parser's accounting.
101            let sum = get("input") + get("output") + get("reasoning");
102            total_tokens = Some(total_tokens.unwrap_or(0) + sum);
103        }
104    }
105
106    let duration_ms = match (first_ts, last_ts) {
107        (Some(f), Some(l)) if timestamp_count >= 2 => Some(l - f),
108        _ => None,
109    };
110
111    Ok(TranscriptSummary {
112        tool_invocations: extract_invocations(&records),
113        total_tokens,
114        duration_ms,
115        final_text,
116    })
117}
118
119#[cfg(test)]
120mod tests {
121    use super::{parse_opencode_events, parse_opencode_events_full};
122    use crate::core::ToolInvocation;
123    use serde_json::{Value, json};
124    use std::fs;
125    use std::path::Path;
126    use tempfile::TempDir;
127
128    fn write_jsonl(path: &Path, lines: &[Value]) {
129        let body = lines
130            .iter()
131            .map(|l| l.to_string())
132            .collect::<Vec<_>>()
133            .join("\n");
134        fs::write(path, format!("{body}\n")).unwrap();
135    }
136
137    /// A `tool_use` envelope wrapping a tool part with the given state.
138    fn tool_use(ts: i64, tool: &str, state: Value) -> Value {
139        json!({
140            "type": "tool_use",
141            "timestamp": ts,
142            "sessionID": "ses_1",
143            "part": {
144                "id": "prt_1",
145                "sessionID": "ses_1",
146                "messageID": "msg_1",
147                "type": "tool",
148                "callID": "call_1",
149                "tool": tool,
150                "state": state,
151            }
152        })
153    }
154
155    #[test]
156    fn extracts_completed_and_errored_tool_calls_with_ordinals_args_and_results() {
157        let dir = TempDir::new().unwrap();
158        let path = dir.path().join("events.jsonl");
159        write_jsonl(
160            &path,
161            &[
162                tool_use(
163                    1_000,
164                    "bash",
165                    json!({"status": "completed", "input": {"command": "bun test", "description": "run tests"}, "output": "2 pass\n0 fail", "title": "bun test", "metadata": {}, "time": {"start": 900, "end": 1_000}}),
166                ),
167                tool_use(
168                    2_000,
169                    "edit",
170                    json!({"status": "error", "input": {"filePath": "/tmp/x.md", "oldString": "a", "newString": "b"}, "error": "permission denied", "time": {"start": 1_900, "end": 2_000}}),
171                ),
172                tool_use(
173                    3_000,
174                    "read",
175                    json!({"status": "completed", "output": "file contents", "title": "read", "metadata": {}, "time": {"start": 2_900, "end": 3_000}}),
176                ),
177            ],
178        );
179
180        let result = parse_opencode_events(&path).unwrap();
181        assert_eq!(
182            result,
183            vec![
184                ToolInvocation {
185                    name: "bash".into(),
186                    args: Some(json!({"command": "bun test", "description": "run tests"})),
187                    result: Some(Value::String("2 pass\n0 fail".into())),
188                    ordinal: 0,
189                },
190                ToolInvocation {
191                    name: "edit".into(),
192                    args: Some(
193                        json!({"filePath": "/tmp/x.md", "oldString": "a", "newString": "b"})
194                    ),
195                    result: Some(Value::String("permission denied".into())),
196                    ordinal: 1,
197                },
198                // No `input` object on the state → null args.
199                ToolInvocation {
200                    name: "read".into(),
201                    args: None,
202                    result: Some(Value::String("file contents".into())),
203                    ordinal: 2,
204                },
205            ]
206        );
207    }
208
209    #[test]
210    fn skill_tool_calls_keep_the_name_argument_for_the_meta_check() {
211        let dir = TempDir::new().unwrap();
212        let path = dir.path().join("events.jsonl");
213        write_jsonl(
214            &path,
215            &[tool_use(
216                1_000,
217                "skill",
218                json!({"status": "completed", "input": {"name": "slow-powers-eval-1-with-skill-mr-review"}, "output": "<skill_content/>", "title": "skill", "metadata": {}, "time": {"start": 900, "end": 1_000}}),
219            )],
220        );
221
222        let invocations = parse_opencode_events(&path).unwrap();
223        assert_eq!(invocations.len(), 1);
224        assert_eq!(invocations[0].name, "skill");
225        assert_eq!(
226            invocations[0].args.as_ref().and_then(|a| a.get("name")),
227            Some(&json!("slow-powers-eval-1-with-skill-mr-review"))
228        );
229    }
230
231    #[test]
232    fn non_tool_events_produce_no_invocations() {
233        let dir = TempDir::new().unwrap();
234        let path = dir.path().join("events.jsonl");
235        write_jsonl(
236            &path,
237            &[
238                json!({"type": "step_start", "timestamp": 1_000, "sessionID": "ses_1", "part": {"id": "p1", "type": "step-start"}}),
239                json!({"type": "reasoning", "timestamp": 2_000, "sessionID": "ses_1", "part": {"id": "p2", "type": "reasoning", "text": "thinking"}}),
240                json!({"type": "text", "timestamp": 3_000, "sessionID": "ses_1", "part": {"id": "p3", "type": "text", "text": "hello"}}),
241                json!({"type": "step_finish", "timestamp": 4_000, "sessionID": "ses_1", "part": {"id": "p4", "type": "step-finish", "reason": "stop", "cost": 0.0, "tokens": {"input": 1, "output": 1, "reasoning": 0, "cache": {"read": 0, "write": 0}}}}),
242                json!({"type": "error", "timestamp": 5_000, "sessionID": "ses_1", "error": {"name": "UnknownError", "data": {"message": "boom"}}}),
243            ],
244        );
245        assert_eq!(parse_opencode_events(&path).unwrap(), vec![]);
246    }
247
248    #[test]
249    fn malformed_tool_use_parts_are_skipped() {
250        let dir = TempDir::new().unwrap();
251        let path = dir.path().join("events.jsonl");
252        write_jsonl(
253            &path,
254            &[
255                json!({"type": "tool_use", "timestamp": 1_000, "sessionID": "ses_1"}),
256                json!({"type": "tool_use", "timestamp": 2_000, "sessionID": "ses_1", "part": "not an object"}),
257                json!({"type": "tool_use", "timestamp": 3_000, "sessionID": "ses_1", "part": {"id": "p1", "type": "tool"}}),
258                json!({"type": "tool_use", "timestamp": 4_000, "sessionID": "ses_1", "part": {"id": "p2", "type": "tool", "tool": 7}}),
259            ],
260        );
261        assert_eq!(parse_opencode_events(&path).unwrap(), vec![]);
262    }
263
264    #[test]
265    fn skips_malformed_jsonl_lines() {
266        let dir = TempDir::new().unwrap();
267        let path = dir.path().join("events.jsonl");
268        let good = tool_use(
269            1_000,
270            "grep",
271            json!({"status": "completed", "input": {"pattern": "todo"}, "output": "3 matches", "title": "grep", "metadata": {}, "time": {"start": 900, "end": 1_000}}),
272        );
273        fs::write(&path, format!("{good}\nnot valid json\n")).unwrap();
274        assert_eq!(parse_opencode_events(&path).unwrap().len(), 1);
275    }
276
277    #[test]
278    fn full_summary_extracts_final_text_token_sum_and_timestamp_duration() {
279        let dir = TempDir::new().unwrap();
280        let path = dir.path().join("events.jsonl");
281        write_jsonl(
282            &path,
283            &[
284                json!({"type": "step_start", "timestamp": 1_000, "sessionID": "ses_1", "part": {"id": "p1", "type": "step-start"}}),
285                tool_use(
286                    2_000,
287                    "bash",
288                    json!({"status": "completed", "input": {"command": "ls"}, "output": "README.md", "title": "ls", "metadata": {}, "time": {"start": 1_900, "end": 2_000}}),
289                ),
290                json!({"type": "text", "timestamp": 3_000, "sessionID": "ses_1", "part": {"id": "p3", "type": "text", "text": "First."}}),
291                json!({"type": "text", "timestamp": 4_000, "sessionID": "ses_1", "part": {"id": "p4", "type": "text", "text": "Final."}}),
292                json!({"type": "step_finish", "timestamp": 5_000, "sessionID": "ses_1", "part": {"id": "p5", "type": "step-finish", "reason": "stop", "cost": 0.002, "tokens": {"input": 100, "output": 20, "reasoning": 5, "cache": {"read": 75, "write": 0}}}}),
293                json!({"type": "step_finish", "timestamp": 6_000, "sessionID": "ses_1", "part": {"id": "p6", "type": "step-finish", "reason": "stop", "cost": 0.001, "tokens": {"input": 40, "output": 2, "reasoning": 0, "cache": {"read": 0, "write": 0}}}}),
294            ],
295        );
296
297        let full = parse_opencode_events_full(&path).unwrap();
298        assert_eq!(full.tool_invocations.len(), 1);
299        assert_eq!(full.tool_invocations[0].name, "bash");
300        assert_eq!(full.final_text, Some("Final.".into()));
301        // cache.read (75) excluded, matching the codex parser's accounting.
302        assert_eq!(full.total_tokens, Some(167));
303        assert_eq!(full.duration_ms, Some(5_000));
304    }
305
306    #[test]
307    fn returns_null_usage_and_duration_when_sparse() {
308        let dir = TempDir::new().unwrap();
309        let path = dir.path().join("events.jsonl");
310        write_jsonl(
311            &path,
312            &[
313                json!({"type": "text", "timestamp": 1_000, "sessionID": "ses_1", "part": {"id": "p1", "type": "text", "text": "Done."}}),
314            ],
315        );
316        let full = parse_opencode_events_full(&path).unwrap();
317        assert_eq!(full.final_text, Some("Done.".into()));
318        assert_eq!(full.total_tokens, None);
319        assert_eq!(full.duration_ms, None);
320    }
321
322    #[test]
323    fn step_finish_without_tokens_leaves_the_total_untouched() {
324        let dir = TempDir::new().unwrap();
325        let path = dir.path().join("events.jsonl");
326        write_jsonl(
327            &path,
328            &[
329                json!({"type": "step_finish", "timestamp": 1_000, "sessionID": "ses_1", "part": {"id": "p1", "type": "step-finish", "reason": "tool-calls", "cost": 0.0}}),
330                json!({"type": "step_finish", "timestamp": 2_000, "sessionID": "ses_1", "part": {"id": "p2", "type": "step-finish", "reason": "stop", "cost": 0.001, "tokens": {"input": 10, "output": 2, "reasoning": 0, "cache": {"read": 0, "write": 0}}}}),
331            ],
332        );
333        assert_eq!(
334            parse_opencode_events_full(&path).unwrap().total_tokens,
335            Some(12)
336        );
337    }
338}
eval_magic/adapters/opencode/transcript.rs

eval_magic/adapters/opencode/
transcript.rs
