eval_magic/adapters/
transcript.rs

1//! Harness-neutral transcript types.
2//!
3//! Every harness's transcript parser reduces its native events file to a
4//! [`TranscriptSummary`]; the pipeline consumes only this shape, never a
5//! harness's raw record types.
6
7use serde::{Deserialize, Serialize};
8use std::fs;
9use std::io;
10use std::path::Path;
11
12use crate::core::ToolInvocation;
13
14/// One ordered, user-visible or tool event parsed from a harness transcript.
15#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
16#[serde(tag = "type", rename_all = "snake_case")]
17pub enum TranscriptEvent {
18    AssistantMessage {
19        ordinal: u32,
20        text: String,
21    },
22    ToolInvocation {
23        ordinal: u32,
24        name: String,
25        #[serde(skip_serializing_if = "Option::is_none")]
26        args: Option<serde_json::Value>,
27        #[serde(skip_serializing_if = "Option::is_none")]
28        result: Option<serde_json::Value>,
29    },
30}
31
32/// Read a JSONL file, deserializing each non-blank line as `T` and silently
33/// skipping malformed lines (a partial transcript still yields its parseable
34/// records).
35pub(crate) fn read_jsonl<T: serde::de::DeserializeOwned>(path: &Path) -> io::Result<Vec<T>> {
36    let raw = fs::read_to_string(path)?;
37    Ok(raw
38        .split('\n')
39        .filter(|line| !line.trim().is_empty())
40        .filter_map(|line| serde_json::from_str::<T>(line).ok())
41        .collect())
42}
43
44/// A transcript boiled down to the artifacts the pipeline needs.
45#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
46pub struct TranscriptSummary {
47    pub tool_invocations: Vec<ToolInvocation>,
48    /// Ordered assistant-message and tool events, for multi-turn gating and
49    /// cross-event grading.
50    #[serde(default)]
51    pub events: Vec<TranscriptEvent>,
52    /// Native conversation/session identifier used to resume the next turn.
53    #[serde(default)]
54    pub session_id: Option<String>,
55    /// Total token usage (input + output + cache creation/read), as reported by
56    /// the run's terminal `result` event.
57    pub total_tokens: Option<i64>,
58    /// Wall-clock duration, as reported by the run's terminal `result` event.
59    pub duration_ms: Option<i64>,
60    /// Concatenated text blocks of the last assistant message.
61    pub final_text: Option<String>,
62}
63
64#[cfg(test)]
65mod tests {
66    use super::read_jsonl;
67    use serde_json::Value;
68    use std::fs;
69    use tempfile::TempDir;
70
71    #[test]
72    fn read_jsonl_skips_malformed_lines() {
73        let dir = TempDir::new().unwrap();
74        let path = dir.path().join("events.jsonl");
75        fs::write(&path, "{\"a\":1}\nnot json\n{\"a\":2}\n").unwrap();
76        let values: Vec<Value> = read_jsonl(&path).unwrap();
77        assert_eq!(values.len(), 2);
78        assert_eq!(values[0]["a"], 1);
79        assert_eq!(values[1]["a"], 2);
80    }
81
82    #[test]
83    fn read_jsonl_skips_blank_and_whitespace_only_lines() {
84        let dir = TempDir::new().unwrap();
85        let path = dir.path().join("events.jsonl");
86        fs::write(&path, "{\"a\":1}\n\n   \n\t\n{\"a\":2}\n").unwrap();
87        let values: Vec<Value> = read_jsonl(&path).unwrap();
88        assert_eq!(values.len(), 2);
89    }
90
91    #[test]
92    fn read_jsonl_errors_on_a_missing_file() {
93        let dir = TempDir::new().unwrap();
94        let missing = dir.path().join("absent.jsonl");
95        assert!(read_jsonl::<Value>(&missing).is_err());
96    }
97}
eval_magic/adapters/transcript.rs

eval_magic/adapters/
transcript.rs
