eval_magic/adapters/
transcript.rs

1//! Harness-neutral transcript types.
2//!
3//! Every harness's transcript parser reduces its native events file to a
4//! [`TranscriptSummary`]; the pipeline consumes only this shape, never a
5//! harness's raw record types.
6
7use serde::{Deserialize, Serialize};
8use std::fs;
9use std::io;
10use std::path::Path;
11
12use crate::core::ToolInvocation;
13
14/// One ordered, user-visible or tool event parsed from a harness transcript.
15#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
16#[serde(tag = "type", rename_all = "snake_case")]
17pub enum TranscriptEvent {
18    AssistantMessage {
19        ordinal: u32,
20        text: String,
21    },
22    ToolInvocation {
23        ordinal: u32,
24        name: String,
25        #[serde(skip_serializing_if = "Option::is_none")]
26        args: Option<serde_json::Value>,
27        #[serde(skip_serializing_if = "Option::is_none")]
28        result: Option<serde_json::Value>,
29    },
30}
31
32/// Read a JSONL file, deserializing each non-blank line as `T` and silently
33/// skipping malformed lines (a partial transcript still yields its parseable
34/// records).
35pub(crate) fn read_jsonl<T: serde::de::DeserializeOwned>(path: &Path) -> io::Result<Vec<T>> {
36    let raw = fs::read_to_string(path)?;
37    Ok(raw
38        .split('\n')
39        .filter(|line| !line.trim().is_empty())
40        .filter_map(|line| serde_json::from_str::<T>(line).ok())
41        .collect())
42}
43
44/// A transcript boiled down to the artifacts the pipeline needs.
45#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
46pub struct TranscriptSummary {
47    pub tool_invocations: Vec<ToolInvocation>,
48    /// Ordered assistant-message and tool events, for multi-turn gating and
49    /// cross-event grading.
50    #[serde(default)]
51    pub events: Vec<TranscriptEvent>,
52    /// Native conversation/session identifier used to resume the next turn.
53    #[serde(default)]
54    pub session_id: Option<String>,
55    /// Harness-normalized total token usage, as reported by the persisted
56    /// transcript.
57    pub total_tokens: Option<i64>,
58    /// Wall-clock duration when the transcript exposes a reliable duration or
59    /// enough timestamps to derive one.
60    pub duration_ms: Option<i64>,
61    /// Concatenated text blocks of the last assistant message.
62    pub final_text: Option<String>,
63}
64
65#[cfg(test)]
66mod tests {
67    use super::read_jsonl;
68    use serde_json::Value;
69    use std::fs;
70    use tempfile::TempDir;
71
72    #[test]
73    fn read_jsonl_skips_malformed_lines() {
74        let dir = TempDir::new().unwrap();
75        let path = dir.path().join("events.jsonl");
76        fs::write(&path, "{\"a\":1}\nnot json\n{\"a\":2}\n").unwrap();
77        let values: Vec<Value> = read_jsonl(&path).unwrap();
78        assert_eq!(values.len(), 2);
79        assert_eq!(values[0]["a"], 1);
80        assert_eq!(values[1]["a"], 2);
81    }
82
83    #[test]
84    fn read_jsonl_skips_blank_and_whitespace_only_lines() {
85        let dir = TempDir::new().unwrap();
86        let path = dir.path().join("events.jsonl");
87        fs::write(&path, "{\"a\":1}\n\n   \n\t\n{\"a\":2}\n").unwrap();
88        let values: Vec<Value> = read_jsonl(&path).unwrap();
89        assert_eq!(values.len(), 2);
90    }
91
92    #[test]
93    fn read_jsonl_errors_on_a_missing_file() {
94        let dir = TempDir::new().unwrap();
95        let missing = dir.path().join("absent.jsonl");
96        assert!(read_jsonl::<Value>(&missing).is_err());
97    }
98}
eval_magic/adapters/transcript.rs

eval_magic/adapters/
transcript.rs
