Skip to main content

mnemo_engine/
transcript.rs

1//! Phase 12.7a transcript parsers.
2//!
3//! Turns a file on disk (or an in-memory reader) into a `Vec<ThreadTurn>`
4//! the [`crate::Mnemo::ingest_thread`] path expects. Two formats land in
5//! v0.9.0; the general shape is one JSON object per line.
6//!
7//! * [`TranscriptFormat::JsonlGeneric`] — the documented shape anyone
8//!   can produce:
9//!   ```json
10//!   {"turn_id": "...", "role": "user", "content": "...", "timestamp": 1234567890}
11//!   ```
12//!   `timestamp` is optional (unix seconds). Unknown keys are ignored.
13//!
14//! * [`TranscriptFormat::ClaudeCodeJsonl`] — Claude Code writes its
15//!   session transcripts at `~/.claude/projects/<project>/<session>.jsonl`
16//!   with one JSON object per line. Each line is tagged with a `type`
17//!   field (`"message"`, `"system"`, `"tool_use"`, …) and carries a
18//!   `uuid` the parser uses as `turn_id`. Only `type == "user"` or
19//!   `type == "assistant"` lines yield turns; every other line type is
20//!   skipped so new line types don't break the parser.
21
22use std::io::{BufRead, BufReader, Read};
23
24use serde_json::Value;
25
26use crate::error::{MnemoError, Result};
27use crate::memory::ThreadTurn;
28
29/// The supported transcript shapes for [`read_transcript`] /
30/// `mnemo thread ingest --format …`.
31#[derive(Debug, Clone, Copy, PartialEq, Eq)]
32pub enum TranscriptFormat {
33    /// `{"turn_id": …, "role": …, "content": …, "timestamp"?: …}` per line.
34    JsonlGeneric,
35    /// Claude Code's `~/.claude/projects/<p>/<s>.jsonl` shape; see the
36    /// module docs.
37    ClaudeCodeJsonl,
38}
39
40impl TranscriptFormat {
41    /// Parse a `--format` string. Case-insensitive; accepts `jsonl` and
42    /// `claude-code-jsonl` (or `claude_code_jsonl` for CLI convenience).
43    pub fn parse(s: &str) -> Option<TranscriptFormat> {
44        match s.to_ascii_lowercase().as_str() {
45            "jsonl" => Some(TranscriptFormat::JsonlGeneric),
46            "claude-code-jsonl" | "claude_code_jsonl" => Some(TranscriptFormat::ClaudeCodeJsonl),
47            _ => None,
48        }
49    }
50}
51
52/// What a Claude Code transcript's tool results keep (12.2 step 4).
53#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
54pub enum ToolResults {
55    /// The body, as `[tool_result ok|error] <body>` (what ingest always
56    /// kept).
57    #[default]
58    Full,
59    /// Only a marker with the body's size, `[tool_result ok|error, N
60    /// bytes]`: tool output is where most secrets land, and most of a
61    /// transcript's size. The hooks' ingest uses it.
62    Omit,
63}
64
65/// How [`read_transcript_with`] reads.
66#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
67pub struct TranscriptOptions {
68    /// What tool results keep.
69    pub tool_results: ToolResults,
70    /// Skip a line that isn't JSON or isn't a valid turn, instead of
71    /// failing: the hooks read a transcript the client may be writing, so
72    /// its last line can be half written.
73    pub lenient: bool,
74}
75
76/// Read a transcript from any `Read` in the given format and return the
77/// turns in file order. Non-matching / unrecognized line types are
78/// skipped silently (Claude Code adds new `type` values over time).
79pub fn read_transcript<R: Read>(reader: R, format: TranscriptFormat) -> Result<Vec<ThreadTurn>> {
80    read_transcript_with(reader, format, TranscriptOptions::default())
81}
82
83/// [`read_transcript`] with [`TranscriptOptions`].
84pub fn read_transcript_with<R: Read>(
85    reader: R,
86    format: TranscriptFormat,
87    opts: TranscriptOptions,
88) -> Result<Vec<ThreadTurn>> {
89    let mut out = Vec::new();
90    for (lineno, line) in BufReader::new(reader).lines().enumerate() {
91        let line = match line {
92            Ok(l) => l,
93            // Not UTF-8 (a cut in the middle of a character, say).
94            Err(e) if opts.lenient && e.kind() == std::io::ErrorKind::InvalidData => continue,
95            Err(e) => return Err(MnemoError::Io(e)),
96        };
97        let trimmed = line.trim();
98        if trimmed.is_empty() || trimmed.starts_with('#') {
99            continue;
100        }
101        let v: Value = match serde_json::from_str(trimmed) {
102            Ok(v) => v,
103            Err(_) if opts.lenient => continue,
104            Err(e) => {
105                return Err(MnemoError::Invalid(format!(
106                    "transcript line {}: invalid JSON: {e}",
107                    lineno + 1
108                )))
109            }
110        };
111        let turn = match format {
112            TranscriptFormat::JsonlGeneric => parse_generic_line(&v),
113            TranscriptFormat::ClaudeCodeJsonl => parse_claude_code_line(&v, opts.tool_results),
114        };
115        match turn {
116            Ok(Some(t)) => out.push(t),
117            Ok(None) => {}
118            Err(_) if opts.lenient => {}
119            Err(e) => return Err(e),
120        }
121    }
122    Ok(out)
123}
124
125fn parse_generic_line(v: &Value) -> Result<Option<ThreadTurn>> {
126    let obj = v.as_object().ok_or_else(|| {
127        MnemoError::Invalid("transcript: generic jsonl line must be a JSON object".into())
128    })?;
129    let turn_id = obj
130        .get("turn_id")
131        .and_then(|v| v.as_str())
132        .ok_or_else(|| {
133            MnemoError::Invalid("transcript: generic jsonl line missing 'turn_id' (string)".into())
134        })?
135        .to_string();
136    let role = obj
137        .get("role")
138        .and_then(|v| v.as_str())
139        .ok_or_else(|| {
140            MnemoError::Invalid("transcript: generic jsonl line missing 'role' (string)".into())
141        })?
142        .to_string();
143    let content = obj
144        .get("content")
145        .and_then(|v| v.as_str())
146        .ok_or_else(|| {
147            MnemoError::Invalid("transcript: generic jsonl line missing 'content' (string)".into())
148        })?
149        .to_string();
150    let timestamp = obj.get("timestamp").and_then(|v| v.as_i64());
151    Ok(Some(ThreadTurn {
152        turn_id,
153        role,
154        content,
155        timestamp,
156    }))
157}
158
159/// Parse one Claude Code jsonl line. Returns `None` for any line that
160/// isn't a user/assistant message (system init records, tool_use
161/// scaffolding, etc.) so new line types added upstream don't break
162/// ingest.
163///
164/// Expected shape (trimmed to the fields we read):
165/// ```json
166/// {"type": "user", "uuid": "…", "message": {"role": "user", "content": "…"}}
167/// {"type": "assistant", "uuid": "…", "message": {"role": "assistant", "content": [{"type":"text","text":"…"}]}}
168/// ```
169///
170/// `message.content` can be either a plain string (older captures) or a
171/// list of content blocks each with `{"type": "text", "text": "…"}`.
172/// Non-text blocks (tool_use, tool_result, image) are flattened into a
173/// readable text summary so the archive stays browsable without being
174/// lossless about the original block structure.
175fn parse_claude_code_line(v: &Value, tool_results: ToolResults) -> Result<Option<ThreadTurn>> {
176    let obj = match v.as_object() {
177        Some(o) => o,
178        None => return Ok(None),
179    };
180    let type_tag = obj.get("type").and_then(|v| v.as_str()).unwrap_or("");
181    if type_tag != "user" && type_tag != "assistant" {
182        // Skip system init, summary, and other non-message types.
183        return Ok(None);
184    }
185    let turn_id = obj
186        .get("uuid")
187        .and_then(|v| v.as_str())
188        .map(|s| s.to_string())
189        .ok_or_else(|| {
190            MnemoError::Invalid("transcript: claude-code-jsonl message missing 'uuid'".into())
191        })?;
192    let message = obj
193        .get("message")
194        .and_then(|v| v.as_object())
195        .ok_or_else(|| {
196            MnemoError::Invalid(
197                "transcript: claude-code-jsonl message missing 'message' object".into(),
198            )
199        })?;
200    let role = message
201        .get("role")
202        .and_then(|v| v.as_str())
203        .unwrap_or(type_tag)
204        .to_string();
205    let content = match message.get("content") {
206        Some(Value::String(s)) => s.clone(),
207        Some(Value::Array(blocks)) => flatten_blocks(blocks, tool_results),
208        _ => String::new(),
209    };
210    // Timestamps in Claude Code jsonl live under `message.timestamp`
211    // (ISO 8601) or the line's top-level `timestamp`. Convert ISO 8601
212    // to unix seconds with a minimal parser; on any failure, let the
213    // ingest path pick `now()`.
214    let timestamp = obj
215        .get("timestamp")
216        .and_then(|v| v.as_str())
217        .and_then(parse_iso8601_secs)
218        .or_else(|| {
219            message
220                .get("timestamp")
221                .and_then(|v| v.as_str())
222                .and_then(parse_iso8601_secs)
223        });
224    Ok(Some(ThreadTurn {
225        turn_id,
226        role,
227        content,
228        timestamp,
229    }))
230}
231
232/// Flatten a Claude Code `message.content` array of blocks into a
233/// single string. Text blocks come through verbatim; other block types
234/// are rendered as short readable summaries so the archive is
235/// grep-able (`[tool_use name=Bash]`, `[tool_result ok]`, `[image]`).
236fn flatten_blocks(blocks: &[Value], tool_results: ToolResults) -> String {
237    let mut out = String::new();
238    for b in blocks {
239        let Some(obj) = b.as_object() else {
240            continue;
241        };
242        let t = obj.get("type").and_then(|v| v.as_str()).unwrap_or("");
243        match t {
244            "text" => {
245                if let Some(s) = obj.get("text").and_then(|v| v.as_str()) {
246                    if !out.is_empty() {
247                        out.push('\n');
248                    }
249                    out.push_str(s);
250                }
251            }
252            "tool_use" => {
253                let name = obj.get("name").and_then(|v| v.as_str()).unwrap_or("?");
254                if !out.is_empty() {
255                    out.push('\n');
256                }
257                out.push_str(&format!("[tool_use name={name}]"));
258            }
259            "tool_result" => {
260                let is_error = obj
261                    .get("is_error")
262                    .and_then(|v| v.as_bool())
263                    .unwrap_or(false);
264                let status = if is_error { "error" } else { "ok" };
265                let inner = obj.get("content").and_then(|c| match c {
266                    Value::String(s) => Some(s.clone()),
267                    Value::Array(nested) => Some(flatten_blocks(nested, tool_results)),
268                    _ => None,
269                });
270                if !out.is_empty() {
271                    out.push('\n');
272                }
273                match (inner, tool_results) {
274                    (Some(body), ToolResults::Full) => {
275                        out.push_str(&format!("[tool_result {status}] {body}"))
276                    }
277                    (inner, ToolResults::Omit) => out.push_str(&format!(
278                        "[tool_result {status}, {} bytes]",
279                        inner.map_or(0, |b| b.len())
280                    )),
281                    (None, ToolResults::Full) => out.push_str(&format!("[tool_result {status}]")),
282                }
283            }
284            "image" => {
285                if !out.is_empty() {
286                    out.push('\n');
287                }
288                out.push_str("[image]");
289            }
290            other => {
291                if !out.is_empty() {
292                    out.push('\n');
293                }
294                out.push_str(&format!("[{other}]"));
295            }
296        }
297    }
298    out
299}
300
301/// Minimal ISO 8601 parser — enough for Claude Code's
302/// `"2026-11-07T01:23:45.678Z"` shape. Returns unix seconds when the
303/// string parses cleanly; `None` otherwise (the caller falls back to
304/// `now()`, which keeps ordering correct within a single ingest).
305fn parse_iso8601_secs(s: &str) -> Option<i64> {
306    // Accept "YYYY-MM-DDTHH:MM:SS[.fraction][Z|±HH:MM]".
307    let bytes = s.as_bytes();
308    if bytes.len() < 19 || bytes[4] != b'-' || bytes[7] != b'-' || bytes[10] != b'T' {
309        return None;
310    }
311    let year: i64 = s.get(0..4)?.parse().ok()?;
312    let month: u32 = s.get(5..7)?.parse().ok()?;
313    let day: u32 = s.get(8..10)?.parse().ok()?;
314    let hour: u32 = s.get(11..13)?.parse().ok()?;
315    let minute: u32 = s.get(14..16)?.parse().ok()?;
316    let second: u32 = s.get(17..19)?.parse().ok()?;
317    Some(
318        days_from_civil(year, month, day) * 86_400
319            + hour as i64 * 3600
320            + minute as i64 * 60
321            + second as i64,
322    )
323}
324
325/// Howard Hinnant's `days_from_civil` — days since 1970-01-01 (Unix
326/// epoch) for a civil date. Used by the ISO 8601 parser above; keeps
327/// us out of any chrono / time dependency for v0.9.0.
328fn days_from_civil(y: i64, m: u32, d: u32) -> i64 {
329    let y = if m <= 2 { y - 1 } else { y };
330    let era = if y >= 0 { y } else { y - 399 } / 400;
331    let yoe = y - era * 400;
332    let doy = (153 * (if m > 2 { m - 3 } else { m + 9 }) as i64 + 2) / 5 + d as i64 - 1;
333    let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
334    era * 146_097 + doe - 719_468
335}
336
337#[cfg(test)]
338mod tests {
339    use super::*;
340
341    #[test]
342    fn jsonl_generic_round_trip() {
343        let src = r#"
344{"turn_id":"a","role":"user","content":"hi","timestamp":1700000000}
345{"turn_id":"b","role":"assistant","content":"hello"}
346"#;
347        let turns = read_transcript(src.as_bytes(), TranscriptFormat::JsonlGeneric).unwrap();
348        assert_eq!(turns.len(), 2);
349        assert_eq!(turns[0].turn_id, "a");
350        assert_eq!(turns[0].role, "user");
351        assert_eq!(turns[0].timestamp, Some(1_700_000_000));
352        assert_eq!(turns[1].timestamp, None);
353    }
354
355    #[test]
356    fn claude_code_jsonl_skips_non_message_types() {
357        let src = r#"{"type":"system","uuid":"sys-1","content":"init"}
358{"type":"user","uuid":"u-1","message":{"role":"user","content":"hi"}}
359{"type":"assistant","uuid":"a-1","message":{"role":"assistant","content":[{"type":"text","text":"hey"}]}}
360"#;
361        let turns = read_transcript(src.as_bytes(), TranscriptFormat::ClaudeCodeJsonl).unwrap();
362        assert_eq!(turns.len(), 2);
363        assert_eq!(turns[0].turn_id, "u-1");
364        assert_eq!(turns[1].content, "hey");
365    }
366
367    #[test]
368    fn claude_code_flattens_tool_use_and_tool_result() {
369        let src = r#"{"type":"assistant","uuid":"a","message":{"role":"assistant","content":[{"type":"text","text":"let me run:"},{"type":"tool_use","id":"t","name":"Bash","input":{}}]}}
370{"type":"user","uuid":"u","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"t","content":"ok\n","is_error":false}]}}
371"#;
372        let turns = read_transcript(src.as_bytes(), TranscriptFormat::ClaudeCodeJsonl).unwrap();
373        assert_eq!(turns.len(), 2);
374        assert!(turns[0].content.contains("let me run:"));
375        assert!(turns[0].content.contains("[tool_use name=Bash]"));
376        assert!(turns[1].content.contains("[tool_result ok]"));
377    }
378}