mnemo_engine/
transcript.rs

1//! Phase 12.7a transcript parsers.
2//!
3//! Turns a file on disk (or an in-memory reader) into a `Vec<ThreadTurn>`
4//! the [`crate::Mnemo::ingest_thread`] path expects. Two formats land in
5//! v0.9.0; the general shape is one JSON object per line.
6//!
7//! * [`TranscriptFormat::JsonlGeneric`] — the documented shape anyone
8//!   can produce:
9//!   ```json
10//!   {"turn_id": "...", "role": "user", "content": "...", "timestamp": 1234567890}
11//!   ```
12//!   `timestamp` is optional (unix seconds). Unknown keys are ignored.
13//!
14//! * [`TranscriptFormat::ClaudeCodeJsonl`] — Claude Code writes its
15//!   session transcripts at `~/.claude/projects/<project>/<session>.jsonl`
16//!   with one JSON object per line. Each line is tagged with a `type`
17//!   field (`"message"`, `"system"`, `"tool_use"`, …) and carries a
18//!   `uuid` the parser uses as `turn_id`. Only `type == "user"` or
19//!   `type == "assistant"` lines yield turns; every other line type is
20//!   skipped so new line types don't break the parser.
21
22use std::io::{BufRead, BufReader, Read};
23
24use serde_json::Value;
25
26use crate::error::{MnemoError, Result};
27use crate::memory::ThreadTurn;
28
29/// The supported transcript shapes for [`read_transcript`] /
30/// `mnemo thread ingest --format …`.
31#[derive(Debug, Clone, Copy, PartialEq, Eq)]
32pub enum TranscriptFormat {
33    /// `{"turn_id": …, "role": …, "content": …, "timestamp"?: …}` per line.
34    JsonlGeneric,
35    /// Claude Code's `~/.claude/projects/<p>/<s>.jsonl` shape; see the
36    /// module docs.
37    ClaudeCodeJsonl,
38}
39
40impl TranscriptFormat {
41    /// Parse a `--format` string. Case-insensitive; accepts `jsonl` and
42    /// `claude-code-jsonl` (or `claude_code_jsonl` for CLI convenience).
43    pub fn parse(s: &str) -> Option<TranscriptFormat> {
44        match s.to_ascii_lowercase().as_str() {
45            "jsonl" => Some(TranscriptFormat::JsonlGeneric),
46            "claude-code-jsonl" | "claude_code_jsonl" => Some(TranscriptFormat::ClaudeCodeJsonl),
47            _ => None,
48        }
49    }
50}
51
52/// Read a transcript from any `Read` in the given format and return the
53/// turns in file order. Non-matching / unrecognized line types are
54/// skipped silently (Claude Code adds new `type` values over time).
55pub fn read_transcript<R: Read>(reader: R, format: TranscriptFormat) -> Result<Vec<ThreadTurn>> {
56    let mut out = Vec::new();
57    for (lineno, line) in BufReader::new(reader).lines().enumerate() {
58        let line = line.map_err(MnemoError::Io)?;
59        let trimmed = line.trim();
60        if trimmed.is_empty() || trimmed.starts_with('#') {
61            continue;
62        }
63        let v: Value = serde_json::from_str(trimmed).map_err(|e| {
64            MnemoError::Invalid(format!("transcript line {}: invalid JSON: {e}", lineno + 1))
65        })?;
66        let turn = match format {
67            TranscriptFormat::JsonlGeneric => parse_generic_line(&v)?,
68            TranscriptFormat::ClaudeCodeJsonl => parse_claude_code_line(&v)?,
69        };
70        if let Some(t) = turn {
71            out.push(t);
72        }
73    }
74    Ok(out)
75}
76
77fn parse_generic_line(v: &Value) -> Result<Option<ThreadTurn>> {
78    let obj = v.as_object().ok_or_else(|| {
79        MnemoError::Invalid("transcript: generic jsonl line must be a JSON object".into())
80    })?;
81    let turn_id = obj
82        .get("turn_id")
83        .and_then(|v| v.as_str())
84        .ok_or_else(|| {
85            MnemoError::Invalid("transcript: generic jsonl line missing 'turn_id' (string)".into())
86        })?
87        .to_string();
88    let role = obj
89        .get("role")
90        .and_then(|v| v.as_str())
91        .ok_or_else(|| {
92            MnemoError::Invalid("transcript: generic jsonl line missing 'role' (string)".into())
93        })?
94        .to_string();
95    let content = obj
96        .get("content")
97        .and_then(|v| v.as_str())
98        .ok_or_else(|| {
99            MnemoError::Invalid("transcript: generic jsonl line missing 'content' (string)".into())
100        })?
101        .to_string();
102    let timestamp = obj.get("timestamp").and_then(|v| v.as_i64());
103    Ok(Some(ThreadTurn {
104        turn_id,
105        role,
106        content,
107        timestamp,
108    }))
109}
110
111/// Parse one Claude Code jsonl line. Returns `None` for any line that
112/// isn't a user/assistant message (system init records, tool_use
113/// scaffolding, etc.) so new line types added upstream don't break
114/// ingest.
115///
116/// Expected shape (trimmed to the fields we read):
117/// ```json
118/// {"type": "user", "uuid": "…", "message": {"role": "user", "content": "…"}}
119/// {"type": "assistant", "uuid": "…", "message": {"role": "assistant", "content": [{"type":"text","text":"…"}]}}
120/// ```
121///
122/// `message.content` can be either a plain string (older captures) or a
123/// list of content blocks each with `{"type": "text", "text": "…"}`.
124/// Non-text blocks (tool_use, tool_result, image) are flattened into a
125/// readable text summary so the archive stays browsable without being
126/// lossless about the original block structure.
127fn parse_claude_code_line(v: &Value) -> Result<Option<ThreadTurn>> {
128    let obj = match v.as_object() {
129        Some(o) => o,
130        None => return Ok(None),
131    };
132    let type_tag = obj.get("type").and_then(|v| v.as_str()).unwrap_or("");
133    if type_tag != "user" && type_tag != "assistant" {
134        // Skip system init, summary, and other non-message types.
135        return Ok(None);
136    }
137    let turn_id = obj
138        .get("uuid")
139        .and_then(|v| v.as_str())
140        .map(|s| s.to_string())
141        .ok_or_else(|| {
142            MnemoError::Invalid("transcript: claude-code-jsonl message missing 'uuid'".into())
143        })?;
144    let message = obj
145        .get("message")
146        .and_then(|v| v.as_object())
147        .ok_or_else(|| {
148            MnemoError::Invalid(
149                "transcript: claude-code-jsonl message missing 'message' object".into(),
150            )
151        })?;
152    let role = message
153        .get("role")
154        .and_then(|v| v.as_str())
155        .unwrap_or(type_tag)
156        .to_string();
157    let content = match message.get("content") {
158        Some(Value::String(s)) => s.clone(),
159        Some(Value::Array(blocks)) => flatten_blocks(blocks),
160        _ => String::new(),
161    };
162    // Timestamps in Claude Code jsonl live under `message.timestamp`
163    // (ISO 8601) or the line's top-level `timestamp`. Convert ISO 8601
164    // to unix seconds with a minimal parser; on any failure, let the
165    // ingest path pick `now()`.
166    let timestamp = obj
167        .get("timestamp")
168        .and_then(|v| v.as_str())
169        .and_then(parse_iso8601_secs)
170        .or_else(|| {
171            message
172                .get("timestamp")
173                .and_then(|v| v.as_str())
174                .and_then(parse_iso8601_secs)
175        });
176    Ok(Some(ThreadTurn {
177        turn_id,
178        role,
179        content,
180        timestamp,
181    }))
182}
183
184/// Flatten a Claude Code `message.content` array of blocks into a
185/// single string. Text blocks come through verbatim; other block types
186/// are rendered as short readable summaries so the archive is
187/// grep-able (`[tool_use name=Bash]`, `[tool_result ok]`, `[image]`).
188fn flatten_blocks(blocks: &[Value]) -> String {
189    let mut out = String::new();
190    for b in blocks {
191        let Some(obj) = b.as_object() else {
192            continue;
193        };
194        let t = obj.get("type").and_then(|v| v.as_str()).unwrap_or("");
195        match t {
196            "text" => {
197                if let Some(s) = obj.get("text").and_then(|v| v.as_str()) {
198                    if !out.is_empty() {
199                        out.push('\n');
200                    }
201                    out.push_str(s);
202                }
203            }
204            "tool_use" => {
205                let name = obj.get("name").and_then(|v| v.as_str()).unwrap_or("?");
206                if !out.is_empty() {
207                    out.push('\n');
208                }
209                out.push_str(&format!("[tool_use name={name}]"));
210            }
211            "tool_result" => {
212                let is_error = obj
213                    .get("is_error")
214                    .and_then(|v| v.as_bool())
215                    .unwrap_or(false);
216                let status = if is_error { "error" } else { "ok" };
217                let inner = obj.get("content").and_then(|c| match c {
218                    Value::String(s) => Some(s.clone()),
219                    Value::Array(nested) => Some(flatten_blocks(nested)),
220                    _ => None,
221                });
222                if !out.is_empty() {
223                    out.push('\n');
224                }
225                match inner {
226                    Some(body) => out.push_str(&format!("[tool_result {status}] {body}")),
227                    None => out.push_str(&format!("[tool_result {status}]")),
228                }
229            }
230            "image" => {
231                if !out.is_empty() {
232                    out.push('\n');
233                }
234                out.push_str("[image]");
235            }
236            other => {
237                if !out.is_empty() {
238                    out.push('\n');
239                }
240                out.push_str(&format!("[{other}]"));
241            }
242        }
243    }
244    out
245}
246
247/// Minimal ISO 8601 parser — enough for Claude Code's
248/// `"2026-11-07T01:23:45.678Z"` shape. Returns unix seconds when the
249/// string parses cleanly; `None` otherwise (the caller falls back to
250/// `now()`, which keeps ordering correct within a single ingest).
251fn parse_iso8601_secs(s: &str) -> Option<i64> {
252    // Accept "YYYY-MM-DDTHH:MM:SS[.fraction][Z|±HH:MM]".
253    let bytes = s.as_bytes();
254    if bytes.len() < 19 || bytes[4] != b'-' || bytes[7] != b'-' || bytes[10] != b'T' {
255        return None;
256    }
257    let year: i64 = s.get(0..4)?.parse().ok()?;
258    let month: u32 = s.get(5..7)?.parse().ok()?;
259    let day: u32 = s.get(8..10)?.parse().ok()?;
260    let hour: u32 = s.get(11..13)?.parse().ok()?;
261    let minute: u32 = s.get(14..16)?.parse().ok()?;
262    let second: u32 = s.get(17..19)?.parse().ok()?;
263    Some(
264        days_from_civil(year, month, day) * 86_400
265            + hour as i64 * 3600
266            + minute as i64 * 60
267            + second as i64,
268    )
269}
270
271/// Howard Hinnant's `days_from_civil` — days since 1970-01-01 (Unix
272/// epoch) for a civil date. Used by the ISO 8601 parser above; keeps
273/// us out of any chrono / time dependency for v0.9.0.
274fn days_from_civil(y: i64, m: u32, d: u32) -> i64 {
275    let y = if m <= 2 { y - 1 } else { y };
276    let era = if y >= 0 { y } else { y - 399 } / 400;
277    let yoe = y - era * 400;
278    let doy = (153 * (if m > 2 { m - 3 } else { m + 9 }) as i64 + 2) / 5 + d as i64 - 1;
279    let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
280    era * 146_097 + doe - 719_468
281}
282
283#[cfg(test)]
284mod tests {
285    use super::*;
286
287    #[test]
288    fn jsonl_generic_round_trip() {
289        let src = r#"
290{"turn_id":"a","role":"user","content":"hi","timestamp":1700000000}
291{"turn_id":"b","role":"assistant","content":"hello"}
292"#;
293        let turns = read_transcript(src.as_bytes(), TranscriptFormat::JsonlGeneric).unwrap();
294        assert_eq!(turns.len(), 2);
295        assert_eq!(turns[0].turn_id, "a");
296        assert_eq!(turns[0].role, "user");
297        assert_eq!(turns[0].timestamp, Some(1_700_000_000));
298        assert_eq!(turns[1].timestamp, None);
299    }
300
301    #[test]
302    fn claude_code_jsonl_skips_non_message_types() {
303        let src = r#"{"type":"system","uuid":"sys-1","content":"init"}
304{"type":"user","uuid":"u-1","message":{"role":"user","content":"hi"}}
305{"type":"assistant","uuid":"a-1","message":{"role":"assistant","content":[{"type":"text","text":"hey"}]}}
306"#;
307        let turns = read_transcript(src.as_bytes(), TranscriptFormat::ClaudeCodeJsonl).unwrap();
308        assert_eq!(turns.len(), 2);
309        assert_eq!(turns[0].turn_id, "u-1");
310        assert_eq!(turns[1].content, "hey");
311    }
312
313    #[test]
314    fn claude_code_flattens_tool_use_and_tool_result() {
315        let src = r#"{"type":"assistant","uuid":"a","message":{"role":"assistant","content":[{"type":"text","text":"let me run:"},{"type":"tool_use","id":"t","name":"Bash","input":{}}]}}
316{"type":"user","uuid":"u","message":{"role":"user","content":[{"type":"tool_result","tool_use_id":"t","content":"ok\n","is_error":false}]}}
317"#;
318        let turns = read_transcript(src.as_bytes(), TranscriptFormat::ClaudeCodeJsonl).unwrap();
319        assert_eq!(turns.len(), 2);
320        assert!(turns[0].content.contains("let me run:"));
321        assert!(turns[0].content.contains("[tool_use name=Bash]"));
322        assert!(turns[1].content.contains("[tool_result ok]"));
323    }
324}
mnemo_engine/transcript.rs

mnemo_engine/
transcript.rs
