Skip to main content

second_brain_adapters/
raw_text.rs

1use std::path::Path;
2
3use chrono::{DateTime, Utc};
4use second_brain_core::redact::redact_for_store;
5use uuid::Uuid;
6
7use crate::claude_code::{
8    assistant_blocks, chunk_text, conversation_id, raw_memory_id, session_key_or_stem,
9    transcript_lines, typed_text,
10};
11
12const PASSAGE_CHARS: usize = 1500;
13
14#[derive(Debug, Clone, Copy, PartialEq, Eq)]
15pub enum RawRole {
16    User,
17    Assistant,
18    ToolCall,
19}
20
21#[derive(Debug, Clone, PartialEq, Eq)]
22pub struct RawText {
23    pub text: String,
24    pub at: Option<DateTime<Utc>>,
25    pub role: RawRole,
26    /// For assistant text, the id ingest gives the raw memory cut from it.
27    pub memory: Option<Uuid>,
28}
29
30fn capped(text: &str) -> String {
31    text.chars().take(PASSAGE_CHARS).collect()
32}
33
34/// Passages of the Claude Code transcript (JSONL) at `source` for the raw
35/// fallback index: what the user typed, every assistant text cut as ingest
36/// cuts it but unscreened, and each tool call's input, all redacted and in
37/// transcript order. User and tool-call passages are capped at 1500
38/// characters. `content` is the archive's copy, redacted when ingest read
39/// the original, so an assistant chunk's id is hashed from it as stored and
40/// matches ingest's even after the redaction rules change.
41pub fn raw_passages(source: &Path, content: &str) -> Vec<RawText> {
42    let conversation = conversation_id(&session_key_or_stem(source, content));
43    let mut passages = Vec::new();
44    for (msg, at) in transcript_lines(content) {
45        if let Some(text) = typed_text(&msg) {
46            passages.extend(
47                chunk_text(&redact_for_store(&text))
48                    .iter()
49                    .map(|chunk| RawText {
50                        text: capped(chunk),
51                        at,
52                        role: RawRole::User,
53                        memory: None,
54                    }),
55            );
56            continue;
57        }
58        let Some(blocks) = assistant_blocks(&msg) else {
59            continue;
60        };
61        for block in blocks {
62            match block.get("type").and_then(|t| t.as_str()) {
63                Some("text") => {
64                    let Some(text) = block.get("text").and_then(|t| t.as_str()) else {
65                        continue;
66                    };
67                    if text.trim().is_empty() {
68                        continue;
69                    }
70                    passages.extend(chunk_text(text).into_iter().map(|chunk| RawText {
71                        memory: Some(raw_memory_id(conversation, &chunk)),
72                        text: redact_for_store(&chunk),
73                        at,
74                        role: RawRole::Assistant,
75                    }));
76                }
77                Some("tool_use") => {
78                    let name = block.get("name").and_then(|n| n.as_str()).unwrap_or("");
79                    let input = block.get("input").cloned().unwrap_or_default();
80                    let rendered = redact_for_store(&format!("tool {name}: {input}"));
81                    passages.push(RawText {
82                        text: capped(&rendered),
83                        at,
84                        role: RawRole::ToolCall,
85                        memory: None,
86                    });
87                }
88                _ => {}
89            }
90        }
91    }
92    passages
93}
94
95#[cfg(test)]
96mod tests {
97    use super::*;
98    use crate::claude_code::{chunk_text, screen_transcript};
99    use second_brain_core::redact::redact;
100    use serde_json::{Value, json};
101
102    const SOURCE: &str = "projects/-srv-garden/s1.jsonl";
103
104    fn user(text: &str) -> Value {
105        json!({"type": "user", "message": {"content": text}})
106    }
107
108    fn assistant(blocks: Value) -> Value {
109        json!({"type": "assistant", "message": {"content": blocks}})
110    }
111
112    fn text_block(text: &str) -> Value {
113        json!({"type": "text", "text": text})
114    }
115
116    fn tool_use(name: &str, input: Value) -> Value {
117        json!({"type": "tool_use", "id": "call-1", "name": name, "input": input})
118    }
119
120    fn transcript(lines: &[Value]) -> String {
121        lines
122            .iter()
123            .map(Value::to_string)
124            .collect::<Vec<_>>()
125            .join("\n")
126    }
127
128    fn roles_and_texts(content: &str) -> Vec<(RawRole, String)> {
129        raw_passages(Path::new(SOURCE), content)
130            .into_iter()
131            .map(|p| (p.role, p.text))
132            .collect()
133    }
134
135    fn token_like() -> String {
136        ["gh", "p_"].concat() + &"a1B2".repeat(9)
137    }
138
139    fn long_block() -> String {
140        (0..12)
141            .map(|i| {
142                format!("Step {i} moves one batch of cache files to the new layout. ").repeat(4)
143            })
144            .collect::<Vec<_>>()
145            .join("\n\n")
146    }
147
148    struct Case {
149        name: &'static str,
150        content: String,
151        want: Vec<(RawRole, String)>,
152    }
153
154    fn cases() -> Vec<Case> {
155        use RawRole::{Assistant, ToolCall, User};
156        let mut meta = user("a skill body injected by the harness");
157        meta["isMeta"] = json!(true);
158        let tool_result = json!({"type": "user", "message": {"content": [
159            {"type": "tool_result", "tool_use_id": "call-1", "content": "file listing output"}
160        ]}});
161        let typed_blocks = json!({"type": "user", "message": {"content": [
162            {"type": "text", "text": "first part"},
163            {"type": "image", "source": {}},
164            {"type": "text", "text": "second part"}
165        ]}});
166        let queued = json!({"type": "attachment", "attachment": {
167            "type": "queued_command", "prompt": "also check the archive"
168        }});
169        let mut sidechain = assistant(json!([text_block("subagent report body")]));
170        sidechain["isSidechain"] = json!(true);
171
172        let long = long_block();
173        let long_chunks = chunk_text(&long);
174
175        let wide = "\u{e9}".repeat(3000);
176        let wide_rendered: String = format!("tool Write: {}", json!({"body": wide}))
177            .chars()
178            .take(1500)
179            .collect();
180
181        let token = token_like();
182
183        vec![
184            Case {
185                name: "typed user text vs tool result, meta, reminder-only and harness lines",
186                content: transcript(&[
187                    user("how do we rotate the cache files"),
188                    tool_result,
189                    meta,
190                    user("<system-reminder>context only</system-reminder>\n"),
191                    user("<command-name>/clear</command-name>"),
192                    user("<system-reminder>context</system-reminder>\nplease keep the old layout"),
193                    typed_blocks,
194                    queued,
195                ]),
196                want: vec![
197                    (User, "how do we rotate the cache files".into()),
198                    (User, "please keep the old layout".into()),
199                    (User, "first part\nsecond part".into()),
200                    (User, "also check the archive".into()),
201                ],
202            },
203            Case {
204                name: "assistant text and tool_use in block order, blank text dropped",
205                content: transcript(&[assistant(json!([
206                    text_block("ok"),
207                    tool_use("Read", json!({"path": "notes.txt", "limit": 20})),
208                    text_block("   \n  "),
209                    text_block("the layout is unchanged"),
210                ]))]),
211                want: vec![
212                    (Assistant, "ok".into()),
213                    (
214                        ToolCall,
215                        r#"tool Read: {"limit":20,"path":"notes.txt"}"#.into(),
216                    ),
217                    (Assistant, "the layout is unchanged".into()),
218                ],
219            },
220            Case {
221                name: "a long assistant block is cut by chunk_text",
222                content: transcript(&[assistant(json!([text_block(&long)]))]),
223                want: long_chunks.into_iter().map(|c| (Assistant, c)).collect(),
224            },
225            Case {
226                name: "a tool call is capped at 1500 characters on a char boundary",
227                content: transcript(&[assistant(json!([tool_use(
228                    "Write",
229                    json!({"body": wide})
230                )]))]),
231                want: vec![(ToolCall, wide_rendered)],
232            },
233            Case {
234                name: "redaction applies to every role",
235                content: transcript(&[
236                    user(&format!("use the token {token} for the mirror")),
237                    assistant(json!([
238                        text_block(&format!("the mirror reads {token} from its config")),
239                        tool_use(
240                            "Bash",
241                            json!({"command": format!("mirror --token {token}")})
242                        ),
243                    ])),
244                ]),
245                want: vec![
246                    (
247                        User,
248                        "use the token [REDACTED:github_token] for the mirror".into(),
249                    ),
250                    (
251                        Assistant,
252                        "the mirror reads [REDACTED:github_token] from its config".into(),
253                    ),
254                    (
255                        ToolCall,
256                        r#"tool Bash: {"command":"mirror --token [REDACTED:github_token]"}"#.into(),
257                    ),
258                ],
259            },
260            Case {
261                name: "malformed lines are skipped",
262                content: [
263                    "{not json".to_string(),
264                    String::new(),
265                    "[1, 2".to_string(),
266                    user("still read after a bad line").to_string(),
267                    "plain words".to_string(),
268                ]
269                .join("\n"),
270                want: vec![(User, "still read after a bad line".into())],
271            },
272            Case {
273                name: "sidechain lines are read like any other",
274                content: transcript(&[sidechain]),
275                want: vec![(Assistant, "subagent report body".into())],
276            },
277        ]
278    }
279
280    #[test]
281    fn passages_by_case() {
282        for case in cases() {
283            assert_eq!(roles_and_texts(&case.content), case.want, "{}", case.name);
284        }
285    }
286
287    #[test]
288    fn long_block_fixture_spans_several_chunks() {
289        assert!(chunk_text(&long_block()).len() > 1);
290    }
291
292    #[test]
293    fn tool_call_cap_counts_chars() {
294        let content = transcript(&[assistant(json!([tool_use(
295            "Write",
296            json!({"body": "\u{e9}".repeat(3000)})
297        )]))]);
298        let passages = raw_passages(Path::new(SOURCE), &content);
299        assert_eq!(passages.len(), 1);
300        assert_eq!(passages[0].text.chars().count(), 1500);
301    }
302
303    #[test]
304    fn at_is_the_line_timestamp() {
305        let mut stamped = user("a stamped question");
306        stamped["timestamp"] = json!("2026-03-04T05:06:07Z");
307        let mut answer = assistant(json!([
308            text_block("a stamped answer"),
309            tool_use("Read", json!({}))
310        ]));
311        answer["timestamp"] = json!("2026-03-04T05:06:09.500Z");
312        let content = transcript(&[stamped, answer, user("an unstamped question")]);
313
314        let at: Vec<Option<String>> = raw_passages(Path::new(SOURCE), &content)
315            .into_iter()
316            .map(|p| p.at.map(|t| t.to_rfc3339()))
317            .collect();
318        assert_eq!(
319            at,
320            vec![
321                Some("2026-03-04T05:06:07+00:00".to_string()),
322                Some("2026-03-04T05:06:09.500+00:00".to_string()),
323                Some("2026-03-04T05:06:09.500+00:00".to_string()),
324                None,
325            ]
326        );
327    }
328
329    #[test]
330    fn a_long_typed_message_is_cut_into_capped_passages() {
331        let pasted = format!("{}\n\n{}", "x".repeat(4000), "short tail");
332        let passages = raw_passages(Path::new(SOURCE), &transcript(&[user(&pasted)]));
333        let lengths: Vec<usize> = passages.iter().map(|p| p.text.chars().count()).collect();
334        assert_eq!(lengths, vec![1500, 10]);
335        assert!(
336            passages
337                .iter()
338                .all(|p| p.role == RawRole::User && p.memory.is_none())
339        );
340    }
341
342    #[test]
343    fn assistant_passages_carry_the_raw_memory_id_ingest_gives_them() {
344        let mut first = user("how do we rotate the cache files");
345        first["sessionId"] = json!("session-one");
346        let content = transcript(&[
347            first,
348            assistant(json!([
349                text_block("Rotate them weekly."),
350                tool_use("Read", json!({}))
351            ])),
352        ]);
353        let conversation = conversation_id(&session_key_or_stem(Path::new(SOURCE), &content));
354
355        let memories: Vec<Option<Uuid>> = raw_passages(Path::new(SOURCE), &content)
356            .into_iter()
357            .map(|p| p.memory)
358            .collect();
359        assert_eq!(
360            memories,
361            vec![
362                None,
363                Some(raw_memory_id(conversation, "Rotate them weekly.")),
364                None
365            ]
366        );
367    }
368
369    #[test]
370    fn assistant_passages_equal_ingest_chunks() {
371        let token = token_like();
372        let content = transcript(&[
373            user("how do we rotate the cache files"),
374            assistant(json!([
375                text_block("ok"),
376                tool_use("Read", json!({"path": "notes.txt"})),
377                text_block(&format!("{}\n\nThe mirror reads {token}.", long_block())),
378            ])),
379            assistant(json!([text_block("Let me check the archive next.")])),
380        ]);
381
382        let archived = redact(&content).text;
383        let conversation = conversation_id(&session_key_or_stem(Path::new(SOURCE), &content));
384
385        let assistant: Vec<(String, Option<Uuid>)> = raw_passages(Path::new(SOURCE), &archived)
386            .into_iter()
387            .filter(|p| p.role == RawRole::Assistant)
388            .map(|p| (p.text, p.memory))
389            .collect();
390        let ingest: Vec<(String, Option<Uuid>)> = screen_transcript(&content)
391            .into_iter()
392            .map(|c| {
393                let id = raw_memory_id(conversation, &c.text);
394                (c.text, Some(id))
395            })
396            .collect();
397
398        assert!(ingest.len() > 2);
399        assert_eq!(assistant, ingest);
400    }
401}