1use std::path::Path;
2
3use chrono::{DateTime, Utc};
4use second_brain_core::redact::redact_for_store;
5use uuid::Uuid;
6
7use crate::claude_code::{
8 assistant_blocks, chunk_text, conversation_id, raw_memory_id, session_key_or_stem,
9 transcript_lines, typed_text,
10};
11
12const PASSAGE_CHARS: usize = 1500;
13
14#[derive(Debug, Clone, Copy, PartialEq, Eq)]
15pub enum RawRole {
16 User,
17 Assistant,
18 ToolCall,
19}
20
21#[derive(Debug, Clone, PartialEq, Eq)]
22pub struct RawText {
23 pub text: String,
24 pub at: Option<DateTime<Utc>>,
25 pub role: RawRole,
26 pub memory: Option<Uuid>,
28}
29
30fn capped(text: &str) -> String {
31 text.chars().take(PASSAGE_CHARS).collect()
32}
33
34pub fn raw_passages(source: &Path, content: &str) -> Vec<RawText> {
42 let conversation = conversation_id(&session_key_or_stem(source, content));
43 let mut passages = Vec::new();
44 for (msg, at) in transcript_lines(content) {
45 if let Some(text) = typed_text(&msg) {
46 passages.extend(
47 chunk_text(&redact_for_store(&text))
48 .iter()
49 .map(|chunk| RawText {
50 text: capped(chunk),
51 at,
52 role: RawRole::User,
53 memory: None,
54 }),
55 );
56 continue;
57 }
58 let Some(blocks) = assistant_blocks(&msg) else {
59 continue;
60 };
61 for block in blocks {
62 match block.get("type").and_then(|t| t.as_str()) {
63 Some("text") => {
64 let Some(text) = block.get("text").and_then(|t| t.as_str()) else {
65 continue;
66 };
67 if text.trim().is_empty() {
68 continue;
69 }
70 passages.extend(chunk_text(text).into_iter().map(|chunk| RawText {
71 memory: Some(raw_memory_id(conversation, &chunk)),
72 text: redact_for_store(&chunk),
73 at,
74 role: RawRole::Assistant,
75 }));
76 }
77 Some("tool_use") => {
78 let name = block.get("name").and_then(|n| n.as_str()).unwrap_or("");
79 let input = block.get("input").cloned().unwrap_or_default();
80 let rendered = redact_for_store(&format!("tool {name}: {input}"));
81 passages.push(RawText {
82 text: capped(&rendered),
83 at,
84 role: RawRole::ToolCall,
85 memory: None,
86 });
87 }
88 _ => {}
89 }
90 }
91 }
92 passages
93}
94
95#[cfg(test)]
96mod tests {
97 use super::*;
98 use crate::claude_code::{chunk_text, screen_transcript};
99 use second_brain_core::redact::redact;
100 use serde_json::{Value, json};
101
102 const SOURCE: &str = "projects/-srv-garden/s1.jsonl";
103
104 fn user(text: &str) -> Value {
105 json!({"type": "user", "message": {"content": text}})
106 }
107
108 fn assistant(blocks: Value) -> Value {
109 json!({"type": "assistant", "message": {"content": blocks}})
110 }
111
112 fn text_block(text: &str) -> Value {
113 json!({"type": "text", "text": text})
114 }
115
116 fn tool_use(name: &str, input: Value) -> Value {
117 json!({"type": "tool_use", "id": "call-1", "name": name, "input": input})
118 }
119
120 fn transcript(lines: &[Value]) -> String {
121 lines
122 .iter()
123 .map(Value::to_string)
124 .collect::<Vec<_>>()
125 .join("\n")
126 }
127
128 fn roles_and_texts(content: &str) -> Vec<(RawRole, String)> {
129 raw_passages(Path::new(SOURCE), content)
130 .into_iter()
131 .map(|p| (p.role, p.text))
132 .collect()
133 }
134
135 fn token_like() -> String {
136 ["gh", "p_"].concat() + &"a1B2".repeat(9)
137 }
138
139 fn long_block() -> String {
140 (0..12)
141 .map(|i| {
142 format!("Step {i} moves one batch of cache files to the new layout. ").repeat(4)
143 })
144 .collect::<Vec<_>>()
145 .join("\n\n")
146 }
147
148 struct Case {
149 name: &'static str,
150 content: String,
151 want: Vec<(RawRole, String)>,
152 }
153
154 fn cases() -> Vec<Case> {
155 use RawRole::{Assistant, ToolCall, User};
156 let mut meta = user("a skill body injected by the harness");
157 meta["isMeta"] = json!(true);
158 let tool_result = json!({"type": "user", "message": {"content": [
159 {"type": "tool_result", "tool_use_id": "call-1", "content": "file listing output"}
160 ]}});
161 let typed_blocks = json!({"type": "user", "message": {"content": [
162 {"type": "text", "text": "first part"},
163 {"type": "image", "source": {}},
164 {"type": "text", "text": "second part"}
165 ]}});
166 let queued = json!({"type": "attachment", "attachment": {
167 "type": "queued_command", "prompt": "also check the archive"
168 }});
169 let mut sidechain = assistant(json!([text_block("subagent report body")]));
170 sidechain["isSidechain"] = json!(true);
171
172 let long = long_block();
173 let long_chunks = chunk_text(&long);
174
175 let wide = "\u{e9}".repeat(3000);
176 let wide_rendered: String = format!("tool Write: {}", json!({"body": wide}))
177 .chars()
178 .take(1500)
179 .collect();
180
181 let token = token_like();
182
183 vec![
184 Case {
185 name: "typed user text vs tool result, meta, reminder-only and harness lines",
186 content: transcript(&[
187 user("how do we rotate the cache files"),
188 tool_result,
189 meta,
190 user("<system-reminder>context only</system-reminder>\n"),
191 user("<command-name>/clear</command-name>"),
192 user("<system-reminder>context</system-reminder>\nplease keep the old layout"),
193 typed_blocks,
194 queued,
195 ]),
196 want: vec![
197 (User, "how do we rotate the cache files".into()),
198 (User, "please keep the old layout".into()),
199 (User, "first part\nsecond part".into()),
200 (User, "also check the archive".into()),
201 ],
202 },
203 Case {
204 name: "assistant text and tool_use in block order, blank text dropped",
205 content: transcript(&[assistant(json!([
206 text_block("ok"),
207 tool_use("Read", json!({"path": "notes.txt", "limit": 20})),
208 text_block(" \n "),
209 text_block("the layout is unchanged"),
210 ]))]),
211 want: vec![
212 (Assistant, "ok".into()),
213 (
214 ToolCall,
215 r#"tool Read: {"limit":20,"path":"notes.txt"}"#.into(),
216 ),
217 (Assistant, "the layout is unchanged".into()),
218 ],
219 },
220 Case {
221 name: "a long assistant block is cut by chunk_text",
222 content: transcript(&[assistant(json!([text_block(&long)]))]),
223 want: long_chunks.into_iter().map(|c| (Assistant, c)).collect(),
224 },
225 Case {
226 name: "a tool call is capped at 1500 characters on a char boundary",
227 content: transcript(&[assistant(json!([tool_use(
228 "Write",
229 json!({"body": wide})
230 )]))]),
231 want: vec![(ToolCall, wide_rendered)],
232 },
233 Case {
234 name: "redaction applies to every role",
235 content: transcript(&[
236 user(&format!("use the token {token} for the mirror")),
237 assistant(json!([
238 text_block(&format!("the mirror reads {token} from its config")),
239 tool_use(
240 "Bash",
241 json!({"command": format!("mirror --token {token}")})
242 ),
243 ])),
244 ]),
245 want: vec![
246 (
247 User,
248 "use the token [REDACTED:github_token] for the mirror".into(),
249 ),
250 (
251 Assistant,
252 "the mirror reads [REDACTED:github_token] from its config".into(),
253 ),
254 (
255 ToolCall,
256 r#"tool Bash: {"command":"mirror --token [REDACTED:github_token]"}"#.into(),
257 ),
258 ],
259 },
260 Case {
261 name: "malformed lines are skipped",
262 content: [
263 "{not json".to_string(),
264 String::new(),
265 "[1, 2".to_string(),
266 user("still read after a bad line").to_string(),
267 "plain words".to_string(),
268 ]
269 .join("\n"),
270 want: vec![(User, "still read after a bad line".into())],
271 },
272 Case {
273 name: "sidechain lines are read like any other",
274 content: transcript(&[sidechain]),
275 want: vec![(Assistant, "subagent report body".into())],
276 },
277 ]
278 }
279
280 #[test]
281 fn passages_by_case() {
282 for case in cases() {
283 assert_eq!(roles_and_texts(&case.content), case.want, "{}", case.name);
284 }
285 }
286
287 #[test]
288 fn long_block_fixture_spans_several_chunks() {
289 assert!(chunk_text(&long_block()).len() > 1);
290 }
291
292 #[test]
293 fn tool_call_cap_counts_chars() {
294 let content = transcript(&[assistant(json!([tool_use(
295 "Write",
296 json!({"body": "\u{e9}".repeat(3000)})
297 )]))]);
298 let passages = raw_passages(Path::new(SOURCE), &content);
299 assert_eq!(passages.len(), 1);
300 assert_eq!(passages[0].text.chars().count(), 1500);
301 }
302
303 #[test]
304 fn at_is_the_line_timestamp() {
305 let mut stamped = user("a stamped question");
306 stamped["timestamp"] = json!("2026-03-04T05:06:07Z");
307 let mut answer = assistant(json!([
308 text_block("a stamped answer"),
309 tool_use("Read", json!({}))
310 ]));
311 answer["timestamp"] = json!("2026-03-04T05:06:09.500Z");
312 let content = transcript(&[stamped, answer, user("an unstamped question")]);
313
314 let at: Vec<Option<String>> = raw_passages(Path::new(SOURCE), &content)
315 .into_iter()
316 .map(|p| p.at.map(|t| t.to_rfc3339()))
317 .collect();
318 assert_eq!(
319 at,
320 vec![
321 Some("2026-03-04T05:06:07+00:00".to_string()),
322 Some("2026-03-04T05:06:09.500+00:00".to_string()),
323 Some("2026-03-04T05:06:09.500+00:00".to_string()),
324 None,
325 ]
326 );
327 }
328
329 #[test]
330 fn a_long_typed_message_is_cut_into_capped_passages() {
331 let pasted = format!("{}\n\n{}", "x".repeat(4000), "short tail");
332 let passages = raw_passages(Path::new(SOURCE), &transcript(&[user(&pasted)]));
333 let lengths: Vec<usize> = passages.iter().map(|p| p.text.chars().count()).collect();
334 assert_eq!(lengths, vec![1500, 10]);
335 assert!(
336 passages
337 .iter()
338 .all(|p| p.role == RawRole::User && p.memory.is_none())
339 );
340 }
341
342 #[test]
343 fn assistant_passages_carry_the_raw_memory_id_ingest_gives_them() {
344 let mut first = user("how do we rotate the cache files");
345 first["sessionId"] = json!("session-one");
346 let content = transcript(&[
347 first,
348 assistant(json!([
349 text_block("Rotate them weekly."),
350 tool_use("Read", json!({}))
351 ])),
352 ]);
353 let conversation = conversation_id(&session_key_or_stem(Path::new(SOURCE), &content));
354
355 let memories: Vec<Option<Uuid>> = raw_passages(Path::new(SOURCE), &content)
356 .into_iter()
357 .map(|p| p.memory)
358 .collect();
359 assert_eq!(
360 memories,
361 vec![
362 None,
363 Some(raw_memory_id(conversation, "Rotate them weekly.")),
364 None
365 ]
366 );
367 }
368
369 #[test]
370 fn assistant_passages_equal_ingest_chunks() {
371 let token = token_like();
372 let content = transcript(&[
373 user("how do we rotate the cache files"),
374 assistant(json!([
375 text_block("ok"),
376 tool_use("Read", json!({"path": "notes.txt"})),
377 text_block(&format!("{}\n\nThe mirror reads {token}.", long_block())),
378 ])),
379 assistant(json!([text_block("Let me check the archive next.")])),
380 ]);
381
382 let archived = redact(&content).text;
383 let conversation = conversation_id(&session_key_or_stem(Path::new(SOURCE), &content));
384
385 let assistant: Vec<(String, Option<Uuid>)> = raw_passages(Path::new(SOURCE), &archived)
386 .into_iter()
387 .filter(|p| p.role == RawRole::Assistant)
388 .map(|p| (p.text, p.memory))
389 .collect();
390 let ingest: Vec<(String, Option<Uuid>)> = screen_transcript(&content)
391 .into_iter()
392 .map(|c| {
393 let id = raw_memory_id(conversation, &c.text);
394 (c.text, Some(id))
395 })
396 .collect();
397
398 assert!(ingest.len() > 2);
399 assert_eq!(assistant, ingest);
400 }
401}