use std::path::Path;
use chrono::{DateTime, Utc};
use second_brain_core::redact::redact_for_store;
use uuid::Uuid;
use crate::claude_code::{
assistant_blocks, chunk_text, conversation_id, raw_memory_id, session_key_or_stem,
transcript_lines, typed_text,
};
const PASSAGE_CHARS: usize = 1500;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RawRole {
User,
Assistant,
ToolCall,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RawText {
pub text: String,
pub at: Option<DateTime<Utc>>,
pub role: RawRole,
pub memory: Option<Uuid>,
}
fn capped(text: &str) -> String {
text.chars().take(PASSAGE_CHARS).collect()
}
pub fn raw_passages(source: &Path, content: &str) -> Vec<RawText> {
let conversation = conversation_id(&session_key_or_stem(source, content));
let mut passages = Vec::new();
for (msg, at) in transcript_lines(content) {
if let Some(text) = typed_text(&msg) {
passages.extend(
chunk_text(&redact_for_store(&text))
.iter()
.map(|chunk| RawText {
text: capped(chunk),
at,
role: RawRole::User,
memory: None,
}),
);
continue;
}
let Some(blocks) = assistant_blocks(&msg) else {
continue;
};
for block in blocks {
match block.get("type").and_then(|t| t.as_str()) {
Some("text") => {
let Some(text) = block.get("text").and_then(|t| t.as_str()) else {
continue;
};
if text.trim().is_empty() {
continue;
}
passages.extend(chunk_text(text).into_iter().map(|chunk| RawText {
memory: Some(raw_memory_id(conversation, &chunk)),
text: redact_for_store(&chunk),
at,
role: RawRole::Assistant,
}));
}
Some("tool_use") => {
let name = block.get("name").and_then(|n| n.as_str()).unwrap_or("");
let input = block.get("input").cloned().unwrap_or_default();
let rendered = redact_for_store(&format!("tool {name}: {input}"));
passages.push(RawText {
text: capped(&rendered),
at,
role: RawRole::ToolCall,
memory: None,
});
}
_ => {}
}
}
}
passages
}
#[cfg(test)]
mod tests {
use super::*;
use crate::claude_code::{chunk_text, screen_transcript};
use second_brain_core::redact::redact;
use serde_json::{Value, json};
const SOURCE: &str = "projects/-srv-garden/s1.jsonl";
fn user(text: &str) -> Value {
json!({"type": "user", "message": {"content": text}})
}
fn assistant(blocks: Value) -> Value {
json!({"type": "assistant", "message": {"content": blocks}})
}
fn text_block(text: &str) -> Value {
json!({"type": "text", "text": text})
}
fn tool_use(name: &str, input: Value) -> Value {
json!({"type": "tool_use", "id": "call-1", "name": name, "input": input})
}
fn transcript(lines: &[Value]) -> String {
lines
.iter()
.map(Value::to_string)
.collect::<Vec<_>>()
.join("\n")
}
fn roles_and_texts(content: &str) -> Vec<(RawRole, String)> {
raw_passages(Path::new(SOURCE), content)
.into_iter()
.map(|p| (p.role, p.text))
.collect()
}
fn token_like() -> String {
["gh", "p_"].concat() + &"a1B2".repeat(9)
}
fn long_block() -> String {
(0..12)
.map(|i| {
format!("Step {i} moves one batch of cache files to the new layout. ").repeat(4)
})
.collect::<Vec<_>>()
.join("\n\n")
}
struct Case {
name: &'static str,
content: String,
want: Vec<(RawRole, String)>,
}
fn cases() -> Vec<Case> {
use RawRole::{Assistant, ToolCall, User};
let mut meta = user("a skill body injected by the harness");
meta["isMeta"] = json!(true);
let tool_result = json!({"type": "user", "message": {"content": [
{"type": "tool_result", "tool_use_id": "call-1", "content": "file listing output"}
]}});
let typed_blocks = json!({"type": "user", "message": {"content": [
{"type": "text", "text": "first part"},
{"type": "image", "source": {}},
{"type": "text", "text": "second part"}
]}});
let queued = json!({"type": "attachment", "attachment": {
"type": "queued_command", "prompt": "also check the archive"
}});
let mut sidechain = assistant(json!([text_block("subagent report body")]));
sidechain["isSidechain"] = json!(true);
let long = long_block();
let long_chunks = chunk_text(&long);
let wide = "\u{e9}".repeat(3000);
let wide_rendered: String = format!("tool Write: {}", json!({"body": wide}))
.chars()
.take(1500)
.collect();
let token = token_like();
vec![
Case {
name: "typed user text vs tool result, meta, reminder-only and harness lines",
content: transcript(&[
user("how do we rotate the cache files"),
tool_result,
meta,
user("<system-reminder>context only</system-reminder>\n"),
user("<command-name>/clear</command-name>"),
user("<system-reminder>context</system-reminder>\nplease keep the old layout"),
typed_blocks,
queued,
]),
want: vec![
(User, "how do we rotate the cache files".into()),
(User, "please keep the old layout".into()),
(User, "first part\nsecond part".into()),
(User, "also check the archive".into()),
],
},
Case {
name: "assistant text and tool_use in block order, blank text dropped",
content: transcript(&[assistant(json!([
text_block("ok"),
tool_use("Read", json!({"path": "notes.txt", "limit": 20})),
text_block(" \n "),
text_block("the layout is unchanged"),
]))]),
want: vec![
(Assistant, "ok".into()),
(
ToolCall,
r#"tool Read: {"limit":20,"path":"notes.txt"}"#.into(),
),
(Assistant, "the layout is unchanged".into()),
],
},
Case {
name: "a long assistant block is cut by chunk_text",
content: transcript(&[assistant(json!([text_block(&long)]))]),
want: long_chunks.into_iter().map(|c| (Assistant, c)).collect(),
},
Case {
name: "a tool call is capped at 1500 characters on a char boundary",
content: transcript(&[assistant(json!([tool_use(
"Write",
json!({"body": wide})
)]))]),
want: vec![(ToolCall, wide_rendered)],
},
Case {
name: "redaction applies to every role",
content: transcript(&[
user(&format!("use the token {token} for the mirror")),
assistant(json!([
text_block(&format!("the mirror reads {token} from its config")),
tool_use(
"Bash",
json!({"command": format!("mirror --token {token}")})
),
])),
]),
want: vec![
(
User,
"use the token [REDACTED:github_token] for the mirror".into(),
),
(
Assistant,
"the mirror reads [REDACTED:github_token] from its config".into(),
),
(
ToolCall,
r#"tool Bash: {"command":"mirror --token [REDACTED:github_token]"}"#.into(),
),
],
},
Case {
name: "malformed lines are skipped",
content: [
"{not json".to_string(),
String::new(),
"[1, 2".to_string(),
user("still read after a bad line").to_string(),
"plain words".to_string(),
]
.join("\n"),
want: vec![(User, "still read after a bad line".into())],
},
Case {
name: "sidechain lines are read like any other",
content: transcript(&[sidechain]),
want: vec![(Assistant, "subagent report body".into())],
},
]
}
#[test]
fn passages_by_case() {
for case in cases() {
assert_eq!(roles_and_texts(&case.content), case.want, "{}", case.name);
}
}
#[test]
fn long_block_fixture_spans_several_chunks() {
assert!(chunk_text(&long_block()).len() > 1);
}
#[test]
fn tool_call_cap_counts_chars() {
let content = transcript(&[assistant(json!([tool_use(
"Write",
json!({"body": "\u{e9}".repeat(3000)})
)]))]);
let passages = raw_passages(Path::new(SOURCE), &content);
assert_eq!(passages.len(), 1);
assert_eq!(passages[0].text.chars().count(), 1500);
}
#[test]
fn at_is_the_line_timestamp() {
let mut stamped = user("a stamped question");
stamped["timestamp"] = json!("2026-03-04T05:06:07Z");
let mut answer = assistant(json!([
text_block("a stamped answer"),
tool_use("Read", json!({}))
]));
answer["timestamp"] = json!("2026-03-04T05:06:09.500Z");
let content = transcript(&[stamped, answer, user("an unstamped question")]);
let at: Vec<Option<String>> = raw_passages(Path::new(SOURCE), &content)
.into_iter()
.map(|p| p.at.map(|t| t.to_rfc3339()))
.collect();
assert_eq!(
at,
vec![
Some("2026-03-04T05:06:07+00:00".to_string()),
Some("2026-03-04T05:06:09.500+00:00".to_string()),
Some("2026-03-04T05:06:09.500+00:00".to_string()),
None,
]
);
}
#[test]
fn a_long_typed_message_is_cut_into_capped_passages() {
let pasted = format!("{}\n\n{}", "x".repeat(4000), "short tail");
let passages = raw_passages(Path::new(SOURCE), &transcript(&[user(&pasted)]));
let lengths: Vec<usize> = passages.iter().map(|p| p.text.chars().count()).collect();
assert_eq!(lengths, vec![1500, 10]);
assert!(
passages
.iter()
.all(|p| p.role == RawRole::User && p.memory.is_none())
);
}
#[test]
fn assistant_passages_carry_the_raw_memory_id_ingest_gives_them() {
let mut first = user("how do we rotate the cache files");
first["sessionId"] = json!("session-one");
let content = transcript(&[
first,
assistant(json!([
text_block("Rotate them weekly."),
tool_use("Read", json!({}))
])),
]);
let conversation = conversation_id(&session_key_or_stem(Path::new(SOURCE), &content));
let memories: Vec<Option<Uuid>> = raw_passages(Path::new(SOURCE), &content)
.into_iter()
.map(|p| p.memory)
.collect();
assert_eq!(
memories,
vec![
None,
Some(raw_memory_id(conversation, "Rotate them weekly.")),
None
]
);
}
#[test]
fn assistant_passages_equal_ingest_chunks() {
let token = token_like();
let content = transcript(&[
user("how do we rotate the cache files"),
assistant(json!([
text_block("ok"),
tool_use("Read", json!({"path": "notes.txt"})),
text_block(&format!("{}\n\nThe mirror reads {token}.", long_block())),
])),
assistant(json!([text_block("Let me check the archive next.")])),
]);
let archived = redact(&content).text;
let conversation = conversation_id(&session_key_or_stem(Path::new(SOURCE), &content));
let assistant: Vec<(String, Option<Uuid>)> = raw_passages(Path::new(SOURCE), &archived)
.into_iter()
.filter(|p| p.role == RawRole::Assistant)
.map(|p| (p.text, p.memory))
.collect();
let ingest: Vec<(String, Option<Uuid>)> = screen_transcript(&content)
.into_iter()
.map(|c| {
let id = raw_memory_id(conversation, &c.text);
(c.text, Some(id))
})
.collect();
assert!(ingest.len() > 2);
assert_eq!(assistant, ingest);
}
}