second-brain-adapters 0.12.0

Conversation parsers for second-brain: Claude Code and Cursor JSONL adapters
Documentation
use std::path::Path;

use chrono::{DateTime, Utc};
use second_brain_core::redact::redact_for_store;
use uuid::Uuid;

use crate::claude_code::{
    assistant_blocks, chunk_text, conversation_id, raw_memory_id, session_key_or_stem,
    transcript_lines, typed_text,
};

const PASSAGE_CHARS: usize = 1500;

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RawRole {
    User,
    Assistant,
    ToolCall,
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RawText {
    pub text: String,
    pub at: Option<DateTime<Utc>>,
    pub role: RawRole,
    /// For assistant text, the id ingest gives the raw memory cut from it.
    pub memory: Option<Uuid>,
}

fn capped(text: &str) -> String {
    text.chars().take(PASSAGE_CHARS).collect()
}

/// Passages of the Claude Code transcript (JSONL) at `source` for the raw
/// fallback index: what the user typed, every assistant text cut as ingest
/// cuts it but unscreened, and each tool call's input, all redacted and in
/// transcript order. User and tool-call passages are capped at 1500
/// characters. `content` is the archive's copy, redacted when ingest read
/// the original, so an assistant chunk's id is hashed from it as stored and
/// matches ingest's even after the redaction rules change.
pub fn raw_passages(source: &Path, content: &str) -> Vec<RawText> {
    let conversation = conversation_id(&session_key_or_stem(source, content));
    let mut passages = Vec::new();
    for (msg, at) in transcript_lines(content) {
        if let Some(text) = typed_text(&msg) {
            passages.extend(
                chunk_text(&redact_for_store(&text))
                    .iter()
                    .map(|chunk| RawText {
                        text: capped(chunk),
                        at,
                        role: RawRole::User,
                        memory: None,
                    }),
            );
            continue;
        }
        let Some(blocks) = assistant_blocks(&msg) else {
            continue;
        };
        for block in blocks {
            match block.get("type").and_then(|t| t.as_str()) {
                Some("text") => {
                    let Some(text) = block.get("text").and_then(|t| t.as_str()) else {
                        continue;
                    };
                    if text.trim().is_empty() {
                        continue;
                    }
                    passages.extend(chunk_text(text).into_iter().map(|chunk| RawText {
                        memory: Some(raw_memory_id(conversation, &chunk)),
                        text: redact_for_store(&chunk),
                        at,
                        role: RawRole::Assistant,
                    }));
                }
                Some("tool_use") => {
                    let name = block.get("name").and_then(|n| n.as_str()).unwrap_or("");
                    let input = block.get("input").cloned().unwrap_or_default();
                    let rendered = redact_for_store(&format!("tool {name}: {input}"));
                    passages.push(RawText {
                        text: capped(&rendered),
                        at,
                        role: RawRole::ToolCall,
                        memory: None,
                    });
                }
                _ => {}
            }
        }
    }
    passages
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::claude_code::{chunk_text, screen_transcript};
    use second_brain_core::redact::redact;
    use serde_json::{Value, json};

    const SOURCE: &str = "projects/-srv-garden/s1.jsonl";

    fn user(text: &str) -> Value {
        json!({"type": "user", "message": {"content": text}})
    }

    fn assistant(blocks: Value) -> Value {
        json!({"type": "assistant", "message": {"content": blocks}})
    }

    fn text_block(text: &str) -> Value {
        json!({"type": "text", "text": text})
    }

    fn tool_use(name: &str, input: Value) -> Value {
        json!({"type": "tool_use", "id": "call-1", "name": name, "input": input})
    }

    fn transcript(lines: &[Value]) -> String {
        lines
            .iter()
            .map(Value::to_string)
            .collect::<Vec<_>>()
            .join("\n")
    }

    fn roles_and_texts(content: &str) -> Vec<(RawRole, String)> {
        raw_passages(Path::new(SOURCE), content)
            .into_iter()
            .map(|p| (p.role, p.text))
            .collect()
    }

    fn token_like() -> String {
        ["gh", "p_"].concat() + &"a1B2".repeat(9)
    }

    fn long_block() -> String {
        (0..12)
            .map(|i| {
                format!("Step {i} moves one batch of cache files to the new layout. ").repeat(4)
            })
            .collect::<Vec<_>>()
            .join("\n\n")
    }

    struct Case {
        name: &'static str,
        content: String,
        want: Vec<(RawRole, String)>,
    }

    fn cases() -> Vec<Case> {
        use RawRole::{Assistant, ToolCall, User};
        let mut meta = user("a skill body injected by the harness");
        meta["isMeta"] = json!(true);
        let tool_result = json!({"type": "user", "message": {"content": [
            {"type": "tool_result", "tool_use_id": "call-1", "content": "file listing output"}
        ]}});
        let typed_blocks = json!({"type": "user", "message": {"content": [
            {"type": "text", "text": "first part"},
            {"type": "image", "source": {}},
            {"type": "text", "text": "second part"}
        ]}});
        let queued = json!({"type": "attachment", "attachment": {
            "type": "queued_command", "prompt": "also check the archive"
        }});
        let mut sidechain = assistant(json!([text_block("subagent report body")]));
        sidechain["isSidechain"] = json!(true);

        let long = long_block();
        let long_chunks = chunk_text(&long);

        let wide = "\u{e9}".repeat(3000);
        let wide_rendered: String = format!("tool Write: {}", json!({"body": wide}))
            .chars()
            .take(1500)
            .collect();

        let token = token_like();

        vec![
            Case {
                name: "typed user text vs tool result, meta, reminder-only and harness lines",
                content: transcript(&[
                    user("how do we rotate the cache files"),
                    tool_result,
                    meta,
                    user("<system-reminder>context only</system-reminder>\n"),
                    user("<command-name>/clear</command-name>"),
                    user("<system-reminder>context</system-reminder>\nplease keep the old layout"),
                    typed_blocks,
                    queued,
                ]),
                want: vec![
                    (User, "how do we rotate the cache files".into()),
                    (User, "please keep the old layout".into()),
                    (User, "first part\nsecond part".into()),
                    (User, "also check the archive".into()),
                ],
            },
            Case {
                name: "assistant text and tool_use in block order, blank text dropped",
                content: transcript(&[assistant(json!([
                    text_block("ok"),
                    tool_use("Read", json!({"path": "notes.txt", "limit": 20})),
                    text_block("   \n  "),
                    text_block("the layout is unchanged"),
                ]))]),
                want: vec![
                    (Assistant, "ok".into()),
                    (
                        ToolCall,
                        r#"tool Read: {"limit":20,"path":"notes.txt"}"#.into(),
                    ),
                    (Assistant, "the layout is unchanged".into()),
                ],
            },
            Case {
                name: "a long assistant block is cut by chunk_text",
                content: transcript(&[assistant(json!([text_block(&long)]))]),
                want: long_chunks.into_iter().map(|c| (Assistant, c)).collect(),
            },
            Case {
                name: "a tool call is capped at 1500 characters on a char boundary",
                content: transcript(&[assistant(json!([tool_use(
                    "Write",
                    json!({"body": wide})
                )]))]),
                want: vec![(ToolCall, wide_rendered)],
            },
            Case {
                name: "redaction applies to every role",
                content: transcript(&[
                    user(&format!("use the token {token} for the mirror")),
                    assistant(json!([
                        text_block(&format!("the mirror reads {token} from its config")),
                        tool_use(
                            "Bash",
                            json!({"command": format!("mirror --token {token}")})
                        ),
                    ])),
                ]),
                want: vec![
                    (
                        User,
                        "use the token [REDACTED:github_token] for the mirror".into(),
                    ),
                    (
                        Assistant,
                        "the mirror reads [REDACTED:github_token] from its config".into(),
                    ),
                    (
                        ToolCall,
                        r#"tool Bash: {"command":"mirror --token [REDACTED:github_token]"}"#.into(),
                    ),
                ],
            },
            Case {
                name: "malformed lines are skipped",
                content: [
                    "{not json".to_string(),
                    String::new(),
                    "[1, 2".to_string(),
                    user("still read after a bad line").to_string(),
                    "plain words".to_string(),
                ]
                .join("\n"),
                want: vec![(User, "still read after a bad line".into())],
            },
            Case {
                name: "sidechain lines are read like any other",
                content: transcript(&[sidechain]),
                want: vec![(Assistant, "subagent report body".into())],
            },
        ]
    }

    #[test]
    fn passages_by_case() {
        for case in cases() {
            assert_eq!(roles_and_texts(&case.content), case.want, "{}", case.name);
        }
    }

    #[test]
    fn long_block_fixture_spans_several_chunks() {
        assert!(chunk_text(&long_block()).len() > 1);
    }

    #[test]
    fn tool_call_cap_counts_chars() {
        let content = transcript(&[assistant(json!([tool_use(
            "Write",
            json!({"body": "\u{e9}".repeat(3000)})
        )]))]);
        let passages = raw_passages(Path::new(SOURCE), &content);
        assert_eq!(passages.len(), 1);
        assert_eq!(passages[0].text.chars().count(), 1500);
    }

    #[test]
    fn at_is_the_line_timestamp() {
        let mut stamped = user("a stamped question");
        stamped["timestamp"] = json!("2026-03-04T05:06:07Z");
        let mut answer = assistant(json!([
            text_block("a stamped answer"),
            tool_use("Read", json!({}))
        ]));
        answer["timestamp"] = json!("2026-03-04T05:06:09.500Z");
        let content = transcript(&[stamped, answer, user("an unstamped question")]);

        let at: Vec<Option<String>> = raw_passages(Path::new(SOURCE), &content)
            .into_iter()
            .map(|p| p.at.map(|t| t.to_rfc3339()))
            .collect();
        assert_eq!(
            at,
            vec![
                Some("2026-03-04T05:06:07+00:00".to_string()),
                Some("2026-03-04T05:06:09.500+00:00".to_string()),
                Some("2026-03-04T05:06:09.500+00:00".to_string()),
                None,
            ]
        );
    }

    #[test]
    fn a_long_typed_message_is_cut_into_capped_passages() {
        let pasted = format!("{}\n\n{}", "x".repeat(4000), "short tail");
        let passages = raw_passages(Path::new(SOURCE), &transcript(&[user(&pasted)]));
        let lengths: Vec<usize> = passages.iter().map(|p| p.text.chars().count()).collect();
        assert_eq!(lengths, vec![1500, 10]);
        assert!(
            passages
                .iter()
                .all(|p| p.role == RawRole::User && p.memory.is_none())
        );
    }

    #[test]
    fn assistant_passages_carry_the_raw_memory_id_ingest_gives_them() {
        let mut first = user("how do we rotate the cache files");
        first["sessionId"] = json!("session-one");
        let content = transcript(&[
            first,
            assistant(json!([
                text_block("Rotate them weekly."),
                tool_use("Read", json!({}))
            ])),
        ]);
        let conversation = conversation_id(&session_key_or_stem(Path::new(SOURCE), &content));

        let memories: Vec<Option<Uuid>> = raw_passages(Path::new(SOURCE), &content)
            .into_iter()
            .map(|p| p.memory)
            .collect();
        assert_eq!(
            memories,
            vec![
                None,
                Some(raw_memory_id(conversation, "Rotate them weekly.")),
                None
            ]
        );
    }

    #[test]
    fn assistant_passages_equal_ingest_chunks() {
        let token = token_like();
        let content = transcript(&[
            user("how do we rotate the cache files"),
            assistant(json!([
                text_block("ok"),
                tool_use("Read", json!({"path": "notes.txt"})),
                text_block(&format!("{}\n\nThe mirror reads {token}.", long_block())),
            ])),
            assistant(json!([text_block("Let me check the archive next.")])),
        ]);

        let archived = redact(&content).text;
        let conversation = conversation_id(&session_key_or_stem(Path::new(SOURCE), &content));

        let assistant: Vec<(String, Option<Uuid>)> = raw_passages(Path::new(SOURCE), &archived)
            .into_iter()
            .filter(|p| p.role == RawRole::Assistant)
            .map(|p| (p.text, p.memory))
            .collect();
        let ingest: Vec<(String, Option<Uuid>)> = screen_transcript(&content)
            .into_iter()
            .map(|c| {
                let id = raw_memory_id(conversation, &c.text);
                (c.text, Some(id))
            })
            .collect();

        assert!(ingest.len() > 2);
        assert_eq!(assistant, ingest);
    }
}