Skip to main content

second_brain_adapters/
claude_code.rs

1use std::collections::HashSet;
2use std::path::{Path, PathBuf};
3
4use anyhow::{Context, Result};
5use chrono::{DateTime, Utc};
6use sha2::{Digest, Sha256};
7use uuid::Uuid;
8
9use second_brain_core::redact::redact_for_store;
10use second_brain_core::schema::{Conversation, Memory, MemoryType};
11
12use crate::{Adapter, ConversationSource, ExtractedEntity, IngestResult};
13
14pub struct ClaudeCodeAdapter {
15    base_path: PathBuf,
16    machine_id: String,
17}
18
19impl ClaudeCodeAdapter {
20    pub fn new(machine_id: String) -> Self {
21        let base_path = dirs_home().join(".claude");
22        Self {
23            base_path,
24            machine_id,
25        }
26    }
27
28    pub fn with_path(base_path: PathBuf, machine_id: String) -> Self {
29        Self {
30            base_path,
31            machine_id,
32        }
33    }
34
35    #[allow(clippy::type_complexity)]
36    fn parse_conversation(
37        content: &str,
38        source_id: &str,
39        conversation_id: Uuid,
40    ) -> (
41        Vec<Memory>,
42        Vec<(ExtractedEntity, Uuid)>,
43        Option<DateTime<Utc>>,
44    ) {
45        let transcript = read_transcript(content);
46
47        let mut memories = Vec::new();
48        let mut all_entities: Vec<(ExtractedEntity, Uuid)> = Vec::new();
49        let mut seen_entities: HashSet<String> = HashSet::new();
50
51        for chunk in transcript.screened_chunks() {
52            if !chunk.kept() {
53                continue;
54            }
55            let memory_type = classify_memory(&chunk.text);
56            let mut memory = Memory::new(
57                chunk.text.clone(),
58                memory_type,
59                "claude-code".to_string(),
60                source_id.to_string(),
61            );
62            memory.id = raw_memory_id(conversation_id, &chunk.text);
63            memory.session_at = chunk.at;
64            let memory_id = memory.id;
65            memories.push(memory);
66
67            for entity in extract_entities(&chunk.text) {
68                let key = entity.name.to_lowercase();
69                if seen_entities.insert(key) {
70                    all_entities.push((entity, memory_id));
71                }
72            }
73        }
74
75        (memories, all_entities, transcript.first_timestamp)
76    }
77}
78
79/// The transcript's `sessionId`, plus `/<agentId>` for a subagent file, each
80/// from the first line carrying it as a top-level field. None when no line
81/// carries a `sessionId`.
82pub fn session_key(content: &str) -> Option<String> {
83    let mut session: Option<String> = None;
84    let mut agent: Option<String> = None;
85    for line in content.lines() {
86        let wants_session = session.is_none() && line.contains("\"sessionId\"");
87        let wants_agent = agent.is_none() && line.contains("\"agentId\"");
88        if !wants_session && !wants_agent {
89            continue;
90        }
91        let Ok(msg) = serde_json::from_str::<serde_json::Value>(line) else {
92            continue;
93        };
94        let field = |name: &str| {
95            msg.get(name)
96                .and_then(|v| v.as_str())
97                .filter(|v| !v.is_empty())
98                .map(str::to_string)
99        };
100        if wants_session {
101            session = field("sessionId");
102        }
103        if wants_agent {
104            agent = field("agentId");
105        }
106    }
107    let session = session?;
108    Some(match agent {
109        Some(agent) => format!("{session}/{agent}"),
110        None => session,
111    })
112}
113
114pub(crate) fn session_key_or_stem(path: &Path, content: &str) -> String {
115    session_key(content).unwrap_or_else(|| {
116        path.file_stem()
117            .map(|s| s.to_string_lossy().to_string())
118            .unwrap_or_default()
119    })
120}
121
122pub fn conversation_id(session_key: &str) -> Uuid {
123    Conversation::claude_code_id(session_key)
124}
125
126/// Hashes the chunk as stored, which is redacted: a raw secret never feeds
127/// a synced id, so a chunk holding one keeps its id only while the redactor
128/// rewrites it the same way.
129pub fn raw_memory_id(conversation_id: Uuid, chunk_text: &str) -> Uuid {
130    let digest = Sha256::digest(chunk_text.as_bytes());
131    Uuid::new_v5(&conversation_id, hex(&digest).as_bytes())
132}
133
134fn hex(bytes: &[u8]) -> String {
135    bytes.iter().map(|b| format!("{b:02x}")).collect()
136}
137
138/// An assistant text block and the time of the transcript line it is on.
139pub(crate) struct AssistantText {
140    pub(crate) at: Option<DateTime<Utc>>,
141    pub(crate) text: String,
142    /// The last non-blank assistant text before the next real user message,
143    /// or the last one in the file: the closing answer the user read.
144    pub(crate) turn_final: bool,
145}
146
147pub(crate) struct Transcript {
148    /// The first timestamp on any line, which is when the session started.
149    pub(crate) first_timestamp: Option<DateTime<Utc>>,
150    pub(crate) assistant_texts: Vec<AssistantText>,
151}
152
153impl Transcript {
154    pub(crate) fn screened_chunks(&self) -> Vec<ScreenedChunk> {
155        self.assistant_texts
156            .iter()
157            .flat_map(|block| {
158                chunk_text(&block.text)
159                    .into_iter()
160                    .map(|text| ScreenedChunk {
161                        failed: failed_rules(&text),
162                        text,
163                        at: block.at,
164                        turn_final: block.turn_final,
165                    })
166            })
167            .collect()
168    }
169}
170
171/// A chunk cut from one assistant text block, with every filter rule it
172/// fails, so a probe can see why a chunk was dropped and not only whether.
173#[derive(Debug, Clone)]
174pub struct ScreenedChunk {
175    pub text: String,
176    pub at: Option<DateTime<Utc>>,
177    pub turn_final: bool,
178    pub failed: Vec<FilterRule>,
179}
180
181impl ScreenedChunk {
182    pub fn kept(&self) -> bool {
183        self.failed.is_empty()
184    }
185}
186
187/// Reads, chunks and screens a transcript exactly as ingest does, for
188/// measuring the filter without a store.
189pub fn screen_transcript(content: &str) -> Vec<ScreenedChunk> {
190    read_transcript(content).screened_chunks()
191}
192
193const HARNESS_PREFIXES: [&str; 4] = [
194    "<local-command",
195    "<command-name>",
196    "<command-message>",
197    "Caveat:",
198];
199
200fn strip_system_reminders(text: &str) -> String {
201    let mut out = String::with_capacity(text.len());
202    let mut rest = text;
203    while let Some(start) = rest.find("<system-reminder>") {
204        out.push_str(&rest[..start]);
205        match rest[start..].find("</system-reminder>") {
206            Some(end) => rest = &rest[start + end + "</system-reminder>".len()..],
207            None => {
208                rest = "";
209                break;
210            }
211        }
212    }
213    out.push_str(rest);
214    out
215}
216
217fn typed_prompt(text: &str) -> Option<String> {
218    let text = strip_system_reminders(text);
219    let text = text.trim();
220    (!text.is_empty() && !HARNESS_PREFIXES.iter().any(|p| text.starts_with(p)))
221        .then(|| text.to_string())
222}
223
224/// The text of a line the user typed, which starts a new turn, with system
225/// reminders stripped. The definition matches
226/// evals/capture_pilot/transcript.py, which measures facts against the
227/// answers it bounds: tool results, meta lines, harness command output and
228/// reminder-only lines belong to the running turn, and a command queued while
229/// the agent was working starts a new one. Sidechain lines are not skipped, so
230/// a subagent file's report is final in its own file.
231pub(crate) fn typed_text(msg: &serde_json::Value) -> Option<String> {
232    if msg.get("isMeta").and_then(|m| m.as_bool()) == Some(true) {
233        return None;
234    }
235    match msg.get("type").and_then(|t| t.as_str()) {
236        Some("user") => match msg.get("message").and_then(|m| m.get("content")) {
237            Some(serde_json::Value::String(text)) => typed_prompt(text),
238            Some(serde_json::Value::Array(blocks)) => {
239                if blocks
240                    .iter()
241                    .any(|b| b.get("type").and_then(|t| t.as_str()) == Some("tool_result"))
242                {
243                    return None;
244                }
245                let text: Vec<&str> = blocks
246                    .iter()
247                    .filter(|b| b.get("type").and_then(|t| t.as_str()) == Some("text"))
248                    .filter_map(|b| b.get("text").and_then(|t| t.as_str()))
249                    .collect();
250                typed_prompt(&text.join("\n"))
251            }
252            _ => None,
253        },
254        Some("attachment") => {
255            let attachment = msg.get("attachment")?;
256            if attachment.get("type").and_then(|t| t.as_str()) != Some("queued_command") {
257                return None;
258            }
259            let prompt = strip_system_reminders(attachment.get("prompt")?.as_str()?);
260            let prompt = prompt.trim();
261            (!prompt.is_empty()).then(|| prompt.to_string())
262        }
263        _ => None,
264    }
265}
266
267/// Each parseable, non-blank line with its `timestamp`; malformed lines are
268/// skipped.
269pub(crate) fn transcript_lines(
270    content: &str,
271) -> impl Iterator<Item = (serde_json::Value, Option<DateTime<Utc>>)> + '_ {
272    content.lines().filter_map(|line| {
273        if line.trim().is_empty() {
274            return None;
275        }
276        let msg: serde_json::Value = serde_json::from_str(line).ok()?;
277        let at = msg
278            .get("timestamp")
279            .and_then(|t| t.as_str())
280            .and_then(|ts| DateTime::parse_from_rfc3339(ts).ok())
281            .map(|dt| dt.with_timezone(&Utc));
282        Some((msg, at))
283    })
284}
285
286pub(crate) fn assistant_blocks(msg: &serde_json::Value) -> Option<&Vec<serde_json::Value>> {
287    if msg.get("type").and_then(|t| t.as_str()) != Some("assistant") {
288        return None;
289    }
290    msg.get("message")
291        .and_then(|m| m.get("content"))
292        .and_then(|c| c.as_array())
293}
294
295/// The one reading of Claude Code's JSONL format, shared by ingest and the
296/// session date backfill so both attribute a raw to the same line.
297pub(crate) fn read_transcript(content: &str) -> Transcript {
298    let mut transcript = Transcript {
299        first_timestamp: None,
300        assistant_texts: Vec::new(),
301    };
302    let mut last_in_turn: Option<usize> = None;
303    for (msg, at) in transcript_lines(content) {
304        if transcript.first_timestamp.is_none() {
305            transcript.first_timestamp = at;
306        }
307
308        if typed_text(&msg).is_some() {
309            if let Some(i) = last_in_turn.take() {
310                transcript.assistant_texts[i].turn_final = true;
311            }
312            continue;
313        }
314        let Some(blocks) = assistant_blocks(&msg) else {
315            continue;
316        };
317        for block in blocks {
318            if block.get("type").and_then(|t| t.as_str()) != Some("text") {
319                continue;
320            }
321            if let Some(text) = block.get("text").and_then(|t| t.as_str()) {
322                if !text.trim().is_empty() {
323                    last_in_turn = Some(transcript.assistant_texts.len());
324                }
325                transcript.assistant_texts.push(AssistantText {
326                    at,
327                    text: redact_for_store(text),
328                    turn_final: false,
329                });
330            }
331        }
332    }
333    if let Some(i) = last_in_turn {
334        transcript.assistant_texts[i].turn_final = true;
335    }
336    transcript
337}
338
339impl Adapter for ClaudeCodeAdapter {
340    fn name(&self) -> &str {
341        "claude-code"
342    }
343
344    fn discover_conversations(&self) -> Result<Vec<ConversationSource>> {
345        let projects_dir = self.base_path.join("projects");
346        let mut sources = Vec::new();
347
348        if !projects_dir.exists() {
349            return Ok(sources);
350        }
351
352        let mut dirs = vec![projects_dir];
353        while let Some(dir) = dirs.pop() {
354            for entry in std::fs::read_dir(&dir)? {
355                let entry = entry?;
356                let path = entry.path();
357                if path.is_dir() {
358                    dirs.push(path);
359                } else if path.extension().is_some_and(|e| e == "jsonl") {
360                    sources.push(ConversationSource {
361                        id: path.to_string_lossy().to_string(),
362                        path: path.to_string_lossy().to_string(),
363                        source_name: "claude-code".to_string(),
364                        machine_id: self.machine_id.clone(),
365                    });
366                }
367            }
368        }
369
370        Ok(sources)
371    }
372
373    fn session_key(&self, path: &Path, content: &str) -> Option<String> {
374        Some(session_key_or_stem(path, content))
375    }
376
377    fn ingest_conversation(&self, source: &ConversationSource) -> Result<IngestResult> {
378        let path = PathBuf::from(&source.path);
379        let content = std::fs::read_to_string(&path)
380            .with_context(|| format!("reading conversation file: {}", path.display()))?;
381        let conversation_id = conversation_id(&session_key_or_stem(&path, &content));
382        let (memories, entities, started_at) =
383            Self::parse_conversation(&content, &path.to_string_lossy(), conversation_id);
384
385        let project_dir = crate::project_slug(&self.base_path.join("projects"), &path);
386
387        let conversation = Conversation {
388            id: conversation_id,
389            source: "claude-code".to_string(),
390            machine_id: source.machine_id.clone(),
391            started_at: started_at.unwrap_or_else(Utc::now),
392            project_path: project_dir.clone(),
393        };
394
395        let mut memories = memories;
396        for mem in &mut memories {
397            mem.project_path = project_dir.clone();
398        }
399
400        let entity_count = entities.len();
401
402        Ok(IngestResult {
403            conversation,
404            memories,
405            entities,
406            entities_extracted: entity_count,
407        })
408    }
409}
410
411fn dirs_home() -> PathBuf {
412    std::env::var("HOME")
413        .map(PathBuf::from)
414        .unwrap_or_else(|_| PathBuf::from("/tmp"))
415}
416
417pub(crate) fn chunk_text(text: &str) -> Vec<String> {
418    let mut chunks = Vec::new();
419    let mut current = String::new();
420
421    for paragraph in text.split("\n\n") {
422        let trimmed = paragraph.trim();
423        if trimmed.is_empty() {
424            continue;
425        }
426
427        if current.len() + trimmed.len() > 1500 && !current.is_empty() {
428            chunks.push(std::mem::take(&mut current));
429        }
430
431        if !current.is_empty() {
432            current.push_str("\n\n");
433        }
434        current.push_str(trimmed);
435
436        if current.len() >= 800 {
437            chunks.push(std::mem::take(&mut current));
438        }
439    }
440
441    if !current.is_empty() {
442        chunks.push(current);
443    }
444
445    if chunks.is_empty() && !text.trim().is_empty() {
446        chunks.push(text.trim().to_string());
447    }
448
449    chunks
450}
451
452#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
453pub enum FilterRule {
454    TooShort,
455    CodeHeavy,
456    TooFewWords,
457    NoiseOpening,
458    NoisePhrases,
459    Narration,
460}
461
462impl FilterRule {
463    pub fn name(self) -> &'static str {
464        match self {
465            FilterRule::TooShort => "too_short",
466            FilterRule::CodeHeavy => "code_heavy",
467            FilterRule::TooFewWords => "too_few_words",
468            FilterRule::NoiseOpening => "noise_opening",
469            FilterRule::NoisePhrases => "noise_phrases",
470            FilterRule::Narration => "narration",
471        }
472    }
473}
474
475#[cfg(test)]
476fn is_memorable(text: &str) -> bool {
477    failed_rules(text).is_empty()
478}
479
480/// Every rule the text fails, not only the first, so a probe can tell what
481/// dropping one rule would let through.
482fn failed_rules(text: &str) -> Vec<FilterRule> {
483    let mut failed = Vec::new();
484    let trimmed = text.trim();
485
486    if trimmed.len() < 80 {
487        failed.push(FilterRule::TooShort);
488    }
489
490    let code_block_count = trimmed.matches("```").count() / 2;
491    if code_block_count > 8 {
492        failed.push(FilterRule::CodeHeavy);
493    }
494
495    let lines: Vec<&str> = trimmed.lines().collect();
496    let non_code_lines: Vec<&str> = lines
497        .iter()
498        .filter(|l| {
499            let t = l.trim();
500            !t.is_empty()
501                && !t.starts_with("```")
502                && !t.starts_with('\t')
503                && !t.starts_with("//")
504                && !t.starts_with("  ")
505        })
506        .copied()
507        .collect();
508
509    let word_count = trimmed.split_whitespace().count();
510    if non_code_lines.len() < 2 && word_count < 15 {
511        failed.push(FilterRule::TooFewWords);
512    }
513
514    let lower = trimmed.to_lowercase();
515
516    let first_line_lower = non_code_lines
517        .first()
518        .map(|l| l.trim().to_lowercase())
519        .unwrap_or_default();
520
521    if is_noise_opening(&first_line_lower) && !has_substance(&lower) {
522        failed.push(FilterRule::NoiseOpening);
523    }
524
525    let noise_count = count_noise_phrases(&lower);
526    if noise_count >= 2 || (noise_count == 1 && non_code_lines.len() < 4) {
527        failed.push(FilterRule::NoisePhrases);
528    }
529
530    if is_pure_narration(&lower) && non_code_lines.len() < 5 {
531        failed.push(FilterRule::Narration);
532    }
533
534    failed
535}
536
537fn is_noise_opening(first_line: &str) -> bool {
538    let patterns = [
539        "let me ",
540        "i'll ",
541        "i will ",
542        "now let me",
543        "now i'll",
544        "let's ",
545        "good question",
546        "good, ",
547        "good ",
548        "great, ",
549        "perfect",
550        "excellent",
551        "alright",
552        "ok, ",
553        "okay, ",
554        "sure, ",
555        "certainly",
556        "absolutely",
557        "interesting",
558        "i see",
559        "i notice",
560        "i can see",
561        "looking at",
562        "checking ",
563        "exploring ",
564        "examining ",
565        "navigating ",
566        "reading ",
567    ];
568    patterns.iter().any(|p| first_line.starts_with(p))
569}
570
571fn has_substance(text: &str) -> bool {
572    let markers = [
573        "because",
574        "the fix",
575        "the solution",
576        "the issue",
577        "the problem",
578        "decided",
579        "trade-off",
580        "tradeoff",
581        "architecture",
582        "we chose",
583        "root cause",
584        "the reason",
585        "this means",
586        "this works",
587        "the approach",
588        "conclusion",
589        "in summary",
590    ];
591    markers.iter().any(|m| text.contains(m))
592}
593
594fn count_noise_phrases(text: &str) -> usize {
595    let phrases = [
596        "let me check",
597        "let me look",
598        "let me explore",
599        "let me examine",
600        "let me navigate",
601        "let me read",
602        "let me search",
603        "let me find",
604        "let me verify",
605        "let me investigate",
606        "let me continue",
607        "let me review",
608        "let me start by",
609        "let me run",
610        "i'll check",
611        "i'll look",
612        "i'll explore",
613        "i'll examine",
614        "i'll search",
615        "i'll read",
616        "i'll investigate",
617        "i'll start by",
618        "i'll run",
619        "now let me",
620    ];
621    phrases.iter().filter(|p| text.contains(*p)).count()
622}
623
624fn is_pure_narration(text: &str) -> bool {
625    let patterns = [
626        "i have the project context",
627        "i have a complete picture",
628        "mark the exploration task",
629        "mark the task",
630        "committed changes:",
631        "let me mark",
632        "task done",
633        "task complete",
634    ];
635    patterns.iter().any(|p| text.contains(p))
636}
637
638fn classify_memory(text: &str) -> MemoryType {
639    let lower = text.to_lowercase();
640
641    let task_markers = [
642        "todo:",
643        "todo ",
644        "fixme",
645        "action item",
646        "remaining:",
647        "left to do",
648        "need to implement",
649        "needs to be done",
650        "next step:",
651        "next steps:",
652        "backlog:",
653    ];
654    if task_markers.iter().any(|m| lower.contains(m)) {
655        return MemoryType::Task;
656    }
657
658    let decision_markers = [
659        "decided",
660        "chose",
661        "went with",
662        "picked",
663        "agreed",
664        "choosing",
665        "selected",
666        "switched to",
667        "we'll use",
668        "we'll go with",
669        "the decision",
670        "opting for",
671    ];
672    if decision_markers.iter().any(|m| lower.contains(m)) {
673        return MemoryType::Decision;
674    }
675
676    let architecture_markers = [
677        "architecture",
678        "data model",
679        "schema design",
680        "the approach is",
681        "system design",
682        "component design",
683        "abstraction",
684        "design pattern",
685        "data flow",
686        "the pattern",
687        "module structure",
688        "separation of concern",
689    ];
690    if architecture_markers.iter().any(|m| lower.contains(m)) {
691        return MemoryType::Architecture;
692    }
693
694    let debugging_markers = [
695        "root cause",
696        "the bug",
697        "the issue is",
698        "the problem is",
699        "stack trace",
700        "error was",
701        "the fix is",
702        "debugging",
703        "regression",
704        "crash",
705        "panicked",
706        "segfault",
707    ];
708    if debugging_markers.iter().any(|m| lower.contains(m)) {
709        return MemoryType::Debugging;
710    }
711
712    if lower.contains('?') {
713        let interrogative = [
714            "how ",
715            "what ",
716            "why ",
717            "whether ",
718            "should we",
719            "could we",
720            "where ",
721            "when ",
722            "which ",
723        ];
724        if interrogative.iter().any(|m| lower.contains(m)) {
725            return MemoryType::Question;
726        }
727    }
728
729    let procedural_markers = [
730        "to do this",
731        "steps:",
732        "step 1",
733        "first,",
734        "then,",
735        "finally,",
736        "run ",
737        "install ",
738        "execute",
739        "command:",
740        "configure",
741        "set up",
742        "to fix",
743        "solution:",
744        "you need to",
745        "you can ",
746        "you should",
747        "here's how",
748    ];
749    if procedural_markers.iter().any(|m| lower.contains(m)) {
750        return MemoryType::Procedural;
751    }
752
753    let episodic_markers = [
754        "migrated",
755        "deployed",
756        "shipped",
757        "found that",
758        "discovered",
759        "realized",
760        "confirmed",
761        "concluded",
762    ];
763    if episodic_markers.iter().any(|m| lower.contains(m)) {
764        return MemoryType::Episodic;
765    }
766
767    MemoryType::Semantic
768}
769
770const TECH_NAMES: &[&str] = &[
771    "rust",
772    "python",
773    "javascript",
774    "typescript",
775    "go",
776    "java",
777    "c++",
778    "ruby",
779    "swift",
780    "kotlin",
781    "react",
782    "vue",
783    "angular",
784    "svelte",
785    "next.js",
786    "nuxt",
787    "django",
788    "flask",
789    "fastapi",
790    "express",
791    "axum",
792    "actix",
793    "tokio",
794    "serde",
795    "docker",
796    "kubernetes",
797    "k8s",
798    "terraform",
799    "ansible",
800    "nginx",
801    "apache",
802    "redis",
803    "postgres",
804    "postgresql",
805    "mysql",
806    "sqlite",
807    "mongodb",
808    "kuzudb",
809    "neo4j",
810    "elasticsearch",
811    "kafka",
812    "rabbitmq",
813    "nats",
814    "grpc",
815    "graphql",
816    "rest",
817    "websocket",
818    "ssh",
819    "tailscale",
820    "wireguard",
821    "git",
822    "github",
823    "gitlab",
824    "linux",
825    "ubuntu",
826    "debian",
827    "macos",
828    "windows",
829    "arm64",
830    "aarch64",
831    "x86",
832    "onnx",
833    "pytorch",
834    "tensorflow",
835    "huggingface",
836    "openai",
837    "anthropic",
838    "claude",
839    "gpt",
840    "llm",
841    "mcp",
842    "lsp",
843    "systemd",
844    "cron",
845    "tmux",
846    "vim",
847    "neovim",
848    "cargo",
849    "npm",
850    "pip",
851    "uv",
852    "bun",
853    "deno",
854    "node.js",
855    "node",
856    "aws",
857    "gcp",
858    "azure",
859    "s3",
860    "ec2",
861    "lambda",
862    "cloudflare",
863    "raspberry pi",
864    "pi",
865    "jetson",
866    "esp32",
867    "arduino",
868    "toml",
869    "yaml",
870    "json",
871    "jsonl",
872    "csv",
873    "protobuf",
874    "msgpack",
875];
876
877pub fn extract_entities(text: &str) -> Vec<ExtractedEntity> {
878    let mut entities = Vec::new();
879    let mut seen = HashSet::new();
880    let lower = text.to_lowercase();
881
882    for &tech in TECH_NAMES {
883        if lower.contains(tech) {
884            let key = tech.to_lowercase();
885            if seen.insert(key) {
886                entities.push(ExtractedEntity {
887                    name: tech.to_string(),
888                    entity_type: "technology".to_string(),
889                });
890            }
891        }
892    }
893
894    for word in text.split_whitespace() {
895        let clean = word.trim_matches(|c: char| {
896            !c.is_alphanumeric() && c != '/' && c != '.' && c != '-' && c != '_'
897        });
898        if clean.contains('/') && clean.contains('.') {
899            let ext_match = clean.rsplit('.').next().unwrap_or("");
900            let is_source_file = matches!(
901                ext_match,
902                "rs" | "py"
903                    | "ts"
904                    | "tsx"
905                    | "js"
906                    | "jsx"
907                    | "go"
908                    | "java"
909                    | "rb"
910                    | "sh"
911                    | "toml"
912                    | "json"
913                    | "yaml"
914                    | "yml"
915                    | "md"
916                    | "sql"
917                    | "html"
918                    | "css"
919            );
920            if is_source_file {
921                let key = clean.to_lowercase();
922                if seen.insert(key) {
923                    entities.push(ExtractedEntity {
924                        name: clean.to_string(),
925                        entity_type: "file_path".to_string(),
926                    });
927                }
928            }
929        }
930    }
931
932    entities
933}
934
935#[cfg(test)]
936mod tests {
937    use super::*;
938
939    #[test]
940    fn classifies_task() {
941        assert_eq!(
942            classify_memory("TODO: implement the retry logic for failed sync batches"),
943            MemoryType::Task
944        );
945        assert_eq!(
946            classify_memory(
947                "There's a FIXME in the auth module that needs addressing before release"
948            ),
949            MemoryType::Task
950        );
951        assert_eq!(
952            classify_memory(
953                "Action item from the review: add rate limiting to the ingest endpoint"
954            ),
955            MemoryType::Task
956        );
957    }
958
959    #[test]
960    fn classifies_decision() {
961        assert_eq!(
962            classify_memory(
963                "We decided to use KuzuDB instead of Neo4j because it embeds and needs no separate server"
964            ),
965            MemoryType::Decision
966        );
967        assert_eq!(
968            classify_memory("Chose axum over actix-web for the HTTP framework"),
969            MemoryType::Decision
970        );
971        assert_eq!(
972            classify_memory(
973                "Went with a daemon architecture so only one process holds the DB lock"
974            ),
975            MemoryType::Decision
976        );
977    }
978
979    #[test]
980    fn classifies_architecture() {
981        assert_eq!(
982            classify_memory(
983                "The architecture uses a single daemon that owns the database and serves queries over HTTP"
984            ),
985            MemoryType::Architecture
986        );
987        assert_eq!(
988            classify_memory(
989                "The data model separates memories from entities with MENTIONS relations between them"
990            ),
991            MemoryType::Architecture
992        );
993        assert_eq!(
994            classify_memory("The design pattern here is event sourcing via the SyncLog table"),
995            MemoryType::Architecture
996        );
997    }
998
999    #[test]
1000    fn classifies_debugging() {
1001        assert_eq!(
1002            classify_memory(
1003                "The root cause was a race condition in the file watcher debounce logic"
1004            ),
1005            MemoryType::Debugging
1006        );
1007        assert_eq!(
1008            classify_memory(
1009                "The issue is that KuzuDB returns an error when the embedding array is empty"
1010            ),
1011            MemoryType::Debugging
1012        );
1013        assert_eq!(
1014            classify_memory(
1015                "The bug manifests as a 413 error because axum has a default 2MB body limit"
1016            ),
1017            MemoryType::Debugging
1018        );
1019    }
1020
1021    #[test]
1022    fn classifies_question() {
1023        assert_eq!(
1024            classify_memory(
1025                "How should we handle the case where the LLM is unreachable during consolidation?"
1026            ),
1027            MemoryType::Question
1028        );
1029        assert_eq!(
1030            classify_memory(
1031                "What happens when two machines sync conflicting edits to the same memory?"
1032            ),
1033            MemoryType::Question
1034        );
1035    }
1036
1037    #[test]
1038    fn classifies_procedural() {
1039        assert_eq!(
1040            classify_memory(
1041                "To fix the auth issue you need to regenerate the token and update the environment variable"
1042            ),
1043            MemoryType::Procedural
1044        );
1045        assert_eq!(
1046            classify_memory(
1047                "Steps: first install the dependency, then configure the TOML file, finally restart the daemon"
1048            ),
1049            MemoryType::Procedural
1050        );
1051    }
1052
1053    #[test]
1054    fn classifies_episodic() {
1055        assert_eq!(
1056            classify_memory(
1057                "Deployed the new sync module to the Raspberry Pi and it handled 50k records without issues"
1058            ),
1059            MemoryType::Episodic
1060        );
1061        assert_eq!(
1062            classify_memory(
1063                "Discovered that the embedding model outputs 384 dimensions, not 1536 as documented"
1064            ),
1065            MemoryType::Episodic
1066        );
1067    }
1068
1069    #[test]
1070    fn classifies_semantic_fallback() {
1071        assert_eq!(
1072            classify_memory(
1073                "KuzuDB stores graph data in a columnar format optimized for analytical queries"
1074            ),
1075            MemoryType::Semantic
1076        );
1077        assert_eq!(
1078            classify_memory(
1079                "BGE-small-en-v1.5 produces 384-dimensional vectors suitable for semantic search"
1080            ),
1081            MemoryType::Semantic
1082        );
1083    }
1084
1085    #[test]
1086    fn rejects_short_text() {
1087        assert!(!is_memorable("short"));
1088        assert!(!is_memorable("This is under eighty characters for sure."));
1089    }
1090
1091    #[test]
1092    fn rejects_noise_openings_without_substance() {
1093        assert!(!is_memorable(
1094            "Let me check the file structure to understand what we're working with here. I'll explore the codebase now."
1095        ));
1096        assert!(!is_memorable(
1097            "I'll look at the configuration files and see what's going on in the project directory structure for this."
1098        ));
1099        assert!(!is_memorable(
1100            "Looking at the codebase now. Let me navigate through the directory to find the relevant source files we need."
1101        ));
1102    }
1103
1104    #[test]
1105    fn keeps_noise_opening_with_substance() {
1106        assert!(is_memorable(
1107            "Let me explain the root cause of this bug. The issue is that the token expiry check uses less-than instead of less-than-or-equal, which means tokens expire one second early. The fix is straightforward."
1108        ));
1109    }
1110
1111    #[test]
1112    fn rejects_multiple_noise_phrases() {
1113        assert!(!is_memorable(
1114            "Let me check the logs. I'll also look at the configuration. Let me explore the routes to understand the full picture of this system."
1115        ));
1116    }
1117
1118    #[test]
1119    fn rejects_pure_narration() {
1120        assert!(!is_memorable(
1121            "Task complete. Committed changes: updated the auth middleware to fix the token validation."
1122        ));
1123        assert!(!is_memorable(
1124            "Let me mark the exploration task as done now that we've reviewed all the relevant files in the project."
1125        ));
1126    }
1127
1128    #[test]
1129    fn keeps_architectural_decisions() {
1130        assert!(is_memorable(
1131            "We decided to use a daemon architecture because KuzuDB takes an exclusive file lock. Only one process can open the database at a time, so we need a single daemon that owns the DB and serves everything else over HTTP."
1132        ));
1133    }
1134
1135    #[test]
1136    fn keeps_debugging_insights() {
1137        assert!(is_memorable(
1138            "Found that the 413 error was caused by axum's default 2MB body limit. The sync payload for 115k records exceeds that. The fix is adding DefaultBodyLimit::max(64MB) to the router layer."
1139        ));
1140    }
1141
1142    #[test]
1143    fn keeps_technical_explanations() {
1144        assert!(is_memorable(
1145            "The paginated export uses raw sequence order instead of sort_for_import because sorting reorders records by type, which breaks cursor-based pagination. The cursor tracks sequence numbers, so records must stay in sequence order."
1146        ));
1147    }
1148
1149    #[test]
1150    fn rejects_code_heavy_blocks() {
1151        let mostly_code = "Here's the fix:\n```rust\nfn main() {\n    println!(\"hello\");\n}\n```\n```rust\nfn other() {\n    println!(\"world\");\n}\n```\n```rust\nfn third() {}\n```\n```rust\nfn fourth() {}\n```\n```rust\nfn fifth() {}\n```\n```rust\nfn sixth() {}\n```\n```rust\nfn seventh() {}\n```\n```rust\nfn eighth() {}\n```\n```rust\nfn ninth() {}\n```";
1152        assert!(!is_memorable(mostly_code));
1153    }
1154
1155    #[test]
1156    fn lists_every_rule_a_chunk_fails() {
1157        assert_eq!(
1158            failed_rules("Let me check the logs. Task complete."),
1159            vec![
1160                FilterRule::TooShort,
1161                FilterRule::TooFewWords,
1162                FilterRule::NoiseOpening,
1163                FilterRule::NoisePhrases,
1164                FilterRule::Narration,
1165            ]
1166        );
1167    }
1168
1169    #[test]
1170    fn rejects_single_noise_phrase_short_message() {
1171        assert!(!is_memorable(
1172            "Let me verify that the build works correctly before we move on to testing."
1173        ));
1174    }
1175}