Skip to main content

second_brain_api/
ingest.rs

1use std::path::Path;
2use std::sync::Arc;
3
4use anyhow::{Context, Result, bail};
5use sha2::{Digest, Sha256};
6
7use second_brain_adapters::claude_code::ClaudeCodeAdapter;
8use second_brain_adapters::claude_memory::ClaudeMemoryAdapter;
9use second_brain_adapters::cursor::CursorAdapter;
10use second_brain_adapters::{Adapter, ConversationSource, store_ingest_result};
11use second_brain_core::embedding::{DocumentEmbedder, Embedder};
12use second_brain_core::schema::IngestRecord;
13use second_brain_core::store::StoreAdmin;
14use second_brain_core::transcript_archive::TranscriptArchive;
15
16#[derive(Debug, Default)]
17pub struct IngestStats {
18    pub new_memories: usize,
19    pub new_conversations: usize,
20    pub new_entities: usize,
21    pub skipped_dupes: usize,
22    pub unchanged: usize,
23    pub errors: usize,
24    pub files_processed: usize,
25}
26
27/// Every adapter an ingest can run, by the `source` name it stores.
28pub const SOURCES: [&str; 3] = ["claude-code", "claude-memory", "cursor"];
29
30fn adapter_named(source: &str, machine_id: &str) -> Option<Box<dyn Adapter>> {
31    let machine_id = machine_id.to_string();
32    match source {
33        "claude-code" => Some(Box::new(ClaudeCodeAdapter::new(machine_id))),
34        "claude-memory" => Some(Box::new(ClaudeMemoryAdapter::new(machine_id))),
35        "cursor" => Some(Box::new(CursorAdapter::new(machine_id))),
36        _ => None,
37    }
38}
39
40/// The adapters an ingest of `source` runs: the one named, or all of them
41/// when no source is given. An unknown name is the caller's mistake and is
42/// answered with a message listing the sources that exist.
43pub fn adapters_for(
44    source: Option<&str>,
45    machine_id: &str,
46) -> Result<Vec<Box<dyn Adapter>>, String> {
47    match source {
48        None => Ok(SOURCES
49            .iter()
50            .filter_map(|name| adapter_named(name, machine_id))
51            .collect()),
52        Some(name) => adapter_named(name, machine_id)
53            .map(|a| vec![a])
54            .ok_or_else(|| {
55                format!(
56                    "unknown ingest source '{name}'; valid values: {}",
57                    SOURCES.join(", ")
58                )
59            }),
60    }
61}
62
63pub fn run_ingest(
64    store: &Arc<dyn StoreAdmin>,
65    embedder: Option<&Embedder>,
66    machine_id: &str,
67    source: Option<&str>,
68) -> Result<IngestStats> {
69    let adapters = adapters_for(source, machine_id).map_err(anyhow::Error::msg)?;
70    let embedder = embedder.map(|e| e as &dyn DocumentEmbedder);
71    Ok(ingest_with_adapters(
72        store.as_ref(),
73        embedder,
74        machine_id,
75        &adapters,
76    ))
77}
78
79pub fn ingest_with_adapters(
80    store: &dyn StoreAdmin,
81    embedder: Option<&dyn DocumentEmbedder>,
82    machine_id: &str,
83    adapters: &[Box<dyn Adapter>],
84) -> IngestStats {
85    let mut total = IngestStats::default();
86    for adapter in adapters {
87        let sources = match adapter.discover_conversations() {
88            Ok(s) => s,
89            Err(e) => {
90                tracing::warn!("ingest: {} discovery failed: {e}", adapter.name());
91                continue;
92            }
93        };
94        total.files_processed += sources.len();
95        ingest_sources(
96            adapter.as_ref(),
97            &sources,
98            store,
99            embedder,
100            machine_id,
101            &mut total,
102        );
103    }
104    total
105}
106
107/// Re-runs every adapter after forgetting which files it saw, except files
108/// ingested before raw ids became deterministic: re-parsing one would bring
109/// back, under new ids, raws that were deleted under their old random ones.
110/// Returns the number of forgotten files with the ingest stats.
111pub fn reingest(
112    store: &Arc<dyn StoreAdmin>,
113    embedder: Option<&Embedder>,
114    machine_id: &str,
115) -> Result<(usize, IngestStats)> {
116    let adapters = adapters_for(None, machine_id).map_err(anyhow::Error::msg)?;
117    reingest_with_adapters(
118        store.as_ref(),
119        embedder.map(|e| e as &dyn DocumentEmbedder),
120        machine_id,
121        &adapters,
122    )
123}
124
125pub fn reingest_with_adapters(
126    store: &dyn StoreAdmin,
127    embedder: Option<&dyn DocumentEmbedder>,
128    machine_id: &str,
129    adapters: &[Box<dyn Adapter>],
130) -> Result<(usize, IngestStats)> {
131    let cleared = store.clear_reparseable_ingest_log()?;
132    Ok((
133        cleared,
134        ingest_with_adapters(store, embedder, machine_id, adapters),
135    ))
136}
137
138/// What a watcher-triggered ingest did with one file. Write failures are
139/// returned as errors, without the path, so they can be reported as health.
140#[derive(Debug, Clone, Copy, PartialEq, Eq)]
141pub enum FileOutcome {
142    /// Stored, or already stored with the same content.
143    Captured,
144    /// Unreadable or not a parseable conversation.
145    Skipped,
146}
147
148pub fn capture_transcript(
149    store: &Arc<dyn StoreAdmin>,
150    embedder: Option<&Embedder>,
151    archive: Option<&TranscriptArchive>,
152    path: &Path,
153    machine_id: &str,
154) -> Result<FileOutcome> {
155    if store.session_deleted(&path.to_string_lossy())? {
156        return Ok(FileOutcome::Skipped);
157    }
158    let archived = match archive.map(|a| a.archive(path)) {
159        Some(Err(_)) if std::fs::File::open(path).is_err() => return Ok(FileOutcome::Skipped),
160        archived => archived,
161    };
162    let outcome = ingest_file(store, embedder, path, machine_id)?;
163    if let Some(Err(e)) = archived {
164        return Err(e.context("archiving a transcript"));
165    }
166    Ok(outcome)
167}
168
169pub fn ingest_file(
170    store: &Arc<dyn StoreAdmin>,
171    embedder: Option<&Embedder>,
172    path: &std::path::Path,
173    machine_id: &str,
174) -> Result<FileOutcome> {
175    let path_str = path.to_string_lossy().to_string();
176    let adapter = ClaudeCodeAdapter::new(machine_id.to_string());
177
178    let changed = match check_file(store.as_ref(), &adapter, path) {
179        Some(FileCheck::Changed(changed)) => changed,
180        Some(FileCheck::Unchanged) => return Ok(FileOutcome::Captured),
181        None => return Ok(FileOutcome::Skipped),
182    };
183
184    let source = ConversationSource {
185        id: path_str.clone(),
186        path: path_str.clone(),
187        source_name: "claude-code".to_string(),
188        machine_id: machine_id.to_string(),
189    };
190
191    let result = match adapter.ingest_conversation(&source) {
192        Ok(r) => r,
193        Err(e) => {
194            tracing::debug!("watcher: failed to parse {}: {e}", path_str);
195            changed
196                .record_unreadable(store.as_ref(), path, "claude-code")
197                .context("recording a transcript that cannot be parsed")?;
198            return Ok(FileOutcome::Skipped);
199        }
200    };
201
202    let outcome = store_ingest_result(
203        store.as_ref(),
204        embedder.map(|e| e as &dyn DocumentEmbedder),
205        &result,
206        machine_id,
207    );
208
209    if outcome.errors > 0 {
210        tracing::warn!(
211            "watcher: ingested {} with {} write errors ({} memories stored)",
212            path_str,
213            outcome.errors,
214            outcome.stored
215        );
216        bail!(
217            "{} memory writes failed while ingesting a transcript",
218            outcome.errors
219        );
220    }
221    changed
222        .record(store.as_ref(), &path_str, outcome.stored, "claude-code")
223        .context("recording an ingested transcript")?;
224    if outcome.stored > 0 {
225        tracing::info!(
226            "watcher: ingested {} ({} memories)",
227            path_str,
228            outcome.stored
229        );
230    }
231    Ok(FileOutcome::Captured)
232}
233
234/// A Claude Code memory note: a `.md` file in a `memory` directory, other
235/// than the `MEMORY.md` index.
236pub(crate) fn is_memory_file(path: &Path) -> bool {
237    path.extension().is_some_and(|e| e == "md")
238        && path
239            .parent()
240            .is_some_and(|p| p.file_name().is_some_and(|n| n == "memory"))
241        && !path
242            .file_name()
243            .is_some_and(|n| n.eq_ignore_ascii_case("MEMORY.md"))
244}
245
246/// True unless ingest_log holds this file's current content; an unreadable
247/// file has nothing to ingest.
248pub(crate) fn changed_since_ingest(store: &dyn StoreAdmin, path: &Path) -> bool {
249    let Ok(bytes) = std::fs::read(path) else {
250        return false;
251    };
252    !ingested_as(store, path, &sha256_hex(&bytes))
253}
254
255/// True when ingest_log records `content_hash` for `path`.
256pub(crate) fn ingested_as(store: &dyn StoreAdmin, path: &Path, content_hash: &str) -> bool {
257    let path_str = path.to_string_lossy();
258    match store.is_file_changed(&path_str, content_hash) {
259        Ok(changed) => !changed,
260        Err(e) => {
261            tracing::warn!("ingest: reading the ingest log for {path_str} failed: {e}");
262            false
263        }
264    }
265}
266
267pub fn ingest_memory_file(
268    store: &Arc<dyn StoreAdmin>,
269    embedder: Option<&Embedder>,
270    path: &std::path::Path,
271    machine_id: &str,
272) -> Result<FileOutcome> {
273    let path_str = path.to_string_lossy().to_string();
274    let adapter = ClaudeMemoryAdapter::new(machine_id.to_string());
275
276    let changed = match check_file(store.as_ref(), &adapter, path) {
277        Some(FileCheck::Changed(changed)) => changed,
278        Some(FileCheck::Unchanged) => return Ok(FileOutcome::Captured),
279        None => return Ok(FileOutcome::Skipped),
280    };
281
282    // remove the previous memory for this file before re-ingesting so stale
283    // content does not accumulate alongside the updated version
284    store
285        .delete_memories_by_source_id(&path_str)
286        .context("removing the previous memory of a memory file")?;
287
288    let source = ConversationSource {
289        id: path_str.clone(),
290        path: path_str.clone(),
291        source_name: "claude-memory".to_string(),
292        machine_id: machine_id.to_string(),
293    };
294
295    let result = match adapter.ingest_conversation(&source) {
296        Ok(r) => r,
297        Err(e) => {
298            tracing::debug!("watcher: failed to parse memory file {}: {e}", path_str);
299            changed
300                .record_unreadable(store.as_ref(), path, "claude-memory")
301                .context("recording a memory file that cannot be parsed")?;
302            return Ok(FileOutcome::Skipped);
303        }
304    };
305
306    let outcome = store_ingest_result(
307        store.as_ref(),
308        embedder.map(|e| e as &dyn DocumentEmbedder),
309        &result,
310        machine_id,
311    );
312
313    if outcome.stored > 0 || outcome.errors > 0 {
314        tracing::info!(
315            "watcher: ingested memory file {} ({} memories, {} errors)",
316            path_str,
317            outcome.stored,
318            outcome.errors
319        );
320    }
321    if outcome.errors > 0 {
322        bail!(
323            "{} memory writes failed while ingesting a memory file",
324            outcome.errors
325        );
326    }
327    changed
328        .record(store.as_ref(), &path_str, outcome.stored, "claude-memory")
329        .context("recording an ingested memory file")?;
330    Ok(FileOutcome::Captured)
331}
332
333fn ingest_sources(
334    adapter: &dyn Adapter,
335    sources: &[ConversationSource],
336    store: &dyn StoreAdmin,
337    embedder: Option<&dyn DocumentEmbedder>,
338    machine_id: &str,
339    stats: &mut IngestStats,
340) {
341    for source in sources {
342        let changed = match check_file(store, adapter, std::path::Path::new(&source.path)) {
343            Some(FileCheck::Changed(changed)) => changed,
344            Some(FileCheck::Unchanged) => {
345                stats.unchanged += 1;
346                continue;
347            }
348            None => {
349                stats.errors += 1;
350                continue;
351            }
352        };
353
354        let result = match adapter.ingest_conversation(source) {
355            Ok(r) => r,
356            Err(e) => {
357                tracing::warn!("ingest: failed to parse {}: {e}", source.path);
358                stats.errors += 1;
359                continue;
360            }
361        };
362
363        let outcome = store_ingest_result(store, embedder, &result, machine_id);
364        if !result.memories.is_empty() {
365            stats.new_conversations += 1;
366        }
367        stats.new_memories += outcome.stored;
368        stats.skipped_dupes += outcome.dupes;
369        stats.new_entities += outcome.entities;
370        stats.errors += outcome.errors;
371
372        if outcome.errors > 0 {
373            continue;
374        }
375        if let Err(e) = changed.record(store, &source.path, outcome.stored, adapter.name()) {
376            tracing::warn!("ingest: mark_ingested failed for {}: {e}", source.path);
377            stats.errors += 1;
378        }
379    }
380}
381
382enum FileCheck {
383    Unchanged,
384    Changed(ChangedFile),
385}
386
387struct ChangedFile {
388    content_hash: String,
389    session_key: Option<String>,
390}
391
392impl ChangedFile {
393    fn record(
394        &self,
395        store: &dyn StoreAdmin,
396        path: &str,
397        memory_count: usize,
398        source: &str,
399    ) -> Result<()> {
400        store.record_ingested(&IngestRecord {
401            file_path: path,
402            content_hash: &self.content_hash,
403            session_key: self.session_key.as_deref(),
404            legacy_ids: false,
405            memory_count,
406            source,
407        })
408    }
409
410    /// True when recorded: only bytes that are still the hashed ones and are
411    /// not text, so a rewrite, unlink or transient read error is retried.
412    fn record_unreadable(&self, store: &dyn StoreAdmin, path: &Path, source: &str) -> Result<bool> {
413        let Ok(bytes) = std::fs::read(path) else {
414            return Ok(false);
415        };
416        if sha256_hex(&bytes) != self.content_hash || std::str::from_utf8(&bytes).is_ok() {
417            return Ok(false);
418        }
419        self.record(store, &path.to_string_lossy(), 0, source)?;
420        Ok(true)
421    }
422}
423
424/// Whether `path` needs parsing. A file is unchanged when its own row holds
425/// its content, when any row holds the same session and content (a copy), or
426/// when its bytes match a pre-SHA-256 legacy row at this path or any path
427/// with the same stem; those legacy rows are upgraded in place on the way.
428/// None when the file can't be read.
429fn check_file(store: &dyn StoreAdmin, adapter: &dyn Adapter, path: &Path) -> Option<FileCheck> {
430    let bytes = std::fs::read(path).ok()?;
431    let content_hash = sha256_hex(&bytes);
432    let path_str = path.to_string_lossy();
433    if adapter.name() == "claude-code" {
434        match store.session_deleted(&path_str) {
435            Ok(true) => return Some(FileCheck::Unchanged),
436            Ok(false) => {}
437            Err(e) => {
438                tracing::warn!("ingest: checking whether {path_str} was deleted failed: {e}");
439                return None;
440            }
441        }
442    }
443
444    if let Ok(false) = store.is_file_changed(&path_str, &content_hash) {
445        return Some(FileCheck::Unchanged);
446    }
447    let session_key = adapter.session_key(path, &String::from_utf8_lossy(&bytes));
448    let legacy_hash = legacy_file_hash(&bytes);
449    let changed = || {
450        Some(FileCheck::Changed(ChangedFile {
451            content_hash: content_hash.clone(),
452            session_key: session_key.clone(),
453        }))
454    };
455
456    let own_row = store.ingest_log_entry(&path_str).unwrap_or_else(|e| {
457        tracing::warn!("ingest: reading the ingest log for {path_str} failed: {e}");
458        None
459    });
460    if let Some(row) = own_row.filter(|r| r.content_hash.is_none() && r.file_hash == legacy_hash) {
461        upgrade(store, &row.file_path, &content_hash, session_key.as_deref());
462        return Some(FileCheck::Unchanged);
463    }
464
465    let Some(key) = session_key.as_deref() else {
466        return changed();
467    };
468    match store.ingested_session(key, &content_hash) {
469        Ok(Some(copy_of)) => {
470            record_copy(
471                store,
472                &path_str,
473                &content_hash,
474                key,
475                copy_of.legacy_ids,
476                adapter.name(),
477            );
478            return Some(FileCheck::Unchanged);
479        }
480        Ok(None) => {}
481        Err(e) => tracing::warn!("ingest: session lookup for {path_str} failed: {e}"),
482    }
483
484    let Some(stem) = path.file_stem() else {
485        return changed();
486    };
487    let legacy_rows = store
488        .legacy_ingest_entries_with_stem(&stem.to_string_lossy())
489        .unwrap_or_else(|e| {
490            tracing::warn!("ingest: legacy lookup for {path_str} failed: {e}");
491            Vec::new()
492        });
493    if let Some(row) = legacy_rows.into_iter().find(|r| r.file_hash == legacy_hash) {
494        upgrade(store, &row.file_path, &content_hash, Some(key));
495        record_copy(store, &path_str, &content_hash, key, true, adapter.name());
496        return Some(FileCheck::Unchanged);
497    }
498    changed()
499}
500
501fn upgrade(store: &dyn StoreAdmin, path: &str, content_hash: &str, session_key: Option<&str>) {
502    if let Err(e) = store.upgrade_ingest_hash(path, content_hash, session_key) {
503        tracing::warn!("ingest: upgrading the ingest log hash for {path} failed: {e}");
504    }
505}
506
507fn record_copy(
508    store: &dyn StoreAdmin,
509    path: &str,
510    content_hash: &str,
511    session_key: &str,
512    legacy_ids: bool,
513    source: &str,
514) {
515    let record = IngestRecord {
516        file_path: path,
517        content_hash,
518        session_key: Some(session_key),
519        legacy_ids,
520        memory_count: 0,
521        source,
522    };
523    if let Err(e) = store.record_ingested(&record) {
524        tracing::warn!("ingest: recording copied transcript {path} failed: {e}");
525    }
526}
527
528fn sha256_hex(bytes: &[u8]) -> String {
529    Sha256::digest(bytes)
530        .iter()
531        .map(|b| format!("{b:02x}"))
532        .collect()
533}
534
535/// The hash ingest_log held before SHA-256: std's `DefaultHasher`, whose
536/// output is not stable across Rust releases, so it is only ever compared
537/// against rows written by an older binary.
538pub fn legacy_file_hash(bytes: &[u8]) -> String {
539    use std::hash::Hasher;
540    let mut hasher = std::collections::hash_map::DefaultHasher::new();
541    for chunk in bytes.chunks(8192) {
542        hasher.write(chunk);
543    }
544    format!("{:016x}", hasher.finish())
545}
546
547#[cfg(test)]
548mod tests {
549    use super::*;
550
551    #[test]
552    fn a_failed_read_is_recorded_only_when_the_checked_bytes_are_not_text() {
553        let store =
554            second_brain_core::sqlite_store::SqliteStore::in_memory("m".to_string()).unwrap();
555        let dir = tempfile::tempdir().unwrap();
556        let path = dir.path().join("note.md");
557        let checked = |bytes: &[u8]| ChangedFile {
558            content_hash: sha256_hex(bytes),
559            session_key: None,
560        };
561        let cut = [b'c', b'a', b'f', 0xC3];
562
563        std::fs::write(&path, [b'x', 0xC3]).unwrap();
564        assert!(
565            !checked(&cut)
566                .record_unreadable(&store, &path, "claude-memory")
567                .unwrap()
568        );
569        assert!(!ingested_as(&store, &path, &sha256_hex(&cut)));
570
571        std::fs::write(&path, b"valid text").unwrap();
572        assert!(
573            !checked(b"valid text")
574                .record_unreadable(&store, &path, "claude-memory")
575                .unwrap()
576        );
577        assert!(!ingested_as(&store, &path, &sha256_hex(b"valid text")));
578
579        std::fs::remove_file(&path).unwrap();
580        assert!(
581            !checked(&cut)
582                .record_unreadable(&store, &path, "claude-memory")
583                .unwrap()
584        );
585
586        std::fs::write(&path, cut).unwrap();
587        assert!(
588            checked(&cut)
589                .record_unreadable(&store, &path, "claude-memory")
590                .unwrap()
591        );
592        assert!(ingested_as(&store, &path, &sha256_hex(&cut)));
593    }
594
595    fn names(source: Option<&str>) -> Result<Vec<String>, String> {
596        adapters_for(source, "m")
597            .map(|adapters| adapters.iter().map(|a| a.name().to_string()).collect())
598    }
599
600    #[test]
601    fn a_named_source_runs_only_that_adapter() {
602        for source in SOURCES {
603            assert_eq!(names(Some(source)).unwrap(), vec![source.to_string()]);
604        }
605    }
606
607    #[test]
608    fn no_source_runs_every_adapter() {
609        assert_eq!(names(None).unwrap(), SOURCES.map(str::to_string).to_vec());
610    }
611
612    #[test]
613    fn an_unknown_source_is_refused_with_the_valid_names() {
614        let error = names(Some("bogus")).unwrap_err();
615        assert!(error.contains("bogus"), "{error}");
616        for source in SOURCES {
617            assert!(error.contains(source), "{error}");
618        }
619    }
620}