Skip to main content

second_brain_api/
ingest.rs

1use std::path::Path;
2use std::sync::Arc;
3
4use anyhow::{Context, Result, bail};
5use sha2::{Digest, Sha256};
6
7use second_brain_adapters::claude_code::ClaudeCodeAdapter;
8use second_brain_adapters::claude_memory::ClaudeMemoryAdapter;
9use second_brain_adapters::cursor::CursorAdapter;
10use second_brain_adapters::{Adapter, ConversationSource, store_ingest_result};
11use second_brain_core::embedding::{DocumentEmbedder, Embedder};
12use second_brain_core::schema::IngestRecord;
13use second_brain_core::store::StoreAdmin;
14use second_brain_core::transcript_archive::TranscriptArchive;
15
16#[derive(Debug, Default)]
17pub struct IngestStats {
18    pub new_memories: usize,
19    pub new_conversations: usize,
20    pub new_entities: usize,
21    pub skipped_dupes: usize,
22    pub unchanged: usize,
23    pub errors: usize,
24    pub files_processed: usize,
25}
26
27/// Every adapter an ingest can run, by the `source` name it stores.
28pub const SOURCES: [&str; 3] = ["claude-code", "claude-memory", "cursor"];
29
30fn adapter_named(source: &str, machine_id: &str) -> Option<Box<dyn Adapter>> {
31    let machine_id = machine_id.to_string();
32    match source {
33        "claude-code" => Some(Box::new(ClaudeCodeAdapter::new(machine_id))),
34        "claude-memory" => Some(Box::new(ClaudeMemoryAdapter::new(machine_id))),
35        "cursor" => Some(Box::new(CursorAdapter::new(machine_id))),
36        _ => None,
37    }
38}
39
40/// The adapters an ingest of `source` runs: the one named, or all of them
41/// when no source is given. An unknown name is the caller's mistake and is
42/// answered with a message listing the sources that exist.
43pub fn adapters_for(
44    source: Option<&str>,
45    machine_id: &str,
46) -> Result<Vec<Box<dyn Adapter>>, String> {
47    match source {
48        None => Ok(SOURCES
49            .iter()
50            .filter_map(|name| adapter_named(name, machine_id))
51            .collect()),
52        Some(name) => adapter_named(name, machine_id)
53            .map(|a| vec![a])
54            .ok_or_else(|| {
55                format!(
56                    "unknown ingest source '{name}'; valid values: {}",
57                    SOURCES.join(", ")
58                )
59            }),
60    }
61}
62
63pub fn run_ingest(
64    store: &Arc<dyn StoreAdmin>,
65    embedder: Option<&Embedder>,
66    machine_id: &str,
67    source: Option<&str>,
68) -> Result<IngestStats> {
69    let adapters = adapters_for(source, machine_id).map_err(anyhow::Error::msg)?;
70    let embedder = embedder.map(|e| e as &dyn DocumentEmbedder);
71    Ok(ingest_with_adapters(
72        store.as_ref(),
73        embedder,
74        machine_id,
75        &adapters,
76    ))
77}
78
79pub fn ingest_with_adapters(
80    store: &dyn StoreAdmin,
81    embedder: Option<&dyn DocumentEmbedder>,
82    machine_id: &str,
83    adapters: &[Box<dyn Adapter>],
84) -> IngestStats {
85    let mut total = IngestStats::default();
86    for adapter in adapters {
87        let sources = match adapter.discover_conversations() {
88            Ok(s) => s,
89            Err(e) => {
90                tracing::warn!("ingest: {} discovery failed: {e}", adapter.name());
91                continue;
92            }
93        };
94        total.files_processed += sources.len();
95        ingest_sources(
96            adapter.as_ref(),
97            &sources,
98            store,
99            embedder,
100            machine_id,
101            &mut total,
102        );
103    }
104    total
105}
106
107/// Re-runs every adapter after forgetting which files it saw, except files
108/// ingested before raw ids became deterministic: re-parsing one would bring
109/// back, under new ids, raws that were deleted under their old random ones.
110/// Returns the number of forgotten files with the ingest stats.
111pub fn reingest(
112    store: &Arc<dyn StoreAdmin>,
113    embedder: Option<&Embedder>,
114    machine_id: &str,
115) -> Result<(usize, IngestStats)> {
116    let adapters = adapters_for(None, machine_id).map_err(anyhow::Error::msg)?;
117    reingest_with_adapters(
118        store.as_ref(),
119        embedder.map(|e| e as &dyn DocumentEmbedder),
120        machine_id,
121        &adapters,
122    )
123}
124
125pub fn reingest_with_adapters(
126    store: &dyn StoreAdmin,
127    embedder: Option<&dyn DocumentEmbedder>,
128    machine_id: &str,
129    adapters: &[Box<dyn Adapter>],
130) -> Result<(usize, IngestStats)> {
131    let cleared = store.clear_reparseable_ingest_log()?;
132    Ok((
133        cleared,
134        ingest_with_adapters(store, embedder, machine_id, adapters),
135    ))
136}
137
138/// What a watcher-triggered ingest did with one file. Write failures are
139/// returned as errors, without the path, so they can be reported as health.
140#[derive(Debug, Clone, Copy, PartialEq, Eq)]
141pub enum FileOutcome {
142    /// Stored, or already stored with the same content.
143    Captured,
144    /// Unreadable or not a parseable conversation.
145    Skipped,
146}
147
148pub fn capture_transcript(
149    store: &Arc<dyn StoreAdmin>,
150    embedder: Option<&Embedder>,
151    archive: Option<&TranscriptArchive>,
152    path: &Path,
153    machine_id: &str,
154) -> Result<FileOutcome> {
155    let archived = match archive.map(|a| a.archive(path)) {
156        Some(Err(_)) if std::fs::File::open(path).is_err() => return Ok(FileOutcome::Skipped),
157        archived => archived,
158    };
159    let outcome = ingest_file(store, embedder, path, machine_id)?;
160    if let Some(Err(e)) = archived {
161        return Err(e.context("archiving a transcript"));
162    }
163    Ok(outcome)
164}
165
166pub fn ingest_file(
167    store: &Arc<dyn StoreAdmin>,
168    embedder: Option<&Embedder>,
169    path: &std::path::Path,
170    machine_id: &str,
171) -> Result<FileOutcome> {
172    let path_str = path.to_string_lossy().to_string();
173    let adapter = ClaudeCodeAdapter::new(machine_id.to_string());
174
175    let changed = match check_file(store.as_ref(), &adapter, path) {
176        Some(FileCheck::Changed(changed)) => changed,
177        Some(FileCheck::Unchanged) => return Ok(FileOutcome::Captured),
178        None => return Ok(FileOutcome::Skipped),
179    };
180
181    let source = ConversationSource {
182        id: path_str.clone(),
183        path: path_str.clone(),
184        source_name: "claude-code".to_string(),
185        machine_id: machine_id.to_string(),
186    };
187
188    let result = match adapter.ingest_conversation(&source) {
189        Ok(r) => r,
190        Err(e) => {
191            tracing::debug!("watcher: failed to parse {}: {e}", path_str);
192            return Ok(FileOutcome::Skipped);
193        }
194    };
195
196    let outcome = store_ingest_result(
197        store.as_ref(),
198        embedder.map(|e| e as &dyn DocumentEmbedder),
199        &result,
200        machine_id,
201    );
202
203    let marked = changed.record(store.as_ref(), &path_str, outcome.stored, "claude-code");
204    if let Err(e) = &marked {
205        tracing::warn!("watcher: mark_ingested failed for {}: {e}", path_str);
206    }
207    if outcome.errors > 0 {
208        tracing::warn!(
209            "watcher: ingested {} with {} write errors ({} memories stored)",
210            path_str,
211            outcome.errors,
212            outcome.stored
213        );
214        bail!(
215            "{} memory writes failed while ingesting a transcript",
216            outcome.errors
217        );
218    } else if outcome.stored > 0 {
219        tracing::info!(
220            "watcher: ingested {} ({} memories)",
221            path_str,
222            outcome.stored
223        );
224    }
225    marked.context("recording an ingested transcript")?;
226    Ok(FileOutcome::Captured)
227}
228
229pub fn ingest_memory_file(
230    store: &Arc<dyn StoreAdmin>,
231    embedder: Option<&Embedder>,
232    path: &std::path::Path,
233    machine_id: &str,
234) -> Result<FileOutcome> {
235    let path_str = path.to_string_lossy().to_string();
236    let adapter = ClaudeMemoryAdapter::new(machine_id.to_string());
237
238    let changed = match check_file(store.as_ref(), &adapter, path) {
239        Some(FileCheck::Changed(changed)) => changed,
240        Some(FileCheck::Unchanged) => return Ok(FileOutcome::Captured),
241        None => return Ok(FileOutcome::Skipped),
242    };
243
244    // remove the previous memory for this file before re-ingesting so stale
245    // content does not accumulate alongside the updated version
246    if let Err(e) = store.delete_memories_by_source_id(&path_str) {
247        tracing::warn!(
248            "watcher: delete_memories_by_source_id failed for {}: {e}",
249            path_str
250        );
251    }
252
253    let source = ConversationSource {
254        id: path_str.clone(),
255        path: path_str.clone(),
256        source_name: "claude-memory".to_string(),
257        machine_id: machine_id.to_string(),
258    };
259
260    let result = match adapter.ingest_conversation(&source) {
261        Ok(r) => r,
262        Err(e) => {
263            tracing::debug!("watcher: failed to parse memory file {}: {e}", path_str);
264            return Ok(FileOutcome::Skipped);
265        }
266    };
267
268    let outcome = store_ingest_result(
269        store.as_ref(),
270        embedder.map(|e| e as &dyn DocumentEmbedder),
271        &result,
272        machine_id,
273    );
274
275    let marked = changed.record(store.as_ref(), &path_str, outcome.stored, "claude-memory");
276    if let Err(e) = &marked {
277        tracing::warn!("watcher: mark_ingested failed for {}: {e}", path_str);
278    }
279    if outcome.stored > 0 || outcome.errors > 0 {
280        tracing::info!(
281            "watcher: ingested memory file {} ({} memories, {} errors)",
282            path_str,
283            outcome.stored,
284            outcome.errors
285        );
286    }
287    if outcome.errors > 0 {
288        bail!(
289            "{} memory writes failed while ingesting a memory file",
290            outcome.errors
291        );
292    }
293    marked.context("recording an ingested memory file")?;
294    Ok(FileOutcome::Captured)
295}
296
297fn ingest_sources(
298    adapter: &dyn Adapter,
299    sources: &[ConversationSource],
300    store: &dyn StoreAdmin,
301    embedder: Option<&dyn DocumentEmbedder>,
302    machine_id: &str,
303    stats: &mut IngestStats,
304) {
305    for source in sources {
306        let changed = match check_file(store, adapter, std::path::Path::new(&source.path)) {
307            Some(FileCheck::Changed(changed)) => changed,
308            Some(FileCheck::Unchanged) => {
309                stats.unchanged += 1;
310                continue;
311            }
312            None => {
313                stats.errors += 1;
314                continue;
315            }
316        };
317
318        let result = match adapter.ingest_conversation(source) {
319            Ok(r) => r,
320            Err(e) => {
321                tracing::warn!("ingest: failed to parse {}: {e}", source.path);
322                stats.errors += 1;
323                continue;
324            }
325        };
326
327        let outcome = store_ingest_result(store, embedder, &result, machine_id);
328        if !result.memories.is_empty() {
329            stats.new_conversations += 1;
330        }
331        stats.new_memories += outcome.stored;
332        stats.skipped_dupes += outcome.dupes;
333        stats.new_entities += outcome.entities;
334        stats.errors += outcome.errors;
335
336        if let Err(e) = changed.record(store, &source.path, outcome.stored, adapter.name()) {
337            tracing::warn!("ingest: mark_ingested failed for {}: {e}", source.path);
338            stats.errors += 1;
339        }
340    }
341}
342
343enum FileCheck {
344    Unchanged,
345    Changed(ChangedFile),
346}
347
348struct ChangedFile {
349    content_hash: String,
350    session_key: Option<String>,
351}
352
353impl ChangedFile {
354    fn record(
355        &self,
356        store: &dyn StoreAdmin,
357        path: &str,
358        memory_count: usize,
359        source: &str,
360    ) -> Result<()> {
361        store.record_ingested(&IngestRecord {
362            file_path: path,
363            content_hash: &self.content_hash,
364            session_key: self.session_key.as_deref(),
365            legacy_ids: false,
366            memory_count,
367            source,
368        })
369    }
370}
371
372/// Whether `path` needs parsing. A file is unchanged when its own row holds
373/// its content, when any row holds the same session and content (a copy), or
374/// when its bytes match a pre-SHA-256 legacy row at this path or any path
375/// with the same stem; those legacy rows are upgraded in place on the way.
376/// None when the file can't be read.
377fn check_file(store: &dyn StoreAdmin, adapter: &dyn Adapter, path: &Path) -> Option<FileCheck> {
378    let bytes = std::fs::read(path).ok()?;
379    let content_hash = sha256_hex(&bytes);
380    let path_str = path.to_string_lossy();
381
382    if let Ok(false) = store.is_file_changed(&path_str, &content_hash) {
383        return Some(FileCheck::Unchanged);
384    }
385    let session_key = adapter.session_key(path, &String::from_utf8_lossy(&bytes));
386    let legacy_hash = legacy_file_hash(&bytes);
387    let changed = || {
388        Some(FileCheck::Changed(ChangedFile {
389            content_hash: content_hash.clone(),
390            session_key: session_key.clone(),
391        }))
392    };
393
394    let own_row = store.ingest_log_entry(&path_str).unwrap_or_else(|e| {
395        tracing::warn!("ingest: reading the ingest log for {path_str} failed: {e}");
396        None
397    });
398    if let Some(row) = own_row.filter(|r| r.content_hash.is_none() && r.file_hash == legacy_hash) {
399        upgrade(store, &row.file_path, &content_hash, session_key.as_deref());
400        return Some(FileCheck::Unchanged);
401    }
402
403    let Some(key) = session_key.as_deref() else {
404        return changed();
405    };
406    match store.ingested_session(key, &content_hash) {
407        Ok(Some(copy_of)) => {
408            record_copy(
409                store,
410                &path_str,
411                &content_hash,
412                key,
413                copy_of.legacy_ids,
414                adapter.name(),
415            );
416            return Some(FileCheck::Unchanged);
417        }
418        Ok(None) => {}
419        Err(e) => tracing::warn!("ingest: session lookup for {path_str} failed: {e}"),
420    }
421
422    let Some(stem) = path.file_stem() else {
423        return changed();
424    };
425    let legacy_rows = store
426        .legacy_ingest_entries_with_stem(&stem.to_string_lossy())
427        .unwrap_or_else(|e| {
428            tracing::warn!("ingest: legacy lookup for {path_str} failed: {e}");
429            Vec::new()
430        });
431    if let Some(row) = legacy_rows.into_iter().find(|r| r.file_hash == legacy_hash) {
432        upgrade(store, &row.file_path, &content_hash, Some(key));
433        record_copy(store, &path_str, &content_hash, key, true, adapter.name());
434        return Some(FileCheck::Unchanged);
435    }
436    changed()
437}
438
439fn upgrade(store: &dyn StoreAdmin, path: &str, content_hash: &str, session_key: Option<&str>) {
440    if let Err(e) = store.upgrade_ingest_hash(path, content_hash, session_key) {
441        tracing::warn!("ingest: upgrading the ingest log hash for {path} failed: {e}");
442    }
443}
444
445fn record_copy(
446    store: &dyn StoreAdmin,
447    path: &str,
448    content_hash: &str,
449    session_key: &str,
450    legacy_ids: bool,
451    source: &str,
452) {
453    let record = IngestRecord {
454        file_path: path,
455        content_hash,
456        session_key: Some(session_key),
457        legacy_ids,
458        memory_count: 0,
459        source,
460    };
461    if let Err(e) = store.record_ingested(&record) {
462        tracing::warn!("ingest: recording copied transcript {path} failed: {e}");
463    }
464}
465
466fn sha256_hex(bytes: &[u8]) -> String {
467    Sha256::digest(bytes)
468        .iter()
469        .map(|b| format!("{b:02x}"))
470        .collect()
471}
472
473/// The hash ingest_log held before SHA-256: std's `DefaultHasher`, whose
474/// output is not stable across Rust releases, so it is only ever compared
475/// against rows written by an older binary.
476pub fn legacy_file_hash(bytes: &[u8]) -> String {
477    use std::hash::Hasher;
478    let mut hasher = std::collections::hash_map::DefaultHasher::new();
479    for chunk in bytes.chunks(8192) {
480        hasher.write(chunk);
481    }
482    format!("{:016x}", hasher.finish())
483}
484
485#[cfg(test)]
486mod tests {
487    use super::*;
488
489    fn names(source: Option<&str>) -> Result<Vec<String>, String> {
490        adapters_for(source, "m")
491            .map(|adapters| adapters.iter().map(|a| a.name().to_string()).collect())
492    }
493
494    #[test]
495    fn a_named_source_runs_only_that_adapter() {
496        for source in SOURCES {
497            assert_eq!(names(Some(source)).unwrap(), vec![source.to_string()]);
498        }
499    }
500
501    #[test]
502    fn no_source_runs_every_adapter() {
503        assert_eq!(names(None).unwrap(), SOURCES.map(str::to_string).to_vec());
504    }
505
506    #[test]
507    fn an_unknown_source_is_refused_with_the_valid_names() {
508        let error = names(Some("bogus")).unwrap_err();
509        assert!(error.contains("bogus"), "{error}");
510        for source in SOURCES {
511            assert!(error.contains(source), "{error}");
512        }
513    }
514}