Skip to main content

second_brain_api/
ingest.rs

1use std::path::Path;
2use std::sync::Arc;
3
4use anyhow::{Context, Result, bail};
5use sha2::{Digest, Sha256};
6
7use second_brain_adapters::claude_code::ClaudeCodeAdapter;
8use second_brain_adapters::claude_memory::ClaudeMemoryAdapter;
9use second_brain_adapters::cursor::CursorAdapter;
10use second_brain_adapters::{Adapter, ConversationSource, store_ingest_result};
11use second_brain_core::embedding::{DocumentEmbedder, Embedder};
12use second_brain_core::schema::IngestRecord;
13use second_brain_core::store::StoreAdmin;
14
15#[derive(Debug, Default)]
16pub struct IngestStats {
17    pub new_memories: usize,
18    pub new_conversations: usize,
19    pub new_entities: usize,
20    pub skipped_dupes: usize,
21    pub unchanged: usize,
22    pub errors: usize,
23    pub files_processed: usize,
24}
25
26/// Every adapter an ingest can run, by the `source` name it stores.
27pub const SOURCES: [&str; 3] = ["claude-code", "claude-memory", "cursor"];
28
29fn adapter_named(source: &str, machine_id: &str) -> Option<Box<dyn Adapter>> {
30    let machine_id = machine_id.to_string();
31    match source {
32        "claude-code" => Some(Box::new(ClaudeCodeAdapter::new(machine_id))),
33        "claude-memory" => Some(Box::new(ClaudeMemoryAdapter::new(machine_id))),
34        "cursor" => Some(Box::new(CursorAdapter::new(machine_id))),
35        _ => None,
36    }
37}
38
39/// The adapters an ingest of `source` runs: the one named, or all of them
40/// when no source is given. An unknown name is the caller's mistake and is
41/// answered with a message listing the sources that exist.
42pub fn adapters_for(
43    source: Option<&str>,
44    machine_id: &str,
45) -> Result<Vec<Box<dyn Adapter>>, String> {
46    match source {
47        None => Ok(SOURCES
48            .iter()
49            .filter_map(|name| adapter_named(name, machine_id))
50            .collect()),
51        Some(name) => adapter_named(name, machine_id)
52            .map(|a| vec![a])
53            .ok_or_else(|| {
54                format!(
55                    "unknown ingest source '{name}'; valid values: {}",
56                    SOURCES.join(", ")
57                )
58            }),
59    }
60}
61
62pub fn run_ingest(
63    store: &Arc<dyn StoreAdmin>,
64    embedder: Option<&Embedder>,
65    machine_id: &str,
66    source: Option<&str>,
67) -> Result<IngestStats> {
68    let adapters = adapters_for(source, machine_id).map_err(anyhow::Error::msg)?;
69    let embedder = embedder.map(|e| e as &dyn DocumentEmbedder);
70    Ok(ingest_with_adapters(
71        store.as_ref(),
72        embedder,
73        machine_id,
74        &adapters,
75    ))
76}
77
78pub fn ingest_with_adapters(
79    store: &dyn StoreAdmin,
80    embedder: Option<&dyn DocumentEmbedder>,
81    machine_id: &str,
82    adapters: &[Box<dyn Adapter>],
83) -> IngestStats {
84    let mut total = IngestStats::default();
85    for adapter in adapters {
86        let sources = match adapter.discover_conversations() {
87            Ok(s) => s,
88            Err(e) => {
89                tracing::warn!("ingest: {} discovery failed: {e}", adapter.name());
90                continue;
91            }
92        };
93        total.files_processed += sources.len();
94        ingest_sources(
95            adapter.as_ref(),
96            &sources,
97            store,
98            embedder,
99            machine_id,
100            &mut total,
101        );
102    }
103    total
104}
105
106/// Re-runs every adapter after forgetting which files it saw, except files
107/// ingested before raw ids became deterministic: re-parsing one would bring
108/// back, under new ids, raws that were deleted under their old random ones.
109/// Returns the number of forgotten files with the ingest stats.
110pub fn reingest(
111    store: &Arc<dyn StoreAdmin>,
112    embedder: Option<&Embedder>,
113    machine_id: &str,
114) -> Result<(usize, IngestStats)> {
115    let adapters = adapters_for(None, machine_id).map_err(anyhow::Error::msg)?;
116    reingest_with_adapters(
117        store.as_ref(),
118        embedder.map(|e| e as &dyn DocumentEmbedder),
119        machine_id,
120        &adapters,
121    )
122}
123
124pub fn reingest_with_adapters(
125    store: &dyn StoreAdmin,
126    embedder: Option<&dyn DocumentEmbedder>,
127    machine_id: &str,
128    adapters: &[Box<dyn Adapter>],
129) -> Result<(usize, IngestStats)> {
130    let cleared = store.clear_reparseable_ingest_log()?;
131    Ok((
132        cleared,
133        ingest_with_adapters(store, embedder, machine_id, adapters),
134    ))
135}
136
137/// What a watcher-triggered ingest did with one file. Write failures are
138/// returned as errors, without the path, so they can be reported as health.
139pub enum FileOutcome {
140    /// Stored, or already stored with the same content.
141    Captured,
142    /// Unreadable or not a parseable conversation.
143    Skipped,
144}
145
146pub fn ingest_file(
147    store: &Arc<dyn StoreAdmin>,
148    embedder: Option<&Embedder>,
149    path: &std::path::Path,
150    machine_id: &str,
151) -> Result<FileOutcome> {
152    let path_str = path.to_string_lossy().to_string();
153    let adapter = ClaudeCodeAdapter::new(machine_id.to_string());
154
155    let changed = match check_file(store.as_ref(), &adapter, path) {
156        Some(FileCheck::Changed(changed)) => changed,
157        Some(FileCheck::Unchanged) => return Ok(FileOutcome::Captured),
158        None => return Ok(FileOutcome::Skipped),
159    };
160
161    let source = ConversationSource {
162        id: path_str.clone(),
163        path: path_str.clone(),
164        source_name: "claude-code".to_string(),
165        machine_id: machine_id.to_string(),
166    };
167
168    let result = match adapter.ingest_conversation(&source) {
169        Ok(r) => r,
170        Err(e) => {
171            tracing::debug!("watcher: failed to parse {}: {e}", path_str);
172            return Ok(FileOutcome::Skipped);
173        }
174    };
175
176    let outcome = store_ingest_result(
177        store.as_ref(),
178        embedder.map(|e| e as &dyn DocumentEmbedder),
179        &result,
180        machine_id,
181    );
182
183    let marked = changed.record(store.as_ref(), &path_str, outcome.stored, "claude-code");
184    if let Err(e) = &marked {
185        tracing::warn!("watcher: mark_ingested failed for {}: {e}", path_str);
186    }
187    if outcome.errors > 0 {
188        tracing::warn!(
189            "watcher: ingested {} with {} write errors ({} memories stored)",
190            path_str,
191            outcome.errors,
192            outcome.stored
193        );
194        bail!(
195            "{} memory writes failed while ingesting a transcript",
196            outcome.errors
197        );
198    } else if outcome.stored > 0 {
199        tracing::info!(
200            "watcher: ingested {} ({} memories)",
201            path_str,
202            outcome.stored
203        );
204    }
205    marked.context("recording an ingested transcript")?;
206    Ok(FileOutcome::Captured)
207}
208
209pub fn ingest_memory_file(
210    store: &Arc<dyn StoreAdmin>,
211    embedder: Option<&Embedder>,
212    path: &std::path::Path,
213    machine_id: &str,
214) -> Result<FileOutcome> {
215    let path_str = path.to_string_lossy().to_string();
216    let adapter = ClaudeMemoryAdapter::new(machine_id.to_string());
217
218    let changed = match check_file(store.as_ref(), &adapter, path) {
219        Some(FileCheck::Changed(changed)) => changed,
220        Some(FileCheck::Unchanged) => return Ok(FileOutcome::Captured),
221        None => return Ok(FileOutcome::Skipped),
222    };
223
224    // remove the previous memory for this file before re-ingesting so stale
225    // content does not accumulate alongside the updated version
226    if let Err(e) = store.delete_memories_by_source_id(&path_str) {
227        tracing::warn!(
228            "watcher: delete_memories_by_source_id failed for {}: {e}",
229            path_str
230        );
231    }
232
233    let source = ConversationSource {
234        id: path_str.clone(),
235        path: path_str.clone(),
236        source_name: "claude-memory".to_string(),
237        machine_id: machine_id.to_string(),
238    };
239
240    let result = match adapter.ingest_conversation(&source) {
241        Ok(r) => r,
242        Err(e) => {
243            tracing::debug!("watcher: failed to parse memory file {}: {e}", path_str);
244            return Ok(FileOutcome::Skipped);
245        }
246    };
247
248    let outcome = store_ingest_result(
249        store.as_ref(),
250        embedder.map(|e| e as &dyn DocumentEmbedder),
251        &result,
252        machine_id,
253    );
254
255    let marked = changed.record(store.as_ref(), &path_str, outcome.stored, "claude-memory");
256    if let Err(e) = &marked {
257        tracing::warn!("watcher: mark_ingested failed for {}: {e}", path_str);
258    }
259    if outcome.stored > 0 || outcome.errors > 0 {
260        tracing::info!(
261            "watcher: ingested memory file {} ({} memories, {} errors)",
262            path_str,
263            outcome.stored,
264            outcome.errors
265        );
266    }
267    if outcome.errors > 0 {
268        bail!(
269            "{} memory writes failed while ingesting a memory file",
270            outcome.errors
271        );
272    }
273    marked.context("recording an ingested memory file")?;
274    Ok(FileOutcome::Captured)
275}
276
277fn ingest_sources(
278    adapter: &dyn Adapter,
279    sources: &[ConversationSource],
280    store: &dyn StoreAdmin,
281    embedder: Option<&dyn DocumentEmbedder>,
282    machine_id: &str,
283    stats: &mut IngestStats,
284) {
285    for source in sources {
286        let changed = match check_file(store, adapter, std::path::Path::new(&source.path)) {
287            Some(FileCheck::Changed(changed)) => changed,
288            Some(FileCheck::Unchanged) => {
289                stats.unchanged += 1;
290                continue;
291            }
292            None => {
293                stats.errors += 1;
294                continue;
295            }
296        };
297
298        let result = match adapter.ingest_conversation(source) {
299            Ok(r) => r,
300            Err(e) => {
301                tracing::warn!("ingest: failed to parse {}: {e}", source.path);
302                stats.errors += 1;
303                continue;
304            }
305        };
306
307        let outcome = store_ingest_result(store, embedder, &result, machine_id);
308        if !result.memories.is_empty() {
309            stats.new_conversations += 1;
310        }
311        stats.new_memories += outcome.stored;
312        stats.skipped_dupes += outcome.dupes;
313        stats.new_entities += outcome.entities;
314        stats.errors += outcome.errors;
315
316        if let Err(e) = changed.record(store, &source.path, outcome.stored, adapter.name()) {
317            tracing::warn!("ingest: mark_ingested failed for {}: {e}", source.path);
318            stats.errors += 1;
319        }
320    }
321}
322
323enum FileCheck {
324    Unchanged,
325    Changed(ChangedFile),
326}
327
328struct ChangedFile {
329    content_hash: String,
330    session_key: Option<String>,
331}
332
333impl ChangedFile {
334    fn record(
335        &self,
336        store: &dyn StoreAdmin,
337        path: &str,
338        memory_count: usize,
339        source: &str,
340    ) -> Result<()> {
341        store.record_ingested(&IngestRecord {
342            file_path: path,
343            content_hash: &self.content_hash,
344            session_key: self.session_key.as_deref(),
345            legacy_ids: false,
346            memory_count,
347            source,
348        })
349    }
350}
351
352/// Whether `path` needs parsing. A file is unchanged when its own row holds
353/// its content, when any row holds the same session and content (a copy), or
354/// when its bytes match a pre-SHA-256 legacy row at this path or any path
355/// with the same stem; those legacy rows are upgraded in place on the way.
356/// None when the file can't be read.
357fn check_file(store: &dyn StoreAdmin, adapter: &dyn Adapter, path: &Path) -> Option<FileCheck> {
358    let bytes = std::fs::read(path).ok()?;
359    let content_hash = sha256_hex(&bytes);
360    let path_str = path.to_string_lossy();
361
362    if let Ok(false) = store.is_file_changed(&path_str, &content_hash) {
363        return Some(FileCheck::Unchanged);
364    }
365    let session_key = adapter.session_key(path, &String::from_utf8_lossy(&bytes));
366    let legacy_hash = legacy_file_hash(&bytes);
367    let changed = || {
368        Some(FileCheck::Changed(ChangedFile {
369            content_hash: content_hash.clone(),
370            session_key: session_key.clone(),
371        }))
372    };
373
374    let own_row = store.ingest_log_entry(&path_str).unwrap_or_else(|e| {
375        tracing::warn!("ingest: reading the ingest log for {path_str} failed: {e}");
376        None
377    });
378    if let Some(row) = own_row.filter(|r| r.content_hash.is_none() && r.file_hash == legacy_hash) {
379        upgrade(store, &row.file_path, &content_hash, session_key.as_deref());
380        return Some(FileCheck::Unchanged);
381    }
382
383    let Some(key) = session_key.as_deref() else {
384        return changed();
385    };
386    match store.ingested_session(key, &content_hash) {
387        Ok(Some(copy_of)) => {
388            record_copy(
389                store,
390                &path_str,
391                &content_hash,
392                key,
393                copy_of.legacy_ids,
394                adapter.name(),
395            );
396            return Some(FileCheck::Unchanged);
397        }
398        Ok(None) => {}
399        Err(e) => tracing::warn!("ingest: session lookup for {path_str} failed: {e}"),
400    }
401
402    let Some(stem) = path.file_stem() else {
403        return changed();
404    };
405    let legacy_rows = store
406        .legacy_ingest_entries_with_stem(&stem.to_string_lossy())
407        .unwrap_or_else(|e| {
408            tracing::warn!("ingest: legacy lookup for {path_str} failed: {e}");
409            Vec::new()
410        });
411    if let Some(row) = legacy_rows.into_iter().find(|r| r.file_hash == legacy_hash) {
412        upgrade(store, &row.file_path, &content_hash, Some(key));
413        record_copy(store, &path_str, &content_hash, key, true, adapter.name());
414        return Some(FileCheck::Unchanged);
415    }
416    changed()
417}
418
419fn upgrade(store: &dyn StoreAdmin, path: &str, content_hash: &str, session_key: Option<&str>) {
420    if let Err(e) = store.upgrade_ingest_hash(path, content_hash, session_key) {
421        tracing::warn!("ingest: upgrading the ingest log hash for {path} failed: {e}");
422    }
423}
424
425fn record_copy(
426    store: &dyn StoreAdmin,
427    path: &str,
428    content_hash: &str,
429    session_key: &str,
430    legacy_ids: bool,
431    source: &str,
432) {
433    let record = IngestRecord {
434        file_path: path,
435        content_hash,
436        session_key: Some(session_key),
437        legacy_ids,
438        memory_count: 0,
439        source,
440    };
441    if let Err(e) = store.record_ingested(&record) {
442        tracing::warn!("ingest: recording copied transcript {path} failed: {e}");
443    }
444}
445
446fn sha256_hex(bytes: &[u8]) -> String {
447    Sha256::digest(bytes)
448        .iter()
449        .map(|b| format!("{b:02x}"))
450        .collect()
451}
452
453/// The hash ingest_log held before SHA-256: std's `DefaultHasher`, whose
454/// output is not stable across Rust releases, so it is only ever compared
455/// against rows written by an older binary.
456pub fn legacy_file_hash(bytes: &[u8]) -> String {
457    use std::hash::Hasher;
458    let mut hasher = std::collections::hash_map::DefaultHasher::new();
459    for chunk in bytes.chunks(8192) {
460        hasher.write(chunk);
461    }
462    format!("{:016x}", hasher.finish())
463}
464
465#[cfg(test)]
466mod tests {
467    use super::*;
468
469    fn names(source: Option<&str>) -> Result<Vec<String>, String> {
470        adapters_for(source, "m")
471            .map(|adapters| adapters.iter().map(|a| a.name().to_string()).collect())
472    }
473
474    #[test]
475    fn a_named_source_runs_only_that_adapter() {
476        for source in SOURCES {
477            assert_eq!(names(Some(source)).unwrap(), vec![source.to_string()]);
478        }
479    }
480
481    #[test]
482    fn no_source_runs_every_adapter() {
483        assert_eq!(names(None).unwrap(), SOURCES.map(str::to_string).to_vec());
484    }
485
486    #[test]
487    fn an_unknown_source_is_refused_with_the_valid_names() {
488        let error = names(Some("bogus")).unwrap_err();
489        assert!(error.contains("bogus"), "{error}");
490        for source in SOURCES {
491            assert!(error.contains(source), "{error}");
492        }
493    }
494}