Skip to main content

second_brain_api/
ingest.rs

1use std::sync::Arc;
2
3use anyhow::Result;
4
5use second_brain_adapters::claude_code::ClaudeCodeAdapter;
6use second_brain_adapters::claude_memory::ClaudeMemoryAdapter;
7use second_brain_adapters::cursor::CursorAdapter;
8use second_brain_adapters::{Adapter, ConversationSource, store_ingest_result};
9use second_brain_core::embedding::{DocumentEmbedder, Embedder};
10use second_brain_core::store::StoreAdmin;
11
12pub struct IngestStats {
13    pub new_memories: usize,
14    pub new_conversations: usize,
15    pub new_entities: usize,
16    pub skipped_dupes: usize,
17    pub unchanged: usize,
18    pub errors: usize,
19    pub files_processed: usize,
20}
21
22/// Every adapter an ingest can run, by the `source` name it stores.
23pub const SOURCES: [&str; 3] = ["claude-code", "claude-memory", "cursor"];
24
25fn adapter_named(source: &str, machine_id: &str) -> Option<Box<dyn Adapter>> {
26    let machine_id = machine_id.to_string();
27    match source {
28        "claude-code" => Some(Box::new(ClaudeCodeAdapter::new(machine_id))),
29        "claude-memory" => Some(Box::new(ClaudeMemoryAdapter::new(machine_id))),
30        "cursor" => Some(Box::new(CursorAdapter::new(machine_id))),
31        _ => None,
32    }
33}
34
35/// The adapters an ingest of `source` runs: the one named, or all of them
36/// when no source is given. An unknown name is the caller's mistake and is
37/// answered with a message listing the sources that exist.
38pub fn adapters_for(
39    source: Option<&str>,
40    machine_id: &str,
41) -> Result<Vec<Box<dyn Adapter>>, String> {
42    match source {
43        None => Ok(SOURCES
44            .iter()
45            .filter_map(|name| adapter_named(name, machine_id))
46            .collect()),
47        Some(name) => adapter_named(name, machine_id)
48            .map(|a| vec![a])
49            .ok_or_else(|| {
50                format!(
51                    "unknown ingest source '{name}'; valid values: {}",
52                    SOURCES.join(", ")
53                )
54            }),
55    }
56}
57
58pub fn run_ingest(
59    store: &Arc<dyn StoreAdmin>,
60    embedder: Option<&Embedder>,
61    machine_id: &str,
62    source: Option<&str>,
63) -> Result<IngestStats> {
64    let adapters = adapters_for(source, machine_id).map_err(anyhow::Error::msg)?;
65    let embedder = embedder.map(|e| e as &dyn DocumentEmbedder);
66
67    let mut total = IngestStats {
68        new_memories: 0,
69        new_conversations: 0,
70        new_entities: 0,
71        skipped_dupes: 0,
72        unchanged: 0,
73        errors: 0,
74        files_processed: 0,
75    };
76
77    for adapter in &adapters {
78        let sources = match adapter.discover_conversations() {
79            Ok(s) => s,
80            Err(e) => {
81                tracing::warn!("ingest: {} discovery failed: {e}", adapter.name());
82                continue;
83            }
84        };
85
86        let source_name = adapter.name().to_string();
87        total.files_processed += sources.len();
88        ingest_sources(
89            adapter.as_ref(),
90            &sources,
91            store.as_ref(),
92            embedder,
93            &source_name,
94            machine_id,
95            &mut total,
96        );
97    }
98
99    Ok(total)
100}
101
102pub fn ingest_file(
103    store: &Arc<dyn StoreAdmin>,
104    embedder: Option<&Embedder>,
105    path: &std::path::Path,
106    machine_id: &str,
107) {
108    let path_str = path.to_string_lossy().to_string();
109
110    let hash = match file_hash(path) {
111        Some(h) => h,
112        None => return,
113    };
114
115    if let Ok(false) = store.is_file_changed(&path_str, &hash) {
116        return;
117    }
118
119    let adapter = ClaudeCodeAdapter::new(machine_id.to_string());
120    let source = ConversationSource {
121        id: path_str.clone(),
122        path: path_str.clone(),
123        source_name: "claude-code".to_string(),
124        machine_id: machine_id.to_string(),
125    };
126
127    let result = match adapter.ingest_conversation(&source) {
128        Ok(r) => r,
129        Err(e) => {
130            tracing::debug!("watcher: failed to parse {}: {e}", path_str);
131            return;
132        }
133    };
134
135    let outcome = store_ingest_result(
136        store.as_ref(),
137        embedder.map(|e| e as &dyn DocumentEmbedder),
138        &result,
139        machine_id,
140    );
141
142    if let Err(e) = store.mark_ingested(&path_str, &hash, outcome.stored, "claude-code") {
143        tracing::warn!("watcher: mark_ingested failed for {}: {e}", path_str);
144    }
145    if outcome.errors > 0 {
146        tracing::warn!(
147            "watcher: ingested {} with {} write errors ({} memories stored)",
148            path_str,
149            outcome.errors,
150            outcome.stored
151        );
152    } else if outcome.stored > 0 {
153        tracing::info!(
154            "watcher: ingested {} ({} memories)",
155            path_str,
156            outcome.stored
157        );
158    }
159}
160
161pub fn ingest_memory_file(
162    store: &Arc<dyn StoreAdmin>,
163    embedder: Option<&Embedder>,
164    path: &std::path::Path,
165    machine_id: &str,
166) {
167    let path_str = path.to_string_lossy().to_string();
168
169    let hash = match file_hash(path) {
170        Some(h) => h,
171        None => return,
172    };
173
174    if let Ok(false) = store.is_file_changed(&path_str, &hash) {
175        return;
176    }
177
178    // remove the previous memory for this file before re-ingesting so stale
179    // content does not accumulate alongside the updated version
180    if let Err(e) = store.delete_memories_by_source_id(&path_str) {
181        tracing::warn!(
182            "watcher: delete_memories_by_source_id failed for {}: {e}",
183            path_str
184        );
185    }
186
187    let adapter = ClaudeMemoryAdapter::new(machine_id.to_string());
188    let source = ConversationSource {
189        id: path_str.clone(),
190        path: path_str.clone(),
191        source_name: "claude-memory".to_string(),
192        machine_id: machine_id.to_string(),
193    };
194
195    let result = match adapter.ingest_conversation(&source) {
196        Ok(r) => r,
197        Err(e) => {
198            tracing::debug!("watcher: failed to parse memory file {}: {e}", path_str);
199            return;
200        }
201    };
202
203    let outcome = store_ingest_result(
204        store.as_ref(),
205        embedder.map(|e| e as &dyn DocumentEmbedder),
206        &result,
207        machine_id,
208    );
209
210    if let Err(e) = store.mark_ingested(&path_str, &hash, outcome.stored, "claude-memory") {
211        tracing::warn!("watcher: mark_ingested failed for {}: {e}", path_str);
212    }
213    if outcome.stored > 0 || outcome.errors > 0 {
214        tracing::info!(
215            "watcher: ingested memory file {} ({} memories, {} errors)",
216            path_str,
217            outcome.stored,
218            outcome.errors
219        );
220    }
221}
222
223fn ingest_sources(
224    adapter: &dyn Adapter,
225    sources: &[ConversationSource],
226    store: &dyn StoreAdmin,
227    embedder: Option<&dyn DocumentEmbedder>,
228    source_name: &str,
229    machine_id: &str,
230    stats: &mut IngestStats,
231) {
232    for source in sources {
233        let path = std::path::Path::new(&source.path);
234        let hash = match file_hash(path) {
235            Some(h) => h,
236            None => {
237                stats.errors += 1;
238                continue;
239            }
240        };
241
242        if let Ok(false) = store.is_file_changed(&source.path, &hash) {
243            stats.unchanged += 1;
244            continue;
245        }
246
247        let result = match adapter.ingest_conversation(source) {
248            Ok(r) => r,
249            Err(e) => {
250                tracing::warn!("ingest: failed to parse {}: {e}", source.path);
251                stats.errors += 1;
252                continue;
253            }
254        };
255
256        let outcome = store_ingest_result(store, embedder, &result, machine_id);
257        if !result.memories.is_empty() {
258            stats.new_conversations += 1;
259        }
260        stats.new_memories += outcome.stored;
261        stats.skipped_dupes += outcome.dupes;
262        stats.new_entities += outcome.entities;
263        stats.errors += outcome.errors;
264
265        if let Err(e) = store.mark_ingested(&source.path, &hash, outcome.stored, source_name) {
266            tracing::warn!("ingest: mark_ingested failed for {}: {e}", source.path);
267            stats.errors += 1;
268        }
269    }
270}
271
272fn file_hash(path: &std::path::Path) -> Option<String> {
273    use std::hash::Hasher;
274    use std::io::Read;
275    let mut file = std::fs::File::open(path).ok()?;
276    let mut hasher = std::collections::hash_map::DefaultHasher::new();
277    let mut buf = [0u8; 8192];
278    loop {
279        let n = file.read(&mut buf).ok()?;
280        if n == 0 {
281            break;
282        }
283        hasher.write(&buf[..n]);
284    }
285    Some(format!("{:016x}", hasher.finish()))
286}
287
288#[cfg(test)]
289mod tests {
290    use super::*;
291
292    fn names(source: Option<&str>) -> Result<Vec<String>, String> {
293        adapters_for(source, "m")
294            .map(|adapters| adapters.iter().map(|a| a.name().to_string()).collect())
295    }
296
297    #[test]
298    fn a_named_source_runs_only_that_adapter() {
299        for source in SOURCES {
300            assert_eq!(names(Some(source)).unwrap(), vec![source.to_string()]);
301        }
302    }
303
304    #[test]
305    fn no_source_runs_every_adapter() {
306        assert_eq!(names(None).unwrap(), SOURCES.map(str::to_string).to_vec());
307    }
308
309    #[test]
310    fn an_unknown_source_is_refused_with_the_valid_names() {
311        let error = names(Some("bogus")).unwrap_err();
312        assert!(error.contains("bogus"), "{error}");
313        for source in SOURCES {
314            assert!(error.contains(source), "{error}");
315        }
316    }
317}