1use std::sync::Arc;
2
3use anyhow::Result;
4
5use second_brain_adapters::claude_code::ClaudeCodeAdapter;
6use second_brain_adapters::claude_memory::ClaudeMemoryAdapter;
7use second_brain_adapters::cursor::CursorAdapter;
8use second_brain_adapters::{Adapter, ConversationSource, store_ingest_result};
9use second_brain_core::embedding::{DocumentEmbedder, Embedder};
10use second_brain_core::store::StoreAdmin;
11
12pub struct IngestStats {
13 pub new_memories: usize,
14 pub new_conversations: usize,
15 pub new_entities: usize,
16 pub skipped_dupes: usize,
17 pub unchanged: usize,
18 pub errors: usize,
19 pub files_processed: usize,
20}
21
22pub const SOURCES: [&str; 3] = ["claude-code", "claude-memory", "cursor"];
24
25fn adapter_named(source: &str, machine_id: &str) -> Option<Box<dyn Adapter>> {
26 let machine_id = machine_id.to_string();
27 match source {
28 "claude-code" => Some(Box::new(ClaudeCodeAdapter::new(machine_id))),
29 "claude-memory" => Some(Box::new(ClaudeMemoryAdapter::new(machine_id))),
30 "cursor" => Some(Box::new(CursorAdapter::new(machine_id))),
31 _ => None,
32 }
33}
34
35pub fn adapters_for(
39 source: Option<&str>,
40 machine_id: &str,
41) -> Result<Vec<Box<dyn Adapter>>, String> {
42 match source {
43 None => Ok(SOURCES
44 .iter()
45 .filter_map(|name| adapter_named(name, machine_id))
46 .collect()),
47 Some(name) => adapter_named(name, machine_id)
48 .map(|a| vec![a])
49 .ok_or_else(|| {
50 format!(
51 "unknown ingest source '{name}'; valid values: {}",
52 SOURCES.join(", ")
53 )
54 }),
55 }
56}
57
58pub fn run_ingest(
59 store: &Arc<dyn StoreAdmin>,
60 embedder: Option<&Embedder>,
61 machine_id: &str,
62 source: Option<&str>,
63) -> Result<IngestStats> {
64 let adapters = adapters_for(source, machine_id).map_err(anyhow::Error::msg)?;
65 let embedder = embedder.map(|e| e as &dyn DocumentEmbedder);
66
67 let mut total = IngestStats {
68 new_memories: 0,
69 new_conversations: 0,
70 new_entities: 0,
71 skipped_dupes: 0,
72 unchanged: 0,
73 errors: 0,
74 files_processed: 0,
75 };
76
77 for adapter in &adapters {
78 let sources = match adapter.discover_conversations() {
79 Ok(s) => s,
80 Err(e) => {
81 tracing::warn!("ingest: {} discovery failed: {e}", adapter.name());
82 continue;
83 }
84 };
85
86 let source_name = adapter.name().to_string();
87 total.files_processed += sources.len();
88 ingest_sources(
89 adapter.as_ref(),
90 &sources,
91 store.as_ref(),
92 embedder,
93 &source_name,
94 machine_id,
95 &mut total,
96 );
97 }
98
99 Ok(total)
100}
101
102pub fn ingest_file(
103 store: &Arc<dyn StoreAdmin>,
104 embedder: Option<&Embedder>,
105 path: &std::path::Path,
106 machine_id: &str,
107) {
108 let path_str = path.to_string_lossy().to_string();
109
110 let hash = match file_hash(path) {
111 Some(h) => h,
112 None => return,
113 };
114
115 if let Ok(false) = store.is_file_changed(&path_str, &hash) {
116 return;
117 }
118
119 let adapter = ClaudeCodeAdapter::new(machine_id.to_string());
120 let source = ConversationSource {
121 id: path_str.clone(),
122 path: path_str.clone(),
123 source_name: "claude-code".to_string(),
124 machine_id: machine_id.to_string(),
125 };
126
127 let result = match adapter.ingest_conversation(&source) {
128 Ok(r) => r,
129 Err(e) => {
130 tracing::debug!("watcher: failed to parse {}: {e}", path_str);
131 return;
132 }
133 };
134
135 let outcome = store_ingest_result(
136 store.as_ref(),
137 embedder.map(|e| e as &dyn DocumentEmbedder),
138 &result,
139 machine_id,
140 );
141
142 if let Err(e) = store.mark_ingested(&path_str, &hash, outcome.stored, "claude-code") {
143 tracing::warn!("watcher: mark_ingested failed for {}: {e}", path_str);
144 }
145 if outcome.errors > 0 {
146 tracing::warn!(
147 "watcher: ingested {} with {} write errors ({} memories stored)",
148 path_str,
149 outcome.errors,
150 outcome.stored
151 );
152 } else if outcome.stored > 0 {
153 tracing::info!(
154 "watcher: ingested {} ({} memories)",
155 path_str,
156 outcome.stored
157 );
158 }
159}
160
161pub fn ingest_memory_file(
162 store: &Arc<dyn StoreAdmin>,
163 embedder: Option<&Embedder>,
164 path: &std::path::Path,
165 machine_id: &str,
166) {
167 let path_str = path.to_string_lossy().to_string();
168
169 let hash = match file_hash(path) {
170 Some(h) => h,
171 None => return,
172 };
173
174 if let Ok(false) = store.is_file_changed(&path_str, &hash) {
175 return;
176 }
177
178 if let Err(e) = store.delete_memories_by_source_id(&path_str) {
181 tracing::warn!(
182 "watcher: delete_memories_by_source_id failed for {}: {e}",
183 path_str
184 );
185 }
186
187 let adapter = ClaudeMemoryAdapter::new(machine_id.to_string());
188 let source = ConversationSource {
189 id: path_str.clone(),
190 path: path_str.clone(),
191 source_name: "claude-memory".to_string(),
192 machine_id: machine_id.to_string(),
193 };
194
195 let result = match adapter.ingest_conversation(&source) {
196 Ok(r) => r,
197 Err(e) => {
198 tracing::debug!("watcher: failed to parse memory file {}: {e}", path_str);
199 return;
200 }
201 };
202
203 let outcome = store_ingest_result(
204 store.as_ref(),
205 embedder.map(|e| e as &dyn DocumentEmbedder),
206 &result,
207 machine_id,
208 );
209
210 if let Err(e) = store.mark_ingested(&path_str, &hash, outcome.stored, "claude-memory") {
211 tracing::warn!("watcher: mark_ingested failed for {}: {e}", path_str);
212 }
213 if outcome.stored > 0 || outcome.errors > 0 {
214 tracing::info!(
215 "watcher: ingested memory file {} ({} memories, {} errors)",
216 path_str,
217 outcome.stored,
218 outcome.errors
219 );
220 }
221}
222
223fn ingest_sources(
224 adapter: &dyn Adapter,
225 sources: &[ConversationSource],
226 store: &dyn StoreAdmin,
227 embedder: Option<&dyn DocumentEmbedder>,
228 source_name: &str,
229 machine_id: &str,
230 stats: &mut IngestStats,
231) {
232 for source in sources {
233 let path = std::path::Path::new(&source.path);
234 let hash = match file_hash(path) {
235 Some(h) => h,
236 None => {
237 stats.errors += 1;
238 continue;
239 }
240 };
241
242 if let Ok(false) = store.is_file_changed(&source.path, &hash) {
243 stats.unchanged += 1;
244 continue;
245 }
246
247 let result = match adapter.ingest_conversation(source) {
248 Ok(r) => r,
249 Err(e) => {
250 tracing::warn!("ingest: failed to parse {}: {e}", source.path);
251 stats.errors += 1;
252 continue;
253 }
254 };
255
256 let outcome = store_ingest_result(store, embedder, &result, machine_id);
257 if !result.memories.is_empty() {
258 stats.new_conversations += 1;
259 }
260 stats.new_memories += outcome.stored;
261 stats.skipped_dupes += outcome.dupes;
262 stats.new_entities += outcome.entities;
263 stats.errors += outcome.errors;
264
265 if let Err(e) = store.mark_ingested(&source.path, &hash, outcome.stored, source_name) {
266 tracing::warn!("ingest: mark_ingested failed for {}: {e}", source.path);
267 stats.errors += 1;
268 }
269 }
270}
271
272fn file_hash(path: &std::path::Path) -> Option<String> {
273 use std::hash::Hasher;
274 use std::io::Read;
275 let mut file = std::fs::File::open(path).ok()?;
276 let mut hasher = std::collections::hash_map::DefaultHasher::new();
277 let mut buf = [0u8; 8192];
278 loop {
279 let n = file.read(&mut buf).ok()?;
280 if n == 0 {
281 break;
282 }
283 hasher.write(&buf[..n]);
284 }
285 Some(format!("{:016x}", hasher.finish()))
286}
287
288#[cfg(test)]
289mod tests {
290 use super::*;
291
292 fn names(source: Option<&str>) -> Result<Vec<String>, String> {
293 adapters_for(source, "m")
294 .map(|adapters| adapters.iter().map(|a| a.name().to_string()).collect())
295 }
296
297 #[test]
298 fn a_named_source_runs_only_that_adapter() {
299 for source in SOURCES {
300 assert_eq!(names(Some(source)).unwrap(), vec![source.to_string()]);
301 }
302 }
303
304 #[test]
305 fn no_source_runs_every_adapter() {
306 assert_eq!(names(None).unwrap(), SOURCES.map(str::to_string).to_vec());
307 }
308
309 #[test]
310 fn an_unknown_source_is_refused_with_the_valid_names() {
311 let error = names(Some("bogus")).unwrap_err();
312 assert!(error.contains("bogus"), "{error}");
313 for source in SOURCES {
314 assert!(error.contains(source), "{error}");
315 }
316 }
317}