1use std::path::Path;
2use std::sync::Arc;
3
4use anyhow::{Context, Result, bail};
5use sha2::{Digest, Sha256};
6
7use second_brain_adapters::claude_code::ClaudeCodeAdapter;
8use second_brain_adapters::claude_memory::ClaudeMemoryAdapter;
9use second_brain_adapters::cursor::CursorAdapter;
10use second_brain_adapters::{Adapter, ConversationSource, store_ingest_result};
11use second_brain_core::embedding::{DocumentEmbedder, Embedder};
12use second_brain_core::schema::IngestRecord;
13use second_brain_core::store::StoreAdmin;
14
15#[derive(Debug, Default)]
16pub struct IngestStats {
17 pub new_memories: usize,
18 pub new_conversations: usize,
19 pub new_entities: usize,
20 pub skipped_dupes: usize,
21 pub unchanged: usize,
22 pub errors: usize,
23 pub files_processed: usize,
24}
25
26pub const SOURCES: [&str; 3] = ["claude-code", "claude-memory", "cursor"];
28
29fn adapter_named(source: &str, machine_id: &str) -> Option<Box<dyn Adapter>> {
30 let machine_id = machine_id.to_string();
31 match source {
32 "claude-code" => Some(Box::new(ClaudeCodeAdapter::new(machine_id))),
33 "claude-memory" => Some(Box::new(ClaudeMemoryAdapter::new(machine_id))),
34 "cursor" => Some(Box::new(CursorAdapter::new(machine_id))),
35 _ => None,
36 }
37}
38
39pub fn adapters_for(
43 source: Option<&str>,
44 machine_id: &str,
45) -> Result<Vec<Box<dyn Adapter>>, String> {
46 match source {
47 None => Ok(SOURCES
48 .iter()
49 .filter_map(|name| adapter_named(name, machine_id))
50 .collect()),
51 Some(name) => adapter_named(name, machine_id)
52 .map(|a| vec![a])
53 .ok_or_else(|| {
54 format!(
55 "unknown ingest source '{name}'; valid values: {}",
56 SOURCES.join(", ")
57 )
58 }),
59 }
60}
61
62pub fn run_ingest(
63 store: &Arc<dyn StoreAdmin>,
64 embedder: Option<&Embedder>,
65 machine_id: &str,
66 source: Option<&str>,
67) -> Result<IngestStats> {
68 let adapters = adapters_for(source, machine_id).map_err(anyhow::Error::msg)?;
69 let embedder = embedder.map(|e| e as &dyn DocumentEmbedder);
70 Ok(ingest_with_adapters(
71 store.as_ref(),
72 embedder,
73 machine_id,
74 &adapters,
75 ))
76}
77
78pub fn ingest_with_adapters(
79 store: &dyn StoreAdmin,
80 embedder: Option<&dyn DocumentEmbedder>,
81 machine_id: &str,
82 adapters: &[Box<dyn Adapter>],
83) -> IngestStats {
84 let mut total = IngestStats::default();
85 for adapter in adapters {
86 let sources = match adapter.discover_conversations() {
87 Ok(s) => s,
88 Err(e) => {
89 tracing::warn!("ingest: {} discovery failed: {e}", adapter.name());
90 continue;
91 }
92 };
93 total.files_processed += sources.len();
94 ingest_sources(
95 adapter.as_ref(),
96 &sources,
97 store,
98 embedder,
99 machine_id,
100 &mut total,
101 );
102 }
103 total
104}
105
106pub fn reingest(
111 store: &Arc<dyn StoreAdmin>,
112 embedder: Option<&Embedder>,
113 machine_id: &str,
114) -> Result<(usize, IngestStats)> {
115 let adapters = adapters_for(None, machine_id).map_err(anyhow::Error::msg)?;
116 reingest_with_adapters(
117 store.as_ref(),
118 embedder.map(|e| e as &dyn DocumentEmbedder),
119 machine_id,
120 &adapters,
121 )
122}
123
124pub fn reingest_with_adapters(
125 store: &dyn StoreAdmin,
126 embedder: Option<&dyn DocumentEmbedder>,
127 machine_id: &str,
128 adapters: &[Box<dyn Adapter>],
129) -> Result<(usize, IngestStats)> {
130 let cleared = store.clear_reparseable_ingest_log()?;
131 Ok((
132 cleared,
133 ingest_with_adapters(store, embedder, machine_id, adapters),
134 ))
135}
136
137pub enum FileOutcome {
140 Captured,
142 Skipped,
144}
145
146pub fn ingest_file(
147 store: &Arc<dyn StoreAdmin>,
148 embedder: Option<&Embedder>,
149 path: &std::path::Path,
150 machine_id: &str,
151) -> Result<FileOutcome> {
152 let path_str = path.to_string_lossy().to_string();
153 let adapter = ClaudeCodeAdapter::new(machine_id.to_string());
154
155 let changed = match check_file(store.as_ref(), &adapter, path) {
156 Some(FileCheck::Changed(changed)) => changed,
157 Some(FileCheck::Unchanged) => return Ok(FileOutcome::Captured),
158 None => return Ok(FileOutcome::Skipped),
159 };
160
161 let source = ConversationSource {
162 id: path_str.clone(),
163 path: path_str.clone(),
164 source_name: "claude-code".to_string(),
165 machine_id: machine_id.to_string(),
166 };
167
168 let result = match adapter.ingest_conversation(&source) {
169 Ok(r) => r,
170 Err(e) => {
171 tracing::debug!("watcher: failed to parse {}: {e}", path_str);
172 return Ok(FileOutcome::Skipped);
173 }
174 };
175
176 let outcome = store_ingest_result(
177 store.as_ref(),
178 embedder.map(|e| e as &dyn DocumentEmbedder),
179 &result,
180 machine_id,
181 );
182
183 let marked = changed.record(store.as_ref(), &path_str, outcome.stored, "claude-code");
184 if let Err(e) = &marked {
185 tracing::warn!("watcher: mark_ingested failed for {}: {e}", path_str);
186 }
187 if outcome.errors > 0 {
188 tracing::warn!(
189 "watcher: ingested {} with {} write errors ({} memories stored)",
190 path_str,
191 outcome.errors,
192 outcome.stored
193 );
194 bail!(
195 "{} memory writes failed while ingesting a transcript",
196 outcome.errors
197 );
198 } else if outcome.stored > 0 {
199 tracing::info!(
200 "watcher: ingested {} ({} memories)",
201 path_str,
202 outcome.stored
203 );
204 }
205 marked.context("recording an ingested transcript")?;
206 Ok(FileOutcome::Captured)
207}
208
209pub fn ingest_memory_file(
210 store: &Arc<dyn StoreAdmin>,
211 embedder: Option<&Embedder>,
212 path: &std::path::Path,
213 machine_id: &str,
214) -> Result<FileOutcome> {
215 let path_str = path.to_string_lossy().to_string();
216 let adapter = ClaudeMemoryAdapter::new(machine_id.to_string());
217
218 let changed = match check_file(store.as_ref(), &adapter, path) {
219 Some(FileCheck::Changed(changed)) => changed,
220 Some(FileCheck::Unchanged) => return Ok(FileOutcome::Captured),
221 None => return Ok(FileOutcome::Skipped),
222 };
223
224 if let Err(e) = store.delete_memories_by_source_id(&path_str) {
227 tracing::warn!(
228 "watcher: delete_memories_by_source_id failed for {}: {e}",
229 path_str
230 );
231 }
232
233 let source = ConversationSource {
234 id: path_str.clone(),
235 path: path_str.clone(),
236 source_name: "claude-memory".to_string(),
237 machine_id: machine_id.to_string(),
238 };
239
240 let result = match adapter.ingest_conversation(&source) {
241 Ok(r) => r,
242 Err(e) => {
243 tracing::debug!("watcher: failed to parse memory file {}: {e}", path_str);
244 return Ok(FileOutcome::Skipped);
245 }
246 };
247
248 let outcome = store_ingest_result(
249 store.as_ref(),
250 embedder.map(|e| e as &dyn DocumentEmbedder),
251 &result,
252 machine_id,
253 );
254
255 let marked = changed.record(store.as_ref(), &path_str, outcome.stored, "claude-memory");
256 if let Err(e) = &marked {
257 tracing::warn!("watcher: mark_ingested failed for {}: {e}", path_str);
258 }
259 if outcome.stored > 0 || outcome.errors > 0 {
260 tracing::info!(
261 "watcher: ingested memory file {} ({} memories, {} errors)",
262 path_str,
263 outcome.stored,
264 outcome.errors
265 );
266 }
267 if outcome.errors > 0 {
268 bail!(
269 "{} memory writes failed while ingesting a memory file",
270 outcome.errors
271 );
272 }
273 marked.context("recording an ingested memory file")?;
274 Ok(FileOutcome::Captured)
275}
276
277fn ingest_sources(
278 adapter: &dyn Adapter,
279 sources: &[ConversationSource],
280 store: &dyn StoreAdmin,
281 embedder: Option<&dyn DocumentEmbedder>,
282 machine_id: &str,
283 stats: &mut IngestStats,
284) {
285 for source in sources {
286 let changed = match check_file(store, adapter, std::path::Path::new(&source.path)) {
287 Some(FileCheck::Changed(changed)) => changed,
288 Some(FileCheck::Unchanged) => {
289 stats.unchanged += 1;
290 continue;
291 }
292 None => {
293 stats.errors += 1;
294 continue;
295 }
296 };
297
298 let result = match adapter.ingest_conversation(source) {
299 Ok(r) => r,
300 Err(e) => {
301 tracing::warn!("ingest: failed to parse {}: {e}", source.path);
302 stats.errors += 1;
303 continue;
304 }
305 };
306
307 let outcome = store_ingest_result(store, embedder, &result, machine_id);
308 if !result.memories.is_empty() {
309 stats.new_conversations += 1;
310 }
311 stats.new_memories += outcome.stored;
312 stats.skipped_dupes += outcome.dupes;
313 stats.new_entities += outcome.entities;
314 stats.errors += outcome.errors;
315
316 if let Err(e) = changed.record(store, &source.path, outcome.stored, adapter.name()) {
317 tracing::warn!("ingest: mark_ingested failed for {}: {e}", source.path);
318 stats.errors += 1;
319 }
320 }
321}
322
323enum FileCheck {
324 Unchanged,
325 Changed(ChangedFile),
326}
327
328struct ChangedFile {
329 content_hash: String,
330 session_key: Option<String>,
331}
332
333impl ChangedFile {
334 fn record(
335 &self,
336 store: &dyn StoreAdmin,
337 path: &str,
338 memory_count: usize,
339 source: &str,
340 ) -> Result<()> {
341 store.record_ingested(&IngestRecord {
342 file_path: path,
343 content_hash: &self.content_hash,
344 session_key: self.session_key.as_deref(),
345 legacy_ids: false,
346 memory_count,
347 source,
348 })
349 }
350}
351
352fn check_file(store: &dyn StoreAdmin, adapter: &dyn Adapter, path: &Path) -> Option<FileCheck> {
358 let bytes = std::fs::read(path).ok()?;
359 let content_hash = sha256_hex(&bytes);
360 let path_str = path.to_string_lossy();
361
362 if let Ok(false) = store.is_file_changed(&path_str, &content_hash) {
363 return Some(FileCheck::Unchanged);
364 }
365 let session_key = adapter.session_key(path, &String::from_utf8_lossy(&bytes));
366 let legacy_hash = legacy_file_hash(&bytes);
367 let changed = || {
368 Some(FileCheck::Changed(ChangedFile {
369 content_hash: content_hash.clone(),
370 session_key: session_key.clone(),
371 }))
372 };
373
374 let own_row = store.ingest_log_entry(&path_str).unwrap_or_else(|e| {
375 tracing::warn!("ingest: reading the ingest log for {path_str} failed: {e}");
376 None
377 });
378 if let Some(row) = own_row.filter(|r| r.content_hash.is_none() && r.file_hash == legacy_hash) {
379 upgrade(store, &row.file_path, &content_hash, session_key.as_deref());
380 return Some(FileCheck::Unchanged);
381 }
382
383 let Some(key) = session_key.as_deref() else {
384 return changed();
385 };
386 match store.ingested_session(key, &content_hash) {
387 Ok(Some(copy_of)) => {
388 record_copy(
389 store,
390 &path_str,
391 &content_hash,
392 key,
393 copy_of.legacy_ids,
394 adapter.name(),
395 );
396 return Some(FileCheck::Unchanged);
397 }
398 Ok(None) => {}
399 Err(e) => tracing::warn!("ingest: session lookup for {path_str} failed: {e}"),
400 }
401
402 let Some(stem) = path.file_stem() else {
403 return changed();
404 };
405 let legacy_rows = store
406 .legacy_ingest_entries_with_stem(&stem.to_string_lossy())
407 .unwrap_or_else(|e| {
408 tracing::warn!("ingest: legacy lookup for {path_str} failed: {e}");
409 Vec::new()
410 });
411 if let Some(row) = legacy_rows.into_iter().find(|r| r.file_hash == legacy_hash) {
412 upgrade(store, &row.file_path, &content_hash, Some(key));
413 record_copy(store, &path_str, &content_hash, key, true, adapter.name());
414 return Some(FileCheck::Unchanged);
415 }
416 changed()
417}
418
419fn upgrade(store: &dyn StoreAdmin, path: &str, content_hash: &str, session_key: Option<&str>) {
420 if let Err(e) = store.upgrade_ingest_hash(path, content_hash, session_key) {
421 tracing::warn!("ingest: upgrading the ingest log hash for {path} failed: {e}");
422 }
423}
424
425fn record_copy(
426 store: &dyn StoreAdmin,
427 path: &str,
428 content_hash: &str,
429 session_key: &str,
430 legacy_ids: bool,
431 source: &str,
432) {
433 let record = IngestRecord {
434 file_path: path,
435 content_hash,
436 session_key: Some(session_key),
437 legacy_ids,
438 memory_count: 0,
439 source,
440 };
441 if let Err(e) = store.record_ingested(&record) {
442 tracing::warn!("ingest: recording copied transcript {path} failed: {e}");
443 }
444}
445
446fn sha256_hex(bytes: &[u8]) -> String {
447 Sha256::digest(bytes)
448 .iter()
449 .map(|b| format!("{b:02x}"))
450 .collect()
451}
452
453pub fn legacy_file_hash(bytes: &[u8]) -> String {
457 use std::hash::Hasher;
458 let mut hasher = std::collections::hash_map::DefaultHasher::new();
459 for chunk in bytes.chunks(8192) {
460 hasher.write(chunk);
461 }
462 format!("{:016x}", hasher.finish())
463}
464
465#[cfg(test)]
466mod tests {
467 use super::*;
468
469 fn names(source: Option<&str>) -> Result<Vec<String>, String> {
470 adapters_for(source, "m")
471 .map(|adapters| adapters.iter().map(|a| a.name().to_string()).collect())
472 }
473
474 #[test]
475 fn a_named_source_runs_only_that_adapter() {
476 for source in SOURCES {
477 assert_eq!(names(Some(source)).unwrap(), vec![source.to_string()]);
478 }
479 }
480
481 #[test]
482 fn no_source_runs_every_adapter() {
483 assert_eq!(names(None).unwrap(), SOURCES.map(str::to_string).to_vec());
484 }
485
486 #[test]
487 fn an_unknown_source_is_refused_with_the_valid_names() {
488 let error = names(Some("bogus")).unwrap_err();
489 assert!(error.contains("bogus"), "{error}");
490 for source in SOURCES {
491 assert!(error.contains(source), "{error}");
492 }
493 }
494}