1use std::path::Path;
2use std::sync::Arc;
3
4use anyhow::{Context, Result, bail};
5use sha2::{Digest, Sha256};
6
7use second_brain_adapters::claude_code::ClaudeCodeAdapter;
8use second_brain_adapters::claude_memory::ClaudeMemoryAdapter;
9use second_brain_adapters::cursor::CursorAdapter;
10use second_brain_adapters::{Adapter, ConversationSource, store_ingest_result};
11use second_brain_core::embedding::{DocumentEmbedder, Embedder};
12use second_brain_core::schema::IngestRecord;
13use second_brain_core::store::StoreAdmin;
14use second_brain_core::transcript_archive::TranscriptArchive;
15
16#[derive(Debug, Default)]
17pub struct IngestStats {
18 pub new_memories: usize,
19 pub new_conversations: usize,
20 pub new_entities: usize,
21 pub skipped_dupes: usize,
22 pub unchanged: usize,
23 pub errors: usize,
24 pub files_processed: usize,
25}
26
27pub const SOURCES: [&str; 3] = ["claude-code", "claude-memory", "cursor"];
29
30fn adapter_named(source: &str, machine_id: &str) -> Option<Box<dyn Adapter>> {
31 let machine_id = machine_id.to_string();
32 match source {
33 "claude-code" => Some(Box::new(ClaudeCodeAdapter::new(machine_id))),
34 "claude-memory" => Some(Box::new(ClaudeMemoryAdapter::new(machine_id))),
35 "cursor" => Some(Box::new(CursorAdapter::new(machine_id))),
36 _ => None,
37 }
38}
39
40pub fn adapters_for(
44 source: Option<&str>,
45 machine_id: &str,
46) -> Result<Vec<Box<dyn Adapter>>, String> {
47 match source {
48 None => Ok(SOURCES
49 .iter()
50 .filter_map(|name| adapter_named(name, machine_id))
51 .collect()),
52 Some(name) => adapter_named(name, machine_id)
53 .map(|a| vec![a])
54 .ok_or_else(|| {
55 format!(
56 "unknown ingest source '{name}'; valid values: {}",
57 SOURCES.join(", ")
58 )
59 }),
60 }
61}
62
63pub fn run_ingest(
64 store: &Arc<dyn StoreAdmin>,
65 embedder: Option<&Embedder>,
66 machine_id: &str,
67 source: Option<&str>,
68) -> Result<IngestStats> {
69 let adapters = adapters_for(source, machine_id).map_err(anyhow::Error::msg)?;
70 let embedder = embedder.map(|e| e as &dyn DocumentEmbedder);
71 Ok(ingest_with_adapters(
72 store.as_ref(),
73 embedder,
74 machine_id,
75 &adapters,
76 ))
77}
78
79pub fn ingest_with_adapters(
80 store: &dyn StoreAdmin,
81 embedder: Option<&dyn DocumentEmbedder>,
82 machine_id: &str,
83 adapters: &[Box<dyn Adapter>],
84) -> IngestStats {
85 let mut total = IngestStats::default();
86 for adapter in adapters {
87 let sources = match adapter.discover_conversations() {
88 Ok(s) => s,
89 Err(e) => {
90 tracing::warn!("ingest: {} discovery failed: {e}", adapter.name());
91 continue;
92 }
93 };
94 total.files_processed += sources.len();
95 ingest_sources(
96 adapter.as_ref(),
97 &sources,
98 store,
99 embedder,
100 machine_id,
101 &mut total,
102 );
103 }
104 total
105}
106
107pub fn reingest(
112 store: &Arc<dyn StoreAdmin>,
113 embedder: Option<&Embedder>,
114 machine_id: &str,
115) -> Result<(usize, IngestStats)> {
116 let adapters = adapters_for(None, machine_id).map_err(anyhow::Error::msg)?;
117 reingest_with_adapters(
118 store.as_ref(),
119 embedder.map(|e| e as &dyn DocumentEmbedder),
120 machine_id,
121 &adapters,
122 )
123}
124
125pub fn reingest_with_adapters(
126 store: &dyn StoreAdmin,
127 embedder: Option<&dyn DocumentEmbedder>,
128 machine_id: &str,
129 adapters: &[Box<dyn Adapter>],
130) -> Result<(usize, IngestStats)> {
131 let cleared = store.clear_reparseable_ingest_log()?;
132 Ok((
133 cleared,
134 ingest_with_adapters(store, embedder, machine_id, adapters),
135 ))
136}
137
138#[derive(Debug, Clone, Copy, PartialEq, Eq)]
141pub enum FileOutcome {
142 Captured,
144 Skipped,
146}
147
148pub fn capture_transcript(
149 store: &Arc<dyn StoreAdmin>,
150 embedder: Option<&Embedder>,
151 archive: Option<&TranscriptArchive>,
152 path: &Path,
153 machine_id: &str,
154) -> Result<FileOutcome> {
155 if store.session_deleted(&path.to_string_lossy())? {
156 return Ok(FileOutcome::Skipped);
157 }
158 let archived = match archive.map(|a| a.archive(path)) {
159 Some(Err(_)) if std::fs::File::open(path).is_err() => return Ok(FileOutcome::Skipped),
160 archived => archived,
161 };
162 let outcome = ingest_file(store, embedder, path, machine_id)?;
163 if let Some(Err(e)) = archived {
164 return Err(e.context("archiving a transcript"));
165 }
166 Ok(outcome)
167}
168
169pub fn ingest_file(
170 store: &Arc<dyn StoreAdmin>,
171 embedder: Option<&Embedder>,
172 path: &std::path::Path,
173 machine_id: &str,
174) -> Result<FileOutcome> {
175 let path_str = path.to_string_lossy().to_string();
176 let adapter = ClaudeCodeAdapter::new(machine_id.to_string());
177
178 let changed = match check_file(store.as_ref(), &adapter, path) {
179 Some(FileCheck::Changed(changed)) => changed,
180 Some(FileCheck::Unchanged) => return Ok(FileOutcome::Captured),
181 None => return Ok(FileOutcome::Skipped),
182 };
183
184 let source = ConversationSource {
185 id: path_str.clone(),
186 path: path_str.clone(),
187 source_name: "claude-code".to_string(),
188 machine_id: machine_id.to_string(),
189 };
190
191 let result = match adapter.ingest_conversation(&source) {
192 Ok(r) => r,
193 Err(e) => {
194 tracing::debug!("watcher: failed to parse {}: {e}", path_str);
195 changed
196 .record_unreadable(store.as_ref(), path, "claude-code")
197 .context("recording a transcript that cannot be parsed")?;
198 return Ok(FileOutcome::Skipped);
199 }
200 };
201
202 let outcome = store_ingest_result(
203 store.as_ref(),
204 embedder.map(|e| e as &dyn DocumentEmbedder),
205 &result,
206 machine_id,
207 );
208
209 if outcome.errors > 0 {
210 tracing::warn!(
211 "watcher: ingested {} with {} write errors ({} memories stored)",
212 path_str,
213 outcome.errors,
214 outcome.stored
215 );
216 bail!(
217 "{} memory writes failed while ingesting a transcript",
218 outcome.errors
219 );
220 }
221 changed
222 .record(store.as_ref(), &path_str, outcome.stored, "claude-code")
223 .context("recording an ingested transcript")?;
224 if outcome.stored > 0 {
225 tracing::info!(
226 "watcher: ingested {} ({} memories)",
227 path_str,
228 outcome.stored
229 );
230 }
231 Ok(FileOutcome::Captured)
232}
233
234pub(crate) fn is_memory_file(path: &Path) -> bool {
237 path.extension().is_some_and(|e| e == "md")
238 && path
239 .parent()
240 .is_some_and(|p| p.file_name().is_some_and(|n| n == "memory"))
241 && !path
242 .file_name()
243 .is_some_and(|n| n.eq_ignore_ascii_case("MEMORY.md"))
244}
245
246pub(crate) fn changed_since_ingest(store: &dyn StoreAdmin, path: &Path) -> bool {
249 let Ok(bytes) = std::fs::read(path) else {
250 return false;
251 };
252 !ingested_as(store, path, &sha256_hex(&bytes))
253}
254
255pub(crate) fn ingested_as(store: &dyn StoreAdmin, path: &Path, content_hash: &str) -> bool {
257 let path_str = path.to_string_lossy();
258 match store.is_file_changed(&path_str, content_hash) {
259 Ok(changed) => !changed,
260 Err(e) => {
261 tracing::warn!("ingest: reading the ingest log for {path_str} failed: {e}");
262 false
263 }
264 }
265}
266
267pub fn ingest_memory_file(
268 store: &Arc<dyn StoreAdmin>,
269 embedder: Option<&Embedder>,
270 path: &std::path::Path,
271 machine_id: &str,
272) -> Result<FileOutcome> {
273 let path_str = path.to_string_lossy().to_string();
274 let adapter = ClaudeMemoryAdapter::new(machine_id.to_string());
275
276 let changed = match check_file(store.as_ref(), &adapter, path) {
277 Some(FileCheck::Changed(changed)) => changed,
278 Some(FileCheck::Unchanged) => return Ok(FileOutcome::Captured),
279 None => return Ok(FileOutcome::Skipped),
280 };
281
282 store
285 .delete_memories_by_source_id(&path_str)
286 .context("removing the previous memory of a memory file")?;
287
288 let source = ConversationSource {
289 id: path_str.clone(),
290 path: path_str.clone(),
291 source_name: "claude-memory".to_string(),
292 machine_id: machine_id.to_string(),
293 };
294
295 let result = match adapter.ingest_conversation(&source) {
296 Ok(r) => r,
297 Err(e) => {
298 tracing::debug!("watcher: failed to parse memory file {}: {e}", path_str);
299 changed
300 .record_unreadable(store.as_ref(), path, "claude-memory")
301 .context("recording a memory file that cannot be parsed")?;
302 return Ok(FileOutcome::Skipped);
303 }
304 };
305
306 let outcome = store_ingest_result(
307 store.as_ref(),
308 embedder.map(|e| e as &dyn DocumentEmbedder),
309 &result,
310 machine_id,
311 );
312
313 if outcome.stored > 0 || outcome.errors > 0 {
314 tracing::info!(
315 "watcher: ingested memory file {} ({} memories, {} errors)",
316 path_str,
317 outcome.stored,
318 outcome.errors
319 );
320 }
321 if outcome.errors > 0 {
322 bail!(
323 "{} memory writes failed while ingesting a memory file",
324 outcome.errors
325 );
326 }
327 changed
328 .record(store.as_ref(), &path_str, outcome.stored, "claude-memory")
329 .context("recording an ingested memory file")?;
330 Ok(FileOutcome::Captured)
331}
332
333fn ingest_sources(
334 adapter: &dyn Adapter,
335 sources: &[ConversationSource],
336 store: &dyn StoreAdmin,
337 embedder: Option<&dyn DocumentEmbedder>,
338 machine_id: &str,
339 stats: &mut IngestStats,
340) {
341 for source in sources {
342 let changed = match check_file(store, adapter, std::path::Path::new(&source.path)) {
343 Some(FileCheck::Changed(changed)) => changed,
344 Some(FileCheck::Unchanged) => {
345 stats.unchanged += 1;
346 continue;
347 }
348 None => {
349 stats.errors += 1;
350 continue;
351 }
352 };
353
354 let result = match adapter.ingest_conversation(source) {
355 Ok(r) => r,
356 Err(e) => {
357 tracing::warn!("ingest: failed to parse {}: {e}", source.path);
358 stats.errors += 1;
359 continue;
360 }
361 };
362
363 let outcome = store_ingest_result(store, embedder, &result, machine_id);
364 if !result.memories.is_empty() {
365 stats.new_conversations += 1;
366 }
367 stats.new_memories += outcome.stored;
368 stats.skipped_dupes += outcome.dupes;
369 stats.new_entities += outcome.entities;
370 stats.errors += outcome.errors;
371
372 if outcome.errors > 0 {
373 continue;
374 }
375 if let Err(e) = changed.record(store, &source.path, outcome.stored, adapter.name()) {
376 tracing::warn!("ingest: mark_ingested failed for {}: {e}", source.path);
377 stats.errors += 1;
378 }
379 }
380}
381
382enum FileCheck {
383 Unchanged,
384 Changed(ChangedFile),
385}
386
387struct ChangedFile {
388 content_hash: String,
389 session_key: Option<String>,
390}
391
392impl ChangedFile {
393 fn record(
394 &self,
395 store: &dyn StoreAdmin,
396 path: &str,
397 memory_count: usize,
398 source: &str,
399 ) -> Result<()> {
400 store.record_ingested(&IngestRecord {
401 file_path: path,
402 content_hash: &self.content_hash,
403 session_key: self.session_key.as_deref(),
404 legacy_ids: false,
405 memory_count,
406 source,
407 })
408 }
409
410 fn record_unreadable(&self, store: &dyn StoreAdmin, path: &Path, source: &str) -> Result<bool> {
413 let Ok(bytes) = std::fs::read(path) else {
414 return Ok(false);
415 };
416 if sha256_hex(&bytes) != self.content_hash || std::str::from_utf8(&bytes).is_ok() {
417 return Ok(false);
418 }
419 self.record(store, &path.to_string_lossy(), 0, source)?;
420 Ok(true)
421 }
422}
423
424fn check_file(store: &dyn StoreAdmin, adapter: &dyn Adapter, path: &Path) -> Option<FileCheck> {
430 let bytes = std::fs::read(path).ok()?;
431 let content_hash = sha256_hex(&bytes);
432 let path_str = path.to_string_lossy();
433 if adapter.name() == "claude-code" {
434 match store.session_deleted(&path_str) {
435 Ok(true) => return Some(FileCheck::Unchanged),
436 Ok(false) => {}
437 Err(e) => {
438 tracing::warn!("ingest: checking whether {path_str} was deleted failed: {e}");
439 return None;
440 }
441 }
442 }
443
444 if let Ok(false) = store.is_file_changed(&path_str, &content_hash) {
445 return Some(FileCheck::Unchanged);
446 }
447 let session_key = adapter.session_key(path, &String::from_utf8_lossy(&bytes));
448 let legacy_hash = legacy_file_hash(&bytes);
449 let changed = || {
450 Some(FileCheck::Changed(ChangedFile {
451 content_hash: content_hash.clone(),
452 session_key: session_key.clone(),
453 }))
454 };
455
456 let own_row = store.ingest_log_entry(&path_str).unwrap_or_else(|e| {
457 tracing::warn!("ingest: reading the ingest log for {path_str} failed: {e}");
458 None
459 });
460 if let Some(row) = own_row.filter(|r| r.content_hash.is_none() && r.file_hash == legacy_hash) {
461 upgrade(store, &row.file_path, &content_hash, session_key.as_deref());
462 return Some(FileCheck::Unchanged);
463 }
464
465 let Some(key) = session_key.as_deref() else {
466 return changed();
467 };
468 match store.ingested_session(key, &content_hash) {
469 Ok(Some(copy_of)) => {
470 record_copy(
471 store,
472 &path_str,
473 &content_hash,
474 key,
475 copy_of.legacy_ids,
476 adapter.name(),
477 );
478 return Some(FileCheck::Unchanged);
479 }
480 Ok(None) => {}
481 Err(e) => tracing::warn!("ingest: session lookup for {path_str} failed: {e}"),
482 }
483
484 let Some(stem) = path.file_stem() else {
485 return changed();
486 };
487 let legacy_rows = store
488 .legacy_ingest_entries_with_stem(&stem.to_string_lossy())
489 .unwrap_or_else(|e| {
490 tracing::warn!("ingest: legacy lookup for {path_str} failed: {e}");
491 Vec::new()
492 });
493 if let Some(row) = legacy_rows.into_iter().find(|r| r.file_hash == legacy_hash) {
494 upgrade(store, &row.file_path, &content_hash, Some(key));
495 record_copy(store, &path_str, &content_hash, key, true, adapter.name());
496 return Some(FileCheck::Unchanged);
497 }
498 changed()
499}
500
501fn upgrade(store: &dyn StoreAdmin, path: &str, content_hash: &str, session_key: Option<&str>) {
502 if let Err(e) = store.upgrade_ingest_hash(path, content_hash, session_key) {
503 tracing::warn!("ingest: upgrading the ingest log hash for {path} failed: {e}");
504 }
505}
506
507fn record_copy(
508 store: &dyn StoreAdmin,
509 path: &str,
510 content_hash: &str,
511 session_key: &str,
512 legacy_ids: bool,
513 source: &str,
514) {
515 let record = IngestRecord {
516 file_path: path,
517 content_hash,
518 session_key: Some(session_key),
519 legacy_ids,
520 memory_count: 0,
521 source,
522 };
523 if let Err(e) = store.record_ingested(&record) {
524 tracing::warn!("ingest: recording copied transcript {path} failed: {e}");
525 }
526}
527
528fn sha256_hex(bytes: &[u8]) -> String {
529 Sha256::digest(bytes)
530 .iter()
531 .map(|b| format!("{b:02x}"))
532 .collect()
533}
534
535pub fn legacy_file_hash(bytes: &[u8]) -> String {
539 use std::hash::Hasher;
540 let mut hasher = std::collections::hash_map::DefaultHasher::new();
541 for chunk in bytes.chunks(8192) {
542 hasher.write(chunk);
543 }
544 format!("{:016x}", hasher.finish())
545}
546
547#[cfg(test)]
548mod tests {
549 use super::*;
550
551 #[test]
552 fn a_failed_read_is_recorded_only_when_the_checked_bytes_are_not_text() {
553 let store =
554 second_brain_core::sqlite_store::SqliteStore::in_memory("m".to_string()).unwrap();
555 let dir = tempfile::tempdir().unwrap();
556 let path = dir.path().join("note.md");
557 let checked = |bytes: &[u8]| ChangedFile {
558 content_hash: sha256_hex(bytes),
559 session_key: None,
560 };
561 let cut = [b'c', b'a', b'f', 0xC3];
562
563 std::fs::write(&path, [b'x', 0xC3]).unwrap();
564 assert!(
565 !checked(&cut)
566 .record_unreadable(&store, &path, "claude-memory")
567 .unwrap()
568 );
569 assert!(!ingested_as(&store, &path, &sha256_hex(&cut)));
570
571 std::fs::write(&path, b"valid text").unwrap();
572 assert!(
573 !checked(b"valid text")
574 .record_unreadable(&store, &path, "claude-memory")
575 .unwrap()
576 );
577 assert!(!ingested_as(&store, &path, &sha256_hex(b"valid text")));
578
579 std::fs::remove_file(&path).unwrap();
580 assert!(
581 !checked(&cut)
582 .record_unreadable(&store, &path, "claude-memory")
583 .unwrap()
584 );
585
586 std::fs::write(&path, cut).unwrap();
587 assert!(
588 checked(&cut)
589 .record_unreadable(&store, &path, "claude-memory")
590 .unwrap()
591 );
592 assert!(ingested_as(&store, &path, &sha256_hex(&cut)));
593 }
594
595 fn names(source: Option<&str>) -> Result<Vec<String>, String> {
596 adapters_for(source, "m")
597 .map(|adapters| adapters.iter().map(|a| a.name().to_string()).collect())
598 }
599
600 #[test]
601 fn a_named_source_runs_only_that_adapter() {
602 for source in SOURCES {
603 assert_eq!(names(Some(source)).unwrap(), vec![source.to_string()]);
604 }
605 }
606
607 #[test]
608 fn no_source_runs_every_adapter() {
609 assert_eq!(names(None).unwrap(), SOURCES.map(str::to_string).to_vec());
610 }
611
612 #[test]
613 fn an_unknown_source_is_refused_with_the_valid_names() {
614 let error = names(Some("bogus")).unwrap_err();
615 assert!(error.contains("bogus"), "{error}");
616 for source in SOURCES {
617 assert!(error.contains(source), "{error}");
618 }
619 }
620}