1use std::path::Path;
2use std::sync::Arc;
3
4use anyhow::{Context, Result, bail};
5use sha2::{Digest, Sha256};
6
7use second_brain_adapters::claude_code::ClaudeCodeAdapter;
8use second_brain_adapters::claude_memory::ClaudeMemoryAdapter;
9use second_brain_adapters::cursor::CursorAdapter;
10use second_brain_adapters::{Adapter, ConversationSource, store_ingest_result};
11use second_brain_core::embedding::{DocumentEmbedder, Embedder};
12use second_brain_core::schema::IngestRecord;
13use second_brain_core::store::StoreAdmin;
14use second_brain_core::transcript_archive::TranscriptArchive;
15
16#[derive(Debug, Default)]
17pub struct IngestStats {
18 pub new_memories: usize,
19 pub new_conversations: usize,
20 pub new_entities: usize,
21 pub skipped_dupes: usize,
22 pub unchanged: usize,
23 pub errors: usize,
24 pub files_processed: usize,
25}
26
27pub const SOURCES: [&str; 3] = ["claude-code", "claude-memory", "cursor"];
29
30fn adapter_named(source: &str, machine_id: &str) -> Option<Box<dyn Adapter>> {
31 let machine_id = machine_id.to_string();
32 match source {
33 "claude-code" => Some(Box::new(ClaudeCodeAdapter::new(machine_id))),
34 "claude-memory" => Some(Box::new(ClaudeMemoryAdapter::new(machine_id))),
35 "cursor" => Some(Box::new(CursorAdapter::new(machine_id))),
36 _ => None,
37 }
38}
39
40pub fn adapters_for(
44 source: Option<&str>,
45 machine_id: &str,
46) -> Result<Vec<Box<dyn Adapter>>, String> {
47 match source {
48 None => Ok(SOURCES
49 .iter()
50 .filter_map(|name| adapter_named(name, machine_id))
51 .collect()),
52 Some(name) => adapter_named(name, machine_id)
53 .map(|a| vec![a])
54 .ok_or_else(|| {
55 format!(
56 "unknown ingest source '{name}'; valid values: {}",
57 SOURCES.join(", ")
58 )
59 }),
60 }
61}
62
63pub fn run_ingest(
64 store: &Arc<dyn StoreAdmin>,
65 embedder: Option<&Embedder>,
66 machine_id: &str,
67 source: Option<&str>,
68) -> Result<IngestStats> {
69 let adapters = adapters_for(source, machine_id).map_err(anyhow::Error::msg)?;
70 let embedder = embedder.map(|e| e as &dyn DocumentEmbedder);
71 Ok(ingest_with_adapters(
72 store.as_ref(),
73 embedder,
74 machine_id,
75 &adapters,
76 ))
77}
78
79pub fn ingest_with_adapters(
80 store: &dyn StoreAdmin,
81 embedder: Option<&dyn DocumentEmbedder>,
82 machine_id: &str,
83 adapters: &[Box<dyn Adapter>],
84) -> IngestStats {
85 let mut total = IngestStats::default();
86 for adapter in adapters {
87 let sources = match adapter.discover_conversations() {
88 Ok(s) => s,
89 Err(e) => {
90 tracing::warn!("ingest: {} discovery failed: {e}", adapter.name());
91 continue;
92 }
93 };
94 total.files_processed += sources.len();
95 ingest_sources(
96 adapter.as_ref(),
97 &sources,
98 store,
99 embedder,
100 machine_id,
101 &mut total,
102 );
103 }
104 total
105}
106
107pub fn reingest(
112 store: &Arc<dyn StoreAdmin>,
113 embedder: Option<&Embedder>,
114 machine_id: &str,
115) -> Result<(usize, IngestStats)> {
116 let adapters = adapters_for(None, machine_id).map_err(anyhow::Error::msg)?;
117 reingest_with_adapters(
118 store.as_ref(),
119 embedder.map(|e| e as &dyn DocumentEmbedder),
120 machine_id,
121 &adapters,
122 )
123}
124
125pub fn reingest_with_adapters(
126 store: &dyn StoreAdmin,
127 embedder: Option<&dyn DocumentEmbedder>,
128 machine_id: &str,
129 adapters: &[Box<dyn Adapter>],
130) -> Result<(usize, IngestStats)> {
131 let cleared = store.clear_reparseable_ingest_log()?;
132 Ok((
133 cleared,
134 ingest_with_adapters(store, embedder, machine_id, adapters),
135 ))
136}
137
138#[derive(Debug, Clone, Copy, PartialEq, Eq)]
141pub enum FileOutcome {
142 Captured,
144 Skipped,
146}
147
148pub fn capture_transcript(
149 store: &Arc<dyn StoreAdmin>,
150 embedder: Option<&Embedder>,
151 archive: Option<&TranscriptArchive>,
152 path: &Path,
153 machine_id: &str,
154) -> Result<FileOutcome> {
155 let archived = match archive.map(|a| a.archive(path)) {
156 Some(Err(_)) if std::fs::File::open(path).is_err() => return Ok(FileOutcome::Skipped),
157 archived => archived,
158 };
159 let outcome = ingest_file(store, embedder, path, machine_id)?;
160 if let Some(Err(e)) = archived {
161 return Err(e.context("archiving a transcript"));
162 }
163 Ok(outcome)
164}
165
166pub fn ingest_file(
167 store: &Arc<dyn StoreAdmin>,
168 embedder: Option<&Embedder>,
169 path: &std::path::Path,
170 machine_id: &str,
171) -> Result<FileOutcome> {
172 let path_str = path.to_string_lossy().to_string();
173 let adapter = ClaudeCodeAdapter::new(machine_id.to_string());
174
175 let changed = match check_file(store.as_ref(), &adapter, path) {
176 Some(FileCheck::Changed(changed)) => changed,
177 Some(FileCheck::Unchanged) => return Ok(FileOutcome::Captured),
178 None => return Ok(FileOutcome::Skipped),
179 };
180
181 let source = ConversationSource {
182 id: path_str.clone(),
183 path: path_str.clone(),
184 source_name: "claude-code".to_string(),
185 machine_id: machine_id.to_string(),
186 };
187
188 let result = match adapter.ingest_conversation(&source) {
189 Ok(r) => r,
190 Err(e) => {
191 tracing::debug!("watcher: failed to parse {}: {e}", path_str);
192 return Ok(FileOutcome::Skipped);
193 }
194 };
195
196 let outcome = store_ingest_result(
197 store.as_ref(),
198 embedder.map(|e| e as &dyn DocumentEmbedder),
199 &result,
200 machine_id,
201 );
202
203 let marked = changed.record(store.as_ref(), &path_str, outcome.stored, "claude-code");
204 if let Err(e) = &marked {
205 tracing::warn!("watcher: mark_ingested failed for {}: {e}", path_str);
206 }
207 if outcome.errors > 0 {
208 tracing::warn!(
209 "watcher: ingested {} with {} write errors ({} memories stored)",
210 path_str,
211 outcome.errors,
212 outcome.stored
213 );
214 bail!(
215 "{} memory writes failed while ingesting a transcript",
216 outcome.errors
217 );
218 } else if outcome.stored > 0 {
219 tracing::info!(
220 "watcher: ingested {} ({} memories)",
221 path_str,
222 outcome.stored
223 );
224 }
225 marked.context("recording an ingested transcript")?;
226 Ok(FileOutcome::Captured)
227}
228
229pub fn ingest_memory_file(
230 store: &Arc<dyn StoreAdmin>,
231 embedder: Option<&Embedder>,
232 path: &std::path::Path,
233 machine_id: &str,
234) -> Result<FileOutcome> {
235 let path_str = path.to_string_lossy().to_string();
236 let adapter = ClaudeMemoryAdapter::new(machine_id.to_string());
237
238 let changed = match check_file(store.as_ref(), &adapter, path) {
239 Some(FileCheck::Changed(changed)) => changed,
240 Some(FileCheck::Unchanged) => return Ok(FileOutcome::Captured),
241 None => return Ok(FileOutcome::Skipped),
242 };
243
244 if let Err(e) = store.delete_memories_by_source_id(&path_str) {
247 tracing::warn!(
248 "watcher: delete_memories_by_source_id failed for {}: {e}",
249 path_str
250 );
251 }
252
253 let source = ConversationSource {
254 id: path_str.clone(),
255 path: path_str.clone(),
256 source_name: "claude-memory".to_string(),
257 machine_id: machine_id.to_string(),
258 };
259
260 let result = match adapter.ingest_conversation(&source) {
261 Ok(r) => r,
262 Err(e) => {
263 tracing::debug!("watcher: failed to parse memory file {}: {e}", path_str);
264 return Ok(FileOutcome::Skipped);
265 }
266 };
267
268 let outcome = store_ingest_result(
269 store.as_ref(),
270 embedder.map(|e| e as &dyn DocumentEmbedder),
271 &result,
272 machine_id,
273 );
274
275 let marked = changed.record(store.as_ref(), &path_str, outcome.stored, "claude-memory");
276 if let Err(e) = &marked {
277 tracing::warn!("watcher: mark_ingested failed for {}: {e}", path_str);
278 }
279 if outcome.stored > 0 || outcome.errors > 0 {
280 tracing::info!(
281 "watcher: ingested memory file {} ({} memories, {} errors)",
282 path_str,
283 outcome.stored,
284 outcome.errors
285 );
286 }
287 if outcome.errors > 0 {
288 bail!(
289 "{} memory writes failed while ingesting a memory file",
290 outcome.errors
291 );
292 }
293 marked.context("recording an ingested memory file")?;
294 Ok(FileOutcome::Captured)
295}
296
297fn ingest_sources(
298 adapter: &dyn Adapter,
299 sources: &[ConversationSource],
300 store: &dyn StoreAdmin,
301 embedder: Option<&dyn DocumentEmbedder>,
302 machine_id: &str,
303 stats: &mut IngestStats,
304) {
305 for source in sources {
306 let changed = match check_file(store, adapter, std::path::Path::new(&source.path)) {
307 Some(FileCheck::Changed(changed)) => changed,
308 Some(FileCheck::Unchanged) => {
309 stats.unchanged += 1;
310 continue;
311 }
312 None => {
313 stats.errors += 1;
314 continue;
315 }
316 };
317
318 let result = match adapter.ingest_conversation(source) {
319 Ok(r) => r,
320 Err(e) => {
321 tracing::warn!("ingest: failed to parse {}: {e}", source.path);
322 stats.errors += 1;
323 continue;
324 }
325 };
326
327 let outcome = store_ingest_result(store, embedder, &result, machine_id);
328 if !result.memories.is_empty() {
329 stats.new_conversations += 1;
330 }
331 stats.new_memories += outcome.stored;
332 stats.skipped_dupes += outcome.dupes;
333 stats.new_entities += outcome.entities;
334 stats.errors += outcome.errors;
335
336 if let Err(e) = changed.record(store, &source.path, outcome.stored, adapter.name()) {
337 tracing::warn!("ingest: mark_ingested failed for {}: {e}", source.path);
338 stats.errors += 1;
339 }
340 }
341}
342
343enum FileCheck {
344 Unchanged,
345 Changed(ChangedFile),
346}
347
348struct ChangedFile {
349 content_hash: String,
350 session_key: Option<String>,
351}
352
353impl ChangedFile {
354 fn record(
355 &self,
356 store: &dyn StoreAdmin,
357 path: &str,
358 memory_count: usize,
359 source: &str,
360 ) -> Result<()> {
361 store.record_ingested(&IngestRecord {
362 file_path: path,
363 content_hash: &self.content_hash,
364 session_key: self.session_key.as_deref(),
365 legacy_ids: false,
366 memory_count,
367 source,
368 })
369 }
370}
371
372fn check_file(store: &dyn StoreAdmin, adapter: &dyn Adapter, path: &Path) -> Option<FileCheck> {
378 let bytes = std::fs::read(path).ok()?;
379 let content_hash = sha256_hex(&bytes);
380 let path_str = path.to_string_lossy();
381
382 if let Ok(false) = store.is_file_changed(&path_str, &content_hash) {
383 return Some(FileCheck::Unchanged);
384 }
385 let session_key = adapter.session_key(path, &String::from_utf8_lossy(&bytes));
386 let legacy_hash = legacy_file_hash(&bytes);
387 let changed = || {
388 Some(FileCheck::Changed(ChangedFile {
389 content_hash: content_hash.clone(),
390 session_key: session_key.clone(),
391 }))
392 };
393
394 let own_row = store.ingest_log_entry(&path_str).unwrap_or_else(|e| {
395 tracing::warn!("ingest: reading the ingest log for {path_str} failed: {e}");
396 None
397 });
398 if let Some(row) = own_row.filter(|r| r.content_hash.is_none() && r.file_hash == legacy_hash) {
399 upgrade(store, &row.file_path, &content_hash, session_key.as_deref());
400 return Some(FileCheck::Unchanged);
401 }
402
403 let Some(key) = session_key.as_deref() else {
404 return changed();
405 };
406 match store.ingested_session(key, &content_hash) {
407 Ok(Some(copy_of)) => {
408 record_copy(
409 store,
410 &path_str,
411 &content_hash,
412 key,
413 copy_of.legacy_ids,
414 adapter.name(),
415 );
416 return Some(FileCheck::Unchanged);
417 }
418 Ok(None) => {}
419 Err(e) => tracing::warn!("ingest: session lookup for {path_str} failed: {e}"),
420 }
421
422 let Some(stem) = path.file_stem() else {
423 return changed();
424 };
425 let legacy_rows = store
426 .legacy_ingest_entries_with_stem(&stem.to_string_lossy())
427 .unwrap_or_else(|e| {
428 tracing::warn!("ingest: legacy lookup for {path_str} failed: {e}");
429 Vec::new()
430 });
431 if let Some(row) = legacy_rows.into_iter().find(|r| r.file_hash == legacy_hash) {
432 upgrade(store, &row.file_path, &content_hash, Some(key));
433 record_copy(store, &path_str, &content_hash, key, true, adapter.name());
434 return Some(FileCheck::Unchanged);
435 }
436 changed()
437}
438
439fn upgrade(store: &dyn StoreAdmin, path: &str, content_hash: &str, session_key: Option<&str>) {
440 if let Err(e) = store.upgrade_ingest_hash(path, content_hash, session_key) {
441 tracing::warn!("ingest: upgrading the ingest log hash for {path} failed: {e}");
442 }
443}
444
445fn record_copy(
446 store: &dyn StoreAdmin,
447 path: &str,
448 content_hash: &str,
449 session_key: &str,
450 legacy_ids: bool,
451 source: &str,
452) {
453 let record = IngestRecord {
454 file_path: path,
455 content_hash,
456 session_key: Some(session_key),
457 legacy_ids,
458 memory_count: 0,
459 source,
460 };
461 if let Err(e) = store.record_ingested(&record) {
462 tracing::warn!("ingest: recording copied transcript {path} failed: {e}");
463 }
464}
465
466fn sha256_hex(bytes: &[u8]) -> String {
467 Sha256::digest(bytes)
468 .iter()
469 .map(|b| format!("{b:02x}"))
470 .collect()
471}
472
473pub fn legacy_file_hash(bytes: &[u8]) -> String {
477 use std::hash::Hasher;
478 let mut hasher = std::collections::hash_map::DefaultHasher::new();
479 for chunk in bytes.chunks(8192) {
480 hasher.write(chunk);
481 }
482 format!("{:016x}", hasher.finish())
483}
484
485#[cfg(test)]
486mod tests {
487 use super::*;
488
489 fn names(source: Option<&str>) -> Result<Vec<String>, String> {
490 adapters_for(source, "m")
491 .map(|adapters| adapters.iter().map(|a| a.name().to_string()).collect())
492 }
493
494 #[test]
495 fn a_named_source_runs_only_that_adapter() {
496 for source in SOURCES {
497 assert_eq!(names(Some(source)).unwrap(), vec![source.to_string()]);
498 }
499 }
500
501 #[test]
502 fn no_source_runs_every_adapter() {
503 assert_eq!(names(None).unwrap(), SOURCES.map(str::to_string).to_vec());
504 }
505
506 #[test]
507 fn an_unknown_source_is_refused_with_the_valid_names() {
508 let error = names(Some("bogus")).unwrap_err();
509 assert!(error.contains("bogus"), "{error}");
510 for source in SOURCES {
511 assert!(error.contains(source), "{error}");
512 }
513 }
514}