use std::path::Path;
use std::sync::Arc;
use anyhow::{Context, Result, bail};
use sha2::{Digest, Sha256};
use second_brain_adapters::claude_code::ClaudeCodeAdapter;
use second_brain_adapters::claude_memory::ClaudeMemoryAdapter;
use second_brain_adapters::cursor::CursorAdapter;
use second_brain_adapters::{Adapter, ConversationSource, store_ingest_result};
use second_brain_core::embedding::{DocumentEmbedder, Embedder};
use second_brain_core::schema::IngestRecord;
use second_brain_core::store::StoreAdmin;
use second_brain_core::transcript_archive::TranscriptArchive;
#[derive(Debug, Default)]
pub struct IngestStats {
pub new_memories: usize,
pub new_conversations: usize,
pub new_entities: usize,
pub skipped_dupes: usize,
pub unchanged: usize,
pub errors: usize,
pub files_processed: usize,
}
pub const SOURCES: [&str; 3] = ["claude-code", "claude-memory", "cursor"];
fn adapter_named(source: &str, machine_id: &str) -> Option<Box<dyn Adapter>> {
let machine_id = machine_id.to_string();
match source {
"claude-code" => Some(Box::new(ClaudeCodeAdapter::new(machine_id))),
"claude-memory" => Some(Box::new(ClaudeMemoryAdapter::new(machine_id))),
"cursor" => Some(Box::new(CursorAdapter::new(machine_id))),
_ => None,
}
}
pub fn adapters_for(
source: Option<&str>,
machine_id: &str,
) -> Result<Vec<Box<dyn Adapter>>, String> {
match source {
None => Ok(SOURCES
.iter()
.filter_map(|name| adapter_named(name, machine_id))
.collect()),
Some(name) => adapter_named(name, machine_id)
.map(|a| vec![a])
.ok_or_else(|| {
format!(
"unknown ingest source '{name}'; valid values: {}",
SOURCES.join(", ")
)
}),
}
}
pub fn run_ingest(
store: &Arc<dyn StoreAdmin>,
embedder: Option<&Embedder>,
machine_id: &str,
source: Option<&str>,
) -> Result<IngestStats> {
let adapters = adapters_for(source, machine_id).map_err(anyhow::Error::msg)?;
let embedder = embedder.map(|e| e as &dyn DocumentEmbedder);
Ok(ingest_with_adapters(
store.as_ref(),
embedder,
machine_id,
&adapters,
))
}
pub fn ingest_with_adapters(
store: &dyn StoreAdmin,
embedder: Option<&dyn DocumentEmbedder>,
machine_id: &str,
adapters: &[Box<dyn Adapter>],
) -> IngestStats {
let mut total = IngestStats::default();
for adapter in adapters {
let sources = match adapter.discover_conversations() {
Ok(s) => s,
Err(e) => {
tracing::warn!("ingest: {} discovery failed: {e}", adapter.name());
continue;
}
};
total.files_processed += sources.len();
ingest_sources(
adapter.as_ref(),
&sources,
store,
embedder,
machine_id,
&mut total,
);
}
total
}
pub fn reingest(
store: &Arc<dyn StoreAdmin>,
embedder: Option<&Embedder>,
machine_id: &str,
) -> Result<(usize, IngestStats)> {
let adapters = adapters_for(None, machine_id).map_err(anyhow::Error::msg)?;
reingest_with_adapters(
store.as_ref(),
embedder.map(|e| e as &dyn DocumentEmbedder),
machine_id,
&adapters,
)
}
pub fn reingest_with_adapters(
store: &dyn StoreAdmin,
embedder: Option<&dyn DocumentEmbedder>,
machine_id: &str,
adapters: &[Box<dyn Adapter>],
) -> Result<(usize, IngestStats)> {
let cleared = store.clear_reparseable_ingest_log()?;
Ok((
cleared,
ingest_with_adapters(store, embedder, machine_id, adapters),
))
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FileOutcome {
Captured,
Skipped,
}
pub fn capture_transcript(
store: &Arc<dyn StoreAdmin>,
embedder: Option<&Embedder>,
archive: Option<&TranscriptArchive>,
path: &Path,
machine_id: &str,
) -> Result<FileOutcome> {
let archived = match archive.map(|a| a.archive(path)) {
Some(Err(_)) if std::fs::File::open(path).is_err() => return Ok(FileOutcome::Skipped),
archived => archived,
};
let outcome = ingest_file(store, embedder, path, machine_id)?;
if let Some(Err(e)) = archived {
return Err(e.context("archiving a transcript"));
}
Ok(outcome)
}
pub fn ingest_file(
store: &Arc<dyn StoreAdmin>,
embedder: Option<&Embedder>,
path: &std::path::Path,
machine_id: &str,
) -> Result<FileOutcome> {
let path_str = path.to_string_lossy().to_string();
let adapter = ClaudeCodeAdapter::new(machine_id.to_string());
let changed = match check_file(store.as_ref(), &adapter, path) {
Some(FileCheck::Changed(changed)) => changed,
Some(FileCheck::Unchanged) => return Ok(FileOutcome::Captured),
None => return Ok(FileOutcome::Skipped),
};
let source = ConversationSource {
id: path_str.clone(),
path: path_str.clone(),
source_name: "claude-code".to_string(),
machine_id: machine_id.to_string(),
};
let result = match adapter.ingest_conversation(&source) {
Ok(r) => r,
Err(e) => {
tracing::debug!("watcher: failed to parse {}: {e}", path_str);
return Ok(FileOutcome::Skipped);
}
};
let outcome = store_ingest_result(
store.as_ref(),
embedder.map(|e| e as &dyn DocumentEmbedder),
&result,
machine_id,
);
let marked = changed.record(store.as_ref(), &path_str, outcome.stored, "claude-code");
if let Err(e) = &marked {
tracing::warn!("watcher: mark_ingested failed for {}: {e}", path_str);
}
if outcome.errors > 0 {
tracing::warn!(
"watcher: ingested {} with {} write errors ({} memories stored)",
path_str,
outcome.errors,
outcome.stored
);
bail!(
"{} memory writes failed while ingesting a transcript",
outcome.errors
);
} else if outcome.stored > 0 {
tracing::info!(
"watcher: ingested {} ({} memories)",
path_str,
outcome.stored
);
}
marked.context("recording an ingested transcript")?;
Ok(FileOutcome::Captured)
}
pub fn ingest_memory_file(
store: &Arc<dyn StoreAdmin>,
embedder: Option<&Embedder>,
path: &std::path::Path,
machine_id: &str,
) -> Result<FileOutcome> {
let path_str = path.to_string_lossy().to_string();
let adapter = ClaudeMemoryAdapter::new(machine_id.to_string());
let changed = match check_file(store.as_ref(), &adapter, path) {
Some(FileCheck::Changed(changed)) => changed,
Some(FileCheck::Unchanged) => return Ok(FileOutcome::Captured),
None => return Ok(FileOutcome::Skipped),
};
if let Err(e) = store.delete_memories_by_source_id(&path_str) {
tracing::warn!(
"watcher: delete_memories_by_source_id failed for {}: {e}",
path_str
);
}
let source = ConversationSource {
id: path_str.clone(),
path: path_str.clone(),
source_name: "claude-memory".to_string(),
machine_id: machine_id.to_string(),
};
let result = match adapter.ingest_conversation(&source) {
Ok(r) => r,
Err(e) => {
tracing::debug!("watcher: failed to parse memory file {}: {e}", path_str);
return Ok(FileOutcome::Skipped);
}
};
let outcome = store_ingest_result(
store.as_ref(),
embedder.map(|e| e as &dyn DocumentEmbedder),
&result,
machine_id,
);
let marked = changed.record(store.as_ref(), &path_str, outcome.stored, "claude-memory");
if let Err(e) = &marked {
tracing::warn!("watcher: mark_ingested failed for {}: {e}", path_str);
}
if outcome.stored > 0 || outcome.errors > 0 {
tracing::info!(
"watcher: ingested memory file {} ({} memories, {} errors)",
path_str,
outcome.stored,
outcome.errors
);
}
if outcome.errors > 0 {
bail!(
"{} memory writes failed while ingesting a memory file",
outcome.errors
);
}
marked.context("recording an ingested memory file")?;
Ok(FileOutcome::Captured)
}
fn ingest_sources(
adapter: &dyn Adapter,
sources: &[ConversationSource],
store: &dyn StoreAdmin,
embedder: Option<&dyn DocumentEmbedder>,
machine_id: &str,
stats: &mut IngestStats,
) {
for source in sources {
let changed = match check_file(store, adapter, std::path::Path::new(&source.path)) {
Some(FileCheck::Changed(changed)) => changed,
Some(FileCheck::Unchanged) => {
stats.unchanged += 1;
continue;
}
None => {
stats.errors += 1;
continue;
}
};
let result = match adapter.ingest_conversation(source) {
Ok(r) => r,
Err(e) => {
tracing::warn!("ingest: failed to parse {}: {e}", source.path);
stats.errors += 1;
continue;
}
};
let outcome = store_ingest_result(store, embedder, &result, machine_id);
if !result.memories.is_empty() {
stats.new_conversations += 1;
}
stats.new_memories += outcome.stored;
stats.skipped_dupes += outcome.dupes;
stats.new_entities += outcome.entities;
stats.errors += outcome.errors;
if let Err(e) = changed.record(store, &source.path, outcome.stored, adapter.name()) {
tracing::warn!("ingest: mark_ingested failed for {}: {e}", source.path);
stats.errors += 1;
}
}
}
enum FileCheck {
Unchanged,
Changed(ChangedFile),
}
struct ChangedFile {
content_hash: String,
session_key: Option<String>,
}
impl ChangedFile {
fn record(
&self,
store: &dyn StoreAdmin,
path: &str,
memory_count: usize,
source: &str,
) -> Result<()> {
store.record_ingested(&IngestRecord {
file_path: path,
content_hash: &self.content_hash,
session_key: self.session_key.as_deref(),
legacy_ids: false,
memory_count,
source,
})
}
}
fn check_file(store: &dyn StoreAdmin, adapter: &dyn Adapter, path: &Path) -> Option<FileCheck> {
let bytes = std::fs::read(path).ok()?;
let content_hash = sha256_hex(&bytes);
let path_str = path.to_string_lossy();
if let Ok(false) = store.is_file_changed(&path_str, &content_hash) {
return Some(FileCheck::Unchanged);
}
let session_key = adapter.session_key(path, &String::from_utf8_lossy(&bytes));
let legacy_hash = legacy_file_hash(&bytes);
let changed = || {
Some(FileCheck::Changed(ChangedFile {
content_hash: content_hash.clone(),
session_key: session_key.clone(),
}))
};
let own_row = store.ingest_log_entry(&path_str).unwrap_or_else(|e| {
tracing::warn!("ingest: reading the ingest log for {path_str} failed: {e}");
None
});
if let Some(row) = own_row.filter(|r| r.content_hash.is_none() && r.file_hash == legacy_hash) {
upgrade(store, &row.file_path, &content_hash, session_key.as_deref());
return Some(FileCheck::Unchanged);
}
let Some(key) = session_key.as_deref() else {
return changed();
};
match store.ingested_session(key, &content_hash) {
Ok(Some(copy_of)) => {
record_copy(
store,
&path_str,
&content_hash,
key,
copy_of.legacy_ids,
adapter.name(),
);
return Some(FileCheck::Unchanged);
}
Ok(None) => {}
Err(e) => tracing::warn!("ingest: session lookup for {path_str} failed: {e}"),
}
let Some(stem) = path.file_stem() else {
return changed();
};
let legacy_rows = store
.legacy_ingest_entries_with_stem(&stem.to_string_lossy())
.unwrap_or_else(|e| {
tracing::warn!("ingest: legacy lookup for {path_str} failed: {e}");
Vec::new()
});
if let Some(row) = legacy_rows.into_iter().find(|r| r.file_hash == legacy_hash) {
upgrade(store, &row.file_path, &content_hash, Some(key));
record_copy(store, &path_str, &content_hash, key, true, adapter.name());
return Some(FileCheck::Unchanged);
}
changed()
}
fn upgrade(store: &dyn StoreAdmin, path: &str, content_hash: &str, session_key: Option<&str>) {
if let Err(e) = store.upgrade_ingest_hash(path, content_hash, session_key) {
tracing::warn!("ingest: upgrading the ingest log hash for {path} failed: {e}");
}
}
fn record_copy(
store: &dyn StoreAdmin,
path: &str,
content_hash: &str,
session_key: &str,
legacy_ids: bool,
source: &str,
) {
let record = IngestRecord {
file_path: path,
content_hash,
session_key: Some(session_key),
legacy_ids,
memory_count: 0,
source,
};
if let Err(e) = store.record_ingested(&record) {
tracing::warn!("ingest: recording copied transcript {path} failed: {e}");
}
}
fn sha256_hex(bytes: &[u8]) -> String {
Sha256::digest(bytes)
.iter()
.map(|b| format!("{b:02x}"))
.collect()
}
pub fn legacy_file_hash(bytes: &[u8]) -> String {
use std::hash::Hasher;
let mut hasher = std::collections::hash_map::DefaultHasher::new();
for chunk in bytes.chunks(8192) {
hasher.write(chunk);
}
format!("{:016x}", hasher.finish())
}
#[cfg(test)]
mod tests {
use super::*;
fn names(source: Option<&str>) -> Result<Vec<String>, String> {
adapters_for(source, "m")
.map(|adapters| adapters.iter().map(|a| a.name().to_string()).collect())
}
#[test]
fn a_named_source_runs_only_that_adapter() {
for source in SOURCES {
assert_eq!(names(Some(source)).unwrap(), vec![source.to_string()]);
}
}
#[test]
fn no_source_runs_every_adapter() {
assert_eq!(names(None).unwrap(), SOURCES.map(str::to_string).to_vec());
}
#[test]
fn an_unknown_source_is_refused_with_the_valid_names() {
let error = names(Some("bogus")).unwrap_err();
assert!(error.contains("bogus"), "{error}");
for source in SOURCES {
assert!(error.contains(source), "{error}");
}
}
}