use std::sync::Arc;
use anyhow::Result;
use second_brain_adapters::claude_code::ClaudeCodeAdapter;
use second_brain_adapters::claude_memory::ClaudeMemoryAdapter;
use second_brain_adapters::cursor::CursorAdapter;
use second_brain_adapters::{Adapter, ConversationSource, store_ingest_result};
use second_brain_core::embedding::{DocumentEmbedder, Embedder};
use second_brain_core::store::StoreAdmin;
pub struct IngestStats {
pub new_memories: usize,
pub new_conversations: usize,
pub new_entities: usize,
pub skipped_dupes: usize,
pub unchanged: usize,
pub errors: usize,
pub files_processed: usize,
}
pub const SOURCES: [&str; 3] = ["claude-code", "claude-memory", "cursor"];
fn adapter_named(source: &str, machine_id: &str) -> Option<Box<dyn Adapter>> {
let machine_id = machine_id.to_string();
match source {
"claude-code" => Some(Box::new(ClaudeCodeAdapter::new(machine_id))),
"claude-memory" => Some(Box::new(ClaudeMemoryAdapter::new(machine_id))),
"cursor" => Some(Box::new(CursorAdapter::new(machine_id))),
_ => None,
}
}
pub fn adapters_for(
source: Option<&str>,
machine_id: &str,
) -> Result<Vec<Box<dyn Adapter>>, String> {
match source {
None => Ok(SOURCES
.iter()
.filter_map(|name| adapter_named(name, machine_id))
.collect()),
Some(name) => adapter_named(name, machine_id)
.map(|a| vec![a])
.ok_or_else(|| {
format!(
"unknown ingest source '{name}'; valid values: {}",
SOURCES.join(", ")
)
}),
}
}
pub fn run_ingest(
store: &Arc<dyn StoreAdmin>,
embedder: Option<&Embedder>,
machine_id: &str,
source: Option<&str>,
) -> Result<IngestStats> {
let adapters = adapters_for(source, machine_id).map_err(anyhow::Error::msg)?;
let embedder = embedder.map(|e| e as &dyn DocumentEmbedder);
let mut total = IngestStats {
new_memories: 0,
new_conversations: 0,
new_entities: 0,
skipped_dupes: 0,
unchanged: 0,
errors: 0,
files_processed: 0,
};
for adapter in &adapters {
let sources = match adapter.discover_conversations() {
Ok(s) => s,
Err(e) => {
tracing::warn!("ingest: {} discovery failed: {e}", adapter.name());
continue;
}
};
let source_name = adapter.name().to_string();
total.files_processed += sources.len();
ingest_sources(
adapter.as_ref(),
&sources,
store.as_ref(),
embedder,
&source_name,
machine_id,
&mut total,
);
}
Ok(total)
}
pub fn ingest_file(
store: &Arc<dyn StoreAdmin>,
embedder: Option<&Embedder>,
path: &std::path::Path,
machine_id: &str,
) {
let path_str = path.to_string_lossy().to_string();
let hash = match file_hash(path) {
Some(h) => h,
None => return,
};
if let Ok(false) = store.is_file_changed(&path_str, &hash) {
return;
}
let adapter = ClaudeCodeAdapter::new(machine_id.to_string());
let source = ConversationSource {
id: path_str.clone(),
path: path_str.clone(),
source_name: "claude-code".to_string(),
machine_id: machine_id.to_string(),
};
let result = match adapter.ingest_conversation(&source) {
Ok(r) => r,
Err(e) => {
tracing::debug!("watcher: failed to parse {}: {e}", path_str);
return;
}
};
let outcome = store_ingest_result(
store.as_ref(),
embedder.map(|e| e as &dyn DocumentEmbedder),
&result,
machine_id,
);
if let Err(e) = store.mark_ingested(&path_str, &hash, outcome.stored, "claude-code") {
tracing::warn!("watcher: mark_ingested failed for {}: {e}", path_str);
}
if outcome.errors > 0 {
tracing::warn!(
"watcher: ingested {} with {} write errors ({} memories stored)",
path_str,
outcome.errors,
outcome.stored
);
} else if outcome.stored > 0 {
tracing::info!(
"watcher: ingested {} ({} memories)",
path_str,
outcome.stored
);
}
}
pub fn ingest_memory_file(
store: &Arc<dyn StoreAdmin>,
embedder: Option<&Embedder>,
path: &std::path::Path,
machine_id: &str,
) {
let path_str = path.to_string_lossy().to_string();
let hash = match file_hash(path) {
Some(h) => h,
None => return,
};
if let Ok(false) = store.is_file_changed(&path_str, &hash) {
return;
}
if let Err(e) = store.delete_memories_by_source_id(&path_str) {
tracing::warn!(
"watcher: delete_memories_by_source_id failed for {}: {e}",
path_str
);
}
let adapter = ClaudeMemoryAdapter::new(machine_id.to_string());
let source = ConversationSource {
id: path_str.clone(),
path: path_str.clone(),
source_name: "claude-memory".to_string(),
machine_id: machine_id.to_string(),
};
let result = match adapter.ingest_conversation(&source) {
Ok(r) => r,
Err(e) => {
tracing::debug!("watcher: failed to parse memory file {}: {e}", path_str);
return;
}
};
let outcome = store_ingest_result(
store.as_ref(),
embedder.map(|e| e as &dyn DocumentEmbedder),
&result,
machine_id,
);
if let Err(e) = store.mark_ingested(&path_str, &hash, outcome.stored, "claude-memory") {
tracing::warn!("watcher: mark_ingested failed for {}: {e}", path_str);
}
if outcome.stored > 0 || outcome.errors > 0 {
tracing::info!(
"watcher: ingested memory file {} ({} memories, {} errors)",
path_str,
outcome.stored,
outcome.errors
);
}
}
fn ingest_sources(
adapter: &dyn Adapter,
sources: &[ConversationSource],
store: &dyn StoreAdmin,
embedder: Option<&dyn DocumentEmbedder>,
source_name: &str,
machine_id: &str,
stats: &mut IngestStats,
) {
for source in sources {
let path = std::path::Path::new(&source.path);
let hash = match file_hash(path) {
Some(h) => h,
None => {
stats.errors += 1;
continue;
}
};
if let Ok(false) = store.is_file_changed(&source.path, &hash) {
stats.unchanged += 1;
continue;
}
let result = match adapter.ingest_conversation(source) {
Ok(r) => r,
Err(e) => {
tracing::warn!("ingest: failed to parse {}: {e}", source.path);
stats.errors += 1;
continue;
}
};
let outcome = store_ingest_result(store, embedder, &result, machine_id);
if !result.memories.is_empty() {
stats.new_conversations += 1;
}
stats.new_memories += outcome.stored;
stats.skipped_dupes += outcome.dupes;
stats.new_entities += outcome.entities;
stats.errors += outcome.errors;
if let Err(e) = store.mark_ingested(&source.path, &hash, outcome.stored, source_name) {
tracing::warn!("ingest: mark_ingested failed for {}: {e}", source.path);
stats.errors += 1;
}
}
}
fn file_hash(path: &std::path::Path) -> Option<String> {
use std::hash::Hasher;
use std::io::Read;
let mut file = std::fs::File::open(path).ok()?;
let mut hasher = std::collections::hash_map::DefaultHasher::new();
let mut buf = [0u8; 8192];
loop {
let n = file.read(&mut buf).ok()?;
if n == 0 {
break;
}
hasher.write(&buf[..n]);
}
Some(format!("{:016x}", hasher.finish()))
}
#[cfg(test)]
mod tests {
use super::*;
fn names(source: Option<&str>) -> Result<Vec<String>, String> {
adapters_for(source, "m")
.map(|adapters| adapters.iter().map(|a| a.name().to_string()).collect())
}
#[test]
fn a_named_source_runs_only_that_adapter() {
for source in SOURCES {
assert_eq!(names(Some(source)).unwrap(), vec![source.to_string()]);
}
}
#[test]
fn no_source_runs_every_adapter() {
assert_eq!(names(None).unwrap(), SOURCES.map(str::to_string).to_vec());
}
#[test]
fn an_unknown_source_is_refused_with_the_valid_names() {
let error = names(Some("bogus")).unwrap_err();
assert!(error.contains("bogus"), "{error}");
for source in SOURCES {
assert!(error.contains(source), "{error}");
}
}
}