use std::sync::Arc;
use anyhow::Result;
use second_brain_adapters::claude_code::ClaudeCodeAdapter;
use second_brain_adapters::claude_memory::ClaudeMemoryAdapter;
use second_brain_adapters::cursor::CursorAdapter;
use second_brain_adapters::{Adapter, ConversationSource, IngestResult};
use second_brain_core::embedding::Embedder;
use second_brain_core::kuzu_store::KuzuStore;
use second_brain_core::schema::{Conversation, Entity, Memory, Relation, RelationType};
use second_brain_core::store::Store;
pub struct IngestStats {
pub new_memories: usize,
pub new_conversations: usize,
pub new_entities: usize,
pub skipped_dupes: usize,
pub unchanged: usize,
pub errors: usize,
pub files_processed: usize,
}
pub(crate) trait IngestStore {
fn store_memory(&self, memory: &Memory) -> Result<()>;
fn store_conversation(&self, conversation: &Conversation) -> Result<()>;
fn store_relation(&self, relation: &Relation) -> Result<()>;
fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity>;
fn memory_content_exists(&self, content_prefix: &str) -> Result<bool>;
fn is_file_changed(&self, file_path: &str, file_hash: &str) -> Result<bool>;
fn mark_ingested(
&self,
file_path: &str,
file_hash: &str,
memory_count: usize,
source: &str,
) -> Result<()>;
fn delete_memories_by_source_id(&self, source_id: &str) -> Result<usize>;
}
impl IngestStore for KuzuStore {
fn store_memory(&self, memory: &Memory) -> Result<()> {
Store::store_memory(self, memory)
}
fn store_conversation(&self, conversation: &Conversation) -> Result<()> {
Store::store_conversation(self, conversation)
}
fn store_relation(&self, relation: &Relation) -> Result<()> {
Store::store_relation(self, relation)
}
fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity> {
KuzuStore::find_or_create_entity(self, name, entity_type)
}
fn memory_content_exists(&self, content_prefix: &str) -> Result<bool> {
KuzuStore::memory_content_exists(self, content_prefix)
}
fn is_file_changed(&self, file_path: &str, file_hash: &str) -> Result<bool> {
KuzuStore::is_file_changed(self, file_path, file_hash)
}
fn mark_ingested(
&self,
file_path: &str,
file_hash: &str,
memory_count: usize,
source: &str,
) -> Result<()> {
KuzuStore::mark_ingested(self, file_path, file_hash, memory_count, source)
}
fn delete_memories_by_source_id(&self, source_id: &str) -> Result<usize> {
KuzuStore::delete_memories_by_source_id(self, source_id)
}
}
#[derive(Debug, Default, PartialEq)]
pub(crate) struct StoreOutcome {
pub stored: usize,
pub dupes: usize,
pub entities: usize,
pub errors: usize,
}
pub fn run_ingest(store: &Arc<KuzuStore>, embedder: Option<&Embedder>, machine_id: &str) -> Result<IngestStats> {
let adapters: Vec<Box<dyn Adapter>> = vec![
Box::new(ClaudeCodeAdapter::new(machine_id.to_string())),
Box::new(ClaudeMemoryAdapter::new(machine_id.to_string())),
Box::new(CursorAdapter::new(machine_id.to_string())),
];
let mut total = IngestStats {
new_memories: 0,
new_conversations: 0,
new_entities: 0,
skipped_dupes: 0,
unchanged: 0,
errors: 0,
files_processed: 0,
};
for adapter in &adapters {
let sources = match adapter.discover_conversations() {
Ok(s) => s,
Err(e) => {
tracing::warn!("ingest: {} discovery failed: {e}", adapter.name());
continue;
}
};
let source_name = adapter.name().to_string();
total.files_processed += sources.len();
ingest_sources(
adapter.as_ref(),
&sources,
store.as_ref(),
embedder,
&source_name,
machine_id,
&mut total,
);
}
Ok(total)
}
pub fn ingest_file(store: &Arc<KuzuStore>, embedder: Option<&Embedder>, path: &std::path::Path, machine_id: &str) {
let path_str = path.to_string_lossy().to_string();
let hash = match file_hash(path) {
Some(h) => h,
None => return,
};
if let Ok(false) = IngestStore::is_file_changed(store.as_ref(), &path_str, &hash) {
return;
}
let adapter = ClaudeCodeAdapter::new(machine_id.to_string());
let source = ConversationSource {
id: path_str.clone(),
path: path_str.clone(),
source_name: "claude-code".to_string(),
machine_id: machine_id.to_string(),
};
let result = match adapter.ingest_conversation(&source) {
Ok(r) => r,
Err(e) => {
tracing::debug!("watcher: failed to parse {}: {e}", path_str);
return;
}
};
let outcome = store_ingest_result(store.as_ref(), embedder, &result, machine_id);
if let Err(e) =
IngestStore::mark_ingested(store.as_ref(), &path_str, &hash, outcome.stored, "claude-code")
{
tracing::warn!("watcher: mark_ingested failed for {}: {e}", path_str);
}
if outcome.errors > 0 {
tracing::warn!(
"watcher: ingested {} with {} write errors ({} memories stored)",
path_str,
outcome.errors,
outcome.stored
);
} else if outcome.stored > 0 {
tracing::info!("watcher: ingested {} ({} memories)", path_str, outcome.stored);
}
}
pub fn ingest_memory_file(store: &Arc<KuzuStore>, embedder: Option<&Embedder>, path: &std::path::Path, machine_id: &str) {
let path_str = path.to_string_lossy().to_string();
let hash = match file_hash(path) {
Some(h) => h,
None => return,
};
if let Ok(false) = IngestStore::is_file_changed(store.as_ref(), &path_str, &hash) {
return;
}
if let Err(e) = IngestStore::delete_memories_by_source_id(store.as_ref(), &path_str) {
tracing::warn!("watcher: delete_memories_by_source_id failed for {}: {e}", path_str);
}
let adapter = ClaudeMemoryAdapter::new(machine_id.to_string());
let source = ConversationSource {
id: path_str.clone(),
path: path_str.clone(),
source_name: "claude-memory".to_string(),
machine_id: machine_id.to_string(),
};
let result = match adapter.ingest_conversation(&source) {
Ok(r) => r,
Err(e) => {
tracing::debug!("watcher: failed to parse memory file {}: {e}", path_str);
return;
}
};
let outcome = store_ingest_result(store.as_ref(), embedder, &result, machine_id);
if let Err(e) =
IngestStore::mark_ingested(store.as_ref(), &path_str, &hash, outcome.stored, "claude-memory")
{
tracing::warn!("watcher: mark_ingested failed for {}: {e}", path_str);
}
if outcome.stored > 0 || outcome.errors > 0 {
tracing::info!(
"watcher: ingested memory file {} ({} memories, {} errors)",
path_str, outcome.stored, outcome.errors
);
}
}
fn ingest_sources<S: IngestStore>(
adapter: &dyn Adapter,
sources: &[ConversationSource],
store: &S,
embedder: Option<&Embedder>,
source_name: &str,
machine_id: &str,
stats: &mut IngestStats,
) {
for source in sources {
let path = std::path::Path::new(&source.path);
let hash = match file_hash(path) {
Some(h) => h,
None => {
stats.errors += 1;
continue;
}
};
if let Ok(false) = store.is_file_changed(&source.path, &hash) {
stats.unchanged += 1;
continue;
}
let result = match adapter.ingest_conversation(source) {
Ok(r) => r,
Err(e) => {
tracing::warn!("ingest: failed to parse {}: {e}", source.path);
stats.errors += 1;
continue;
}
};
let outcome = store_ingest_result(store, embedder, &result, machine_id);
if !result.memories.is_empty() {
stats.new_conversations += 1;
}
stats.new_memories += outcome.stored;
stats.skipped_dupes += outcome.dupes;
stats.new_entities += outcome.entities;
stats.errors += outcome.errors;
if let Err(e) = store.mark_ingested(&source.path, &hash, outcome.stored, source_name) {
tracing::warn!("ingest: mark_ingested failed for {}: {e}", source.path);
stats.errors += 1;
}
}
}
fn store_ingest_result<S: IngestStore>(
store: &S,
embedder: Option<&Embedder>,
result: &IngestResult,
machine_id: &str,
) -> StoreOutcome {
let mut outcome = StoreOutcome::default();
if result.memories.is_empty() {
return outcome;
}
if let Err(e) = store.store_conversation(&result.conversation) {
tracing::warn!("ingest: store_conversation failed: {e}");
outcome.errors += 1;
}
for memory in &result.memories {
let dedup_prefix: String = memory.content.chars().take(128).collect();
match store.memory_content_exists(&dedup_prefix) {
Ok(true) => {
outcome.dupes += 1;
continue;
}
Ok(false) => {}
Err(e) => {
tracing::warn!("ingest: dedup check failed: {e}");
outcome.errors += 1;
}
}
let mut mem = memory.clone();
mem.machine_id = machine_id.to_string();
if let Some(emb) = embedder {
match emb.embed(&mem.content) {
Ok(embedding) => mem.embedding = embedding,
Err(e) => {
tracing::warn!("ingest: embedding failed, storing without vector: {e}");
outcome.errors += 1;
}
}
}
match store.store_memory(&mem) {
Ok(()) => {
let relation = Relation {
from_id: mem.id,
to_id: result.conversation.id,
relation_type: RelationType::DerivedFrom,
strength: 1.0,
context: None,
};
if let Err(e) = store.store_relation(&relation) {
tracing::warn!("ingest: store_relation (derived_from) failed: {e}");
outcome.errors += 1;
}
outcome.stored += 1;
}
Err(e) => {
tracing::warn!("ingest: store_memory failed: {e}");
outcome.errors += 1;
}
}
}
for (entity_info, memory_id) in &result.entities {
match store.find_or_create_entity(&entity_info.name, &entity_info.entity_type) {
Ok(entity) => {
let relation = Relation {
from_id: *memory_id,
to_id: entity.id,
relation_type: RelationType::Mentions,
strength: 1.0,
context: None,
};
if let Err(e) = store.store_relation(&relation) {
tracing::warn!("ingest: store_relation (mentions) failed: {e}");
outcome.errors += 1;
} else {
outcome.entities += 1;
}
}
Err(e) => {
tracing::warn!("ingest: find_or_create_entity failed for {}: {e}", entity_info.name);
outcome.errors += 1;
}
}
}
outcome
}
fn file_hash(path: &std::path::Path) -> Option<String> {
use std::hash::Hasher;
use std::io::Read;
let mut file = std::fs::File::open(path).ok()?;
let mut hasher = std::collections::hash_map::DefaultHasher::new();
let mut buf = [0u8; 8192];
loop {
let n = file.read(&mut buf).ok()?;
if n == 0 {
break;
}
hasher.write(&buf[..n]);
}
Some(format!("{:016x}", hasher.finish()))
}
#[cfg(test)]
mod tests {
use super::*;
use chrono::Utc;
use second_brain_adapters::ExtractedEntity;
use second_brain_core::schema::MemoryType;
use std::cell::RefCell;
use uuid::Uuid;
struct StubStore {
fail_memory_writes: bool,
fail_relation_writes: bool,
existing_prefixes: Vec<String>,
stored_memories: RefCell<Vec<Memory>>,
stored_relations: RefCell<Vec<Relation>>,
}
impl StubStore {
fn ok() -> Self {
Self {
fail_memory_writes: false,
fail_relation_writes: false,
existing_prefixes: Vec::new(),
stored_memories: RefCell::new(Vec::new()),
stored_relations: RefCell::new(Vec::new()),
}
}
}
impl IngestStore for StubStore {
fn store_memory(&self, memory: &Memory) -> Result<()> {
if self.fail_memory_writes {
anyhow::bail!("disk full");
}
self.stored_memories.borrow_mut().push(memory.clone());
Ok(())
}
fn store_conversation(&self, _c: &Conversation) -> Result<()> {
Ok(())
}
fn store_relation(&self, relation: &Relation) -> Result<()> {
if self.fail_relation_writes {
anyhow::bail!("disk full");
}
self.stored_relations.borrow_mut().push(relation.clone());
Ok(())
}
fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity> {
Ok(Entity::new(name.to_string(), entity_type.to_string()))
}
fn memory_content_exists(&self, content_prefix: &str) -> Result<bool> {
Ok(self.existing_prefixes.iter().any(|p| p == content_prefix))
}
fn is_file_changed(&self, _p: &str, _h: &str) -> Result<bool> {
Ok(true)
}
fn mark_ingested(&self, _p: &str, _h: &str, _c: usize, _s: &str) -> Result<()> {
Ok(())
}
fn delete_memories_by_source_id(&self, _source_id: &str) -> Result<usize> {
Ok(0)
}
}
fn fixture(contents: &[&str]) -> IngestResult {
let conversation = Conversation {
id: Uuid::new_v4(),
source: "claude-code".to_string(),
machine_id: "test-machine".to_string(),
started_at: Utc::now(),
project_path: None,
};
let memories: Vec<Memory> = contents
.iter()
.map(|c| {
Memory::new(
c.to_string(),
MemoryType::Semantic,
"claude-code".to_string(),
Uuid::new_v4().to_string(),
)
})
.collect();
let entity_target = memories.first().map(|m| m.id).unwrap_or_else(Uuid::new_v4);
IngestResult {
conversation,
memories,
entities: vec![(
ExtractedEntity {
name: "kuzu".to_string(),
entity_type: "tool".to_string(),
},
entity_target,
)],
entities_extracted: 1,
}
}
#[test]
fn successful_ingest_stores_memories_relations_and_entities() {
let store = StubStore::ok();
let result = fixture(&["fact one", "fact two"]);
let outcome = store_ingest_result(&store, None, &result, "test-machine");
assert_eq!(outcome.stored, 2);
assert_eq!(outcome.errors, 0);
assert_eq!(outcome.entities, 1);
assert_eq!(store.stored_memories.borrow().len(), 2);
assert!(store
.stored_memories
.borrow()
.iter()
.all(|m| m.machine_id == "test-machine"));
assert_eq!(store.stored_relations.borrow().len(), 3);
}
#[test]
fn failed_memory_writes_are_counted_not_swallowed() {
let store = StubStore {
fail_memory_writes: true,
..StubStore::ok()
};
let result = fixture(&["fact one", "fact two"]);
let outcome = store_ingest_result(&store, None, &result, "test-machine");
assert_eq!(outcome.stored, 0);
assert_eq!(outcome.errors, 2);
assert!(store.stored_memories.borrow().is_empty());
assert!(store
.stored_relations
.borrow()
.iter()
.all(|r| r.relation_type != RelationType::DerivedFrom));
}
#[test]
fn failed_relation_writes_are_counted_but_memory_still_stored() {
let store = StubStore {
fail_relation_writes: true,
..StubStore::ok()
};
let result = fixture(&["fact one"]);
let outcome = store_ingest_result(&store, None, &result, "test-machine");
assert_eq!(outcome.stored, 1);
assert_eq!(outcome.errors, 2);
assert_eq!(outcome.entities, 0);
}
#[test]
fn duplicate_prefix_skips_storage_and_counts_dupe() {
let dup = "a".repeat(200);
let store = StubStore {
existing_prefixes: vec![dup.chars().take(128).collect()],
..StubStore::ok()
};
let result = fixture(&[dup.as_str(), "fresh fact"]);
let outcome = store_ingest_result(&store, None, &result, "test-machine");
assert_eq!(outcome.stored, 1);
assert_eq!(outcome.dupes, 1);
assert_eq!(outcome.errors, 0);
}
#[test]
fn empty_memories_is_a_noop() {
let store = StubStore::ok();
let result = fixture(&[]);
let entities_only = IngestResult {
entities: result.entities.clone(),
..result
};
let outcome = store_ingest_result(&store, None, &entities_only, "test-machine");
assert_eq!(outcome, StoreOutcome::default());
assert!(store.stored_relations.borrow().is_empty());
}
}