use std::collections::HashSet;
use anyhow::Result;
use second_brain_core::embedding::DocumentEmbedder;
use second_brain_core::redact::{redact_for_store, redact_memory};
use second_brain_core::schema::{Conversation, Entity, Memory, Relation, RelationType};
use second_brain_core::store::{Store, StoreAdmin};
use uuid::Uuid;
use crate::IngestResult;
pub trait IngestStore {
fn store_memory(&self, memory: &Memory) -> Result<()>;
fn store_conversation(&self, conversation: &Conversation) -> Result<()>;
fn store_relation(&self, relation: &Relation) -> Result<()>;
fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity>;
fn memory_content_exists(&self, content_prefix: &str) -> Result<bool>;
fn store_memory_if_absent(&self, memory: &Memory) -> Result<bool> {
self.store_memory(memory)?;
Ok(true)
}
fn store_conversation_if_absent(&self, conversation: &Conversation) -> Result<bool> {
self.store_conversation(conversation)?;
Ok(true)
}
fn memory_id_taken(&self, _id: Uuid) -> Result<bool> {
Ok(false)
}
}
impl<T: StoreAdmin + ?Sized> IngestStore for T {
fn store_memory(&self, memory: &Memory) -> Result<()> {
Store::store_memory(self, memory)
}
fn store_conversation(&self, conversation: &Conversation) -> Result<()> {
Store::store_conversation(self, conversation)
}
fn store_relation(&self, relation: &Relation) -> Result<()> {
Store::store_relation(self, relation)
}
fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity> {
StoreAdmin::find_or_create_entity(self, name, entity_type)
}
fn memory_content_exists(&self, content_prefix: &str) -> Result<bool> {
StoreAdmin::memory_content_exists(self, content_prefix)
}
fn store_memory_if_absent(&self, memory: &Memory) -> Result<bool> {
StoreAdmin::store_memory_if_absent(self, memory)
}
fn store_conversation_if_absent(&self, conversation: &Conversation) -> Result<bool> {
StoreAdmin::store_conversation_if_absent(self, conversation)
}
fn memory_id_taken(&self, id: Uuid) -> Result<bool> {
StoreAdmin::memory_id_taken(self, id)
}
}
#[derive(Debug, Default, PartialEq, Eq)]
pub struct StoreOutcome {
pub stored: usize,
pub dupes: usize,
pub entities: usize,
pub errors: usize,
}
pub fn store_ingest_result<S: IngestStore + ?Sized>(
store: &S,
embedder: Option<&dyn DocumentEmbedder>,
result: &IngestResult,
machine_id: &str,
) -> StoreOutcome {
let mut outcome = StoreOutcome::default();
if result.memories.is_empty() {
return outcome;
}
let mut stored_ids: HashSet<Uuid> = HashSet::new();
let mut conversation_written = false;
for memory in &result.memories {
match store.memory_id_taken(memory.id) {
Ok(true) => {
outcome.dupes += 1;
continue;
}
Ok(false) => {}
Err(e) => {
tracing::warn!("ingest: id check failed: {e}");
outcome.errors += 1;
}
}
let mut mem = memory.clone();
redact_memory(&mut mem);
let dedup_prefix: String = mem.content.chars().take(128).collect();
match store.memory_content_exists(&dedup_prefix) {
Ok(true) => {
outcome.dupes += 1;
continue;
}
Ok(false) => {}
Err(e) => {
tracing::warn!("ingest: dedup check failed: {e}");
outcome.errors += 1;
}
}
mem.machine_id = machine_id.to_string();
if let Some(emb) = embedder {
match emb.embed_document(&mem.content) {
Ok(embedding) => mem.embedding = embedding,
Err(e) => {
tracing::warn!("ingest: embedding failed, storing without vector: {e}");
outcome.errors += 1;
}
}
}
if !conversation_written {
conversation_written = true;
if let Err(e) = store.store_conversation_if_absent(&result.conversation) {
tracing::warn!("ingest: store_conversation failed: {e}");
outcome.errors += 1;
}
}
match store.store_memory_if_absent(&mem) {
Ok(false) => outcome.dupes += 1,
Ok(true) => {
stored_ids.insert(mem.id);
let relation = Relation {
from_id: mem.id,
to_id: result.conversation.id,
relation_type: RelationType::DerivedFrom,
strength: 1.0,
context: None,
};
if let Err(e) = store.store_relation(&relation) {
tracing::warn!("ingest: store_relation (derived_from) failed: {e}");
outcome.errors += 1;
}
outcome.stored += 1;
}
Err(e) => {
tracing::warn!("ingest: store_memory failed: {e}");
outcome.errors += 1;
}
}
}
for (entity_info, memory_id) in &result.entities {
if !stored_ids.contains(memory_id) {
continue;
}
let name = redact_for_store(&entity_info.name);
match store.find_or_create_entity(&name, &entity_info.entity_type) {
Ok(entity) => {
let relation = Relation {
from_id: *memory_id,
to_id: entity.id,
relation_type: RelationType::Mentions,
strength: 1.0,
context: None,
};
if let Err(e) = store.store_relation(&relation) {
tracing::warn!("ingest: store_relation (mentions) failed: {e}");
outcome.errors += 1;
} else {
outcome.entities += 1;
}
}
Err(e) => {
tracing::warn!("ingest: find_or_create_entity failed for {name}: {e}");
outcome.errors += 1;
}
}
}
outcome
}