Skip to main content

second_brain_adapters/
persist.rs

1use std::collections::HashSet;
2
3use anyhow::Result;
4use second_brain_core::embedding::DocumentEmbedder;
5use second_brain_core::redact::{redact_for_store, redact_memory};
6use second_brain_core::schema::{Conversation, Entity, Memory, Relation, RelationType};
7use second_brain_core::store::{Store, StoreAdmin};
8use uuid::Uuid;
9
10use crate::IngestResult;
11
12/// The store surface `store_ingest_result` writes through. Every
13/// `StoreAdmin` has it through the blanket impl below; tests implement it
14/// directly to inject failing writes.
15pub trait IngestStore {
16    fn store_memory(&self, memory: &Memory) -> Result<()>;
17    fn store_conversation(&self, conversation: &Conversation) -> Result<()>;
18    fn store_relation(&self, relation: &Relation) -> Result<()>;
19    fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity>;
20    fn memory_content_exists(&self, content_prefix: &str) -> Result<bool>;
21    /// True when it wrote. A store that cannot check existence writes.
22    fn store_memory_if_absent(&self, memory: &Memory) -> Result<bool> {
23        self.store_memory(memory)?;
24        Ok(true)
25    }
26    /// True when it wrote. A store that cannot check existence writes.
27    fn store_conversation_if_absent(&self, conversation: &Conversation) -> Result<bool> {
28        self.store_conversation(conversation)?;
29        Ok(true)
30    }
31    /// Whether a memory holds this id in any state or it is tombstoned.
32    fn memory_id_taken(&self, _id: Uuid) -> Result<bool> {
33        Ok(false)
34    }
35}
36
37impl<T: StoreAdmin + ?Sized> IngestStore for T {
38    fn store_memory(&self, memory: &Memory) -> Result<()> {
39        Store::store_memory(self, memory)
40    }
41    fn store_conversation(&self, conversation: &Conversation) -> Result<()> {
42        Store::store_conversation(self, conversation)
43    }
44    fn store_relation(&self, relation: &Relation) -> Result<()> {
45        Store::store_relation(self, relation)
46    }
47    fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity> {
48        StoreAdmin::find_or_create_entity(self, name, entity_type)
49    }
50    fn memory_content_exists(&self, content_prefix: &str) -> Result<bool> {
51        StoreAdmin::memory_content_exists(self, content_prefix)
52    }
53    fn store_memory_if_absent(&self, memory: &Memory) -> Result<bool> {
54        StoreAdmin::store_memory_if_absent(self, memory)
55    }
56    fn store_conversation_if_absent(&self, conversation: &Conversation) -> Result<bool> {
57        StoreAdmin::store_conversation_if_absent(self, conversation)
58    }
59    fn memory_id_taken(&self, id: Uuid) -> Result<bool> {
60        StoreAdmin::memory_id_taken(self, id)
61    }
62}
63
64#[derive(Debug, Default, PartialEq, Eq)]
65pub struct StoreOutcome {
66    pub stored: usize,
67    pub dupes: usize,
68    pub entities: usize,
69    pub errors: usize,
70}
71
72/// Persists one adapter result: every memory whose id is not already stored
73/// or tombstoned and whose first 128 characters are not already stored
74/// (embedded first when an embedder is given), its conversation once one of
75/// them lands and only if absent, a derived_from edge per stored memory, and
76/// a mentions edge per entity whose memory actually landed. Write failures
77/// are counted in the outcome rather than returned, so one bad row does not
78/// cost the rest.
79pub fn store_ingest_result<S: IngestStore + ?Sized>(
80    store: &S,
81    embedder: Option<&dyn DocumentEmbedder>,
82    result: &IngestResult,
83    machine_id: &str,
84) -> StoreOutcome {
85    let mut outcome = StoreOutcome::default();
86
87    if result.memories.is_empty() {
88        return outcome;
89    }
90
91    // Only memories that actually landed can anchor an edge. A deduped or failed
92    // write still has an id in result.entities, and a mentions relation built from
93    // it dangles: the local store has no foreign key to reject it, but the sync
94    // importer does, so the edge can never replicate.
95    let mut stored_ids: HashSet<Uuid> = HashSet::new();
96    let mut conversation_written = false;
97
98    for memory in &result.memories {
99        match store.memory_id_taken(memory.id) {
100            Ok(true) => {
101                outcome.dupes += 1;
102                continue;
103            }
104            Ok(false) => {}
105            Err(e) => {
106                tracing::warn!("ingest: id check failed: {e}");
107                outcome.errors += 1;
108            }
109        }
110        let mut mem = memory.clone();
111        redact_memory(&mut mem);
112        let dedup_prefix: String = mem.content.chars().take(128).collect();
113        match store.memory_content_exists(&dedup_prefix) {
114            Ok(true) => {
115                outcome.dupes += 1;
116                continue;
117            }
118            Ok(false) => {}
119            Err(e) => {
120                // store the memory anyway: a duplicate is recoverable, a dropped memory is not
121                tracing::warn!("ingest: dedup check failed: {e}");
122                outcome.errors += 1;
123            }
124        }
125
126        mem.machine_id = machine_id.to_string();
127        if let Some(emb) = embedder {
128            match emb.embed_document(&mem.content) {
129                Ok(embedding) => mem.embedding = embedding,
130                Err(e) => {
131                    // stored without an embedding the memory is invisible to vector
132                    // recall, so this must be loud
133                    tracing::warn!("ingest: embedding failed, storing without vector: {e}");
134                    outcome.errors += 1;
135                }
136            }
137        }
138
139        if !conversation_written {
140            conversation_written = true;
141            if let Err(e) = store.store_conversation_if_absent(&result.conversation) {
142                tracing::warn!("ingest: store_conversation failed: {e}");
143                outcome.errors += 1;
144            }
145        }
146
147        match store.store_memory_if_absent(&mem) {
148            Ok(false) => outcome.dupes += 1,
149            Ok(true) => {
150                stored_ids.insert(mem.id);
151                let relation = Relation {
152                    from_id: mem.id,
153                    to_id: result.conversation.id,
154                    relation_type: RelationType::DerivedFrom,
155                    strength: 1.0,
156                    context: None,
157                };
158                if let Err(e) = store.store_relation(&relation) {
159                    tracing::warn!("ingest: store_relation (derived_from) failed: {e}");
160                    outcome.errors += 1;
161                }
162                outcome.stored += 1;
163            }
164            Err(e) => {
165                tracing::warn!("ingest: store_memory failed: {e}");
166                outcome.errors += 1;
167            }
168        }
169    }
170
171    for (entity_info, memory_id) in &result.entities {
172        if !stored_ids.contains(memory_id) {
173            continue;
174        }
175        let name = redact_for_store(&entity_info.name);
176        match store.find_or_create_entity(&name, &entity_info.entity_type) {
177            Ok(entity) => {
178                let relation = Relation {
179                    from_id: *memory_id,
180                    to_id: entity.id,
181                    relation_type: RelationType::Mentions,
182                    strength: 1.0,
183                    context: None,
184                };
185                if let Err(e) = store.store_relation(&relation) {
186                    tracing::warn!("ingest: store_relation (mentions) failed: {e}");
187                    outcome.errors += 1;
188                } else {
189                    outcome.entities += 1;
190                }
191            }
192            Err(e) => {
193                tracing::warn!("ingest: find_or_create_entity failed for {name}: {e}");
194                outcome.errors += 1;
195            }
196        }
197    }
198
199    outcome
200}