Skip to main content

second_brain_adapters/
persist.rs

1use std::collections::HashSet;
2
3use anyhow::Result;
4use second_brain_core::embedding::DocumentEmbedder;
5use second_brain_core::redact::{redact_for_store, redact_memory};
6use second_brain_core::schema::{Conversation, Entity, Memory, Relation, RelationType};
7use second_brain_core::store::{Store, StoreAdmin};
8use uuid::Uuid;
9
10use crate::IngestResult;
11
12/// The store surface `store_ingest_result` writes through. Every
13/// `StoreAdmin` has it through the blanket impl below; tests implement it
14/// directly to inject failing writes.
15pub trait IngestStore {
16    fn store_memory(&self, memory: &Memory) -> Result<()>;
17    fn store_conversation(&self, conversation: &Conversation) -> Result<()>;
18    fn store_relation(&self, relation: &Relation) -> Result<()>;
19    fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity>;
20    fn memory_content_exists(&self, content_prefix: &str) -> Result<bool>;
21    /// True when it wrote. A store that cannot check existence writes.
22    fn store_memory_if_absent(&self, memory: &Memory) -> Result<bool> {
23        self.store_memory(memory)?;
24        Ok(true)
25    }
26    /// True when it wrote. A store that cannot check existence writes.
27    fn store_conversation_if_absent(&self, conversation: &Conversation) -> Result<bool> {
28        self.store_conversation(conversation)?;
29        Ok(true)
30    }
31    /// Whether a memory holds this id in any state or it is tombstoned.
32    fn memory_id_taken(&self, _id: Uuid) -> Result<bool> {
33        Ok(false)
34    }
35}
36
37impl<T: StoreAdmin + ?Sized> IngestStore for T {
38    fn store_memory(&self, memory: &Memory) -> Result<()> {
39        Store::store_memory(self, memory)
40    }
41    fn store_conversation(&self, conversation: &Conversation) -> Result<()> {
42        Store::store_conversation(self, conversation)
43    }
44    fn store_relation(&self, relation: &Relation) -> Result<()> {
45        Store::store_relation(self, relation)
46    }
47    fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity> {
48        StoreAdmin::find_or_create_entity(self, name, entity_type)
49    }
50    fn memory_content_exists(&self, content_prefix: &str) -> Result<bool> {
51        StoreAdmin::memory_content_exists(self, content_prefix)
52    }
53    fn store_memory_if_absent(&self, memory: &Memory) -> Result<bool> {
54        StoreAdmin::store_memory_if_absent(self, memory)
55    }
56    fn store_conversation_if_absent(&self, conversation: &Conversation) -> Result<bool> {
57        StoreAdmin::store_conversation_if_absent(self, conversation)
58    }
59    fn memory_id_taken(&self, id: Uuid) -> Result<bool> {
60        StoreAdmin::memory_id_taken(self, id)
61    }
62}
63
64#[derive(Debug, Default, PartialEq, Eq)]
65pub struct StoreOutcome {
66    pub stored: usize,
67    pub dupes: usize,
68    pub entities: usize,
69    pub errors: usize,
70}
71
72/// Persists one adapter result: every memory whose id is not already stored
73/// or tombstoned and whose first 128 characters are not already stored
74/// (embedded first when an embedder is given), its conversation once one of
75/// them lands and only if absent, a derived_from edge per stored memory, and
76/// a mentions edge per entity whose memory actually landed. Write failures
77/// are counted in the outcome rather than returned, so one bad row does not
78/// cost the rest.
79pub fn store_ingest_result<S: IngestStore + ?Sized>(
80    store: &S,
81    embedder: Option<&dyn DocumentEmbedder>,
82    result: &IngestResult,
83    machine_id: &str,
84) -> StoreOutcome {
85    store_result(store, embedder, result, machine_id, ContentDedup::Prefix)
86}
87
88/// `store_ingest_result` for versioned documents, whose memories carry ids
89/// derived from their content: a memory is skipped only when its id is
90/// taken, since an edited document keeps most of its opening text and must
91/// still land as a new version.
92pub fn store_document_result<S: IngestStore + ?Sized>(
93    store: &S,
94    embedder: Option<&dyn DocumentEmbedder>,
95    result: &IngestResult,
96    machine_id: &str,
97) -> StoreOutcome {
98    store_result(store, embedder, result, machine_id, ContentDedup::IdOnly)
99}
100
101#[derive(Clone, Copy, PartialEq, Eq)]
102enum ContentDedup {
103    Prefix,
104    IdOnly,
105}
106
107fn store_result<S: IngestStore + ?Sized>(
108    store: &S,
109    embedder: Option<&dyn DocumentEmbedder>,
110    result: &IngestResult,
111    machine_id: &str,
112    dedup: ContentDedup,
113) -> StoreOutcome {
114    let mut outcome = StoreOutcome::default();
115
116    if result.memories.is_empty() {
117        return outcome;
118    }
119
120    // Only memories that actually landed can anchor an edge. A deduped or failed
121    // write still has an id in result.entities, and a mentions relation built from
122    // it dangles: the local store has no foreign key to reject it, but the sync
123    // importer does, so the edge can never replicate.
124    let mut stored_ids: HashSet<Uuid> = HashSet::new();
125    let mut conversation_written = false;
126
127    for memory in &result.memories {
128        match store.memory_id_taken(memory.id) {
129            Ok(true) => {
130                outcome.dupes += 1;
131                continue;
132            }
133            Ok(false) => {}
134            Err(e) => {
135                tracing::warn!("ingest: id check failed: {e}");
136                outcome.errors += 1;
137            }
138        }
139        let mut mem = memory.clone();
140        redact_memory(&mut mem);
141        if dedup == ContentDedup::Prefix {
142            let dedup_prefix: String = mem.content.chars().take(128).collect();
143            match store.memory_content_exists(&dedup_prefix) {
144                Ok(true) => {
145                    outcome.dupes += 1;
146                    continue;
147                }
148                Ok(false) => {}
149                Err(e) => {
150                    // store the memory anyway: a duplicate is recoverable, a dropped memory is not
151                    tracing::warn!("ingest: dedup check failed: {e}");
152                    outcome.errors += 1;
153                }
154            }
155        }
156
157        mem.machine_id = machine_id.to_string();
158        if let Some(emb) = embedder {
159            match emb.embed_document(&mem.content) {
160                Ok(embedding) => mem.embedding = embedding,
161                Err(e) => {
162                    // stored without an embedding the memory is invisible to vector
163                    // recall, so this must be loud
164                    tracing::warn!("ingest: embedding failed, storing without vector: {e}");
165                    outcome.errors += 1;
166                }
167            }
168        }
169
170        if !conversation_written {
171            conversation_written = true;
172            if let Err(e) = store.store_conversation_if_absent(&result.conversation) {
173                tracing::warn!("ingest: store_conversation failed: {e}");
174                outcome.errors += 1;
175            }
176        }
177
178        match store.store_memory_if_absent(&mem) {
179            Ok(false) => outcome.dupes += 1,
180            Ok(true) => {
181                stored_ids.insert(mem.id);
182                let relation = Relation {
183                    from_id: mem.id,
184                    to_id: result.conversation.id,
185                    relation_type: RelationType::DerivedFrom,
186                    strength: 1.0,
187                    context: None,
188                };
189                if let Err(e) = store.store_relation(&relation) {
190                    tracing::warn!("ingest: store_relation (derived_from) failed: {e}");
191                    outcome.errors += 1;
192                }
193                outcome.stored += 1;
194            }
195            Err(e) => {
196                tracing::warn!("ingest: store_memory failed: {e}");
197                outcome.errors += 1;
198            }
199        }
200    }
201
202    for (entity_info, memory_id) in &result.entities {
203        if !stored_ids.contains(memory_id) {
204            continue;
205        }
206        let name = redact_for_store(&entity_info.name);
207        match store.find_or_create_entity(&name, &entity_info.entity_type) {
208            Ok(entity) => {
209                let relation = Relation {
210                    from_id: *memory_id,
211                    to_id: entity.id,
212                    relation_type: RelationType::Mentions,
213                    strength: 1.0,
214                    context: None,
215                };
216                if let Err(e) = store.store_relation(&relation) {
217                    tracing::warn!("ingest: store_relation (mentions) failed: {e}");
218                    outcome.errors += 1;
219                } else {
220                    outcome.entities += 1;
221                }
222            }
223            Err(e) => {
224                tracing::warn!("ingest: find_or_create_entity failed for {name}: {e}");
225                outcome.errors += 1;
226            }
227        }
228    }
229
230    outcome
231}