Skip to main content

second_brain_adapters/
persist.rs

1use std::collections::HashSet;
2
3use anyhow::Result;
4use second_brain_core::embedding::DocumentEmbedder;
5use second_brain_core::redact::{redact_for_store, redact_memory};
6use second_brain_core::schema::{Conversation, Entity, Memory, Relation, RelationType};
7use second_brain_core::store::{Store, StoreAdmin};
8use uuid::Uuid;
9
10use crate::IngestResult;
11
12/// The store surface `store_ingest_result` writes through. Every
13/// `StoreAdmin` has it through the blanket impl below; tests implement it
14/// directly to inject failing writes.
15pub trait IngestStore {
16    fn store_memory(&self, memory: &Memory) -> Result<()>;
17    fn store_conversation(&self, conversation: &Conversation) -> Result<()>;
18    fn store_relation(&self, relation: &Relation) -> Result<()>;
19    fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity>;
20    fn memory_content_exists(&self, content_prefix: &str) -> Result<bool>;
21    /// True when it wrote. A store that cannot check existence writes.
22    fn store_memory_if_absent(&self, memory: &Memory) -> Result<bool> {
23        self.store_memory(memory)?;
24        Ok(true)
25    }
26    /// True when it wrote. A store that cannot check existence writes.
27    fn store_conversation_if_absent(&self, conversation: &Conversation) -> Result<bool> {
28        self.store_conversation(conversation)?;
29        Ok(true)
30    }
31    /// Whether a memory holds this id in any state or it is tombstoned.
32    fn memory_id_taken(&self, _id: Uuid) -> Result<bool> {
33        Ok(false)
34    }
35    /// Whether a delete removed this memory's text from its source.
36    fn memory_blocked(&self, _memory: &Memory) -> Result<bool> {
37        Ok(false)
38    }
39}
40
41impl<T: StoreAdmin + ?Sized> IngestStore for T {
42    fn store_memory(&self, memory: &Memory) -> Result<()> {
43        Store::store_memory(self, memory)
44    }
45    fn store_conversation(&self, conversation: &Conversation) -> Result<()> {
46        Store::store_conversation(self, conversation)
47    }
48    fn store_relation(&self, relation: &Relation) -> Result<()> {
49        Store::store_relation(self, relation)
50    }
51    fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity> {
52        StoreAdmin::find_or_create_entity(self, name, entity_type)
53    }
54    fn memory_content_exists(&self, content_prefix: &str) -> Result<bool> {
55        StoreAdmin::memory_content_exists(self, content_prefix)
56    }
57    fn store_memory_if_absent(&self, memory: &Memory) -> Result<bool> {
58        StoreAdmin::ingest_memory_if_absent(self, memory)
59    }
60    fn store_conversation_if_absent(&self, conversation: &Conversation) -> Result<bool> {
61        StoreAdmin::store_conversation_if_absent(self, conversation)
62    }
63    fn memory_id_taken(&self, id: Uuid) -> Result<bool> {
64        StoreAdmin::memory_id_taken(self, id)
65    }
66    fn memory_blocked(&self, memory: &Memory) -> Result<bool> {
67        StoreAdmin::ingest_blocked(self, memory)
68    }
69}
70
71#[derive(Debug, Default, PartialEq, Eq)]
72pub struct StoreOutcome {
73    pub stored: usize,
74    pub dupes: usize,
75    pub entities: usize,
76    pub errors: usize,
77}
78
79/// Persists one adapter result: every memory whose id is not already stored
80/// or tombstoned, whose text no delete removed from its source, and whose
81/// first 128 characters are not already stored
82/// (embedded first when an embedder is given), its conversation once one of
83/// them lands and only if absent, a derived_from edge per stored memory, and
84/// a mentions edge per entity whose memory actually landed. Write failures
85/// are counted in the outcome rather than returned, so one bad row does not
86/// cost the rest.
87pub fn store_ingest_result<S: IngestStore + ?Sized>(
88    store: &S,
89    embedder: Option<&dyn DocumentEmbedder>,
90    result: &IngestResult,
91    machine_id: &str,
92) -> StoreOutcome {
93    store_result(store, embedder, result, machine_id, ContentDedup::Prefix)
94}
95
96/// `store_ingest_result` for versioned documents, whose memories carry ids
97/// derived from their content: a memory is skipped only when its id is
98/// taken, since an edited document keeps most of its opening text and must
99/// still land as a new version.
100pub fn store_document_result<S: IngestStore + ?Sized>(
101    store: &S,
102    embedder: Option<&dyn DocumentEmbedder>,
103    result: &IngestResult,
104    machine_id: &str,
105) -> StoreOutcome {
106    store_result(store, embedder, result, machine_id, ContentDedup::IdOnly)
107}
108
109#[derive(Clone, Copy, PartialEq, Eq)]
110enum ContentDedup {
111    Prefix,
112    IdOnly,
113}
114
115fn store_result<S: IngestStore + ?Sized>(
116    store: &S,
117    embedder: Option<&dyn DocumentEmbedder>,
118    result: &IngestResult,
119    machine_id: &str,
120    dedup: ContentDedup,
121) -> StoreOutcome {
122    let mut outcome = StoreOutcome::default();
123
124    if result.memories.is_empty() {
125        return outcome;
126    }
127
128    // Only memories that actually landed can anchor an edge. A deduped or failed
129    // write still has an id in result.entities, and a mentions relation built from
130    // it dangles: the local store has no foreign key to reject it, but the sync
131    // importer does, so the edge can never replicate.
132    let mut stored_ids: HashSet<Uuid> = HashSet::new();
133    let mut conversation_written = false;
134
135    for memory in &result.memories {
136        match store.memory_id_taken(memory.id) {
137            Ok(true) => {
138                outcome.dupes += 1;
139                continue;
140            }
141            Ok(false) => {}
142            Err(e) => {
143                tracing::warn!("ingest: id check failed: {e}");
144                outcome.errors += 1;
145            }
146        }
147        let mut mem = memory.clone();
148        redact_memory(&mut mem);
149        match store.memory_blocked(&mem) {
150            Ok(true) => {
151                outcome.dupes += 1;
152                continue;
153            }
154            Ok(false) => {}
155            Err(e) => {
156                tracing::warn!("ingest: block list check failed: {e}");
157                outcome.errors += 1;
158            }
159        }
160        if dedup == ContentDedup::Prefix {
161            let dedup_prefix: String = mem
162                .content
163                .chars()
164                .take(second_brain_core::store::CONTENT_PREFIX_CHARS)
165                .collect();
166            match store.memory_content_exists(&dedup_prefix) {
167                Ok(true) => {
168                    outcome.dupes += 1;
169                    continue;
170                }
171                Ok(false) => {}
172                Err(e) => {
173                    // store the memory anyway: a duplicate is recoverable, a dropped memory is not
174                    tracing::warn!("ingest: dedup check failed: {e}");
175                    outcome.errors += 1;
176                }
177            }
178        }
179
180        mem.machine_id = machine_id.to_string();
181        if let Some(emb) = embedder {
182            match emb.embed_document(&mem.content) {
183                Ok(embedding) => mem.embedding = embedding,
184                Err(e) => {
185                    // stored without an embedding the memory is invisible to vector
186                    // recall, so this must be loud
187                    tracing::warn!("ingest: embedding failed, storing without vector: {e}");
188                    outcome.errors += 1;
189                }
190            }
191        }
192
193        if !conversation_written {
194            conversation_written = true;
195            if let Err(e) = store.store_conversation_if_absent(&result.conversation) {
196                tracing::warn!("ingest: store_conversation failed: {e}");
197                outcome.errors += 1;
198            }
199        }
200
201        match store.store_memory_if_absent(&mem) {
202            Ok(false) => outcome.dupes += 1,
203            Ok(true) => {
204                stored_ids.insert(mem.id);
205                let relation = Relation {
206                    from_id: mem.id,
207                    to_id: result.conversation.id,
208                    relation_type: RelationType::DerivedFrom,
209                    strength: 1.0,
210                    context: None,
211                };
212                if let Err(e) = store.store_relation(&relation) {
213                    tracing::warn!("ingest: store_relation (derived_from) failed: {e}");
214                    outcome.errors += 1;
215                }
216                outcome.stored += 1;
217            }
218            Err(e) => {
219                tracing::warn!("ingest: store_memory failed: {e}");
220                outcome.errors += 1;
221            }
222        }
223    }
224
225    for (entity_info, memory_id) in &result.entities {
226        if !stored_ids.contains(memory_id) {
227            continue;
228        }
229        let name = redact_for_store(&entity_info.name);
230        match store.find_or_create_entity(&name, &entity_info.entity_type) {
231            Ok(entity) => {
232                let relation = Relation {
233                    from_id: *memory_id,
234                    to_id: entity.id,
235                    relation_type: RelationType::Mentions,
236                    strength: 1.0,
237                    context: None,
238                };
239                if let Err(e) = store.store_relation(&relation) {
240                    tracing::warn!("ingest: store_relation (mentions) failed: {e}");
241                    outcome.errors += 1;
242                } else {
243                    outcome.entities += 1;
244                }
245            }
246            Err(e) => {
247                tracing::warn!("ingest: find_or_create_entity failed for {name}: {e}");
248                outcome.errors += 1;
249            }
250        }
251    }
252
253    outcome
254}