second_brain_adapters/
persist.rs1use std::collections::HashSet;
2
3use anyhow::Result;
4use second_brain_core::embedding::DocumentEmbedder;
5use second_brain_core::redact::{redact_for_store, redact_memory};
6use second_brain_core::schema::{Conversation, Entity, Memory, Relation, RelationType};
7use second_brain_core::store::{Store, StoreAdmin};
8use uuid::Uuid;
9
10use crate::IngestResult;
11
12pub trait IngestStore {
16 fn store_memory(&self, memory: &Memory) -> Result<()>;
17 fn store_conversation(&self, conversation: &Conversation) -> Result<()>;
18 fn store_relation(&self, relation: &Relation) -> Result<()>;
19 fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity>;
20 fn memory_content_exists(&self, content_prefix: &str) -> Result<bool>;
21 fn store_memory_if_absent(&self, memory: &Memory) -> Result<bool> {
23 self.store_memory(memory)?;
24 Ok(true)
25 }
26 fn store_conversation_if_absent(&self, conversation: &Conversation) -> Result<bool> {
28 self.store_conversation(conversation)?;
29 Ok(true)
30 }
31 fn memory_id_taken(&self, _id: Uuid) -> Result<bool> {
33 Ok(false)
34 }
35 fn memory_blocked(&self, _memory: &Memory) -> Result<bool> {
37 Ok(false)
38 }
39}
40
41impl<T: StoreAdmin + ?Sized> IngestStore for T {
42 fn store_memory(&self, memory: &Memory) -> Result<()> {
43 Store::store_memory(self, memory)
44 }
45 fn store_conversation(&self, conversation: &Conversation) -> Result<()> {
46 Store::store_conversation(self, conversation)
47 }
48 fn store_relation(&self, relation: &Relation) -> Result<()> {
49 Store::store_relation(self, relation)
50 }
51 fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity> {
52 StoreAdmin::find_or_create_entity(self, name, entity_type)
53 }
54 fn memory_content_exists(&self, content_prefix: &str) -> Result<bool> {
55 StoreAdmin::memory_content_exists(self, content_prefix)
56 }
57 fn store_memory_if_absent(&self, memory: &Memory) -> Result<bool> {
58 StoreAdmin::ingest_memory_if_absent(self, memory)
59 }
60 fn store_conversation_if_absent(&self, conversation: &Conversation) -> Result<bool> {
61 StoreAdmin::store_conversation_if_absent(self, conversation)
62 }
63 fn memory_id_taken(&self, id: Uuid) -> Result<bool> {
64 StoreAdmin::memory_id_taken(self, id)
65 }
66 fn memory_blocked(&self, memory: &Memory) -> Result<bool> {
67 StoreAdmin::ingest_blocked(self, memory)
68 }
69}
70
71#[derive(Debug, Default, PartialEq, Eq)]
72pub struct StoreOutcome {
73 pub stored: usize,
74 pub dupes: usize,
75 pub entities: usize,
76 pub errors: usize,
77}
78
79pub fn store_ingest_result<S: IngestStore + ?Sized>(
88 store: &S,
89 embedder: Option<&dyn DocumentEmbedder>,
90 result: &IngestResult,
91 machine_id: &str,
92) -> StoreOutcome {
93 store_result(store, embedder, result, machine_id, ContentDedup::Prefix)
94}
95
96pub fn store_document_result<S: IngestStore + ?Sized>(
101 store: &S,
102 embedder: Option<&dyn DocumentEmbedder>,
103 result: &IngestResult,
104 machine_id: &str,
105) -> StoreOutcome {
106 store_result(store, embedder, result, machine_id, ContentDedup::IdOnly)
107}
108
109#[derive(Clone, Copy, PartialEq, Eq)]
110enum ContentDedup {
111 Prefix,
112 IdOnly,
113}
114
115fn store_result<S: IngestStore + ?Sized>(
116 store: &S,
117 embedder: Option<&dyn DocumentEmbedder>,
118 result: &IngestResult,
119 machine_id: &str,
120 dedup: ContentDedup,
121) -> StoreOutcome {
122 let mut outcome = StoreOutcome::default();
123
124 if result.memories.is_empty() {
125 return outcome;
126 }
127
128 let mut stored_ids: HashSet<Uuid> = HashSet::new();
133 let mut conversation_written = false;
134
135 for memory in &result.memories {
136 match store.memory_id_taken(memory.id) {
137 Ok(true) => {
138 outcome.dupes += 1;
139 continue;
140 }
141 Ok(false) => {}
142 Err(e) => {
143 tracing::warn!("ingest: id check failed: {e}");
144 outcome.errors += 1;
145 }
146 }
147 let mut mem = memory.clone();
148 redact_memory(&mut mem);
149 match store.memory_blocked(&mem) {
150 Ok(true) => {
151 outcome.dupes += 1;
152 continue;
153 }
154 Ok(false) => {}
155 Err(e) => {
156 tracing::warn!("ingest: block list check failed: {e}");
157 outcome.errors += 1;
158 }
159 }
160 if dedup == ContentDedup::Prefix {
161 let dedup_prefix: String = mem
162 .content
163 .chars()
164 .take(second_brain_core::store::CONTENT_PREFIX_CHARS)
165 .collect();
166 match store.memory_content_exists(&dedup_prefix) {
167 Ok(true) => {
168 outcome.dupes += 1;
169 continue;
170 }
171 Ok(false) => {}
172 Err(e) => {
173 tracing::warn!("ingest: dedup check failed: {e}");
175 outcome.errors += 1;
176 }
177 }
178 }
179
180 mem.machine_id = machine_id.to_string();
181 if let Some(emb) = embedder {
182 match emb.embed_document(&mem.content) {
183 Ok(embedding) => mem.embedding = embedding,
184 Err(e) => {
185 tracing::warn!("ingest: embedding failed, storing without vector: {e}");
188 outcome.errors += 1;
189 }
190 }
191 }
192
193 if !conversation_written {
194 conversation_written = true;
195 if let Err(e) = store.store_conversation_if_absent(&result.conversation) {
196 tracing::warn!("ingest: store_conversation failed: {e}");
197 outcome.errors += 1;
198 }
199 }
200
201 match store.store_memory_if_absent(&mem) {
202 Ok(false) => outcome.dupes += 1,
203 Ok(true) => {
204 stored_ids.insert(mem.id);
205 let relation = Relation {
206 from_id: mem.id,
207 to_id: result.conversation.id,
208 relation_type: RelationType::DerivedFrom,
209 strength: 1.0,
210 context: None,
211 };
212 if let Err(e) = store.store_relation(&relation) {
213 tracing::warn!("ingest: store_relation (derived_from) failed: {e}");
214 outcome.errors += 1;
215 }
216 outcome.stored += 1;
217 }
218 Err(e) => {
219 tracing::warn!("ingest: store_memory failed: {e}");
220 outcome.errors += 1;
221 }
222 }
223 }
224
225 for (entity_info, memory_id) in &result.entities {
226 if !stored_ids.contains(memory_id) {
227 continue;
228 }
229 let name = redact_for_store(&entity_info.name);
230 match store.find_or_create_entity(&name, &entity_info.entity_type) {
231 Ok(entity) => {
232 let relation = Relation {
233 from_id: *memory_id,
234 to_id: entity.id,
235 relation_type: RelationType::Mentions,
236 strength: 1.0,
237 context: None,
238 };
239 if let Err(e) = store.store_relation(&relation) {
240 tracing::warn!("ingest: store_relation (mentions) failed: {e}");
241 outcome.errors += 1;
242 } else {
243 outcome.entities += 1;
244 }
245 }
246 Err(e) => {
247 tracing::warn!("ingest: find_or_create_entity failed for {name}: {e}");
248 outcome.errors += 1;
249 }
250 }
251 }
252
253 outcome
254}