second_brain_adapters/
persist.rs1use std::collections::HashSet;
2
3use anyhow::Result;
4use second_brain_core::embedding::DocumentEmbedder;
5use second_brain_core::redact::{redact_for_store, redact_memory};
6use second_brain_core::schema::{Conversation, Entity, Memory, Relation, RelationType};
7use second_brain_core::store::{Store, StoreAdmin};
8use uuid::Uuid;
9
10use crate::IngestResult;
11
12pub trait IngestStore {
16 fn store_memory(&self, memory: &Memory) -> Result<()>;
17 fn store_conversation(&self, conversation: &Conversation) -> Result<()>;
18 fn store_relation(&self, relation: &Relation) -> Result<()>;
19 fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity>;
20 fn memory_content_exists(&self, content_prefix: &str) -> Result<bool>;
21 fn store_memory_if_absent(&self, memory: &Memory) -> Result<bool> {
23 self.store_memory(memory)?;
24 Ok(true)
25 }
26 fn store_conversation_if_absent(&self, conversation: &Conversation) -> Result<bool> {
28 self.store_conversation(conversation)?;
29 Ok(true)
30 }
31 fn memory_id_taken(&self, _id: Uuid) -> Result<bool> {
33 Ok(false)
34 }
35}
36
37impl<T: StoreAdmin + ?Sized> IngestStore for T {
38 fn store_memory(&self, memory: &Memory) -> Result<()> {
39 Store::store_memory(self, memory)
40 }
41 fn store_conversation(&self, conversation: &Conversation) -> Result<()> {
42 Store::store_conversation(self, conversation)
43 }
44 fn store_relation(&self, relation: &Relation) -> Result<()> {
45 Store::store_relation(self, relation)
46 }
47 fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity> {
48 StoreAdmin::find_or_create_entity(self, name, entity_type)
49 }
50 fn memory_content_exists(&self, content_prefix: &str) -> Result<bool> {
51 StoreAdmin::memory_content_exists(self, content_prefix)
52 }
53 fn store_memory_if_absent(&self, memory: &Memory) -> Result<bool> {
54 StoreAdmin::store_memory_if_absent(self, memory)
55 }
56 fn store_conversation_if_absent(&self, conversation: &Conversation) -> Result<bool> {
57 StoreAdmin::store_conversation_if_absent(self, conversation)
58 }
59 fn memory_id_taken(&self, id: Uuid) -> Result<bool> {
60 StoreAdmin::memory_id_taken(self, id)
61 }
62}
63
64#[derive(Debug, Default, PartialEq, Eq)]
65pub struct StoreOutcome {
66 pub stored: usize,
67 pub dupes: usize,
68 pub entities: usize,
69 pub errors: usize,
70}
71
72pub fn store_ingest_result<S: IngestStore + ?Sized>(
80 store: &S,
81 embedder: Option<&dyn DocumentEmbedder>,
82 result: &IngestResult,
83 machine_id: &str,
84) -> StoreOutcome {
85 store_result(store, embedder, result, machine_id, ContentDedup::Prefix)
86}
87
88pub fn store_document_result<S: IngestStore + ?Sized>(
93 store: &S,
94 embedder: Option<&dyn DocumentEmbedder>,
95 result: &IngestResult,
96 machine_id: &str,
97) -> StoreOutcome {
98 store_result(store, embedder, result, machine_id, ContentDedup::IdOnly)
99}
100
101#[derive(Clone, Copy, PartialEq, Eq)]
102enum ContentDedup {
103 Prefix,
104 IdOnly,
105}
106
107fn store_result<S: IngestStore + ?Sized>(
108 store: &S,
109 embedder: Option<&dyn DocumentEmbedder>,
110 result: &IngestResult,
111 machine_id: &str,
112 dedup: ContentDedup,
113) -> StoreOutcome {
114 let mut outcome = StoreOutcome::default();
115
116 if result.memories.is_empty() {
117 return outcome;
118 }
119
120 let mut stored_ids: HashSet<Uuid> = HashSet::new();
125 let mut conversation_written = false;
126
127 for memory in &result.memories {
128 match store.memory_id_taken(memory.id) {
129 Ok(true) => {
130 outcome.dupes += 1;
131 continue;
132 }
133 Ok(false) => {}
134 Err(e) => {
135 tracing::warn!("ingest: id check failed: {e}");
136 outcome.errors += 1;
137 }
138 }
139 let mut mem = memory.clone();
140 redact_memory(&mut mem);
141 if dedup == ContentDedup::Prefix {
142 let dedup_prefix: String = mem.content.chars().take(128).collect();
143 match store.memory_content_exists(&dedup_prefix) {
144 Ok(true) => {
145 outcome.dupes += 1;
146 continue;
147 }
148 Ok(false) => {}
149 Err(e) => {
150 tracing::warn!("ingest: dedup check failed: {e}");
152 outcome.errors += 1;
153 }
154 }
155 }
156
157 mem.machine_id = machine_id.to_string();
158 if let Some(emb) = embedder {
159 match emb.embed_document(&mem.content) {
160 Ok(embedding) => mem.embedding = embedding,
161 Err(e) => {
162 tracing::warn!("ingest: embedding failed, storing without vector: {e}");
165 outcome.errors += 1;
166 }
167 }
168 }
169
170 if !conversation_written {
171 conversation_written = true;
172 if let Err(e) = store.store_conversation_if_absent(&result.conversation) {
173 tracing::warn!("ingest: store_conversation failed: {e}");
174 outcome.errors += 1;
175 }
176 }
177
178 match store.store_memory_if_absent(&mem) {
179 Ok(false) => outcome.dupes += 1,
180 Ok(true) => {
181 stored_ids.insert(mem.id);
182 let relation = Relation {
183 from_id: mem.id,
184 to_id: result.conversation.id,
185 relation_type: RelationType::DerivedFrom,
186 strength: 1.0,
187 context: None,
188 };
189 if let Err(e) = store.store_relation(&relation) {
190 tracing::warn!("ingest: store_relation (derived_from) failed: {e}");
191 outcome.errors += 1;
192 }
193 outcome.stored += 1;
194 }
195 Err(e) => {
196 tracing::warn!("ingest: store_memory failed: {e}");
197 outcome.errors += 1;
198 }
199 }
200 }
201
202 for (entity_info, memory_id) in &result.entities {
203 if !stored_ids.contains(memory_id) {
204 continue;
205 }
206 let name = redact_for_store(&entity_info.name);
207 match store.find_or_create_entity(&name, &entity_info.entity_type) {
208 Ok(entity) => {
209 let relation = Relation {
210 from_id: *memory_id,
211 to_id: entity.id,
212 relation_type: RelationType::Mentions,
213 strength: 1.0,
214 context: None,
215 };
216 if let Err(e) = store.store_relation(&relation) {
217 tracing::warn!("ingest: store_relation (mentions) failed: {e}");
218 outcome.errors += 1;
219 } else {
220 outcome.entities += 1;
221 }
222 }
223 Err(e) => {
224 tracing::warn!("ingest: find_or_create_entity failed for {name}: {e}");
225 outcome.errors += 1;
226 }
227 }
228 }
229
230 outcome
231}