second_brain_adapters/
persist.rs1use std::collections::HashSet;
2
3use anyhow::Result;
4use second_brain_core::embedding::DocumentEmbedder;
5use second_brain_core::redact::{redact_for_store, redact_memory};
6use second_brain_core::schema::{Conversation, Entity, Memory, Relation, RelationType};
7use second_brain_core::store::{Store, StoreAdmin};
8use uuid::Uuid;
9
10use crate::IngestResult;
11
12pub trait IngestStore {
16 fn store_memory(&self, memory: &Memory) -> Result<()>;
17 fn store_conversation(&self, conversation: &Conversation) -> Result<()>;
18 fn store_relation(&self, relation: &Relation) -> Result<()>;
19 fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity>;
20 fn memory_content_exists(&self, content_prefix: &str) -> Result<bool>;
21 fn store_memory_if_absent(&self, memory: &Memory) -> Result<bool> {
23 self.store_memory(memory)?;
24 Ok(true)
25 }
26 fn store_conversation_if_absent(&self, conversation: &Conversation) -> Result<bool> {
28 self.store_conversation(conversation)?;
29 Ok(true)
30 }
31 fn memory_id_taken(&self, _id: Uuid) -> Result<bool> {
33 Ok(false)
34 }
35}
36
37impl<T: StoreAdmin + ?Sized> IngestStore for T {
38 fn store_memory(&self, memory: &Memory) -> Result<()> {
39 Store::store_memory(self, memory)
40 }
41 fn store_conversation(&self, conversation: &Conversation) -> Result<()> {
42 Store::store_conversation(self, conversation)
43 }
44 fn store_relation(&self, relation: &Relation) -> Result<()> {
45 Store::store_relation(self, relation)
46 }
47 fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity> {
48 StoreAdmin::find_or_create_entity(self, name, entity_type)
49 }
50 fn memory_content_exists(&self, content_prefix: &str) -> Result<bool> {
51 StoreAdmin::memory_content_exists(self, content_prefix)
52 }
53 fn store_memory_if_absent(&self, memory: &Memory) -> Result<bool> {
54 StoreAdmin::store_memory_if_absent(self, memory)
55 }
56 fn store_conversation_if_absent(&self, conversation: &Conversation) -> Result<bool> {
57 StoreAdmin::store_conversation_if_absent(self, conversation)
58 }
59 fn memory_id_taken(&self, id: Uuid) -> Result<bool> {
60 StoreAdmin::memory_id_taken(self, id)
61 }
62}
63
64#[derive(Debug, Default, PartialEq, Eq)]
65pub struct StoreOutcome {
66 pub stored: usize,
67 pub dupes: usize,
68 pub entities: usize,
69 pub errors: usize,
70}
71
72pub fn store_ingest_result<S: IngestStore + ?Sized>(
80 store: &S,
81 embedder: Option<&dyn DocumentEmbedder>,
82 result: &IngestResult,
83 machine_id: &str,
84) -> StoreOutcome {
85 let mut outcome = StoreOutcome::default();
86
87 if result.memories.is_empty() {
88 return outcome;
89 }
90
91 let mut stored_ids: HashSet<Uuid> = HashSet::new();
96 let mut conversation_written = false;
97
98 for memory in &result.memories {
99 match store.memory_id_taken(memory.id) {
100 Ok(true) => {
101 outcome.dupes += 1;
102 continue;
103 }
104 Ok(false) => {}
105 Err(e) => {
106 tracing::warn!("ingest: id check failed: {e}");
107 outcome.errors += 1;
108 }
109 }
110 let mut mem = memory.clone();
111 redact_memory(&mut mem);
112 let dedup_prefix: String = mem.content.chars().take(128).collect();
113 match store.memory_content_exists(&dedup_prefix) {
114 Ok(true) => {
115 outcome.dupes += 1;
116 continue;
117 }
118 Ok(false) => {}
119 Err(e) => {
120 tracing::warn!("ingest: dedup check failed: {e}");
122 outcome.errors += 1;
123 }
124 }
125
126 mem.machine_id = machine_id.to_string();
127 if let Some(emb) = embedder {
128 match emb.embed_document(&mem.content) {
129 Ok(embedding) => mem.embedding = embedding,
130 Err(e) => {
131 tracing::warn!("ingest: embedding failed, storing without vector: {e}");
134 outcome.errors += 1;
135 }
136 }
137 }
138
139 if !conversation_written {
140 conversation_written = true;
141 if let Err(e) = store.store_conversation_if_absent(&result.conversation) {
142 tracing::warn!("ingest: store_conversation failed: {e}");
143 outcome.errors += 1;
144 }
145 }
146
147 match store.store_memory_if_absent(&mem) {
148 Ok(false) => outcome.dupes += 1,
149 Ok(true) => {
150 stored_ids.insert(mem.id);
151 let relation = Relation {
152 from_id: mem.id,
153 to_id: result.conversation.id,
154 relation_type: RelationType::DerivedFrom,
155 strength: 1.0,
156 context: None,
157 };
158 if let Err(e) = store.store_relation(&relation) {
159 tracing::warn!("ingest: store_relation (derived_from) failed: {e}");
160 outcome.errors += 1;
161 }
162 outcome.stored += 1;
163 }
164 Err(e) => {
165 tracing::warn!("ingest: store_memory failed: {e}");
166 outcome.errors += 1;
167 }
168 }
169 }
170
171 for (entity_info, memory_id) in &result.entities {
172 if !stored_ids.contains(memory_id) {
173 continue;
174 }
175 let name = redact_for_store(&entity_info.name);
176 match store.find_or_create_entity(&name, &entity_info.entity_type) {
177 Ok(entity) => {
178 let relation = Relation {
179 from_id: *memory_id,
180 to_id: entity.id,
181 relation_type: RelationType::Mentions,
182 strength: 1.0,
183 context: None,
184 };
185 if let Err(e) = store.store_relation(&relation) {
186 tracing::warn!("ingest: store_relation (mentions) failed: {e}");
187 outcome.errors += 1;
188 } else {
189 outcome.entities += 1;
190 }
191 }
192 Err(e) => {
193 tracing::warn!("ingest: find_or_create_entity failed for {name}: {e}");
194 outcome.errors += 1;
195 }
196 }
197 }
198
199 outcome
200}