Skip to main content

second_brain_adapters/
persist.rs

1use std::collections::HashSet;
2
3use anyhow::Result;
4use second_brain_core::embedding::DocumentEmbedder;
5use second_brain_core::schema::{Conversation, Entity, Memory, Relation, RelationType};
6use second_brain_core::store::{Store, StoreAdmin};
7use uuid::Uuid;
8
9use crate::IngestResult;
10
11/// The store surface `store_ingest_result` writes through. Every
12/// `StoreAdmin` has it through the blanket impl below; tests implement it
13/// directly to inject failing writes.
14pub trait IngestStore {
15    fn store_memory(&self, memory: &Memory) -> Result<()>;
16    fn store_conversation(&self, conversation: &Conversation) -> Result<()>;
17    fn store_relation(&self, relation: &Relation) -> Result<()>;
18    fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity>;
19    fn memory_content_exists(&self, content_prefix: &str) -> Result<bool>;
20}
21
22impl<T: StoreAdmin + ?Sized> IngestStore for T {
23    fn store_memory(&self, memory: &Memory) -> Result<()> {
24        Store::store_memory(self, memory)
25    }
26    fn store_conversation(&self, conversation: &Conversation) -> Result<()> {
27        Store::store_conversation(self, conversation)
28    }
29    fn store_relation(&self, relation: &Relation) -> Result<()> {
30        Store::store_relation(self, relation)
31    }
32    fn find_or_create_entity(&self, name: &str, entity_type: &str) -> Result<Entity> {
33        StoreAdmin::find_or_create_entity(self, name, entity_type)
34    }
35    fn memory_content_exists(&self, content_prefix: &str) -> Result<bool> {
36        StoreAdmin::memory_content_exists(self, content_prefix)
37    }
38}
39
40#[derive(Debug, Default, PartialEq, Eq)]
41pub struct StoreOutcome {
42    pub stored: usize,
43    pub dupes: usize,
44    pub entities: usize,
45    pub errors: usize,
46}
47
48/// Persists one adapter result: its conversation, every memory whose first
49/// 128 characters are not already stored (embedded first when an embedder is
50/// given), a derived_from edge per stored memory, and a mentions edge per
51/// entity whose memory actually landed. Write failures are counted in the
52/// outcome rather than returned, so one bad row does not cost the rest.
53pub fn store_ingest_result<S: IngestStore + ?Sized>(
54    store: &S,
55    embedder: Option<&dyn DocumentEmbedder>,
56    result: &IngestResult,
57    machine_id: &str,
58) -> StoreOutcome {
59    let mut outcome = StoreOutcome::default();
60
61    if result.memories.is_empty() {
62        return outcome;
63    }
64
65    // Only memories that actually landed can anchor an edge. A deduped or failed
66    // write still has an id in result.entities, and a mentions relation built from
67    // it dangles: the local store has no foreign key to reject it, but the sync
68    // importer does, so the edge can never replicate.
69    let mut stored_ids: HashSet<Uuid> = HashSet::new();
70
71    if let Err(e) = store.store_conversation(&result.conversation) {
72        tracing::warn!("ingest: store_conversation failed: {e}");
73        outcome.errors += 1;
74    }
75
76    for memory in &result.memories {
77        let dedup_prefix: String = memory.content.chars().take(128).collect();
78        match store.memory_content_exists(&dedup_prefix) {
79            Ok(true) => {
80                outcome.dupes += 1;
81                continue;
82            }
83            Ok(false) => {}
84            Err(e) => {
85                // store the memory anyway: a duplicate is recoverable, a dropped memory is not
86                tracing::warn!("ingest: dedup check failed: {e}");
87                outcome.errors += 1;
88            }
89        }
90
91        let mut mem = memory.clone();
92        mem.machine_id = machine_id.to_string();
93        if let Some(emb) = embedder {
94            match emb.embed_document(&mem.content) {
95                Ok(embedding) => mem.embedding = embedding,
96                Err(e) => {
97                    // stored without an embedding the memory is invisible to vector
98                    // recall, so this must be loud
99                    tracing::warn!("ingest: embedding failed, storing without vector: {e}");
100                    outcome.errors += 1;
101                }
102            }
103        }
104
105        match store.store_memory(&mem) {
106            Ok(()) => {
107                stored_ids.insert(mem.id);
108                let relation = Relation {
109                    from_id: mem.id,
110                    to_id: result.conversation.id,
111                    relation_type: RelationType::DerivedFrom,
112                    strength: 1.0,
113                    context: None,
114                };
115                if let Err(e) = store.store_relation(&relation) {
116                    tracing::warn!("ingest: store_relation (derived_from) failed: {e}");
117                    outcome.errors += 1;
118                }
119                outcome.stored += 1;
120            }
121            Err(e) => {
122                tracing::warn!("ingest: store_memory failed: {e}");
123                outcome.errors += 1;
124            }
125        }
126    }
127
128    for (entity_info, memory_id) in &result.entities {
129        if !stored_ids.contains(memory_id) {
130            continue;
131        }
132        match store.find_or_create_entity(&entity_info.name, &entity_info.entity_type) {
133            Ok(entity) => {
134                let relation = Relation {
135                    from_id: *memory_id,
136                    to_id: entity.id,
137                    relation_type: RelationType::Mentions,
138                    strength: 1.0,
139                    context: None,
140                };
141                if let Err(e) = store.store_relation(&relation) {
142                    tracing::warn!("ingest: store_relation (mentions) failed: {e}");
143                    outcome.errors += 1;
144                } else {
145                    outcome.entities += 1;
146                }
147            }
148            Err(e) => {
149                tracing::warn!(
150                    "ingest: find_or_create_entity failed for {}: {e}",
151                    entity_info.name
152                );
153                outcome.errors += 1;
154            }
155        }
156    }
157
158    outcome
159}