Skip to main content

second_brain_api/
raw_index.rs

1use std::collections::{HashMap, HashSet};
2use std::path::Path;
3use std::sync::atomic::{AtomicUsize, Ordering};
4use std::sync::{Arc, Mutex, RwLock};
5use std::time::Duration;
6
7use anyhow::Result;
8use chrono::{DateTime, Utc};
9use second_brain_adapters::raw_text::raw_passages;
10use second_brain_core::index::EMBEDDING_DIM;
11use second_brain_core::index::raw::{RawIndex, RawOrigin, RawPassage};
12use second_brain_core::project_path;
13use second_brain_core::residency;
14use second_brain_core::sqlite_store::SqliteStore;
15use second_brain_core::store::Store;
16use second_brain_core::transcript_archive::{ArchiveEntry, TranscriptArchive};
17use uuid::Uuid;
18
19const EMBED_BATCH: usize = 8;
20const COMMIT_EVERY: usize = 20;
21/// Embedding sessions are handed out without fairness, so without this gap
22/// the indexer takes the session straight back and an ingest or recall
23/// waiting for it times out.
24pub const PAUSE_AFTER_EMBED: Duration = Duration::from_millis(100);
25const PASS_INTERVAL: Duration = Duration::from_secs(600);
26
27/// Opens the raw index, or rebuilds it from scratch when its files cannot
28/// be read as an index, for example after a schema change: everything in it
29/// is derived from the transcript archive and the store, so the next pass
30/// refills it. An I/O error opening the directories themselves is returned
31/// instead; tantivy and usearch report their own I/O faults without an I/O
32/// cause, so those still rebuild.
33pub fn open_or_rebuild(dir: &Path) -> Result<RawIndex> {
34    match RawIndex::open(dir) {
35        Ok(index) => Ok(index),
36        Err(e)
37            if e.chain()
38                .any(|c| c.downcast_ref::<std::io::Error>().is_some()) =>
39        {
40            Err(e)
41        }
42        Err(e) => {
43            tracing::warn!(
44                "raw index at {} cannot be opened, rebuilding it: {e:#}",
45                dir.display()
46            );
47            std::fs::remove_dir_all(dir)?;
48            RawIndex::open(dir)
49        }
50    }
51}
52
53pub type EmbedBatch = dyn Fn(&[&str]) -> Result<Vec<Vec<f32>>> + Send + Sync;
54
55#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
56pub struct PassSummary {
57    pub transcripts: usize,
58    pub memories: usize,
59    pub passages: usize,
60    pub restored: usize,
61    /// Sources that could not be read or embedded; they are retried next pass.
62    pub failed: usize,
63    /// Passages left out because their source is older than the age bound.
64    pub skipped_by_age: usize,
65}
66
67/// The age bound an indexer works to and what its last pass left out,
68/// shared with status. It also remembers when each transcript left out
69/// was last active, so later passes skip it without reading it again.
70pub struct RawCoverage {
71    max_age_days: u32,
72    skipped_by_age: AtomicUsize,
73    aged: Mutex<HashMap<String, Dated>>,
74}
75
76#[derive(Clone, Copy)]
77struct Dated {
78    newest: DateTime<Utc>,
79    passages: usize,
80}
81
82impl RawCoverage {
83    pub fn new(max_age_days: u32) -> Self {
84        Self {
85            max_age_days,
86            skipped_by_age: AtomicUsize::new(0),
87            aged: Mutex::new(HashMap::new()),
88        }
89    }
90
91    pub fn max_age_days(&self) -> u32 {
92        self.max_age_days
93    }
94
95    /// Passages the last finished pass left out for age.
96    pub fn skipped_by_age(&self) -> usize {
97        self.skipped_by_age.load(Ordering::Relaxed)
98    }
99
100    fn cutoff(&self) -> DateTime<Utc> {
101        Utc::now()
102            .checked_sub_signed(chrono::Duration::days(i64::from(self.max_age_days)))
103            .unwrap_or(DateTime::<Utc>::MIN_UTC)
104    }
105
106    fn aged(&self) -> std::sync::MutexGuard<'_, HashMap<String, Dated>> {
107        self.aged.lock().unwrap_or_else(|p| p.into_inner())
108    }
109}
110
111/// Feeds the raw fallback index from the transcript archive and from raw
112/// memories consolidation archived, embedding outside the index lock so
113/// recall is never held behind the model.
114pub struct RawIndexer {
115    pub index: Arc<RwLock<RawIndex>>,
116    pub archive: Option<Arc<TranscriptArchive>>,
117    pub store: Arc<SqliteStore>,
118    pub embed: Arc<EmbedBatch>,
119    pub pause: Duration,
120    pub coverage: Arc<RawCoverage>,
121}
122
123impl RawIndexer {
124    pub fn spawn(self) {
125        std::thread::Builder::new()
126            .name("raw-index".to_string())
127            .spawn(move || {
128                loop {
129                    match self.pass() {
130                        Ok(s)
131                            if PassSummary {
132                                skipped_by_age: 0,
133                                ..s
134                            } != PassSummary::default() =>
135                        {
136                            tracing::info!(
137                                "raw index: {} transcripts, {} archived memories, {} passages, {} vectors restored, {} sources failed, {} passages older than {} days skipped",
138                                s.transcripts,
139                                s.memories,
140                                s.passages,
141                                s.restored,
142                                s.failed,
143                                s.skipped_by_age,
144                                self.coverage.max_age_days()
145                            );
146                            residency::release_free_heap();
147                        }
148                        Ok(_) => {}
149                        Err(e) => {
150                            tracing::warn!("raw index pass failed: {e:#}");
151                            residency::release_free_heap();
152                        }
153                    }
154                    std::thread::sleep(PASS_INTERVAL);
155                }
156            })
157            .expect("spawning the raw index thread");
158    }
159
160    /// Indexes every source not indexed yet that is within the age bound,
161    /// newest first. A source that fails is counted, skipped and retried on
162    /// the next pass, so one bad blob or a passing embedder error never stops
163    /// the rest.
164    pub fn pass(&self) -> Result<PassSummary> {
165        let mut summary = PassSummary::default();
166        let mut first_error = None;
167        let mut pending = 0;
168        let cutoff = self.coverage.cutoff();
169        if let Some(archive) = &self.archive {
170            let mut queue = Vec::new();
171            let mut aged = HashMap::new();
172            for entry in archive.entries() {
173                let origin = RawOrigin::Transcript {
174                    sha256: entry.sha256.clone(),
175                };
176                if self.indexed(&origin) {
177                    continue;
178                }
179                match self.date_transcript(archive, &entry) {
180                    Ok(dated) if dated.newest < cutoff => {
181                        summary.skipped_by_age += dated.passages;
182                        aged.insert(entry.sha256.clone(), dated);
183                    }
184                    Ok(dated) => queue.push((dated.newest, entry, origin)),
185                    Err(e) => {
186                        summary.failed += 1;
187                        first_error.get_or_insert(e);
188                    }
189                }
190            }
191            *self.coverage.aged() = aged;
192            queue.sort_by_key(|q| std::cmp::Reverse(q.0));
193            for (_, entry, origin) in queue {
194                match self.add_transcript(archive, &entry, &origin) {
195                    Ok(added) => {
196                        summary.passages += added;
197                        summary.transcripts += 1;
198                        pending = self.commit_every(pending + 1)?;
199                    }
200                    Err(e) => {
201                        summary.failed += 1;
202                        first_error.get_or_insert(e);
203                    }
204                }
205            }
206        }
207
208        let raws = self.store.archived_raws_since(cutoff)?;
209        summary.skipped_by_age += raws
210            .older
211            .into_iter()
212            .filter(|&id| !self.indexed(&RawOrigin::Memory { id }))
213            .count();
214        for id in raws.recent {
215            let origin = RawOrigin::Memory { id };
216            if self.indexed(&origin) {
217                continue;
218            }
219            match self.add_memory(id, &origin) {
220                Ok((added, restored)) => {
221                    summary.passages += added;
222                    summary.restored += usize::from(restored);
223                    summary.memories += 1;
224                    pending = self.commit_every(pending + 1)?;
225                }
226                Err(e) => {
227                    summary.failed += 1;
228                    first_error.get_or_insert(e);
229                }
230            }
231        }
232        if pending > 0 {
233            self.write().commit()?;
234        }
235        self.coverage
236            .skipped_by_age
237            .store(summary.skipped_by_age, Ordering::Relaxed);
238        if let Some(e) = first_error {
239            tracing::warn!(
240                "raw index: {} sources failed and will be retried, first: {e:#}",
241                summary.failed
242            );
243        }
244        Ok(summary)
245    }
246
247    /// When a transcript was last active: its newest passage time, or when
248    /// it was archived if no line carries one. A transcript left out for age
249    /// is dated from memory rather than read again.
250    fn date_transcript(&self, archive: &TranscriptArchive, entry: &ArchiveEntry) -> Result<Dated> {
251        if let Some(dated) = self.coverage.aged().get(&entry.sha256) {
252            return Ok(*dated);
253        }
254        let source = Path::new(&entry.source);
255        let passages = match archive.read(source)? {
256            Some(content) => raw_passages(source, &content),
257            None => Vec::new(),
258        };
259        Ok(Dated {
260            newest: passages
261                .iter()
262                .filter_map(|p| p.at)
263                .max()
264                .unwrap_or(entry.archived_at),
265            passages: passages.len(),
266        })
267    }
268
269    fn add_transcript(
270        &self,
271        archive: &TranscriptArchive,
272        entry: &ArchiveEntry,
273        origin: &RawOrigin,
274    ) -> Result<usize> {
275        let source = Path::new(&entry.source);
276        let Some(content) = archive.read(source)? else {
277            return self.add(origin, Vec::new());
278        };
279        let project = project_path::from_transcript(&entry.source);
280        let passages = raw_passages(source, &content)
281            .into_iter()
282            .map(|p| RawPassage {
283                text: p.text,
284                at: p.at,
285                project: project.clone(),
286                memory: p.memory,
287            })
288            .collect();
289        self.add(origin, passages)
290    }
291
292    fn add_memory(&self, id: Uuid, origin: &RawOrigin) -> Result<(usize, bool)> {
293        let Some(memory) = self.store.get_memory_with_embedding(id)? else {
294            return Ok((0, false));
295        };
296        let mut restored = false;
297        let embedding = if memory.embedding.len() == EMBEDDING_DIM {
298            memory.embedding.clone()
299        } else {
300            let embedding = self
301                .embed_and_yield(&[memory.content.as_str()])?
302                .into_iter()
303                .next()
304                .unwrap_or_default();
305            if embedding.len() == EMBEDDING_DIM {
306                restored = self.store.restore_archived_embedding(id, &embedding)?;
307            }
308            embedding
309        };
310        let passage = RawPassage {
311            text: memory.content.clone(),
312            at: memory.session_at,
313            project: memory.project_path.clone(),
314            memory: Some(id),
315        };
316        let added = self.write().add_source(origin, &[(passage, embedding)])?;
317        Ok((added, restored))
318    }
319
320    /// Embeds only passages whose text is not indexed yet, so a transcript
321    /// archived again as it grows costs only its new text.
322    fn add(&self, origin: &RawOrigin, passages: Vec<RawPassage>) -> Result<usize> {
323        let passages: Vec<RawPassage> = {
324            let index = self.read();
325            let mut seen = HashSet::new();
326            passages
327                .into_iter()
328                .filter(|p| seen.insert(p.text.trim().to_string()) && !index.has_text(&p.text))
329                .collect()
330        };
331        let mut embedded = Vec::with_capacity(passages.len());
332        for batch in passages.chunks(EMBED_BATCH) {
333            let texts: Vec<&str> = batch.iter().map(|p| p.text.as_str()).collect();
334            let vectors = self.embed_and_yield(&texts)?;
335            embedded.extend(batch.iter().cloned().zip(vectors));
336        }
337        self.write().add_source(origin, &embedded)
338    }
339
340    fn embed_and_yield(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
341        let vectors = (self.embed)(texts);
342        std::thread::sleep(self.pause);
343        vectors
344    }
345
346    fn commit_every(&self, pending: usize) -> Result<usize> {
347        if pending < COMMIT_EVERY {
348            return Ok(pending);
349        }
350        self.write().commit()?;
351        Ok(0)
352    }
353
354    fn indexed(&self, origin: &RawOrigin) -> bool {
355        self.read().has_source(origin)
356    }
357
358    fn read(&self) -> std::sync::RwLockReadGuard<'_, RawIndex> {
359        self.index.read().unwrap_or_else(|p| p.into_inner())
360    }
361
362    fn write(&self) -> std::sync::RwLockWriteGuard<'_, RawIndex> {
363        self.index.write().unwrap_or_else(|p| p.into_inner())
364    }
365}