Skip to main content

second_brain_api/
raw_index.rs

1use std::collections::{HashMap, HashSet};
2use std::path::Path;
3use std::sync::atomic::{AtomicUsize, Ordering};
4use std::sync::{Arc, Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard};
5use std::time::Duration;
6
7use anyhow::Result;
8use chrono::{DateTime, Utc};
9use second_brain_adapters::raw_text::raw_passages;
10use second_brain_core::index::EMBEDDING_DIM;
11use second_brain_core::index::raw::{RawHit, RawIndex, RawOrigin, RawPassage, text_hash};
12use second_brain_core::project_path;
13use second_brain_core::residency;
14use second_brain_core::sealed::{SealError, SealKeys, seal_error};
15use second_brain_core::sqlite_store::SqliteStore;
16use second_brain_core::transcript_archive::{ArchiveEntry, TranscriptArchive};
17use uuid::Uuid;
18
19const EMBED_BATCH: usize = 8;
20const COMMIT_EVERY: usize = 20;
21/// Embedding sessions are handed out without fairness, so without this gap
22/// the indexer takes the session straight back and an ingest or recall
23/// waiting for it times out.
24pub const PAUSE_AFTER_EMBED: Duration = Duration::from_millis(100);
25const PASS_INTERVAL: Duration = Duration::from_secs(600);
26
27/// Opens the raw index, or rebuilds it from scratch when its files cannot
28/// be read as an index, for example after a schema change: everything in it
29/// is derived from the transcript archive and the store, so the next pass
30/// refills it. An I/O error opening the directories themselves is returned
31/// instead; tantivy and usearch report their own I/O faults without an I/O
32/// cause, so those still rebuild.
33pub fn open_or_rebuild(dir: &Path) -> Result<RawIndex> {
34    open_or_rebuild_keyed(dir, None)
35}
36
37/// `open_or_rebuild` with the index sealed at rest under `keys` when given.
38/// Files sealed under a key not given, or sealed when no keys are, cannot be
39/// opened, so the index is rebuilt like one with an unreadable schema.
40pub fn open_or_rebuild_keyed(dir: &Path, keys: Option<&SealKeys>) -> Result<RawIndex> {
41    let open = || match keys {
42        Some(keys) => RawIndex::open_keyed(dir, keys),
43        None => RawIndex::open(dir),
44    };
45    match open() {
46        Ok(index) => Ok(index),
47        Err(e) if seal_error(&e) == Some(&SealError::KeyRequired) => Err(e),
48        Err(e)
49            if seal_error(&e).is_none()
50                && e.chain()
51                    .any(|c| c.downcast_ref::<std::io::Error>().is_some()) =>
52        {
53            Err(e)
54        }
55        Err(e) => {
56            tracing::warn!(
57                "raw index at {} cannot be opened, rebuilding it: {e:#}",
58                dir.display()
59            );
60            std::fs::remove_dir_all(dir)?;
61            open()
62        }
63    }
64}
65
66pub type EmbedBatch = dyn Fn(&[&str]) -> Result<Vec<Vec<f32>>> + Send + Sync;
67
68#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
69pub struct PassSummary {
70    pub transcripts: usize,
71    pub memories: usize,
72    pub passages: usize,
73    pub restored: usize,
74    /// Sources that could not be read or embedded; they are retried next pass.
75    pub failed: usize,
76    /// Passages left out because their source is older than the age bound.
77    pub skipped_by_age: usize,
78    /// Passages withdrawn because what they came from was deleted.
79    pub purged: usize,
80}
81
82/// The age bound an indexer works to and what its last pass left out,
83/// shared with status. It also remembers when each transcript left out
84/// was last active, so later passes skip it without reading it again.
85pub struct RawCoverage {
86    max_age_days: u32,
87    skipped_by_age: AtomicUsize,
88    aged: Mutex<HashMap<String, Dated>>,
89}
90
91#[derive(Clone, Copy)]
92struct Dated {
93    newest: DateTime<Utc>,
94    passages: usize,
95}
96
97impl RawCoverage {
98    pub fn new(max_age_days: u32) -> Self {
99        Self {
100            max_age_days,
101            skipped_by_age: AtomicUsize::new(0),
102            aged: Mutex::new(HashMap::new()),
103        }
104    }
105
106    pub fn max_age_days(&self) -> u32 {
107        self.max_age_days
108    }
109
110    /// Passages the last finished pass left out for age.
111    pub fn skipped_by_age(&self) -> usize {
112        self.skipped_by_age.load(Ordering::Relaxed)
113    }
114
115    fn cutoff(&self) -> DateTime<Utc> {
116        Utc::now()
117            .checked_sub_signed(chrono::Duration::days(i64::from(self.max_age_days)))
118            .unwrap_or(DateTime::<Utc>::MIN_UTC)
119    }
120
121    fn aged(&self) -> std::sync::MutexGuard<'_, HashMap<String, Dated>> {
122        self.aged.lock().unwrap_or_else(|p| p.into_inner())
123    }
124}
125
126/// What a purge removed from the raw index and the transcript archive.
127#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
128pub struct RawPurge {
129    pub passages: usize,
130    pub archive_entries: usize,
131}
132
133static PURGES: Mutex<()> = Mutex::new(());
134
135/// Withdraws from the raw index every passage of a tombstoned memory, of an
136/// archived transcript whose session was deleted, or with the text of one
137/// of that transcript's lines (an older copy of a grown transcript holds
138/// them under another source), then drops those transcripts from the
139/// archive. The index stores each text once, so a passage whose text a kept
140/// transcript still holds is filed under that transcript instead. `memories`
141/// are ids the caller just deleted, checked without a store lookup. The
142/// archive is read, and the vector segments and text files rewritten,
143/// without the index's write lock, so recall keeps answering; the lock is
144/// held from the last check for doomed passages until the doomed
145/// transcripts leave the archive, so an indexer pass cannot add one back in
146/// between.
147pub fn purge_deleted(
148    index: Option<&RwLock<RawIndex>>,
149    archive: Option<&TranscriptArchive>,
150    store: &SqliteStore,
151    memories: &[Uuid],
152) -> Result<RawPurge> {
153    let _one_at_a_time = PURGES.lock().unwrap_or_else(|p| p.into_inner());
154    let doomed = match archive {
155        Some(archive) => doomed_entries(archive, store)?,
156        None => Vec::new(),
157    };
158    let mut gone = Gone::new(store, memories);
159    match index {
160        Some(index) => purge_index(index, archive, &doomed, &mut gone),
161        None => Ok(RawPurge {
162            passages: 0,
163            archive_entries: match archive {
164                Some(archive) => remove_entries(archive, &doomed)?,
165                None => 0,
166            },
167        }),
168    }
169}
170
171fn doomed_entries(archive: &TranscriptArchive, store: &SqliteStore) -> Result<Vec<ArchiveEntry>> {
172    let mut doomed = Vec::new();
173    for entry in archive.entries() {
174        if store.session_deleted(&entry.source)? {
175            doomed.push(entry);
176        }
177    }
178    Ok(doomed)
179}
180
181fn purge_index(
182    lock: &RwLock<RawIndex>,
183    archive: Option<&TranscriptArchive>,
184    doomed: &[ArchiveEntry],
185    gone: &mut Gone<'_>,
186) -> Result<RawPurge> {
187    let (memories, recorded_sources) = {
188        let index = read_lock(lock);
189        (index.passage_memories()?, index.sources()?)
190    };
191    let live = live_entries(archive, doomed);
192    let live_shas: HashSet<&str> = live.iter().map(|e| e.sha256.as_str()).collect();
193    let mut stale = Vec::new();
194    for id in memories {
195        if gone.check(id)? {
196            stale.push(id);
197        }
198    }
199    let mut sources: Vec<RawOrigin> = doomed
200        .iter()
201        .filter(|e| !live_shas.contains(e.sha256.as_str()))
202        .map(|e| RawOrigin::Transcript {
203            sha256: e.sha256.clone(),
204        })
205        .collect();
206    for origin in recorded_sources {
207        if let RawOrigin::Memory { id } = origin
208            && gone.check(id)?
209        {
210            sources.push(origin);
211        }
212    }
213    if doomed.is_empty() && sources.is_empty() && stale.is_empty() {
214        return Ok(RawPurge::default());
215    }
216    let texts = transcript_texts(archive, doomed)?;
217    let texts: Vec<&str> = texts.iter().map(String::as_str).collect();
218    let doomed_sources: HashSet<String> = sources.iter().map(RawOrigin::source_key).collect();
219
220    let (candidates, recorded) = {
221        let index = read_lock(lock);
222        (
223            index.matching(&sources, &stale, &texts)?,
224            recorded_entries(&index, &live),
225        )
226    };
227    let mut holders = Holders::default();
228    if let Some(archive) = archive
229        && !candidates.is_empty()
230    {
231        holders.read(archive, &recorded, &candidates, gone)?;
232    }
233
234    let mut index = write_lock(lock);
235    index.commit_without_merging()?;
236    let current = index.matching(&sources, &stale, &texts)?;
237    if let Some(archive) = archive
238        && !current.is_empty()
239    {
240        let late: Vec<ArchiveEntry> = recorded_entries(&index, &live)
241            .into_iter()
242            .filter(|e| !holders.has_read(e))
243            .collect();
244        holders.read(archive, &late, &current, gone)?;
245    }
246    let mut dropped = Vec::new();
247    for hit in &current {
248        let memory_gone = match hit.memory {
249            Some(id) => gone.check(id)?,
250            None => false,
251        };
252        let memory_holder = match hit.memory {
253            Some(id) => !memory_gone && index.has_source(&RawOrigin::Memory { id }),
254            None => false,
255        };
256        match fate(hit, &doomed_sources, &holders, memory_gone, memory_holder) {
257            Fate::Keep => {}
258            Fate::Move(holder) => index.rekey(
259                hit.key,
260                &holder.origin,
261                &RawPassage {
262                    text: holder.text,
263                    at: holder.at,
264                    project: holder.project,
265                    memory: holder.memory,
266                },
267            )?,
268            Fate::Drop => dropped.push(hit.key),
269        }
270    }
271    index.withdraw(&dropped)?;
272    index.forget_sources(&sources)?;
273    let rewrite = index.seal_for_rewrite(&dropped)?;
274    index.commit_without_merging()?;
275    let archive_entries = match archive {
276        Some(archive) => remove_entries(archive, doomed)?,
277        None => 0,
278    };
279    let compaction = index.start_compaction()?;
280    drop(index);
281
282    let rewritten = rewrite.run();
283    let merged = compaction.wait();
284    let mut index = write_lock(lock);
285    index.reload_sealed(&rewrite)?;
286    index.finish_compaction(merged)?;
287    rewritten?;
288    Ok(RawPurge {
289        passages: dropped.len(),
290        archive_entries,
291    })
292}
293
294fn live_entries(archive: Option<&TranscriptArchive>, doomed: &[ArchiveEntry]) -> Vec<ArchiveEntry> {
295    let Some(archive) = archive else {
296        return Vec::new();
297    };
298    let doomed: HashSet<&str> = doomed.iter().map(|e| e.source.as_str()).collect();
299    archive
300        .entries()
301        .into_iter()
302        .filter(|e| !doomed.contains(e.source.as_str()))
303        .collect()
304}
305
306fn transcript_texts(
307    archive: Option<&TranscriptArchive>,
308    entries: &[ArchiveEntry],
309) -> Result<Vec<String>> {
310    let Some(archive) = archive else {
311        return Ok(Vec::new());
312    };
313    let mut texts = Vec::new();
314    for entry in entries {
315        let source = Path::new(&entry.source);
316        if let Some(content) = archive.read(source)? {
317            texts.extend(raw_passages(source, &content).into_iter().map(|p| p.text));
318        }
319    }
320    Ok(texts)
321}
322
323/// The entries whose blob the index records as a source. Only these can
324/// take over a passage: filing one under a transcript not indexed yet would
325/// mark that transcript indexed and the rest of it would never be read.
326fn recorded_entries(index: &RawIndex, entries: &[ArchiveEntry]) -> Vec<ArchiveEntry> {
327    entries
328        .iter()
329        .filter(|e| {
330            index.has_source(&RawOrigin::Transcript {
331                sha256: e.sha256.clone(),
332            })
333        })
334        .cloned()
335        .collect()
336}
337
338/// Whether a memory is deleted, each id looked up in the store once.
339struct Gone<'a> {
340    store: &'a SqliteStore,
341    known: HashMap<Uuid, bool>,
342}
343
344impl<'a> Gone<'a> {
345    fn new(store: &'a SqliteStore, just_deleted: &[Uuid]) -> Self {
346        Self {
347            store,
348            known: just_deleted.iter().map(|id| (*id, true)).collect(),
349        }
350    }
351
352    fn check(&mut self, id: Uuid) -> Result<bool> {
353        if let Some(gone) = self.known.get(&id) {
354            return Ok(*gone);
355        }
356        let gone = self.store.tombstone_exists(id)?;
357        self.known.insert(id, gone);
358        Ok(gone)
359    }
360}
361
362/// Where a passage moves to, with the text it is filed under there: an
363/// archived memory's passage carries no text of its own.
364#[derive(Clone)]
365struct Holder {
366    origin: RawOrigin,
367    text: String,
368    project: Option<String>,
369    at: Option<DateTime<Utc>>,
370    memory: Option<Uuid>,
371}
372
373/// Kept transcripts that hold the text of a passage being purged, by the
374/// text's hash, leaving out lines whose raw memory was deleted.
375#[derive(Default)]
376struct Holders {
377    by_hash: HashMap<String, Vec<Holder>>,
378    read: HashSet<String>,
379}
380
381impl Holders {
382    fn has_read(&self, entry: &ArchiveEntry) -> bool {
383        self.read.contains(&entry.sha256)
384    }
385
386    fn read(
387        &mut self,
388        archive: &TranscriptArchive,
389        entries: &[ArchiveEntry],
390        hits: &[RawHit],
391        gone: &mut Gone<'_>,
392    ) -> Result<()> {
393        let wanted: HashSet<&str> = hits.iter().map(|h| h.text_hash.as_str()).collect();
394        for entry in entries {
395            if !self.read.insert(entry.sha256.clone()) {
396                continue;
397            }
398            let source = Path::new(&entry.source);
399            let Some(content) = archive.read(source)? else {
400                continue;
401            };
402            let project = project_path::from_transcript(&entry.source);
403            for passage in raw_passages(source, &content) {
404                let hash = text_hash(&passage.text);
405                if !wanted.contains(hash.as_str()) {
406                    continue;
407                }
408                if let Some(id) = passage.memory
409                    && gone.check(id)?
410                {
411                    continue;
412                }
413                self.by_hash.entry(hash).or_default().push(Holder {
414                    origin: RawOrigin::Transcript {
415                        sha256: entry.sha256.clone(),
416                    },
417                    text: passage.text,
418                    project: project.clone(),
419                    at: passage.at,
420                    memory: passage.memory,
421                });
422            }
423        }
424        Ok(())
425    }
426}
427
428enum Fate {
429    Keep,
430    Move(Holder),
431    Drop,
432}
433
434/// A passage under a kept memory or held by its own kept transcript stays,
435/// taking that line's metadata when its own raw memory was deleted; one
436/// whose text another kept transcript holds, or whose kept raw memory the
437/// index records, moves there; the rest go. A text no recorded
438/// transcript was read for has no holder: a transcript the index records
439/// had every text it holds indexed, so a passage added since the holders
440/// were read carries a text none of them holds.
441fn fate(
442    hit: &RawHit,
443    doomed_sources: &HashSet<String>,
444    holders: &Holders,
445    memory_gone: bool,
446    memory_holder: bool,
447) -> Fate {
448    if matches!(hit.origin, RawOrigin::Memory { .. })
449        && !doomed_sources.contains(&hit.origin.source_key())
450    {
451        return Fate::Keep;
452    }
453    if let Some(held) = holders.by_hash.get(&hit.text_hash) {
454        if let Some(own) = held.iter().find(|h| h.origin == hit.origin) {
455            return if memory_gone {
456                Fate::Move(own.clone())
457            } else {
458                Fate::Keep
459            };
460        }
461        if let Some(holder) = held.first() {
462            return Fate::Move(holder.clone());
463        }
464    }
465    match hit.memory {
466        Some(id) if memory_holder => Fate::Move(Holder {
467            origin: RawOrigin::Memory { id },
468            text: hit.text.clone(),
469            project: hit.project.clone(),
470            at: hit.at,
471            memory: Some(id),
472        }),
473        _ => Fate::Drop,
474    }
475}
476
477fn read_lock(index: &RwLock<RawIndex>) -> RwLockReadGuard<'_, RawIndex> {
478    index.read().unwrap_or_else(|p| p.into_inner())
479}
480
481fn write_lock(index: &RwLock<RawIndex>) -> RwLockWriteGuard<'_, RawIndex> {
482    index.write().unwrap_or_else(|p| p.into_inner())
483}
484
485fn remove_entries(archive: &TranscriptArchive, doomed: &[ArchiveEntry]) -> Result<usize> {
486    if doomed.is_empty() {
487        return Ok(0);
488    }
489    let sources: HashSet<&str> = doomed.iter().map(|e| e.source.as_str()).collect();
490    Ok(archive
491        .remove_where(|e| sources.contains(e.source.as_str()))?
492        .len())
493}
494
495/// Feeds the raw fallback index from the transcript archive and from raw
496/// memories consolidation archived, embedding outside the index lock so
497/// recall is never held behind the model.
498pub struct RawIndexer {
499    pub index: Arc<RwLock<RawIndex>>,
500    pub archive: Option<Arc<TranscriptArchive>>,
501    pub store: Arc<SqliteStore>,
502    pub embed: Arc<EmbedBatch>,
503    pub pause: Duration,
504    pub coverage: Arc<RawCoverage>,
505}
506
507impl RawIndexer {
508    pub fn spawn(self) {
509        std::thread::Builder::new()
510            .name("raw-index".to_string())
511            .spawn(move || {
512                loop {
513                    match self.pass() {
514                        Ok(s)
515                            if PassSummary {
516                                skipped_by_age: 0,
517                                ..s
518                            } != PassSummary::default() =>
519                        {
520                            tracing::info!(
521                                "raw index: {} transcripts, {} archived memories, {} passages, {} vectors restored, {} sources failed, {} passages older than {} days skipped, {} deleted passages withdrawn",
522                                s.transcripts,
523                                s.memories,
524                                s.passages,
525                                s.restored,
526                                s.failed,
527                                s.skipped_by_age,
528                                self.coverage.max_age_days(),
529                                s.purged
530                            );
531                            residency::release_free_heap();
532                        }
533                        Ok(_) => {}
534                        Err(e) => {
535                            tracing::warn!("raw index pass failed: {e:#}");
536                            residency::release_free_heap();
537                        }
538                    }
539                    std::thread::sleep(PASS_INTERVAL);
540                }
541            })
542            .expect("spawning the raw index thread");
543    }
544
545    /// Indexes every source not indexed yet that is within the age bound,
546    /// newest first. A source that fails is counted, skipped and retried on
547    /// the next pass, so one bad blob or a passing embedder error never stops
548    /// the rest.
549    pub fn pass(&self) -> Result<PassSummary> {
550        let mut summary = PassSummary::default();
551        match purge_deleted(Some(&self.index), self.archive.as_deref(), &self.store, &[]) {
552            Ok(purge) => summary.purged = purge.passages,
553            Err(e) => tracing::warn!("raw index: purging deleted sources failed: {e:#}"),
554        }
555        let mut first_error = None;
556        let mut pending = 0;
557        let cutoff = self.coverage.cutoff();
558        if let Some(archive) = &self.archive {
559            let mut queue = Vec::new();
560            let mut aged = HashMap::new();
561            for entry in archive.entries() {
562                let origin = RawOrigin::Transcript {
563                    sha256: entry.sha256.clone(),
564                };
565                if self.indexed(&origin) {
566                    continue;
567                }
568                match self.date_transcript(archive, &entry) {
569                    Ok(dated) if dated.newest < cutoff => {
570                        summary.skipped_by_age += dated.passages;
571                        aged.insert(entry.sha256.clone(), dated);
572                    }
573                    Ok(dated) => queue.push((dated.newest, entry, origin)),
574                    Err(e) => {
575                        summary.failed += 1;
576                        first_error.get_or_insert(e);
577                    }
578                }
579            }
580            *self.coverage.aged() = aged;
581            queue.sort_by_key(|q| std::cmp::Reverse(q.0));
582            for (_, entry, origin) in queue {
583                match self.add_transcript(archive, &entry, &origin) {
584                    Ok(added) => {
585                        summary.passages += added;
586                        summary.transcripts += 1;
587                        pending = self.commit_every(pending + 1)?;
588                    }
589                    Err(e) => {
590                        summary.failed += 1;
591                        first_error.get_or_insert(e);
592                    }
593                }
594            }
595        }
596
597        let raws = self.store.archived_raws_since(cutoff)?;
598        summary.skipped_by_age += raws
599            .older
600            .into_iter()
601            .filter(|&id| !self.indexed(&RawOrigin::Memory { id }))
602            .count();
603        for id in raws.recent {
604            let origin = RawOrigin::Memory { id };
605            if self.indexed(&origin) {
606                continue;
607            }
608            match self.add_memory(id, &origin) {
609                Ok((added, restored)) => {
610                    summary.passages += added;
611                    summary.restored += usize::from(restored);
612                    summary.memories += 1;
613                    pending = self.commit_every(pending + 1)?;
614                }
615                Err(e) => {
616                    summary.failed += 1;
617                    first_error.get_or_insert(e);
618                }
619            }
620        }
621        if pending > 0 {
622            self.write().commit()?;
623        }
624        self.coverage
625            .skipped_by_age
626            .store(summary.skipped_by_age, Ordering::Relaxed);
627        if let Some(e) = first_error {
628            tracing::warn!(
629                "raw index: {} sources failed and will be retried, first: {e:#}",
630                summary.failed
631            );
632        }
633        Ok(summary)
634    }
635
636    /// When a transcript was last active: its newest passage time, or when
637    /// it was archived if no line carries one. A transcript left out for age
638    /// is dated from memory rather than read again.
639    fn date_transcript(&self, archive: &TranscriptArchive, entry: &ArchiveEntry) -> Result<Dated> {
640        if let Some(dated) = self.coverage.aged().get(&entry.sha256) {
641            return Ok(*dated);
642        }
643        let source = Path::new(&entry.source);
644        let passages = match archive.read(source)? {
645            Some(content) => raw_passages(source, &content),
646            None => Vec::new(),
647        };
648        Ok(Dated {
649            newest: passages
650                .iter()
651                .filter_map(|p| p.at)
652                .max()
653                .unwrap_or(entry.archived_at),
654            passages: passages.len(),
655        })
656    }
657
658    fn add_transcript(
659        &self,
660        archive: &TranscriptArchive,
661        entry: &ArchiveEntry,
662        origin: &RawOrigin,
663    ) -> Result<usize> {
664        let source = Path::new(&entry.source);
665        let Some(content) = archive.read(source)? else {
666            return self.add(origin, Vec::new(), Some((archive, entry)));
667        };
668        let project = project_path::from_transcript(&entry.source);
669        let passages = raw_passages(source, &content)
670            .into_iter()
671            .map(|p| RawPassage {
672                text: p.text,
673                at: p.at,
674                project: project.clone(),
675                memory: p.memory,
676            })
677            .collect();
678        self.add(origin, passages, Some((archive, entry)))
679    }
680
681    fn add_memory(&self, id: Uuid, origin: &RawOrigin) -> Result<(usize, bool)> {
682        let Some(memory) = self.store.get_memory_full(id)? else {
683            return Ok((0, false));
684        };
685        let mut restored = false;
686        let embedding = if memory.embedding.len() == EMBEDDING_DIM {
687            memory.embedding.clone()
688        } else {
689            let embedding = self
690                .embed_and_yield(&[memory.content.as_str()])?
691                .into_iter()
692                .next()
693                .unwrap_or_default();
694            if embedding.len() == EMBEDDING_DIM {
695                restored = self.store.restore_archived_embedding(id, &embedding)?;
696            }
697            embedding
698        };
699        let passage = RawPassage {
700            text: memory.content.clone(),
701            at: memory.session_at,
702            project: memory.project_path.clone(),
703            memory: Some(id),
704        };
705        let added = self.write().add_source(origin, &[(passage, embedding)])?;
706        Ok((added, restored))
707    }
708
709    /// Embeds only passages whose text is not indexed yet, so a transcript
710    /// archived again as it grows costs only its new text, and none whose
711    /// raw memory was deleted, which a purge would only withdraw again. A
712    /// transcript is added only if the archive still holds it once the
713    /// write lock is taken, since a purge may have removed it while it was
714    /// embedded, and its source is not recorded if a purge withdrew or
715    /// rekeyed passages meanwhile, since a text it skipped as indexed may
716    /// be gone; the next pass reads it again.
717    fn add(
718        &self,
719        origin: &RawOrigin,
720        passages: Vec<RawPassage>,
721        archived: Option<(&TranscriptArchive, &ArchiveEntry)>,
722    ) -> Result<usize> {
723        let (unindexed, generation) = {
724            let index = self.read();
725            let unindexed: Vec<RawPassage> = passages
726                .into_iter()
727                .filter(|p| !index.has_text(&p.text))
728                .collect();
729            (unindexed, index.purge_generation())
730        };
731        let mut seen = HashSet::new();
732        let mut passages = Vec::with_capacity(unindexed.len());
733        for passage in unindexed {
734            if let Some(id) = passage.memory
735                && self.store.tombstone_exists(id)?
736            {
737                continue;
738            }
739            if seen.insert(passage.text.trim().to_string()) {
740                passages.push(passage);
741            }
742        }
743        let mut embedded = Vec::with_capacity(passages.len());
744        for batch in passages.chunks(EMBED_BATCH) {
745            let texts: Vec<&str> = batch.iter().map(|p| p.text.as_str()).collect();
746            let vectors = self.embed_and_yield(&texts)?;
747            embedded.extend(batch.iter().cloned().zip(vectors));
748        }
749        let mut index = self.write();
750        if let Some((archive, entry)) = archived
751            && archive
752                .entry(Path::new(&entry.source))
753                .is_none_or(|held| held.sha256 != entry.sha256)
754        {
755            return Ok(0);
756        }
757        if index.purge_generation() != generation {
758            return Ok(0);
759        }
760        index.add_source(origin, &embedded)
761    }
762
763    fn embed_and_yield(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
764        let vectors = (self.embed)(texts);
765        std::thread::sleep(self.pause);
766        vectors
767    }
768
769    fn commit_every(&self, pending: usize) -> Result<usize> {
770        if pending < COMMIT_EVERY {
771            return Ok(pending);
772        }
773        self.write().commit()?;
774        Ok(0)
775    }
776
777    fn indexed(&self, origin: &RawOrigin) -> bool {
778        self.read().has_source(origin)
779    }
780
781    fn read(&self) -> RwLockReadGuard<'_, RawIndex> {
782        read_lock(&self.index)
783    }
784
785    fn write(&self) -> RwLockWriteGuard<'_, RawIndex> {
786        write_lock(&self.index)
787    }
788}