Skip to main content

second_brain_api/
maintenance.rs

1//! The drain thread: applies the index outbox every few seconds and, when
2//! the outbox is quiet and the interval has passed, repairs the store with
3//! the writable indexes it owns. On-demand repairs arrive over a channel so
4//! the same thread services them; two writers never exist.
5
6use std::path::PathBuf;
7use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender};
8use std::sync::{Arc, PoisonError, RwLock};
9use std::time::{Duration, Instant};
10
11use anyhow::{Result, anyhow};
12use chrono::Utc;
13use second_brain_core::config::MaintenanceConfig;
14use second_brain_core::embedding::Embedder;
15use second_brain_core::index::{IndexClaim, Indexes};
16use second_brain_core::maintenance::{MaintenanceSummary, RepairStats};
17use second_brain_core::residency::{self, IndexMemory};
18use second_brain_core::sealed::{SealKeyId, SealKeys};
19use second_brain_core::sqlite_store::{AtRestPurpose, SqliteStore};
20
21use crate::health::Health;
22use crate::near_duplicates::{NearDupCount, refresh_count};
23
24const DRAIN_INTERVAL: Duration = Duration::from_secs(5);
25/// Peers' deletes ask for a lexical merge, which rewrites every segment,
26/// so a burst of them is served by one merge.
27const DELETE_PURGE_GAP: Duration = Duration::from_secs(600);
28
29const WRITER_IDLE: Duration = Duration::from_secs(60);
30/// How often the drain thread asks the store whether the index files need
31/// sealing: after the store's first encryption, or a store key rotation.
32const SEAL_CHECK_INTERVAL: Duration = Duration::from_secs(60);
33
34/// Finding chunk work splits every live row, so once a scan comes back
35/// empty-handed the next one waits this long unless a drain applies writes.
36const CHUNK_IDLE_RESCAN: Duration = Duration::from_secs(600);
37/// Per idle tick, so the drain and repair behind it are never held up long.
38const CHUNK_TICK_BUDGET: Duration = Duration::from_secs(3);
39/// Bounds what one tick holds in memory: at most 16 chunks per memory.
40const CHUNK_MEMORIES_PER_TICK: usize = 32;
41
42/// When the drain thread may next look for memories needing chunks.
43struct ChunkThrottle {
44    next_scan: Instant,
45}
46
47impl ChunkThrottle {
48    fn new(now: Instant) -> Self {
49        Self { next_scan: now }
50    }
51
52    fn due(&self, now: Instant) -> bool {
53        now >= self.next_scan
54    }
55
56    fn after_backfill(&mut self, now: Instant, progressed: bool) {
57        self.next_scan = if progressed {
58            now
59        } else {
60            now + CHUNK_IDLE_RESCAN
61        };
62    }
63
64    /// Applied outbox rows may carry content changes that need re-chunking.
65    fn after_drain(&mut self, now: Instant, applied: usize) {
66        if applied > 0 {
67            self.next_scan = now;
68        }
69    }
70}
71
72pub struct RepairRequest {
73    pub dry_run: bool,
74    pub reply: Sender<Result<MaintenanceSummary>>,
75}
76
77/// Work only the drain thread may do, because it owns the writable indexes.
78pub enum DrainRequest {
79    Repair(RepairRequest),
80    /// Applies the whole outbox, then rewrites the lexical index into fresh
81    /// segments (see `Indexes::compact`).
82    CompactIndexes(Sender<Result<()>>),
83    /// Holds the drain thread, so nothing writes the index files, until the
84    /// sender of these commands is dropped.
85    Pause(Receiver<PauseCommand>),
86}
87
88pub enum PauseCommand {
89    /// Applies the whole outbox and lets the lexical writer's merges finish,
90    /// so the index files on disk match the store at its watermark.
91    Settle(Sender<Result<()>>),
92}
93
94/// Runs a repair and re-measures the store afterwards, on whatever index
95/// handle the caller owns (the writer on the drain thread, none when the
96/// indexes never opened or a released writer could not reopen).
97pub fn run_repair(
98    store: &SqliteStore,
99    mut indexes: Option<&mut Indexes>,
100    embedder: Option<&Embedder>,
101    dry_run: bool,
102) -> Result<MaintenanceSummary> {
103    let embed = embedder.map(|e| move |text: &str| e.embed(text));
104    let embed_ref = embed
105        .as_ref()
106        .map(|f| f as &dyn Fn(&str) -> Result<Vec<f32>>);
107    let stats = store.repair(indexes.as_deref_mut(), embed_ref, dry_run)?;
108    let report_after = store.integrity_report(indexes.as_deref())?;
109    Ok(MaintenanceSummary {
110        ran_at: Utc::now(),
111        stats,
112        report_after,
113    })
114}
115
116fn log_stats(source: &str, stats: &RepairStats) {
117    if stats.is_clean() {
118        tracing::info!("{source} store repair: nothing to do");
119    } else {
120        tracing::info!("{source} store repair: {}", summarize(stats));
121    }
122}
123
124fn summarize(s: &RepairStats) -> String {
125    let mut parts = Vec::new();
126    let mut push = |name: &str, n: u64| {
127        if n > 0 {
128            parts.push(format!("{name}={n}"));
129        }
130    };
131    push("duplicates_superseded", s.duplicates_superseded);
132    push("markers_inherited", s.distillation_markers_inherited);
133    push("orphaned_markers_dropped", s.orphaned_markers_dropped);
134    push("raws_retired", s.raws_retired);
135    push("relations_pruned", s.relations_pruned);
136    push("vec_keys_deleted", s.vec_keys_deleted);
137    push("sync_log_rows_deleted", s.sync_log_rows_deleted);
138    push("stale_cursors_dropped", s.stale_cursors_dropped);
139    push("embeddings_backfilled", s.embeddings_backfilled);
140    push("embedding_failures", s.embedding_failures);
141    if s.vec_keys_gc_skipped {
142        parts.push("vec_keys_gc=skipped".to_string());
143    }
144    if s.index_rebuilt {
145        parts.push("index=rebuilt".to_string());
146    }
147    if s.dry_run {
148        parts.push("dry_run".to_string());
149    }
150    parts.join(" ")
151}
152
153/// The drain thread's writable indexes. usearch cannot mutate a viewed index,
154/// so an open writer holds its own anonymous copy of the vectors beside the
155/// reader's mmap; the slot drops it once `WRITER_IDLE` has passed since its
156/// last operation finished, and reopens it from the persisted files when work
157/// arrives. A writer holding changes its files lack is kept, because a drain
158/// commits its store-side deletes before it persists, so only that writer
159/// still holds those removals.
160///
161/// Sealed indexes can't be mapped, so recall reads a reader sharing the
162/// writer's vectors; the slot then never drops that writer, and an idle one
163/// gives up only its lexical writer's buffers.
164pub struct WriterSlot {
165    dir: PathBuf,
166    open: Option<Indexes>,
167    last_used: Instant,
168    shared: bool,
169    idle_released: bool,
170    _claim: IndexClaim,
171}
172
173impl WriterSlot {
174    pub fn new(writer: Indexes, claim: IndexClaim) -> Self {
175        crate::heap::record_index_writer(Some(writer.memory()));
176        Self {
177            dir: writer.dir().to_path_buf(),
178            shared: writer.seal_keys().is_some(),
179            open: Some(writer),
180            last_used: Instant::now(),
181            idle_released: false,
182            _claim: claim,
183        }
184    }
185
186    fn open(&mut self) -> Result<&mut Indexes> {
187        if self.open.is_none() {
188            if self.shared {
189                return Err(anyhow!(
190                    "the sealed index writer at {} is gone; the daemon must restart to reopen it",
191                    self.dir.display()
192                ));
193            }
194            let writer = Indexes::open(&self.dir)?;
195            tracing::info!("index writer reopened at {}", self.dir.display());
196            crate::heap::record_index_writer(Some(writer.memory()));
197            self.open = Some(writer);
198        }
199        Ok(self.open.as_mut().expect("writer opened above"))
200    }
201
202    fn with<R>(&mut self, op: impl FnOnce(&mut Indexes) -> Result<R>) -> Result<R> {
203        let result = op(self.open()?);
204        self.last_used = Instant::now();
205        self.idle_released = false;
206        crate::heap::record_index_writer(self.memory());
207        result
208    }
209
210    fn release_if_idle(&mut self, now: Instant) -> bool {
211        let current = self
212            .open
213            .as_ref()
214            .is_some_and(|w| !w.has_unpersisted_changes());
215        if !current || now.saturating_duration_since(self.last_used) < WRITER_IDLE {
216            return false;
217        }
218        if self.shared {
219            return self.release_lexical_writer();
220        }
221        if let Some(writer) = self.open.take()
222            && let Err(e) = writer.wait_for_merges()
223        {
224            tracing::warn!("index writer released without its merges: {e:#}");
225        }
226        crate::heap::record_index_writer(None);
227        residency::release_free_heap();
228        tracing::info!(
229            "index writer released after {}s idle",
230            WRITER_IDLE.as_secs()
231        );
232        true
233    }
234
235    fn release_lexical_writer(&mut self) -> bool {
236        if self.idle_released {
237            return false;
238        }
239        if let Some(writer) = self.open.as_mut()
240            && let Err(e) = writer.close_lexical_writer()
241        {
242            tracing::warn!("lexical writer released without its merges: {e:#}");
243        }
244        self.idle_released = true;
245        residency::release_free_heap();
246        tracing::info!(
247            "lexical index writer released after {}s idle",
248            WRITER_IDLE.as_secs()
249        );
250        true
251    }
252
253    fn memory(&self) -> Option<IndexMemory> {
254        self.open.as_ref().map(Indexes::memory)
255    }
256
257    fn sealed_under(&self) -> Option<SealKeyId> {
258        self.open
259            .as_ref()
260            .and_then(Indexes::seal_keys)
261            .map(|keys| keys.current_id())
262    }
263
264    /// Seals every index file under `keys`' current key. A plaintext writer
265    /// is persisted and reopened sealed, and recall moves to a reader that
266    /// shares its vectors.
267    fn seal_under(&mut self, keys: SealKeys, reader: &RwLock<Indexes>) -> Result<()> {
268        if self.shared {
269            let writer = self
270                .open
271                .as_mut()
272                .ok_or_else(|| anyhow!("the sealed index writer is gone"))?;
273            writer.set_seal_keys(keys)?;
274            return writer.reseal();
275        }
276        if let Some(writer) = self.open.as_mut()
277            && writer.has_unpersisted_changes()
278        {
279            writer.persist()?;
280        }
281        if let Some(writer) = self.open.take() {
282            crate::heap::record_index_writer(None);
283            writer.wait_for_merges()?;
284        }
285        let mut writer = Indexes::open_keyed(&self.dir, &keys)?;
286        writer.reseal()?;
287        let shared = writer.shared_reader()?;
288        *reader.write().unwrap_or_else(PoisonError::into_inner) = shared;
289        crate::heap::record_index_writer(Some(writer.memory()));
290        self.open = Some(writer);
291        self.shared = true;
292        self.idle_released = false;
293        self.last_used = Instant::now();
294        Ok(())
295    }
296}
297
298pub struct DrainLoop {
299    pub store: Arc<SqliteStore>,
300    pub reader: Arc<RwLock<Indexes>>,
301    pub writer: WriterSlot,
302    pub embedder: Option<Arc<Embedder>>,
303    pub summary: Arc<RwLock<Option<MaintenanceSummary>>>,
304    pub near_duplicates: Arc<RwLock<Option<NearDupCount>>>,
305    pub near_duplicate_threshold: f32,
306    pub config: MaintenanceConfig,
307    pub requests: Receiver<DrainRequest>,
308    pub health: Arc<Health>,
309}
310
311impl DrainLoop {
312    pub fn spawn(self) {
313        std::thread::Builder::new()
314            .name("index-drain".to_string())
315            .spawn(move || self.run())
316            .expect("spawning the index drain thread");
317    }
318
319    fn run(mut self) {
320        let mut last_repair: Option<Instant> = None;
321        let mut chunks = ChunkThrottle::new(Instant::now());
322        let mut last_purge: Option<Instant> = None;
323        let mut last_seal_check: Option<Instant> = None;
324        loop {
325            match self.requests.recv_timeout(DRAIN_INTERVAL) {
326                Ok(DrainRequest::Repair(request)) => {
327                    let result = self.repair("requested", request.dry_run);
328                    if result.is_ok() && !request.dry_run {
329                        last_repair = Some(Instant::now());
330                    }
331                    // A caller that gave up waiting is not an error here.
332                    let _ = request.reply.send(result);
333                }
334                Ok(DrainRequest::CompactIndexes(reply)) => {
335                    let _ = reply.send(self.compact_indexes());
336                }
337                Ok(DrainRequest::Pause(commands)) => {
338                    for PauseCommand::Settle(reply) in commands {
339                        let _ = reply.send(self.settle());
340                    }
341                }
342                Err(RecvTimeoutError::Timeout) | Err(RecvTimeoutError::Disconnected) => {
343                    let applied = self.drain();
344                    chunks.after_drain(Instant::now(), applied.unwrap_or(0));
345                    let due = last_repair.is_none_or(|t| {
346                        t.elapsed() >= Duration::from_secs(self.config.interval_secs)
347                    });
348                    if applied == Some(0) && self.config.enabled && due {
349                        if self.repair("scheduled", false).is_ok() {
350                            last_repair = Some(Instant::now());
351                        } else {
352                            // Retry sooner than the full interval, but not
353                            // every tick against a store that keeps failing.
354                            last_repair = Some(
355                                Instant::now()
356                                    - Duration::from_secs(
357                                        self.config.interval_secs.saturating_sub(600),
358                                    ),
359                            );
360                        }
361                    }
362                    if applied == Some(0)
363                        && last_purge.is_none_or(|t| t.elapsed() >= DELETE_PURGE_GAP)
364                        && self.purge_applied_deletes()
365                    {
366                        last_purge = Some(Instant::now());
367                    }
368                    if applied == Some(0) && chunks.due(Instant::now()) {
369                        let progressed = self.backfill_chunks();
370                        chunks.after_backfill(Instant::now(), progressed);
371                    }
372                }
373            }
374            if last_seal_check.is_none_or(|t| t.elapsed() >= SEAL_CHECK_INTERVAL) {
375                self.seal_if_due();
376                last_seal_check = Some(Instant::now());
377            }
378            self.writer.release_if_idle(Instant::now());
379        }
380    }
381
382    /// Seals the index files when the store's keys for them changed: on the
383    /// store's first encryption, after a store key rotation, or when a
384    /// reseal was cut short. Retired keys are forgotten only once every file
385    /// is under the current one.
386    fn seal_if_due(&mut self) {
387        let keys = match self.store.at_rest_keys(AtRestPurpose::Indexes) {
388            Ok(Some(keys)) => keys,
389            Ok(None) => return,
390            Err(e) => {
391                tracing::warn!("reading the search indexes' keys failed: {e:#}");
392                return;
393            }
394        };
395        let current = keys.keys.current_id();
396        if keys.settled && self.writer.sealed_under() == Some(current) {
397            return;
398        }
399        let started = Instant::now();
400        if let Err(e) = self.writer.seal_under(keys.keys, &self.reader) {
401            tracing::warn!("sealing the search indexes failed and will retry: {e:#}");
402            return;
403        }
404        match self
405            .store
406            .settle_at_rest_keys(AtRestPurpose::Indexes, current)
407        {
408            Ok(()) => tracing::info!(
409                "search indexes sealed under key {current} in {:.1}s",
410                started.elapsed().as_secs_f64()
411            ),
412            Err(e) => tracing::warn!("recording the sealed search indexes failed: {e:#}"),
413        }
414    }
415
416    /// Embeds pending chunk sets within one tick's budget; true when any set
417    /// was stored, so the next idle tick keeps going.
418    fn backfill_chunks(&self) -> bool {
419        let Some(embedder) = self.embedder.as_deref() else {
420            return false;
421        };
422        let embed = |texts: &[&str]| embedder.embed_batch(texts);
423        match self.store.backfill_chunk_embeddings(
424            &embed,
425            CHUNK_MEMORIES_PER_TICK,
426            CHUNK_TICK_BUDGET,
427        ) {
428            Ok(r) => {
429                if r.memories > 0 || r.failed > 0 {
430                    tracing::info!(
431                        "chunk backfill: memories={} chunks={} failed={} pending={}",
432                        r.memories,
433                        r.chunks,
434                        r.failed,
435                        r.pending
436                    );
437                }
438                r.memories > 0
439            }
440            Err(e) => {
441                tracing::warn!("chunk backfill failed: {e:#}");
442                false
443            }
444        }
445    }
446
447    /// Applies pending outbox rows; None when the drain itself failed. An
448    /// empty outbox never touches the writer, so a released one stays closed.
449    fn drain(&mut self) -> Option<usize> {
450        let store = &self.store;
451        let writer = &mut self.writer;
452        let result = store.outbox_pending().and_then(|pending| {
453            if pending == 0 {
454                Ok(0)
455            } else {
456                writer.with(|w| w.drain(store))
457            }
458        });
459        match result {
460            Ok(0) => {
461                self.health.drained(0, Utc::now());
462                Some(0)
463            }
464            Ok(n) => {
465                tracing::info!("index drain applied {n} outbox rows");
466                self.reload_reader();
467                match self.store.outbox_pending() {
468                    Ok(pending) => self.health.drained(pending, Utc::now()),
469                    Err(e) => tracing::warn!("counting the index outbox failed: {e:#}"),
470                }
471                Some(n)
472            }
473            Err(e) => {
474                tracing::warn!("index drain failed: {e:#}");
475                self.health.drain_failed(&format!("{e:#}"), Utc::now());
476                None
477            }
478        }
479    }
480
481    fn repair(&mut self, source: &str, dry_run: bool) -> Result<MaintenanceSummary> {
482        let started = Instant::now();
483        let store = &self.store;
484        let embedder = self.embedder.as_deref();
485        let repair = |w: &mut Indexes| run_repair(store, Some(w), embedder, dry_run);
486        let result = match self.writer.open().map(|_| ()) {
487            Ok(()) => self.writer.with(repair),
488            Err(e) => {
489                tracing::warn!(
490                    "index writer unavailable ({e:#}); repairing the store without the indexes"
491                );
492                match run_repair(store, None, embedder, dry_run) {
493                    Ok(summary) => log_stats(source, &summary.stats),
494                    Err(store_error) => {
495                        tracing::warn!("{source} store repair failed: {store_error:#}")
496                    }
497                }
498                Err(e
499                    .context("the index writer could not reopen, so the indexes were not repaired"))
500            }
501        };
502        match &result {
503            Ok(summary) => {
504                self.health.repaired(Utc::now());
505                log_stats(source, &summary.stats);
506                tracing::info!(
507                    "{source} store repair finished in {:.1}s; drift remaining: {:?}",
508                    started.elapsed().as_secs_f64(),
509                    summary.report_after.drift()
510                );
511                if !dry_run {
512                    self.reload_reader();
513                    let mut guard = match self.summary.write() {
514                        Ok(g) => g,
515                        Err(poisoned) => poisoned.into_inner(),
516                    };
517                    *guard = Some(summary.clone());
518                    drop(guard);
519                    let threshold = self.near_duplicate_threshold;
520                    let near_duplicates = &self.near_duplicates;
521                    let counted = self.writer.with(|w| {
522                        refresh_count(store, w, threshold, near_duplicates);
523                        Ok(())
524                    });
525                    if let Err(e) = counted {
526                        tracing::warn!("near-duplicate count failed: {e:#}");
527                    }
528                    if summary.stats.index_rebuilt {
529                        residency::release_free_heap();
530                    }
531                }
532            }
533            Err(e) => {
534                tracing::warn!("{source} store repair failed: {e:#}");
535                self.health.repair_failed(&format!("{e:#}"), Utc::now());
536            }
537        }
538        result
539    }
540
541    /// Runs the lexical merge and cold purge a peer's applied delete asked
542    /// for, so its text leaves this disk as it left the deleting one's;
543    /// true when it ran.
544    fn purge_applied_deletes(&mut self) -> bool {
545        let token = match self.store.delete_purge_requested() {
546            Ok(Some(token)) => token,
547            Ok(None) => return false,
548            Err(e) => {
549                tracing::warn!("reading the delete purge request failed: {e:#}");
550                return false;
551            }
552        };
553        let result = self.compact_indexes().and_then(|()| {
554            let cold = self.store.purge_cold_segments()?;
555            if cold != Default::default() {
556                tracing::info!(
557                    "deleted records purged from the cold tier: {} retired, {} segments rewritten, {} files removed",
558                    cold.retired,
559                    cold.rewritten,
560                    cold.removed_files
561                );
562            }
563            self.store.delete_purge_done(&token)
564        });
565        if let Err(e) = result {
566            tracing::warn!("purging a peer's deletes failed and will retry: {e:#}");
567        }
568        true
569    }
570
571    fn compact_indexes(&mut self) -> Result<()> {
572        let store = &self.store;
573        let took = self.writer.with(|w| {
574            while w.drain(store)? > 0 {}
575            let started = Instant::now();
576            w.compact()?;
577            Ok(started.elapsed())
578        })?;
579        self.reload_reader();
580        residency::release_free_heap();
581        tracing::info!("lexical index compacted in {:.1}s", took.as_secs_f64());
582        Ok(())
583    }
584
585    fn settle(&mut self) -> Result<()> {
586        let store = &self.store;
587        self.writer.with(|w| {
588            while w.drain(store)? > 0 {}
589            if w.has_unpersisted_changes() {
590                w.persist()?;
591            }
592            w.close_lexical_writer()
593        })?;
594        self.reload_reader();
595        Ok(())
596    }
597
598    fn reload_reader(&self) {
599        let mut guard = match self.reader.write() {
600            Ok(g) => g,
601            Err(poisoned) => poisoned.into_inner(),
602        };
603        if let Err(e) = guard.reload_view() {
604            tracing::warn!("index reader reload failed: {e:#}");
605        }
606    }
607}
608
609#[cfg(test)]
610mod tests {
611    use std::fs::Permissions;
612    use std::os::unix::fs::PermissionsExt;
613
614    use super::*;
615    use second_brain_core::schema::{Memory, MemoryType};
616    use second_brain_core::store::Store;
617
618    #[test]
619    fn a_fresh_throttle_scans_on_the_first_idle_tick() {
620        let now = Instant::now();
621        assert!(ChunkThrottle::new(now).due(now));
622    }
623
624    #[test]
625    fn a_scan_that_finds_nothing_waits_out_the_idle_interval() {
626        let now = Instant::now();
627        let mut t = ChunkThrottle::new(now);
628        t.after_backfill(now, false);
629        assert!(!t.due(now + CHUNK_IDLE_RESCAN - Duration::from_secs(1)));
630        assert!(t.due(now + CHUNK_IDLE_RESCAN));
631    }
632
633    #[test]
634    fn progress_keeps_the_next_idle_tick_scanning() {
635        let now = Instant::now();
636        let mut t = ChunkThrottle::new(now);
637        t.after_backfill(now, true);
638        assert!(t.due(now));
639    }
640
641    #[test]
642    fn an_applied_drain_resets_the_idle_wait() {
643        let now = Instant::now();
644        let mut t = ChunkThrottle::new(now);
645        t.after_backfill(now, false);
646        t.after_drain(now, 0);
647        assert!(!t.due(now));
648        t.after_drain(now, 3);
649        assert!(t.due(now));
650    }
651
652    fn remember(store: &SqliteStore, content: &str) -> uuid::Uuid {
653        let m = Memory::new(
654            content.to_string(),
655            MemoryType::Semantic,
656            "test".to_string(),
657            String::new(),
658        );
659        store.store_memory(&m).unwrap();
660        m.id
661    }
662
663    fn drain_loop(
664        store: Arc<SqliteStore>,
665        reader: Arc<RwLock<Indexes>>,
666        writer: Indexes,
667    ) -> DrainLoop {
668        let (_tx, requests) = std::sync::mpsc::channel();
669        let claim = IndexClaim::acquire(writer.dir()).unwrap();
670        DrainLoop {
671            store,
672            reader,
673            writer: WriterSlot::new(writer, claim),
674            embedder: None,
675            summary: Arc::new(RwLock::new(None)),
676            near_duplicates: Arc::new(RwLock::new(None)),
677            near_duplicate_threshold: 0.97,
678            config: MaintenanceConfig {
679                enabled: false,
680                interval_secs: 3600,
681            },
682            requests,
683            health: Arc::new(Health::new(Utc::now(), None)),
684        }
685    }
686
687    fn idle_loop(dir: &std::path::Path) -> DrainLoop {
688        let store = Arc::new(SqliteStore::in_memory("m".to_string()).unwrap());
689        let mut writer = Indexes::open(dir).unwrap();
690        writer.persist().unwrap();
691        let reader = Arc::new(RwLock::new(Indexes::open_readonly(dir).unwrap()));
692        drain_loop(store, reader, writer)
693    }
694
695    #[test]
696    fn idleness_counts_from_when_an_operation_finishes() {
697        let dir = tempfile::tempdir().unwrap();
698        let mut lp = idle_loop(dir.path());
699        let mut finishing = None;
700        lp.writer
701            .with(|_| {
702                finishing = Some(Instant::now());
703                Ok(())
704            })
705            .unwrap();
706        let finishing = finishing.unwrap();
707        assert!(lp.writer.last_used >= finishing);
708        assert!(
709            !lp.writer
710                .release_if_idle(finishing + WRITER_IDLE - Duration::from_secs(1))
711        );
712    }
713
714    #[test]
715    fn a_writer_holding_changes_its_files_lack_is_never_released() {
716        let dir = tempfile::tempdir().unwrap();
717        let mut lp = idle_loop(dir.path());
718        remember(&lp.store, "the zanzibar ferry leaves at noon");
719        let tantivy = dir.path().join("tantivy");
720        let set_mode =
721            |mode| std::fs::set_permissions(&tantivy, Permissions::from_mode(mode)).unwrap();
722        set_mode(0o500);
723        let drained = lp.drain();
724        set_mode(0o700);
725        assert_eq!(
726            drained, None,
727            "the drain should fail to write the lexical index"
728        );
729        assert!(
730            !lp.writer
731                .release_if_idle(lp.writer.last_used + WRITER_IDLE * 10)
732        );
733        lp.repair("test", true).unwrap();
734        assert!(!lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
735
736        assert_eq!(lp.drain(), Some(1));
737        assert!(lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
738    }
739
740    #[test]
741    fn a_failure_that_left_the_files_current_still_releases() {
742        let dir = tempfile::tempdir().unwrap();
743        let mut lp = idle_loop(dir.path());
744        let failed: Result<()> = lp.writer.with(|_| anyhow::bail!("a store step failed"));
745        assert!(failed.is_err());
746        assert!(lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
747    }
748
749    #[test]
750    fn a_writer_that_cannot_reopen_fails_the_repair() {
751        let dir = tempfile::tempdir().unwrap();
752        let mut lp = idle_loop(dir.path());
753        assert!(lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
754        let blocker = dir.path().join("not-a-dir");
755        std::fs::write(&blocker, b"x").unwrap();
756        lp.writer.dir = blocker.join("indexes");
757
758        let err = lp.repair("test", false).unwrap_err();
759        assert!(
760            format!("{err:#}").contains("indexes were not repaired"),
761            "{err:#}"
762        );
763        assert!(lp.writer.memory().is_none());
764    }
765
766    #[test]
767    fn an_idle_writer_is_released_and_reopened_by_the_next_write() {
768        let dir = tempfile::tempdir().unwrap();
769        let store = Arc::new(SqliteStore::in_memory("m".to_string()).unwrap());
770        let mut writer = Indexes::open(dir.path()).unwrap();
771        writer.persist().unwrap();
772        let reader = Arc::new(RwLock::new(Indexes::open_readonly(dir.path()).unwrap()));
773        let mut lp = drain_loop(store.clone(), reader.clone(), writer);
774
775        let first = remember(&store, "the zanzibar ferry leaves at noon");
776        assert_eq!(lp.drain(), Some(1));
777        assert!(lp.writer.memory().is_some());
778
779        let used = lp.writer.last_used;
780        assert!(
781            !lp.writer
782                .release_if_idle(used + WRITER_IDLE - Duration::from_secs(1))
783        );
784        assert!(lp.writer.memory().is_some());
785        assert!(lp.writer.release_if_idle(used + WRITER_IDLE));
786        assert!(lp.writer.memory().is_none());
787
788        assert_eq!(lp.drain(), Some(0));
789        assert!(
790            lp.writer.memory().is_none(),
791            "an empty tick reopened the writer"
792        );
793
794        let second = remember(&store, "the kilimanjaro trail opens in june");
795        assert_eq!(lp.drain(), Some(1));
796        assert!(lp.writer.memory().is_some());
797        assert_eq!(lp.writer.open().unwrap().lexical_len(), 2);
798
799        let guard = reader.read().unwrap();
800        let hits = |q: &str| -> Vec<uuid::Uuid> {
801            guard
802                .bm25_search(q, 5)
803                .unwrap()
804                .into_iter()
805                .map(|(id, _)| id)
806                .collect()
807        };
808        assert_eq!(hits("kilimanjaro"), vec![second]);
809        assert_eq!(hits("zanzibar"), vec![first]);
810    }
811
812    fn keyed_store(dir: &std::path::Path) -> Arc<SqliteStore> {
813        let key = second_brain_core::keys::StoreKey::generate().unwrap();
814        Arc::new(SqliteStore::open_with_key(&dir.join("brain.db"), "m".to_string(), &key).unwrap())
815    }
816
817    fn index_files(dir: &std::path::Path) -> Vec<std::path::PathBuf> {
818        let mut out = Vec::new();
819        let mut stack = vec![dir.to_path_buf()];
820        while let Some(d) = stack.pop() {
821            for entry in std::fs::read_dir(d).unwrap() {
822                let path = entry.unwrap().path();
823                let name = path.file_name().unwrap().to_string_lossy().to_string();
824                if path.is_dir() {
825                    stack.push(path);
826                } else if !name.ends_with(".lock") && name != second_brain_core::index::CLAIM_FILE {
827                    out.push(path);
828                }
829            }
830        }
831        out
832    }
833
834    fn assert_all_sealed_under(dir: &std::path::Path, key: SealKeyId) {
835        let files = index_files(dir);
836        assert!(!files.is_empty());
837        for path in files {
838            assert_eq!(
839                second_brain_core::sealed::sealed_key_id(&path).unwrap(),
840                Some(key),
841                "{}",
842                path.display()
843            );
844        }
845    }
846
847    #[test]
848    fn indexes_opened_before_the_store_was_encrypted_are_sealed_and_recall_moves_to_a_shared_reader()
849     {
850        let data = tempfile::tempdir().unwrap();
851        let store = keyed_store(data.path());
852        let dir = data.path().join("brain.indexes");
853        let mut writer = Indexes::open(&dir).unwrap();
854        writer.persist().unwrap();
855        let reader = Arc::new(RwLock::new(Indexes::open_readonly(&dir).unwrap()));
856        let mut lp = drain_loop(store.clone(), reader.clone(), writer);
857        let first = remember(&store, "the zanzibar ferry leaves at noon");
858        assert_eq!(lp.drain(), Some(1));
859
860        lp.seal_if_due();
861        let keys = store.at_rest_keys(AtRestPurpose::Indexes).unwrap().unwrap();
862        assert!(keys.settled);
863        assert_all_sealed_under(&dir, keys.keys.current_id());
864        assert!(Indexes::open_readonly(&dir).is_err());
865
866        let second = remember(&store, "the kilimanjaro trail opens in june");
867        assert_eq!(lp.drain(), Some(1));
868        let guard = reader.read().unwrap();
869        assert_eq!(guard.bm25_search("zanzibar", 5).unwrap()[0].0, first);
870        assert_eq!(guard.bm25_search("kilimanjaro", 5).unwrap()[0].0, second);
871        assert_eq!(guard.vector_len(), lp.writer.open().unwrap().vector_len());
872    }
873
874    #[test]
875    fn a_store_key_rotation_reseals_the_indexes_under_a_new_key() {
876        let data = tempfile::tempdir().unwrap();
877        let store = keyed_store(data.path());
878        let dir = data.path().join("brain.indexes");
879        let keys = store
880            .at_rest_keys(AtRestPurpose::Indexes)
881            .unwrap()
882            .unwrap()
883            .keys;
884        let handles = crate::models::open_index_handles(&dir, &store, Some(&keys), |_| {}).unwrap();
885        let crate::models::IndexHandles {
886            reader,
887            writer,
888            claim,
889        } = handles;
890        drop(claim);
891        let mut lp = drain_loop(store.clone(), reader.clone(), writer);
892        let id = remember(&store, "the zanzibar ferry leaves at noon");
893        assert_eq!(lp.drain(), Some(1));
894        lp.seal_if_due();
895        let before = store.at_rest_keys(AtRestPurpose::Indexes).unwrap().unwrap();
896        assert!(before.settled);
897
898        let rotated = second_brain_core::keys::StoreKey::generate().unwrap();
899        store
900            .rewrite_under_key(
901                second_brain_core::sqlite_store::encrypt_migration::RewriteTo::Key(&rotated),
902            )
903            .unwrap();
904        lp.seal_if_due();
905        let after = store.at_rest_keys(AtRestPurpose::Indexes).unwrap().unwrap();
906        assert_ne!(after.keys.current_id(), before.keys.current_id());
907        assert!(after.settled);
908        assert!(after.keys.retired_ids().is_empty());
909        assert_all_sealed_under(&dir, after.keys.current_id());
910        assert_eq!(
911            reader.read().unwrap().bm25_search("zanzibar", 5).unwrap()[0].0,
912            id
913        );
914    }
915
916    #[test]
917    fn an_idle_sealed_writer_keeps_its_vectors_for_recall() {
918        let data = tempfile::tempdir().unwrap();
919        let store = keyed_store(data.path());
920        let dir = data.path().join("brain.indexes");
921        let keys = store
922            .at_rest_keys(AtRestPurpose::Indexes)
923            .unwrap()
924            .unwrap()
925            .keys;
926        let handles = crate::models::open_index_handles(&dir, &store, Some(&keys), |_| {}).unwrap();
927        let crate::models::IndexHandles {
928            reader,
929            writer,
930            claim,
931        } = handles;
932        drop(claim);
933        let mut lp = drain_loop(store.clone(), reader.clone(), writer);
934        let first = remember(&store, "the zanzibar ferry leaves at noon");
935        assert_eq!(lp.drain(), Some(1));
936        let used = lp.writer.last_used;
937        assert!(lp.writer.release_if_idle(used + WRITER_IDLE));
938        assert!(!lp.writer.release_if_idle(used + WRITER_IDLE * 2));
939        assert!(lp.writer.memory().is_some());
940
941        let second = remember(&store, "the kilimanjaro trail opens in june");
942        assert_eq!(lp.drain(), Some(1));
943        let guard = reader.read().unwrap();
944        assert_eq!(guard.bm25_search("zanzibar", 5).unwrap()[0].0, first);
945        assert_eq!(guard.bm25_search("kilimanjaro", 5).unwrap()[0].0, second);
946    }
947
948    #[test]
949    fn indexes_that_open_under_no_held_key_are_rebuilt_from_the_store() {
950        let data = tempfile::tempdir().unwrap();
951        let store = keyed_store(data.path());
952        let dir = data.path().join("brain.indexes");
953        let id = remember(&store, "the zanzibar ferry leaves at noon");
954        let foreign = SealKeys::single(second_brain_core::sealed::SealKey::generate().unwrap());
955        let handles =
956            crate::models::open_index_handles(&dir, &store, Some(&foreign), |_| {}).unwrap();
957        drop(handles);
958
959        let keys = store
960            .at_rest_keys(AtRestPurpose::Indexes)
961            .unwrap()
962            .unwrap()
963            .keys;
964        let handles = crate::models::open_index_handles(&dir, &store, Some(&keys), |_| {}).unwrap();
965        assert_eq!(
966            handles
967                .reader
968                .read()
969                .unwrap()
970                .bm25_search("zanzibar", 5)
971                .unwrap()[0]
972                .0,
973            id
974        );
975    }
976}