Skip to main content

second_brain_api/
maintenance.rs

1//! The drain thread: applies the index outbox every few seconds and, when
2//! the outbox is quiet and the interval has passed, repairs the store with
3//! the writable indexes it owns. On-demand repairs arrive over a channel so
4//! the same thread services them; two writers never exist.
5
6use std::path::PathBuf;
7use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender};
8use std::sync::{Arc, PoisonError, RwLock};
9use std::time::{Duration, Instant};
10
11use anyhow::{Result, anyhow};
12use chrono::Utc;
13use second_brain_core::config::MaintenanceConfig;
14use second_brain_core::embedding::Embedder;
15use second_brain_core::index::{IndexClaim, Indexes};
16use second_brain_core::maintenance::{MaintenanceSummary, RepairStats};
17use second_brain_core::residency::{self, IndexMemory};
18use second_brain_core::sealed::{SealKeyId, SealKeys};
19use second_brain_core::sqlite_store::{AtRestPurpose, SqliteStore};
20
21use crate::health::Health;
22use crate::near_duplicates::{NearDupCount, refresh_count};
23
24const DRAIN_INTERVAL: Duration = Duration::from_secs(5);
25/// Peers' deletes ask for a lexical merge, which rewrites every segment,
26/// so a burst of them is served by one merge.
27const DELETE_PURGE_GAP: Duration = Duration::from_secs(600);
28
29const WRITER_IDLE: Duration = Duration::from_secs(60);
30/// How often the drain thread asks the store whether the index files need
31/// sealing: after the store's first encryption, or a store key rotation.
32const SEAL_CHECK_INTERVAL: Duration = Duration::from_secs(60);
33
34/// Finding chunk work splits every live row, so once a scan comes back
35/// empty-handed the next one waits this long unless a drain applies writes.
36const CHUNK_IDLE_RESCAN: Duration = Duration::from_secs(600);
37/// Per idle tick, so the drain and repair behind it are never held up long.
38const CHUNK_TICK_BUDGET: Duration = Duration::from_secs(3);
39/// Bounds what one tick holds in memory: at most 16 chunks per memory.
40const CHUNK_MEMORIES_PER_TICK: usize = 32;
41
42/// When the drain thread may next look for memories needing chunks.
43struct ChunkThrottle {
44    next_scan: Instant,
45}
46
47impl ChunkThrottle {
48    fn new(now: Instant) -> Self {
49        Self { next_scan: now }
50    }
51
52    fn due(&self, now: Instant) -> bool {
53        now >= self.next_scan
54    }
55
56    fn after_backfill(&mut self, now: Instant, progressed: bool) {
57        self.next_scan = if progressed {
58            now
59        } else {
60            now + CHUNK_IDLE_RESCAN
61        };
62    }
63
64    /// Applied outbox rows may carry content changes that need re-chunking.
65    fn after_drain(&mut self, now: Instant, applied: usize) {
66        if applied > 0 {
67            self.next_scan = now;
68        }
69    }
70}
71
72pub struct RepairRequest {
73    pub dry_run: bool,
74    pub reply: Sender<Result<MaintenanceSummary>>,
75}
76
77/// Work only the drain thread may do, because it owns the writable indexes.
78pub enum DrainRequest {
79    Repair(RepairRequest),
80    /// Applies the whole outbox, then rewrites the lexical index into fresh
81    /// segments (see `Indexes::compact`).
82    CompactIndexes(Sender<Result<()>>),
83}
84
85/// Runs a repair and re-measures the store afterwards, on whatever index
86/// handle the caller owns (the writer on the drain thread, none when the
87/// indexes never opened or a released writer could not reopen).
88pub fn run_repair(
89    store: &SqliteStore,
90    mut indexes: Option<&mut Indexes>,
91    embedder: Option<&Embedder>,
92    dry_run: bool,
93) -> Result<MaintenanceSummary> {
94    let embed = embedder.map(|e| move |text: &str| e.embed(text));
95    let embed_ref = embed
96        .as_ref()
97        .map(|f| f as &dyn Fn(&str) -> Result<Vec<f32>>);
98    let stats = store.repair(indexes.as_deref_mut(), embed_ref, dry_run)?;
99    let report_after = store.integrity_report(indexes.as_deref())?;
100    Ok(MaintenanceSummary {
101        ran_at: Utc::now(),
102        stats,
103        report_after,
104    })
105}
106
107fn log_stats(source: &str, stats: &RepairStats) {
108    if stats.is_clean() {
109        tracing::info!("{source} store repair: nothing to do");
110    } else {
111        tracing::info!("{source} store repair: {}", summarize(stats));
112    }
113}
114
115fn summarize(s: &RepairStats) -> String {
116    let mut parts = Vec::new();
117    let mut push = |name: &str, n: u64| {
118        if n > 0 {
119            parts.push(format!("{name}={n}"));
120        }
121    };
122    push("duplicates_superseded", s.duplicates_superseded);
123    push("markers_inherited", s.distillation_markers_inherited);
124    push("orphaned_markers_dropped", s.orphaned_markers_dropped);
125    push("raws_retired", s.raws_retired);
126    push("relations_pruned", s.relations_pruned);
127    push("vec_keys_deleted", s.vec_keys_deleted);
128    push("sync_log_rows_deleted", s.sync_log_rows_deleted);
129    push("stale_cursors_dropped", s.stale_cursors_dropped);
130    push("embeddings_backfilled", s.embeddings_backfilled);
131    push("embedding_failures", s.embedding_failures);
132    if s.vec_keys_gc_skipped {
133        parts.push("vec_keys_gc=skipped".to_string());
134    }
135    if s.index_rebuilt {
136        parts.push("index=rebuilt".to_string());
137    }
138    if s.dry_run {
139        parts.push("dry_run".to_string());
140    }
141    parts.join(" ")
142}
143
144/// The drain thread's writable indexes. usearch cannot mutate a viewed index,
145/// so an open writer holds its own anonymous copy of the vectors beside the
146/// reader's mmap; the slot drops it once `WRITER_IDLE` has passed since its
147/// last operation finished, and reopens it from the persisted files when work
148/// arrives. A writer holding changes its files lack is kept, because a drain
149/// commits its store-side deletes before it persists, so only that writer
150/// still holds those removals.
151///
152/// Sealed indexes can't be mapped, so recall reads a reader sharing the
153/// writer's vectors; the slot then never drops that writer, and an idle one
154/// gives up only its lexical writer's buffers.
155pub struct WriterSlot {
156    dir: PathBuf,
157    open: Option<Indexes>,
158    last_used: Instant,
159    shared: bool,
160    idle_released: bool,
161    _claim: IndexClaim,
162}
163
164impl WriterSlot {
165    pub fn new(writer: Indexes, claim: IndexClaim) -> Self {
166        crate::heap::record_index_writer(Some(writer.memory()));
167        Self {
168            dir: writer.dir().to_path_buf(),
169            shared: writer.seal_keys().is_some(),
170            open: Some(writer),
171            last_used: Instant::now(),
172            idle_released: false,
173            _claim: claim,
174        }
175    }
176
177    fn open(&mut self) -> Result<&mut Indexes> {
178        if self.open.is_none() {
179            if self.shared {
180                return Err(anyhow!(
181                    "the sealed index writer at {} is gone; the daemon must restart to reopen it",
182                    self.dir.display()
183                ));
184            }
185            let writer = Indexes::open(&self.dir)?;
186            tracing::info!("index writer reopened at {}", self.dir.display());
187            crate::heap::record_index_writer(Some(writer.memory()));
188            self.open = Some(writer);
189        }
190        Ok(self.open.as_mut().expect("writer opened above"))
191    }
192
193    fn with<R>(&mut self, op: impl FnOnce(&mut Indexes) -> Result<R>) -> Result<R> {
194        let result = op(self.open()?);
195        self.last_used = Instant::now();
196        self.idle_released = false;
197        crate::heap::record_index_writer(self.memory());
198        result
199    }
200
201    fn release_if_idle(&mut self, now: Instant) -> bool {
202        let current = self
203            .open
204            .as_ref()
205            .is_some_and(|w| !w.has_unpersisted_changes());
206        if !current || now.saturating_duration_since(self.last_used) < WRITER_IDLE {
207            return false;
208        }
209        if self.shared {
210            return self.release_lexical_writer();
211        }
212        if let Some(writer) = self.open.take()
213            && let Err(e) = writer.wait_for_merges()
214        {
215            tracing::warn!("index writer released without its merges: {e:#}");
216        }
217        crate::heap::record_index_writer(None);
218        residency::release_free_heap();
219        tracing::info!(
220            "index writer released after {}s idle",
221            WRITER_IDLE.as_secs()
222        );
223        true
224    }
225
226    fn release_lexical_writer(&mut self) -> bool {
227        if self.idle_released {
228            return false;
229        }
230        if let Some(writer) = self.open.as_mut()
231            && let Err(e) = writer.close_lexical_writer()
232        {
233            tracing::warn!("lexical writer released without its merges: {e:#}");
234        }
235        self.idle_released = true;
236        residency::release_free_heap();
237        tracing::info!(
238            "lexical index writer released after {}s idle",
239            WRITER_IDLE.as_secs()
240        );
241        true
242    }
243
244    fn memory(&self) -> Option<IndexMemory> {
245        self.open.as_ref().map(Indexes::memory)
246    }
247
248    fn sealed_under(&self) -> Option<SealKeyId> {
249        self.open
250            .as_ref()
251            .and_then(Indexes::seal_keys)
252            .map(|keys| keys.current_id())
253    }
254
255    /// Seals every index file under `keys`' current key. A plaintext writer
256    /// is persisted and reopened sealed, and recall moves to a reader that
257    /// shares its vectors.
258    fn seal_under(&mut self, keys: SealKeys, reader: &RwLock<Indexes>) -> Result<()> {
259        if self.shared {
260            let writer = self
261                .open
262                .as_mut()
263                .ok_or_else(|| anyhow!("the sealed index writer is gone"))?;
264            writer.set_seal_keys(keys)?;
265            return writer.reseal();
266        }
267        if let Some(writer) = self.open.as_mut()
268            && writer.has_unpersisted_changes()
269        {
270            writer.persist()?;
271        }
272        if let Some(writer) = self.open.take() {
273            crate::heap::record_index_writer(None);
274            writer.wait_for_merges()?;
275        }
276        let mut writer = Indexes::open_keyed(&self.dir, &keys)?;
277        writer.reseal()?;
278        let shared = writer.shared_reader()?;
279        *reader.write().unwrap_or_else(PoisonError::into_inner) = shared;
280        crate::heap::record_index_writer(Some(writer.memory()));
281        self.open = Some(writer);
282        self.shared = true;
283        self.idle_released = false;
284        self.last_used = Instant::now();
285        Ok(())
286    }
287}
288
289pub struct DrainLoop {
290    pub store: Arc<SqliteStore>,
291    pub reader: Arc<RwLock<Indexes>>,
292    pub writer: WriterSlot,
293    pub embedder: Option<Arc<Embedder>>,
294    pub summary: Arc<RwLock<Option<MaintenanceSummary>>>,
295    pub near_duplicates: Arc<RwLock<Option<NearDupCount>>>,
296    pub near_duplicate_threshold: f32,
297    pub config: MaintenanceConfig,
298    pub requests: Receiver<DrainRequest>,
299    pub health: Arc<Health>,
300}
301
302impl DrainLoop {
303    pub fn spawn(self) {
304        std::thread::Builder::new()
305            .name("index-drain".to_string())
306            .spawn(move || self.run())
307            .expect("spawning the index drain thread");
308    }
309
310    fn run(mut self) {
311        let mut last_repair: Option<Instant> = None;
312        let mut chunks = ChunkThrottle::new(Instant::now());
313        let mut last_purge: Option<Instant> = None;
314        let mut last_seal_check: Option<Instant> = None;
315        loop {
316            match self.requests.recv_timeout(DRAIN_INTERVAL) {
317                Ok(DrainRequest::Repair(request)) => {
318                    let result = self.repair("requested", request.dry_run);
319                    if result.is_ok() && !request.dry_run {
320                        last_repair = Some(Instant::now());
321                    }
322                    // A caller that gave up waiting is not an error here.
323                    let _ = request.reply.send(result);
324                }
325                Ok(DrainRequest::CompactIndexes(reply)) => {
326                    let _ = reply.send(self.compact_indexes());
327                }
328                Err(RecvTimeoutError::Timeout) | Err(RecvTimeoutError::Disconnected) => {
329                    let applied = self.drain();
330                    chunks.after_drain(Instant::now(), applied.unwrap_or(0));
331                    let due = last_repair.is_none_or(|t| {
332                        t.elapsed() >= Duration::from_secs(self.config.interval_secs)
333                    });
334                    if applied == Some(0) && self.config.enabled && due {
335                        if self.repair("scheduled", false).is_ok() {
336                            last_repair = Some(Instant::now());
337                        } else {
338                            // Retry sooner than the full interval, but not
339                            // every tick against a store that keeps failing.
340                            last_repair = Some(
341                                Instant::now()
342                                    - Duration::from_secs(
343                                        self.config.interval_secs.saturating_sub(600),
344                                    ),
345                            );
346                        }
347                    }
348                    if applied == Some(0)
349                        && last_purge.is_none_or(|t| t.elapsed() >= DELETE_PURGE_GAP)
350                        && self.purge_applied_deletes()
351                    {
352                        last_purge = Some(Instant::now());
353                    }
354                    if applied == Some(0) && chunks.due(Instant::now()) {
355                        let progressed = self.backfill_chunks();
356                        chunks.after_backfill(Instant::now(), progressed);
357                    }
358                }
359            }
360            if last_seal_check.is_none_or(|t| t.elapsed() >= SEAL_CHECK_INTERVAL) {
361                self.seal_if_due();
362                last_seal_check = Some(Instant::now());
363            }
364            self.writer.release_if_idle(Instant::now());
365        }
366    }
367
368    /// Seals the index files when the store's keys for them changed: on the
369    /// store's first encryption, after a store key rotation, or when a
370    /// reseal was cut short. Retired keys are forgotten only once every file
371    /// is under the current one.
372    fn seal_if_due(&mut self) {
373        let keys = match self.store.at_rest_keys(AtRestPurpose::Indexes) {
374            Ok(Some(keys)) => keys,
375            Ok(None) => return,
376            Err(e) => {
377                tracing::warn!("reading the search indexes' keys failed: {e:#}");
378                return;
379            }
380        };
381        let current = keys.keys.current_id();
382        if keys.settled && self.writer.sealed_under() == Some(current) {
383            return;
384        }
385        let started = Instant::now();
386        if let Err(e) = self.writer.seal_under(keys.keys, &self.reader) {
387            tracing::warn!("sealing the search indexes failed and will retry: {e:#}");
388            return;
389        }
390        match self
391            .store
392            .settle_at_rest_keys(AtRestPurpose::Indexes, current)
393        {
394            Ok(()) => tracing::info!(
395                "search indexes sealed under key {current} in {:.1}s",
396                started.elapsed().as_secs_f64()
397            ),
398            Err(e) => tracing::warn!("recording the sealed search indexes failed: {e:#}"),
399        }
400    }
401
402    /// Embeds pending chunk sets within one tick's budget; true when any set
403    /// was stored, so the next idle tick keeps going.
404    fn backfill_chunks(&self) -> bool {
405        let Some(embedder) = self.embedder.as_deref() else {
406            return false;
407        };
408        let embed = |texts: &[&str]| embedder.embed_batch(texts);
409        match self.store.backfill_chunk_embeddings(
410            &embed,
411            CHUNK_MEMORIES_PER_TICK,
412            CHUNK_TICK_BUDGET,
413        ) {
414            Ok(r) => {
415                if r.memories > 0 || r.failed > 0 {
416                    tracing::info!(
417                        "chunk backfill: memories={} chunks={} failed={} pending={}",
418                        r.memories,
419                        r.chunks,
420                        r.failed,
421                        r.pending
422                    );
423                }
424                r.memories > 0
425            }
426            Err(e) => {
427                tracing::warn!("chunk backfill failed: {e:#}");
428                false
429            }
430        }
431    }
432
433    /// Applies pending outbox rows; None when the drain itself failed. An
434    /// empty outbox never touches the writer, so a released one stays closed.
435    fn drain(&mut self) -> Option<usize> {
436        let store = &self.store;
437        let writer = &mut self.writer;
438        let result = store.outbox_pending().and_then(|pending| {
439            if pending == 0 {
440                Ok(0)
441            } else {
442                writer.with(|w| w.drain(store))
443            }
444        });
445        match result {
446            Ok(0) => {
447                self.health.drained(0, Utc::now());
448                Some(0)
449            }
450            Ok(n) => {
451                tracing::info!("index drain applied {n} outbox rows");
452                self.reload_reader();
453                match self.store.outbox_pending() {
454                    Ok(pending) => self.health.drained(pending, Utc::now()),
455                    Err(e) => tracing::warn!("counting the index outbox failed: {e:#}"),
456                }
457                Some(n)
458            }
459            Err(e) => {
460                tracing::warn!("index drain failed: {e:#}");
461                self.health.drain_failed(&format!("{e:#}"), Utc::now());
462                None
463            }
464        }
465    }
466
467    fn repair(&mut self, source: &str, dry_run: bool) -> Result<MaintenanceSummary> {
468        let started = Instant::now();
469        let store = &self.store;
470        let embedder = self.embedder.as_deref();
471        let repair = |w: &mut Indexes| run_repair(store, Some(w), embedder, dry_run);
472        let result = match self.writer.open().map(|_| ()) {
473            Ok(()) => self.writer.with(repair),
474            Err(e) => {
475                tracing::warn!(
476                    "index writer unavailable ({e:#}); repairing the store without the indexes"
477                );
478                match run_repair(store, None, embedder, dry_run) {
479                    Ok(summary) => log_stats(source, &summary.stats),
480                    Err(store_error) => {
481                        tracing::warn!("{source} store repair failed: {store_error:#}")
482                    }
483                }
484                Err(e
485                    .context("the index writer could not reopen, so the indexes were not repaired"))
486            }
487        };
488        match &result {
489            Ok(summary) => {
490                self.health.repaired(Utc::now());
491                log_stats(source, &summary.stats);
492                tracing::info!(
493                    "{source} store repair finished in {:.1}s; drift remaining: {:?}",
494                    started.elapsed().as_secs_f64(),
495                    summary.report_after.drift()
496                );
497                if !dry_run {
498                    self.reload_reader();
499                    let mut guard = match self.summary.write() {
500                        Ok(g) => g,
501                        Err(poisoned) => poisoned.into_inner(),
502                    };
503                    *guard = Some(summary.clone());
504                    drop(guard);
505                    let threshold = self.near_duplicate_threshold;
506                    let near_duplicates = &self.near_duplicates;
507                    let counted = self.writer.with(|w| {
508                        refresh_count(store, w, threshold, near_duplicates);
509                        Ok(())
510                    });
511                    if let Err(e) = counted {
512                        tracing::warn!("near-duplicate count failed: {e:#}");
513                    }
514                    if summary.stats.index_rebuilt {
515                        residency::release_free_heap();
516                    }
517                }
518            }
519            Err(e) => {
520                tracing::warn!("{source} store repair failed: {e:#}");
521                self.health.repair_failed(&format!("{e:#}"), Utc::now());
522            }
523        }
524        result
525    }
526
527    /// Runs the lexical merge and cold purge a peer's applied delete asked
528    /// for, so its text leaves this disk as it left the deleting one's;
529    /// true when it ran.
530    fn purge_applied_deletes(&mut self) -> bool {
531        let token = match self.store.delete_purge_requested() {
532            Ok(Some(token)) => token,
533            Ok(None) => return false,
534            Err(e) => {
535                tracing::warn!("reading the delete purge request failed: {e:#}");
536                return false;
537            }
538        };
539        let result = self.compact_indexes().and_then(|()| {
540            let cold = self.store.purge_cold_segments()?;
541            if cold != Default::default() {
542                tracing::info!(
543                    "deleted records purged from the cold tier: {} retired, {} segments rewritten, {} files removed",
544                    cold.retired,
545                    cold.rewritten,
546                    cold.removed_files
547                );
548            }
549            self.store.delete_purge_done(&token)
550        });
551        if let Err(e) = result {
552            tracing::warn!("purging a peer's deletes failed and will retry: {e:#}");
553        }
554        true
555    }
556
557    fn compact_indexes(&mut self) -> Result<()> {
558        let store = &self.store;
559        let took = self.writer.with(|w| {
560            while w.drain(store)? > 0 {}
561            let started = Instant::now();
562            w.compact()?;
563            Ok(started.elapsed())
564        })?;
565        self.reload_reader();
566        residency::release_free_heap();
567        tracing::info!("lexical index compacted in {:.1}s", took.as_secs_f64());
568        Ok(())
569    }
570
571    fn reload_reader(&self) {
572        let mut guard = match self.reader.write() {
573            Ok(g) => g,
574            Err(poisoned) => poisoned.into_inner(),
575        };
576        if let Err(e) = guard.reload_view() {
577            tracing::warn!("index reader reload failed: {e:#}");
578        }
579    }
580}
581
582#[cfg(test)]
583mod tests {
584    use std::fs::Permissions;
585    use std::os::unix::fs::PermissionsExt;
586
587    use super::*;
588    use second_brain_core::schema::{Memory, MemoryType};
589    use second_brain_core::store::Store;
590
591    #[test]
592    fn a_fresh_throttle_scans_on_the_first_idle_tick() {
593        let now = Instant::now();
594        assert!(ChunkThrottle::new(now).due(now));
595    }
596
597    #[test]
598    fn a_scan_that_finds_nothing_waits_out_the_idle_interval() {
599        let now = Instant::now();
600        let mut t = ChunkThrottle::new(now);
601        t.after_backfill(now, false);
602        assert!(!t.due(now + CHUNK_IDLE_RESCAN - Duration::from_secs(1)));
603        assert!(t.due(now + CHUNK_IDLE_RESCAN));
604    }
605
606    #[test]
607    fn progress_keeps_the_next_idle_tick_scanning() {
608        let now = Instant::now();
609        let mut t = ChunkThrottle::new(now);
610        t.after_backfill(now, true);
611        assert!(t.due(now));
612    }
613
614    #[test]
615    fn an_applied_drain_resets_the_idle_wait() {
616        let now = Instant::now();
617        let mut t = ChunkThrottle::new(now);
618        t.after_backfill(now, false);
619        t.after_drain(now, 0);
620        assert!(!t.due(now));
621        t.after_drain(now, 3);
622        assert!(t.due(now));
623    }
624
625    fn remember(store: &SqliteStore, content: &str) -> uuid::Uuid {
626        let m = Memory::new(
627            content.to_string(),
628            MemoryType::Semantic,
629            "test".to_string(),
630            String::new(),
631        );
632        store.store_memory(&m).unwrap();
633        m.id
634    }
635
636    fn drain_loop(
637        store: Arc<SqliteStore>,
638        reader: Arc<RwLock<Indexes>>,
639        writer: Indexes,
640    ) -> DrainLoop {
641        let (_tx, requests) = std::sync::mpsc::channel();
642        let claim = IndexClaim::acquire(writer.dir()).unwrap();
643        DrainLoop {
644            store,
645            reader,
646            writer: WriterSlot::new(writer, claim),
647            embedder: None,
648            summary: Arc::new(RwLock::new(None)),
649            near_duplicates: Arc::new(RwLock::new(None)),
650            near_duplicate_threshold: 0.97,
651            config: MaintenanceConfig {
652                enabled: false,
653                interval_secs: 3600,
654            },
655            requests,
656            health: Arc::new(Health::new(Utc::now(), None)),
657        }
658    }
659
660    fn idle_loop(dir: &std::path::Path) -> DrainLoop {
661        let store = Arc::new(SqliteStore::in_memory("m".to_string()).unwrap());
662        let mut writer = Indexes::open(dir).unwrap();
663        writer.persist().unwrap();
664        let reader = Arc::new(RwLock::new(Indexes::open_readonly(dir).unwrap()));
665        drain_loop(store, reader, writer)
666    }
667
668    #[test]
669    fn idleness_counts_from_when_an_operation_finishes() {
670        let dir = tempfile::tempdir().unwrap();
671        let mut lp = idle_loop(dir.path());
672        let mut finishing = None;
673        lp.writer
674            .with(|_| {
675                finishing = Some(Instant::now());
676                Ok(())
677            })
678            .unwrap();
679        let finishing = finishing.unwrap();
680        assert!(lp.writer.last_used >= finishing);
681        assert!(
682            !lp.writer
683                .release_if_idle(finishing + WRITER_IDLE - Duration::from_secs(1))
684        );
685    }
686
687    #[test]
688    fn a_writer_holding_changes_its_files_lack_is_never_released() {
689        let dir = tempfile::tempdir().unwrap();
690        let mut lp = idle_loop(dir.path());
691        remember(&lp.store, "the zanzibar ferry leaves at noon");
692        let tantivy = dir.path().join("tantivy");
693        let set_mode =
694            |mode| std::fs::set_permissions(&tantivy, Permissions::from_mode(mode)).unwrap();
695        set_mode(0o500);
696        let drained = lp.drain();
697        set_mode(0o700);
698        assert_eq!(
699            drained, None,
700            "the drain should fail to write the lexical index"
701        );
702        assert!(
703            !lp.writer
704                .release_if_idle(lp.writer.last_used + WRITER_IDLE * 10)
705        );
706        lp.repair("test", true).unwrap();
707        assert!(!lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
708
709        assert_eq!(lp.drain(), Some(1));
710        assert!(lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
711    }
712
713    #[test]
714    fn a_failure_that_left_the_files_current_still_releases() {
715        let dir = tempfile::tempdir().unwrap();
716        let mut lp = idle_loop(dir.path());
717        let failed: Result<()> = lp.writer.with(|_| anyhow::bail!("a store step failed"));
718        assert!(failed.is_err());
719        assert!(lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
720    }
721
722    #[test]
723    fn a_writer_that_cannot_reopen_fails_the_repair() {
724        let dir = tempfile::tempdir().unwrap();
725        let mut lp = idle_loop(dir.path());
726        assert!(lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
727        let blocker = dir.path().join("not-a-dir");
728        std::fs::write(&blocker, b"x").unwrap();
729        lp.writer.dir = blocker.join("indexes");
730
731        let err = lp.repair("test", false).unwrap_err();
732        assert!(
733            format!("{err:#}").contains("indexes were not repaired"),
734            "{err:#}"
735        );
736        assert!(lp.writer.memory().is_none());
737    }
738
739    #[test]
740    fn an_idle_writer_is_released_and_reopened_by_the_next_write() {
741        let dir = tempfile::tempdir().unwrap();
742        let store = Arc::new(SqliteStore::in_memory("m".to_string()).unwrap());
743        let mut writer = Indexes::open(dir.path()).unwrap();
744        writer.persist().unwrap();
745        let reader = Arc::new(RwLock::new(Indexes::open_readonly(dir.path()).unwrap()));
746        let mut lp = drain_loop(store.clone(), reader.clone(), writer);
747
748        let first = remember(&store, "the zanzibar ferry leaves at noon");
749        assert_eq!(lp.drain(), Some(1));
750        assert!(lp.writer.memory().is_some());
751
752        let used = lp.writer.last_used;
753        assert!(
754            !lp.writer
755                .release_if_idle(used + WRITER_IDLE - Duration::from_secs(1))
756        );
757        assert!(lp.writer.memory().is_some());
758        assert!(lp.writer.release_if_idle(used + WRITER_IDLE));
759        assert!(lp.writer.memory().is_none());
760
761        assert_eq!(lp.drain(), Some(0));
762        assert!(
763            lp.writer.memory().is_none(),
764            "an empty tick reopened the writer"
765        );
766
767        let second = remember(&store, "the kilimanjaro trail opens in june");
768        assert_eq!(lp.drain(), Some(1));
769        assert!(lp.writer.memory().is_some());
770        assert_eq!(lp.writer.open().unwrap().lexical_len(), 2);
771
772        let guard = reader.read().unwrap();
773        let hits = |q: &str| -> Vec<uuid::Uuid> {
774            guard
775                .bm25_search(q, 5)
776                .unwrap()
777                .into_iter()
778                .map(|(id, _)| id)
779                .collect()
780        };
781        assert_eq!(hits("kilimanjaro"), vec![second]);
782        assert_eq!(hits("zanzibar"), vec![first]);
783    }
784
785    fn keyed_store(dir: &std::path::Path) -> Arc<SqliteStore> {
786        let key = second_brain_core::keys::StoreKey::generate().unwrap();
787        Arc::new(SqliteStore::open_with_key(&dir.join("brain.db"), "m".to_string(), &key).unwrap())
788    }
789
790    fn index_files(dir: &std::path::Path) -> Vec<std::path::PathBuf> {
791        let mut out = Vec::new();
792        let mut stack = vec![dir.to_path_buf()];
793        while let Some(d) = stack.pop() {
794            for entry in std::fs::read_dir(d).unwrap() {
795                let path = entry.unwrap().path();
796                let name = path.file_name().unwrap().to_string_lossy().to_string();
797                if path.is_dir() {
798                    stack.push(path);
799                } else if !name.ends_with(".lock") && name != second_brain_core::index::CLAIM_FILE {
800                    out.push(path);
801                }
802            }
803        }
804        out
805    }
806
807    fn assert_all_sealed_under(dir: &std::path::Path, key: SealKeyId) {
808        let files = index_files(dir);
809        assert!(!files.is_empty());
810        for path in files {
811            assert_eq!(
812                second_brain_core::sealed::sealed_key_id(&path).unwrap(),
813                Some(key),
814                "{}",
815                path.display()
816            );
817        }
818    }
819
820    #[test]
821    fn indexes_opened_before_the_store_was_encrypted_are_sealed_and_recall_moves_to_a_shared_reader()
822     {
823        let data = tempfile::tempdir().unwrap();
824        let store = keyed_store(data.path());
825        let dir = data.path().join("brain.indexes");
826        let mut writer = Indexes::open(&dir).unwrap();
827        writer.persist().unwrap();
828        let reader = Arc::new(RwLock::new(Indexes::open_readonly(&dir).unwrap()));
829        let mut lp = drain_loop(store.clone(), reader.clone(), writer);
830        let first = remember(&store, "the zanzibar ferry leaves at noon");
831        assert_eq!(lp.drain(), Some(1));
832
833        lp.seal_if_due();
834        let keys = store.at_rest_keys(AtRestPurpose::Indexes).unwrap().unwrap();
835        assert!(keys.settled);
836        assert_all_sealed_under(&dir, keys.keys.current_id());
837        assert!(Indexes::open_readonly(&dir).is_err());
838
839        let second = remember(&store, "the kilimanjaro trail opens in june");
840        assert_eq!(lp.drain(), Some(1));
841        let guard = reader.read().unwrap();
842        assert_eq!(guard.bm25_search("zanzibar", 5).unwrap()[0].0, first);
843        assert_eq!(guard.bm25_search("kilimanjaro", 5).unwrap()[0].0, second);
844        assert_eq!(guard.vector_len(), lp.writer.open().unwrap().vector_len());
845    }
846
847    #[test]
848    fn a_store_key_rotation_reseals_the_indexes_under_a_new_key() {
849        let data = tempfile::tempdir().unwrap();
850        let store = keyed_store(data.path());
851        let dir = data.path().join("brain.indexes");
852        let keys = store
853            .at_rest_keys(AtRestPurpose::Indexes)
854            .unwrap()
855            .unwrap()
856            .keys;
857        let handles = crate::models::open_index_handles(&dir, &store, Some(&keys), |_| {}).unwrap();
858        let crate::models::IndexHandles {
859            reader,
860            writer,
861            claim,
862        } = handles;
863        drop(claim);
864        let mut lp = drain_loop(store.clone(), reader.clone(), writer);
865        let id = remember(&store, "the zanzibar ferry leaves at noon");
866        assert_eq!(lp.drain(), Some(1));
867        lp.seal_if_due();
868        let before = store.at_rest_keys(AtRestPurpose::Indexes).unwrap().unwrap();
869        assert!(before.settled);
870
871        let rotated = second_brain_core::keys::StoreKey::generate().unwrap();
872        store
873            .rewrite_under_key(
874                second_brain_core::sqlite_store::encrypt_migration::RewriteTo::Key(&rotated),
875            )
876            .unwrap();
877        lp.seal_if_due();
878        let after = store.at_rest_keys(AtRestPurpose::Indexes).unwrap().unwrap();
879        assert_ne!(after.keys.current_id(), before.keys.current_id());
880        assert!(after.settled);
881        assert!(after.keys.retired_ids().is_empty());
882        assert_all_sealed_under(&dir, after.keys.current_id());
883        assert_eq!(
884            reader.read().unwrap().bm25_search("zanzibar", 5).unwrap()[0].0,
885            id
886        );
887    }
888
889    #[test]
890    fn an_idle_sealed_writer_keeps_its_vectors_for_recall() {
891        let data = tempfile::tempdir().unwrap();
892        let store = keyed_store(data.path());
893        let dir = data.path().join("brain.indexes");
894        let keys = store
895            .at_rest_keys(AtRestPurpose::Indexes)
896            .unwrap()
897            .unwrap()
898            .keys;
899        let handles = crate::models::open_index_handles(&dir, &store, Some(&keys), |_| {}).unwrap();
900        let crate::models::IndexHandles {
901            reader,
902            writer,
903            claim,
904        } = handles;
905        drop(claim);
906        let mut lp = drain_loop(store.clone(), reader.clone(), writer);
907        let first = remember(&store, "the zanzibar ferry leaves at noon");
908        assert_eq!(lp.drain(), Some(1));
909        let used = lp.writer.last_used;
910        assert!(lp.writer.release_if_idle(used + WRITER_IDLE));
911        assert!(!lp.writer.release_if_idle(used + WRITER_IDLE * 2));
912        assert!(lp.writer.memory().is_some());
913
914        let second = remember(&store, "the kilimanjaro trail opens in june");
915        assert_eq!(lp.drain(), Some(1));
916        let guard = reader.read().unwrap();
917        assert_eq!(guard.bm25_search("zanzibar", 5).unwrap()[0].0, first);
918        assert_eq!(guard.bm25_search("kilimanjaro", 5).unwrap()[0].0, second);
919    }
920
921    #[test]
922    fn indexes_that_open_under_no_held_key_are_rebuilt_from_the_store() {
923        let data = tempfile::tempdir().unwrap();
924        let store = keyed_store(data.path());
925        let dir = data.path().join("brain.indexes");
926        let id = remember(&store, "the zanzibar ferry leaves at noon");
927        let foreign = SealKeys::single(second_brain_core::sealed::SealKey::generate().unwrap());
928        let handles =
929            crate::models::open_index_handles(&dir, &store, Some(&foreign), |_| {}).unwrap();
930        drop(handles);
931
932        let keys = store
933            .at_rest_keys(AtRestPurpose::Indexes)
934            .unwrap()
935            .unwrap()
936            .keys;
937        let handles = crate::models::open_index_handles(&dir, &store, Some(&keys), |_| {}).unwrap();
938        assert_eq!(
939            handles
940                .reader
941                .read()
942                .unwrap()
943                .bm25_search("zanzibar", 5)
944                .unwrap()[0]
945                .0,
946            id
947        );
948    }
949}