1use std::path::PathBuf;
7use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender};
8use std::sync::{Arc, PoisonError, RwLock};
9use std::time::{Duration, Instant};
10
11use anyhow::{Result, anyhow};
12use chrono::Utc;
13use second_brain_core::config::MaintenanceConfig;
14use second_brain_core::embedding::Embedder;
15use second_brain_core::index::{IndexClaim, Indexes};
16use second_brain_core::maintenance::{MaintenanceSummary, RepairStats};
17use second_brain_core::residency::{self, IndexMemory};
18use second_brain_core::sealed::{SealKeyId, SealKeys};
19use second_brain_core::sqlite_store::{AtRestPurpose, SqliteStore};
20
21use crate::health::Health;
22use crate::near_duplicates::{NearDupCount, refresh_count};
23
24const DRAIN_INTERVAL: Duration = Duration::from_secs(5);
25const DELETE_PURGE_GAP: Duration = Duration::from_secs(600);
28
29const WRITER_IDLE: Duration = Duration::from_secs(60);
30const SEAL_CHECK_INTERVAL: Duration = Duration::from_secs(60);
33
34const CHUNK_IDLE_RESCAN: Duration = Duration::from_secs(600);
37const CHUNK_TICK_BUDGET: Duration = Duration::from_secs(3);
39const CHUNK_MEMORIES_PER_TICK: usize = 32;
41
42struct ChunkThrottle {
44 next_scan: Instant,
45}
46
47impl ChunkThrottle {
48 fn new(now: Instant) -> Self {
49 Self { next_scan: now }
50 }
51
52 fn due(&self, now: Instant) -> bool {
53 now >= self.next_scan
54 }
55
56 fn after_backfill(&mut self, now: Instant, progressed: bool) {
57 self.next_scan = if progressed {
58 now
59 } else {
60 now + CHUNK_IDLE_RESCAN
61 };
62 }
63
64 fn after_drain(&mut self, now: Instant, applied: usize) {
66 if applied > 0 {
67 self.next_scan = now;
68 }
69 }
70}
71
72pub struct RepairRequest {
73 pub dry_run: bool,
74 pub reply: Sender<Result<MaintenanceSummary>>,
75}
76
77pub enum DrainRequest {
79 Repair(RepairRequest),
80 CompactIndexes(Sender<Result<()>>),
83 Pause(Receiver<PauseCommand>),
86}
87
88pub enum PauseCommand {
89 Settle(Sender<Result<()>>),
92}
93
94pub fn run_repair(
98 store: &SqliteStore,
99 mut indexes: Option<&mut Indexes>,
100 embedder: Option<&Embedder>,
101 dry_run: bool,
102) -> Result<MaintenanceSummary> {
103 let embed = embedder.map(|e| move |text: &str| e.embed(text));
104 let embed_ref = embed
105 .as_ref()
106 .map(|f| f as &dyn Fn(&str) -> Result<Vec<f32>>);
107 let stats = store.repair(indexes.as_deref_mut(), embed_ref, dry_run)?;
108 let report_after = store.integrity_report(indexes.as_deref())?;
109 Ok(MaintenanceSummary {
110 ran_at: Utc::now(),
111 stats,
112 report_after,
113 })
114}
115
116fn log_stats(source: &str, stats: &RepairStats) {
117 if stats.is_clean() {
118 tracing::info!("{source} store repair: nothing to do");
119 } else {
120 tracing::info!("{source} store repair: {}", summarize(stats));
121 }
122}
123
124fn summarize(s: &RepairStats) -> String {
125 let mut parts = Vec::new();
126 let mut push = |name: &str, n: u64| {
127 if n > 0 {
128 parts.push(format!("{name}={n}"));
129 }
130 };
131 push("duplicates_superseded", s.duplicates_superseded);
132 push("markers_inherited", s.distillation_markers_inherited);
133 push("orphaned_markers_dropped", s.orphaned_markers_dropped);
134 push("raws_retired", s.raws_retired);
135 push("relations_pruned", s.relations_pruned);
136 push("vec_keys_deleted", s.vec_keys_deleted);
137 push("sync_log_rows_deleted", s.sync_log_rows_deleted);
138 push("stale_cursors_dropped", s.stale_cursors_dropped);
139 push("embeddings_backfilled", s.embeddings_backfilled);
140 push("embedding_failures", s.embedding_failures);
141 if s.vec_keys_gc_skipped {
142 parts.push("vec_keys_gc=skipped".to_string());
143 }
144 if s.index_rebuilt {
145 parts.push("index=rebuilt".to_string());
146 }
147 if s.dry_run {
148 parts.push("dry_run".to_string());
149 }
150 parts.join(" ")
151}
152
153pub struct WriterSlot {
165 dir: PathBuf,
166 open: Option<Indexes>,
167 last_used: Instant,
168 shared: bool,
169 idle_released: bool,
170 _claim: IndexClaim,
171}
172
173impl WriterSlot {
174 pub fn new(writer: Indexes, claim: IndexClaim) -> Self {
175 crate::heap::record_index_writer(Some(writer.memory()));
176 Self {
177 dir: writer.dir().to_path_buf(),
178 shared: writer.seal_keys().is_some(),
179 open: Some(writer),
180 last_used: Instant::now(),
181 idle_released: false,
182 _claim: claim,
183 }
184 }
185
186 fn open(&mut self) -> Result<&mut Indexes> {
187 if self.open.is_none() {
188 if self.shared {
189 return Err(anyhow!(
190 "the sealed index writer at {} is gone; the daemon must restart to reopen it",
191 self.dir.display()
192 ));
193 }
194 let writer = Indexes::open(&self.dir)?;
195 tracing::info!("index writer reopened at {}", self.dir.display());
196 crate::heap::record_index_writer(Some(writer.memory()));
197 self.open = Some(writer);
198 }
199 Ok(self.open.as_mut().expect("writer opened above"))
200 }
201
202 fn with<R>(&mut self, op: impl FnOnce(&mut Indexes) -> Result<R>) -> Result<R> {
203 let result = op(self.open()?);
204 self.last_used = Instant::now();
205 self.idle_released = false;
206 crate::heap::record_index_writer(self.memory());
207 result
208 }
209
210 fn release_if_idle(&mut self, now: Instant) -> bool {
211 let current = self
212 .open
213 .as_ref()
214 .is_some_and(|w| !w.has_unpersisted_changes());
215 if !current || now.saturating_duration_since(self.last_used) < WRITER_IDLE {
216 return false;
217 }
218 if self.shared {
219 return self.release_lexical_writer();
220 }
221 if let Some(writer) = self.open.take()
222 && let Err(e) = writer.wait_for_merges()
223 {
224 tracing::warn!("index writer released without its merges: {e:#}");
225 }
226 crate::heap::record_index_writer(None);
227 residency::release_free_heap();
228 tracing::info!(
229 "index writer released after {}s idle",
230 WRITER_IDLE.as_secs()
231 );
232 true
233 }
234
235 fn release_lexical_writer(&mut self) -> bool {
236 if self.idle_released {
237 return false;
238 }
239 if let Some(writer) = self.open.as_mut()
240 && let Err(e) = writer.close_lexical_writer()
241 {
242 tracing::warn!("lexical writer released without its merges: {e:#}");
243 }
244 self.idle_released = true;
245 residency::release_free_heap();
246 tracing::info!(
247 "lexical index writer released after {}s idle",
248 WRITER_IDLE.as_secs()
249 );
250 true
251 }
252
253 fn memory(&self) -> Option<IndexMemory> {
254 self.open.as_ref().map(Indexes::memory)
255 }
256
257 fn sealed_under(&self) -> Option<SealKeyId> {
258 self.open
259 .as_ref()
260 .and_then(Indexes::seal_keys)
261 .map(|keys| keys.current_id())
262 }
263
264 fn seal_under(&mut self, keys: SealKeys, reader: &RwLock<Indexes>) -> Result<()> {
268 if self.shared {
269 let writer = self
270 .open
271 .as_mut()
272 .ok_or_else(|| anyhow!("the sealed index writer is gone"))?;
273 writer.set_seal_keys(keys)?;
274 return writer.reseal();
275 }
276 if let Some(writer) = self.open.as_mut()
277 && writer.has_unpersisted_changes()
278 {
279 writer.persist()?;
280 }
281 if let Some(writer) = self.open.take() {
282 crate::heap::record_index_writer(None);
283 writer.wait_for_merges()?;
284 }
285 let mut writer = Indexes::open_keyed(&self.dir, &keys)?;
286 writer.reseal()?;
287 let shared = writer.shared_reader()?;
288 *reader.write().unwrap_or_else(PoisonError::into_inner) = shared;
289 crate::heap::record_index_writer(Some(writer.memory()));
290 self.open = Some(writer);
291 self.shared = true;
292 self.idle_released = false;
293 self.last_used = Instant::now();
294 Ok(())
295 }
296}
297
298pub struct DrainLoop {
299 pub store: Arc<SqliteStore>,
300 pub reader: Arc<RwLock<Indexes>>,
301 pub writer: WriterSlot,
302 pub embedder: Option<Arc<Embedder>>,
303 pub summary: Arc<RwLock<Option<MaintenanceSummary>>>,
304 pub near_duplicates: Arc<RwLock<Option<NearDupCount>>>,
305 pub near_duplicate_threshold: f32,
306 pub config: MaintenanceConfig,
307 pub requests: Receiver<DrainRequest>,
308 pub health: Arc<Health>,
309}
310
311impl DrainLoop {
312 pub fn spawn(self) {
313 std::thread::Builder::new()
314 .name("index-drain".to_string())
315 .spawn(move || self.run())
316 .expect("spawning the index drain thread");
317 }
318
319 fn run(mut self) {
320 let mut last_repair: Option<Instant> = None;
321 let mut chunks = ChunkThrottle::new(Instant::now());
322 let mut last_purge: Option<Instant> = None;
323 let mut last_seal_check: Option<Instant> = None;
324 loop {
325 match self.requests.recv_timeout(DRAIN_INTERVAL) {
326 Ok(DrainRequest::Repair(request)) => {
327 let result = self.repair("requested", request.dry_run);
328 if result.is_ok() && !request.dry_run {
329 last_repair = Some(Instant::now());
330 }
331 let _ = request.reply.send(result);
333 }
334 Ok(DrainRequest::CompactIndexes(reply)) => {
335 let _ = reply.send(self.compact_indexes());
336 }
337 Ok(DrainRequest::Pause(commands)) => {
338 for PauseCommand::Settle(reply) in commands {
339 let _ = reply.send(self.settle());
340 }
341 }
342 Err(RecvTimeoutError::Timeout) | Err(RecvTimeoutError::Disconnected) => {
343 let applied = self.drain();
344 chunks.after_drain(Instant::now(), applied.unwrap_or(0));
345 let due = last_repair.is_none_or(|t| {
346 t.elapsed() >= Duration::from_secs(self.config.interval_secs)
347 });
348 if applied == Some(0) && self.config.enabled && due {
349 if self.repair("scheduled", false).is_ok() {
350 last_repair = Some(Instant::now());
351 } else {
352 last_repair = Some(
355 Instant::now()
356 - Duration::from_secs(
357 self.config.interval_secs.saturating_sub(600),
358 ),
359 );
360 }
361 }
362 if applied == Some(0)
363 && last_purge.is_none_or(|t| t.elapsed() >= DELETE_PURGE_GAP)
364 && self.purge_applied_deletes()
365 {
366 last_purge = Some(Instant::now());
367 }
368 if applied == Some(0) && chunks.due(Instant::now()) {
369 let progressed = self.backfill_chunks();
370 chunks.after_backfill(Instant::now(), progressed);
371 }
372 }
373 }
374 if last_seal_check.is_none_or(|t| t.elapsed() >= SEAL_CHECK_INTERVAL) {
375 self.seal_if_due();
376 last_seal_check = Some(Instant::now());
377 }
378 self.writer.release_if_idle(Instant::now());
379 }
380 }
381
382 fn seal_if_due(&mut self) {
387 let keys = match self.store.at_rest_keys(AtRestPurpose::Indexes) {
388 Ok(Some(keys)) => keys,
389 Ok(None) => return,
390 Err(e) => {
391 tracing::warn!("reading the search indexes' keys failed: {e:#}");
392 return;
393 }
394 };
395 let current = keys.keys.current_id();
396 if keys.settled && self.writer.sealed_under() == Some(current) {
397 return;
398 }
399 let started = Instant::now();
400 if let Err(e) = self.writer.seal_under(keys.keys, &self.reader) {
401 tracing::warn!("sealing the search indexes failed and will retry: {e:#}");
402 return;
403 }
404 match self
405 .store
406 .settle_at_rest_keys(AtRestPurpose::Indexes, current)
407 {
408 Ok(()) => tracing::info!(
409 "search indexes sealed under key {current} in {:.1}s",
410 started.elapsed().as_secs_f64()
411 ),
412 Err(e) => tracing::warn!("recording the sealed search indexes failed: {e:#}"),
413 }
414 }
415
416 fn backfill_chunks(&self) -> bool {
419 let Some(embedder) = self.embedder.as_deref() else {
420 return false;
421 };
422 let embed = |texts: &[&str]| embedder.embed_batch(texts);
423 match self.store.backfill_chunk_embeddings(
424 &embed,
425 CHUNK_MEMORIES_PER_TICK,
426 CHUNK_TICK_BUDGET,
427 ) {
428 Ok(r) => {
429 if r.memories > 0 || r.failed > 0 {
430 tracing::info!(
431 "chunk backfill: memories={} chunks={} failed={} pending={}",
432 r.memories,
433 r.chunks,
434 r.failed,
435 r.pending
436 );
437 }
438 r.memories > 0
439 }
440 Err(e) => {
441 tracing::warn!("chunk backfill failed: {e:#}");
442 false
443 }
444 }
445 }
446
447 fn drain(&mut self) -> Option<usize> {
450 let store = &self.store;
451 let writer = &mut self.writer;
452 let result = store.outbox_pending().and_then(|pending| {
453 if pending == 0 {
454 Ok(0)
455 } else {
456 writer.with(|w| w.drain(store))
457 }
458 });
459 match result {
460 Ok(0) => {
461 self.health.drained(0, Utc::now());
462 Some(0)
463 }
464 Ok(n) => {
465 tracing::info!("index drain applied {n} outbox rows");
466 self.reload_reader();
467 match self.store.outbox_pending() {
468 Ok(pending) => self.health.drained(pending, Utc::now()),
469 Err(e) => tracing::warn!("counting the index outbox failed: {e:#}"),
470 }
471 Some(n)
472 }
473 Err(e) => {
474 tracing::warn!("index drain failed: {e:#}");
475 self.health.drain_failed(&format!("{e:#}"), Utc::now());
476 None
477 }
478 }
479 }
480
481 fn repair(&mut self, source: &str, dry_run: bool) -> Result<MaintenanceSummary> {
482 let started = Instant::now();
483 let store = &self.store;
484 let embedder = self.embedder.as_deref();
485 let repair = |w: &mut Indexes| run_repair(store, Some(w), embedder, dry_run);
486 let result = match self.writer.open().map(|_| ()) {
487 Ok(()) => self.writer.with(repair),
488 Err(e) => {
489 tracing::warn!(
490 "index writer unavailable ({e:#}); repairing the store without the indexes"
491 );
492 match run_repair(store, None, embedder, dry_run) {
493 Ok(summary) => log_stats(source, &summary.stats),
494 Err(store_error) => {
495 tracing::warn!("{source} store repair failed: {store_error:#}")
496 }
497 }
498 Err(e
499 .context("the index writer could not reopen, so the indexes were not repaired"))
500 }
501 };
502 match &result {
503 Ok(summary) => {
504 self.health.repaired(Utc::now());
505 log_stats(source, &summary.stats);
506 tracing::info!(
507 "{source} store repair finished in {:.1}s; drift remaining: {:?}",
508 started.elapsed().as_secs_f64(),
509 summary.report_after.drift()
510 );
511 if !dry_run {
512 self.reload_reader();
513 let mut guard = match self.summary.write() {
514 Ok(g) => g,
515 Err(poisoned) => poisoned.into_inner(),
516 };
517 *guard = Some(summary.clone());
518 drop(guard);
519 let threshold = self.near_duplicate_threshold;
520 let near_duplicates = &self.near_duplicates;
521 let counted = self.writer.with(|w| {
522 refresh_count(store, w, threshold, near_duplicates);
523 Ok(())
524 });
525 if let Err(e) = counted {
526 tracing::warn!("near-duplicate count failed: {e:#}");
527 }
528 if summary.stats.index_rebuilt {
529 residency::release_free_heap();
530 }
531 }
532 }
533 Err(e) => {
534 tracing::warn!("{source} store repair failed: {e:#}");
535 self.health.repair_failed(&format!("{e:#}"), Utc::now());
536 }
537 }
538 result
539 }
540
541 fn purge_applied_deletes(&mut self) -> bool {
545 let token = match self.store.delete_purge_requested() {
546 Ok(Some(token)) => token,
547 Ok(None) => return false,
548 Err(e) => {
549 tracing::warn!("reading the delete purge request failed: {e:#}");
550 return false;
551 }
552 };
553 let result = self.compact_indexes().and_then(|()| {
554 let cold = self.store.purge_cold_segments()?;
555 if cold != Default::default() {
556 tracing::info!(
557 "deleted records purged from the cold tier: {} retired, {} segments rewritten, {} files removed",
558 cold.retired,
559 cold.rewritten,
560 cold.removed_files
561 );
562 }
563 self.store.delete_purge_done(&token)
564 });
565 if let Err(e) = result {
566 tracing::warn!("purging a peer's deletes failed and will retry: {e:#}");
567 }
568 true
569 }
570
571 fn compact_indexes(&mut self) -> Result<()> {
572 let store = &self.store;
573 let took = self.writer.with(|w| {
574 while w.drain(store)? > 0 {}
575 let started = Instant::now();
576 w.compact()?;
577 Ok(started.elapsed())
578 })?;
579 self.reload_reader();
580 residency::release_free_heap();
581 tracing::info!("lexical index compacted in {:.1}s", took.as_secs_f64());
582 Ok(())
583 }
584
585 fn settle(&mut self) -> Result<()> {
586 let store = &self.store;
587 self.writer.with(|w| {
588 while w.drain(store)? > 0 {}
589 if w.has_unpersisted_changes() {
590 w.persist()?;
591 }
592 w.close_lexical_writer()
593 })?;
594 self.reload_reader();
595 Ok(())
596 }
597
598 fn reload_reader(&self) {
599 let mut guard = match self.reader.write() {
600 Ok(g) => g,
601 Err(poisoned) => poisoned.into_inner(),
602 };
603 if let Err(e) = guard.reload_view() {
604 tracing::warn!("index reader reload failed: {e:#}");
605 }
606 }
607}
608
609#[cfg(test)]
610mod tests {
611 use std::fs::Permissions;
612 use std::os::unix::fs::PermissionsExt;
613
614 use super::*;
615 use second_brain_core::schema::{Memory, MemoryType};
616 use second_brain_core::store::Store;
617
618 #[test]
619 fn a_fresh_throttle_scans_on_the_first_idle_tick() {
620 let now = Instant::now();
621 assert!(ChunkThrottle::new(now).due(now));
622 }
623
624 #[test]
625 fn a_scan_that_finds_nothing_waits_out_the_idle_interval() {
626 let now = Instant::now();
627 let mut t = ChunkThrottle::new(now);
628 t.after_backfill(now, false);
629 assert!(!t.due(now + CHUNK_IDLE_RESCAN - Duration::from_secs(1)));
630 assert!(t.due(now + CHUNK_IDLE_RESCAN));
631 }
632
633 #[test]
634 fn progress_keeps_the_next_idle_tick_scanning() {
635 let now = Instant::now();
636 let mut t = ChunkThrottle::new(now);
637 t.after_backfill(now, true);
638 assert!(t.due(now));
639 }
640
641 #[test]
642 fn an_applied_drain_resets_the_idle_wait() {
643 let now = Instant::now();
644 let mut t = ChunkThrottle::new(now);
645 t.after_backfill(now, false);
646 t.after_drain(now, 0);
647 assert!(!t.due(now));
648 t.after_drain(now, 3);
649 assert!(t.due(now));
650 }
651
652 fn remember(store: &SqliteStore, content: &str) -> uuid::Uuid {
653 let m = Memory::new(
654 content.to_string(),
655 MemoryType::Semantic,
656 "test".to_string(),
657 String::new(),
658 );
659 store.store_memory(&m).unwrap();
660 m.id
661 }
662
663 fn drain_loop(
664 store: Arc<SqliteStore>,
665 reader: Arc<RwLock<Indexes>>,
666 writer: Indexes,
667 ) -> DrainLoop {
668 let (_tx, requests) = std::sync::mpsc::channel();
669 let claim = IndexClaim::acquire(writer.dir()).unwrap();
670 DrainLoop {
671 store,
672 reader,
673 writer: WriterSlot::new(writer, claim),
674 embedder: None,
675 summary: Arc::new(RwLock::new(None)),
676 near_duplicates: Arc::new(RwLock::new(None)),
677 near_duplicate_threshold: 0.97,
678 config: MaintenanceConfig {
679 enabled: false,
680 interval_secs: 3600,
681 },
682 requests,
683 health: Arc::new(Health::new(Utc::now(), None)),
684 }
685 }
686
687 fn idle_loop(dir: &std::path::Path) -> DrainLoop {
688 let store = Arc::new(SqliteStore::in_memory("m".to_string()).unwrap());
689 let mut writer = Indexes::open(dir).unwrap();
690 writer.persist().unwrap();
691 let reader = Arc::new(RwLock::new(Indexes::open_readonly(dir).unwrap()));
692 drain_loop(store, reader, writer)
693 }
694
695 #[test]
696 fn idleness_counts_from_when_an_operation_finishes() {
697 let dir = tempfile::tempdir().unwrap();
698 let mut lp = idle_loop(dir.path());
699 let mut finishing = None;
700 lp.writer
701 .with(|_| {
702 finishing = Some(Instant::now());
703 Ok(())
704 })
705 .unwrap();
706 let finishing = finishing.unwrap();
707 assert!(lp.writer.last_used >= finishing);
708 assert!(
709 !lp.writer
710 .release_if_idle(finishing + WRITER_IDLE - Duration::from_secs(1))
711 );
712 }
713
714 #[test]
715 fn a_writer_holding_changes_its_files_lack_is_never_released() {
716 let dir = tempfile::tempdir().unwrap();
717 let mut lp = idle_loop(dir.path());
718 remember(&lp.store, "the zanzibar ferry leaves at noon");
719 let tantivy = dir.path().join("tantivy");
720 let set_mode =
721 |mode| std::fs::set_permissions(&tantivy, Permissions::from_mode(mode)).unwrap();
722 set_mode(0o500);
723 let drained = lp.drain();
724 set_mode(0o700);
725 assert_eq!(
726 drained, None,
727 "the drain should fail to write the lexical index"
728 );
729 assert!(
730 !lp.writer
731 .release_if_idle(lp.writer.last_used + WRITER_IDLE * 10)
732 );
733 lp.repair("test", true).unwrap();
734 assert!(!lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
735
736 assert_eq!(lp.drain(), Some(1));
737 assert!(lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
738 }
739
740 #[test]
741 fn a_failure_that_left_the_files_current_still_releases() {
742 let dir = tempfile::tempdir().unwrap();
743 let mut lp = idle_loop(dir.path());
744 let failed: Result<()> = lp.writer.with(|_| anyhow::bail!("a store step failed"));
745 assert!(failed.is_err());
746 assert!(lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
747 }
748
749 #[test]
750 fn a_writer_that_cannot_reopen_fails_the_repair() {
751 let dir = tempfile::tempdir().unwrap();
752 let mut lp = idle_loop(dir.path());
753 assert!(lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
754 let blocker = dir.path().join("not-a-dir");
755 std::fs::write(&blocker, b"x").unwrap();
756 lp.writer.dir = blocker.join("indexes");
757
758 let err = lp.repair("test", false).unwrap_err();
759 assert!(
760 format!("{err:#}").contains("indexes were not repaired"),
761 "{err:#}"
762 );
763 assert!(lp.writer.memory().is_none());
764 }
765
766 #[test]
767 fn an_idle_writer_is_released_and_reopened_by_the_next_write() {
768 let dir = tempfile::tempdir().unwrap();
769 let store = Arc::new(SqliteStore::in_memory("m".to_string()).unwrap());
770 let mut writer = Indexes::open(dir.path()).unwrap();
771 writer.persist().unwrap();
772 let reader = Arc::new(RwLock::new(Indexes::open_readonly(dir.path()).unwrap()));
773 let mut lp = drain_loop(store.clone(), reader.clone(), writer);
774
775 let first = remember(&store, "the zanzibar ferry leaves at noon");
776 assert_eq!(lp.drain(), Some(1));
777 assert!(lp.writer.memory().is_some());
778
779 let used = lp.writer.last_used;
780 assert!(
781 !lp.writer
782 .release_if_idle(used + WRITER_IDLE - Duration::from_secs(1))
783 );
784 assert!(lp.writer.memory().is_some());
785 assert!(lp.writer.release_if_idle(used + WRITER_IDLE));
786 assert!(lp.writer.memory().is_none());
787
788 assert_eq!(lp.drain(), Some(0));
789 assert!(
790 lp.writer.memory().is_none(),
791 "an empty tick reopened the writer"
792 );
793
794 let second = remember(&store, "the kilimanjaro trail opens in june");
795 assert_eq!(lp.drain(), Some(1));
796 assert!(lp.writer.memory().is_some());
797 assert_eq!(lp.writer.open().unwrap().lexical_len(), 2);
798
799 let guard = reader.read().unwrap();
800 let hits = |q: &str| -> Vec<uuid::Uuid> {
801 guard
802 .bm25_search(q, 5)
803 .unwrap()
804 .into_iter()
805 .map(|(id, _)| id)
806 .collect()
807 };
808 assert_eq!(hits("kilimanjaro"), vec![second]);
809 assert_eq!(hits("zanzibar"), vec![first]);
810 }
811
812 fn keyed_store(dir: &std::path::Path) -> Arc<SqliteStore> {
813 let key = second_brain_core::keys::StoreKey::generate().unwrap();
814 Arc::new(SqliteStore::open_with_key(&dir.join("brain.db"), "m".to_string(), &key).unwrap())
815 }
816
817 fn index_files(dir: &std::path::Path) -> Vec<std::path::PathBuf> {
818 let mut out = Vec::new();
819 let mut stack = vec![dir.to_path_buf()];
820 while let Some(d) = stack.pop() {
821 for entry in std::fs::read_dir(d).unwrap() {
822 let path = entry.unwrap().path();
823 let name = path.file_name().unwrap().to_string_lossy().to_string();
824 if path.is_dir() {
825 stack.push(path);
826 } else if !name.ends_with(".lock") && name != second_brain_core::index::CLAIM_FILE {
827 out.push(path);
828 }
829 }
830 }
831 out
832 }
833
834 fn assert_all_sealed_under(dir: &std::path::Path, key: SealKeyId) {
835 let files = index_files(dir);
836 assert!(!files.is_empty());
837 for path in files {
838 assert_eq!(
839 second_brain_core::sealed::sealed_key_id(&path).unwrap(),
840 Some(key),
841 "{}",
842 path.display()
843 );
844 }
845 }
846
847 #[test]
848 fn indexes_opened_before_the_store_was_encrypted_are_sealed_and_recall_moves_to_a_shared_reader()
849 {
850 let data = tempfile::tempdir().unwrap();
851 let store = keyed_store(data.path());
852 let dir = data.path().join("brain.indexes");
853 let mut writer = Indexes::open(&dir).unwrap();
854 writer.persist().unwrap();
855 let reader = Arc::new(RwLock::new(Indexes::open_readonly(&dir).unwrap()));
856 let mut lp = drain_loop(store.clone(), reader.clone(), writer);
857 let first = remember(&store, "the zanzibar ferry leaves at noon");
858 assert_eq!(lp.drain(), Some(1));
859
860 lp.seal_if_due();
861 let keys = store.at_rest_keys(AtRestPurpose::Indexes).unwrap().unwrap();
862 assert!(keys.settled);
863 assert_all_sealed_under(&dir, keys.keys.current_id());
864 assert!(Indexes::open_readonly(&dir).is_err());
865
866 let second = remember(&store, "the kilimanjaro trail opens in june");
867 assert_eq!(lp.drain(), Some(1));
868 let guard = reader.read().unwrap();
869 assert_eq!(guard.bm25_search("zanzibar", 5).unwrap()[0].0, first);
870 assert_eq!(guard.bm25_search("kilimanjaro", 5).unwrap()[0].0, second);
871 assert_eq!(guard.vector_len(), lp.writer.open().unwrap().vector_len());
872 }
873
874 #[test]
875 fn a_store_key_rotation_reseals_the_indexes_under_a_new_key() {
876 let data = tempfile::tempdir().unwrap();
877 let store = keyed_store(data.path());
878 let dir = data.path().join("brain.indexes");
879 let keys = store
880 .at_rest_keys(AtRestPurpose::Indexes)
881 .unwrap()
882 .unwrap()
883 .keys;
884 let handles = crate::models::open_index_handles(&dir, &store, Some(&keys), |_| {}).unwrap();
885 let crate::models::IndexHandles {
886 reader,
887 writer,
888 claim,
889 } = handles;
890 drop(claim);
891 let mut lp = drain_loop(store.clone(), reader.clone(), writer);
892 let id = remember(&store, "the zanzibar ferry leaves at noon");
893 assert_eq!(lp.drain(), Some(1));
894 lp.seal_if_due();
895 let before = store.at_rest_keys(AtRestPurpose::Indexes).unwrap().unwrap();
896 assert!(before.settled);
897
898 let rotated = second_brain_core::keys::StoreKey::generate().unwrap();
899 store
900 .rewrite_under_key(
901 second_brain_core::sqlite_store::encrypt_migration::RewriteTo::Key(&rotated),
902 )
903 .unwrap();
904 lp.seal_if_due();
905 let after = store.at_rest_keys(AtRestPurpose::Indexes).unwrap().unwrap();
906 assert_ne!(after.keys.current_id(), before.keys.current_id());
907 assert!(after.settled);
908 assert!(after.keys.retired_ids().is_empty());
909 assert_all_sealed_under(&dir, after.keys.current_id());
910 assert_eq!(
911 reader.read().unwrap().bm25_search("zanzibar", 5).unwrap()[0].0,
912 id
913 );
914 }
915
916 #[test]
917 fn an_idle_sealed_writer_keeps_its_vectors_for_recall() {
918 let data = tempfile::tempdir().unwrap();
919 let store = keyed_store(data.path());
920 let dir = data.path().join("brain.indexes");
921 let keys = store
922 .at_rest_keys(AtRestPurpose::Indexes)
923 .unwrap()
924 .unwrap()
925 .keys;
926 let handles = crate::models::open_index_handles(&dir, &store, Some(&keys), |_| {}).unwrap();
927 let crate::models::IndexHandles {
928 reader,
929 writer,
930 claim,
931 } = handles;
932 drop(claim);
933 let mut lp = drain_loop(store.clone(), reader.clone(), writer);
934 let first = remember(&store, "the zanzibar ferry leaves at noon");
935 assert_eq!(lp.drain(), Some(1));
936 let used = lp.writer.last_used;
937 assert!(lp.writer.release_if_idle(used + WRITER_IDLE));
938 assert!(!lp.writer.release_if_idle(used + WRITER_IDLE * 2));
939 assert!(lp.writer.memory().is_some());
940
941 let second = remember(&store, "the kilimanjaro trail opens in june");
942 assert_eq!(lp.drain(), Some(1));
943 let guard = reader.read().unwrap();
944 assert_eq!(guard.bm25_search("zanzibar", 5).unwrap()[0].0, first);
945 assert_eq!(guard.bm25_search("kilimanjaro", 5).unwrap()[0].0, second);
946 }
947
948 #[test]
949 fn indexes_that_open_under_no_held_key_are_rebuilt_from_the_store() {
950 let data = tempfile::tempdir().unwrap();
951 let store = keyed_store(data.path());
952 let dir = data.path().join("brain.indexes");
953 let id = remember(&store, "the zanzibar ferry leaves at noon");
954 let foreign = SealKeys::single(second_brain_core::sealed::SealKey::generate().unwrap());
955 let handles =
956 crate::models::open_index_handles(&dir, &store, Some(&foreign), |_| {}).unwrap();
957 drop(handles);
958
959 let keys = store
960 .at_rest_keys(AtRestPurpose::Indexes)
961 .unwrap()
962 .unwrap()
963 .keys;
964 let handles = crate::models::open_index_handles(&dir, &store, Some(&keys), |_| {}).unwrap();
965 assert_eq!(
966 handles
967 .reader
968 .read()
969 .unwrap()
970 .bm25_search("zanzibar", 5)
971 .unwrap()[0]
972 .0,
973 id
974 );
975 }
976}