1use std::path::PathBuf;
7use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender};
8use std::sync::{Arc, PoisonError, RwLock};
9use std::time::{Duration, Instant};
10
11use anyhow::{Result, anyhow};
12use chrono::Utc;
13use second_brain_core::config::MaintenanceConfig;
14use second_brain_core::embedding::Embedder;
15use second_brain_core::index::{IndexClaim, Indexes};
16use second_brain_core::maintenance::{MaintenanceSummary, RepairStats};
17use second_brain_core::residency::{self, IndexMemory};
18use second_brain_core::sealed::{SealKeyId, SealKeys};
19use second_brain_core::sqlite_store::{AtRestPurpose, SqliteStore};
20
21use crate::health::Health;
22use crate::near_duplicates::{NearDupCount, refresh_count};
23
24const DRAIN_INTERVAL: Duration = Duration::from_secs(5);
25const DELETE_PURGE_GAP: Duration = Duration::from_secs(600);
28
29const WRITER_IDLE: Duration = Duration::from_secs(60);
30const SEAL_CHECK_INTERVAL: Duration = Duration::from_secs(60);
33
34const CHUNK_IDLE_RESCAN: Duration = Duration::from_secs(600);
37const CHUNK_TICK_BUDGET: Duration = Duration::from_secs(3);
39const CHUNK_MEMORIES_PER_TICK: usize = 32;
41
42struct ChunkThrottle {
44 next_scan: Instant,
45}
46
47impl ChunkThrottle {
48 fn new(now: Instant) -> Self {
49 Self { next_scan: now }
50 }
51
52 fn due(&self, now: Instant) -> bool {
53 now >= self.next_scan
54 }
55
56 fn after_backfill(&mut self, now: Instant, progressed: bool) {
57 self.next_scan = if progressed {
58 now
59 } else {
60 now + CHUNK_IDLE_RESCAN
61 };
62 }
63
64 fn after_drain(&mut self, now: Instant, applied: usize) {
66 if applied > 0 {
67 self.next_scan = now;
68 }
69 }
70}
71
72pub struct RepairRequest {
73 pub dry_run: bool,
74 pub reply: Sender<Result<MaintenanceSummary>>,
75}
76
77pub enum DrainRequest {
79 Repair(RepairRequest),
80 CompactIndexes(Sender<Result<()>>),
83}
84
85pub fn run_repair(
89 store: &SqliteStore,
90 mut indexes: Option<&mut Indexes>,
91 embedder: Option<&Embedder>,
92 dry_run: bool,
93) -> Result<MaintenanceSummary> {
94 let embed = embedder.map(|e| move |text: &str| e.embed(text));
95 let embed_ref = embed
96 .as_ref()
97 .map(|f| f as &dyn Fn(&str) -> Result<Vec<f32>>);
98 let stats = store.repair(indexes.as_deref_mut(), embed_ref, dry_run)?;
99 let report_after = store.integrity_report(indexes.as_deref())?;
100 Ok(MaintenanceSummary {
101 ran_at: Utc::now(),
102 stats,
103 report_after,
104 })
105}
106
107fn log_stats(source: &str, stats: &RepairStats) {
108 if stats.is_clean() {
109 tracing::info!("{source} store repair: nothing to do");
110 } else {
111 tracing::info!("{source} store repair: {}", summarize(stats));
112 }
113}
114
115fn summarize(s: &RepairStats) -> String {
116 let mut parts = Vec::new();
117 let mut push = |name: &str, n: u64| {
118 if n > 0 {
119 parts.push(format!("{name}={n}"));
120 }
121 };
122 push("duplicates_superseded", s.duplicates_superseded);
123 push("markers_inherited", s.distillation_markers_inherited);
124 push("orphaned_markers_dropped", s.orphaned_markers_dropped);
125 push("raws_retired", s.raws_retired);
126 push("relations_pruned", s.relations_pruned);
127 push("vec_keys_deleted", s.vec_keys_deleted);
128 push("sync_log_rows_deleted", s.sync_log_rows_deleted);
129 push("stale_cursors_dropped", s.stale_cursors_dropped);
130 push("embeddings_backfilled", s.embeddings_backfilled);
131 push("embedding_failures", s.embedding_failures);
132 if s.vec_keys_gc_skipped {
133 parts.push("vec_keys_gc=skipped".to_string());
134 }
135 if s.index_rebuilt {
136 parts.push("index=rebuilt".to_string());
137 }
138 if s.dry_run {
139 parts.push("dry_run".to_string());
140 }
141 parts.join(" ")
142}
143
144pub struct WriterSlot {
156 dir: PathBuf,
157 open: Option<Indexes>,
158 last_used: Instant,
159 shared: bool,
160 idle_released: bool,
161 _claim: IndexClaim,
162}
163
164impl WriterSlot {
165 pub fn new(writer: Indexes, claim: IndexClaim) -> Self {
166 crate::heap::record_index_writer(Some(writer.memory()));
167 Self {
168 dir: writer.dir().to_path_buf(),
169 shared: writer.seal_keys().is_some(),
170 open: Some(writer),
171 last_used: Instant::now(),
172 idle_released: false,
173 _claim: claim,
174 }
175 }
176
177 fn open(&mut self) -> Result<&mut Indexes> {
178 if self.open.is_none() {
179 if self.shared {
180 return Err(anyhow!(
181 "the sealed index writer at {} is gone; the daemon must restart to reopen it",
182 self.dir.display()
183 ));
184 }
185 let writer = Indexes::open(&self.dir)?;
186 tracing::info!("index writer reopened at {}", self.dir.display());
187 crate::heap::record_index_writer(Some(writer.memory()));
188 self.open = Some(writer);
189 }
190 Ok(self.open.as_mut().expect("writer opened above"))
191 }
192
193 fn with<R>(&mut self, op: impl FnOnce(&mut Indexes) -> Result<R>) -> Result<R> {
194 let result = op(self.open()?);
195 self.last_used = Instant::now();
196 self.idle_released = false;
197 crate::heap::record_index_writer(self.memory());
198 result
199 }
200
201 fn release_if_idle(&mut self, now: Instant) -> bool {
202 let current = self
203 .open
204 .as_ref()
205 .is_some_and(|w| !w.has_unpersisted_changes());
206 if !current || now.saturating_duration_since(self.last_used) < WRITER_IDLE {
207 return false;
208 }
209 if self.shared {
210 return self.release_lexical_writer();
211 }
212 if let Some(writer) = self.open.take()
213 && let Err(e) = writer.wait_for_merges()
214 {
215 tracing::warn!("index writer released without its merges: {e:#}");
216 }
217 crate::heap::record_index_writer(None);
218 residency::release_free_heap();
219 tracing::info!(
220 "index writer released after {}s idle",
221 WRITER_IDLE.as_secs()
222 );
223 true
224 }
225
226 fn release_lexical_writer(&mut self) -> bool {
227 if self.idle_released {
228 return false;
229 }
230 if let Some(writer) = self.open.as_mut()
231 && let Err(e) = writer.close_lexical_writer()
232 {
233 tracing::warn!("lexical writer released without its merges: {e:#}");
234 }
235 self.idle_released = true;
236 residency::release_free_heap();
237 tracing::info!(
238 "lexical index writer released after {}s idle",
239 WRITER_IDLE.as_secs()
240 );
241 true
242 }
243
244 fn memory(&self) -> Option<IndexMemory> {
245 self.open.as_ref().map(Indexes::memory)
246 }
247
248 fn sealed_under(&self) -> Option<SealKeyId> {
249 self.open
250 .as_ref()
251 .and_then(Indexes::seal_keys)
252 .map(|keys| keys.current_id())
253 }
254
255 fn seal_under(&mut self, keys: SealKeys, reader: &RwLock<Indexes>) -> Result<()> {
259 if self.shared {
260 let writer = self
261 .open
262 .as_mut()
263 .ok_or_else(|| anyhow!("the sealed index writer is gone"))?;
264 writer.set_seal_keys(keys)?;
265 return writer.reseal();
266 }
267 if let Some(writer) = self.open.as_mut()
268 && writer.has_unpersisted_changes()
269 {
270 writer.persist()?;
271 }
272 if let Some(writer) = self.open.take() {
273 crate::heap::record_index_writer(None);
274 writer.wait_for_merges()?;
275 }
276 let mut writer = Indexes::open_keyed(&self.dir, &keys)?;
277 writer.reseal()?;
278 let shared = writer.shared_reader()?;
279 *reader.write().unwrap_or_else(PoisonError::into_inner) = shared;
280 crate::heap::record_index_writer(Some(writer.memory()));
281 self.open = Some(writer);
282 self.shared = true;
283 self.idle_released = false;
284 self.last_used = Instant::now();
285 Ok(())
286 }
287}
288
289pub struct DrainLoop {
290 pub store: Arc<SqliteStore>,
291 pub reader: Arc<RwLock<Indexes>>,
292 pub writer: WriterSlot,
293 pub embedder: Option<Arc<Embedder>>,
294 pub summary: Arc<RwLock<Option<MaintenanceSummary>>>,
295 pub near_duplicates: Arc<RwLock<Option<NearDupCount>>>,
296 pub near_duplicate_threshold: f32,
297 pub config: MaintenanceConfig,
298 pub requests: Receiver<DrainRequest>,
299 pub health: Arc<Health>,
300}
301
302impl DrainLoop {
303 pub fn spawn(self) {
304 std::thread::Builder::new()
305 .name("index-drain".to_string())
306 .spawn(move || self.run())
307 .expect("spawning the index drain thread");
308 }
309
310 fn run(mut self) {
311 let mut last_repair: Option<Instant> = None;
312 let mut chunks = ChunkThrottle::new(Instant::now());
313 let mut last_purge: Option<Instant> = None;
314 let mut last_seal_check: Option<Instant> = None;
315 loop {
316 match self.requests.recv_timeout(DRAIN_INTERVAL) {
317 Ok(DrainRequest::Repair(request)) => {
318 let result = self.repair("requested", request.dry_run);
319 if result.is_ok() && !request.dry_run {
320 last_repair = Some(Instant::now());
321 }
322 let _ = request.reply.send(result);
324 }
325 Ok(DrainRequest::CompactIndexes(reply)) => {
326 let _ = reply.send(self.compact_indexes());
327 }
328 Err(RecvTimeoutError::Timeout) | Err(RecvTimeoutError::Disconnected) => {
329 let applied = self.drain();
330 chunks.after_drain(Instant::now(), applied.unwrap_or(0));
331 let due = last_repair.is_none_or(|t| {
332 t.elapsed() >= Duration::from_secs(self.config.interval_secs)
333 });
334 if applied == Some(0) && self.config.enabled && due {
335 if self.repair("scheduled", false).is_ok() {
336 last_repair = Some(Instant::now());
337 } else {
338 last_repair = Some(
341 Instant::now()
342 - Duration::from_secs(
343 self.config.interval_secs.saturating_sub(600),
344 ),
345 );
346 }
347 }
348 if applied == Some(0)
349 && last_purge.is_none_or(|t| t.elapsed() >= DELETE_PURGE_GAP)
350 && self.purge_applied_deletes()
351 {
352 last_purge = Some(Instant::now());
353 }
354 if applied == Some(0) && chunks.due(Instant::now()) {
355 let progressed = self.backfill_chunks();
356 chunks.after_backfill(Instant::now(), progressed);
357 }
358 }
359 }
360 if last_seal_check.is_none_or(|t| t.elapsed() >= SEAL_CHECK_INTERVAL) {
361 self.seal_if_due();
362 last_seal_check = Some(Instant::now());
363 }
364 self.writer.release_if_idle(Instant::now());
365 }
366 }
367
368 fn seal_if_due(&mut self) {
373 let keys = match self.store.at_rest_keys(AtRestPurpose::Indexes) {
374 Ok(Some(keys)) => keys,
375 Ok(None) => return,
376 Err(e) => {
377 tracing::warn!("reading the search indexes' keys failed: {e:#}");
378 return;
379 }
380 };
381 let current = keys.keys.current_id();
382 if keys.settled && self.writer.sealed_under() == Some(current) {
383 return;
384 }
385 let started = Instant::now();
386 if let Err(e) = self.writer.seal_under(keys.keys, &self.reader) {
387 tracing::warn!("sealing the search indexes failed and will retry: {e:#}");
388 return;
389 }
390 match self
391 .store
392 .settle_at_rest_keys(AtRestPurpose::Indexes, current)
393 {
394 Ok(()) => tracing::info!(
395 "search indexes sealed under key {current} in {:.1}s",
396 started.elapsed().as_secs_f64()
397 ),
398 Err(e) => tracing::warn!("recording the sealed search indexes failed: {e:#}"),
399 }
400 }
401
402 fn backfill_chunks(&self) -> bool {
405 let Some(embedder) = self.embedder.as_deref() else {
406 return false;
407 };
408 let embed = |texts: &[&str]| embedder.embed_batch(texts);
409 match self.store.backfill_chunk_embeddings(
410 &embed,
411 CHUNK_MEMORIES_PER_TICK,
412 CHUNK_TICK_BUDGET,
413 ) {
414 Ok(r) => {
415 if r.memories > 0 || r.failed > 0 {
416 tracing::info!(
417 "chunk backfill: memories={} chunks={} failed={} pending={}",
418 r.memories,
419 r.chunks,
420 r.failed,
421 r.pending
422 );
423 }
424 r.memories > 0
425 }
426 Err(e) => {
427 tracing::warn!("chunk backfill failed: {e:#}");
428 false
429 }
430 }
431 }
432
433 fn drain(&mut self) -> Option<usize> {
436 let store = &self.store;
437 let writer = &mut self.writer;
438 let result = store.outbox_pending().and_then(|pending| {
439 if pending == 0 {
440 Ok(0)
441 } else {
442 writer.with(|w| w.drain(store))
443 }
444 });
445 match result {
446 Ok(0) => {
447 self.health.drained(0, Utc::now());
448 Some(0)
449 }
450 Ok(n) => {
451 tracing::info!("index drain applied {n} outbox rows");
452 self.reload_reader();
453 match self.store.outbox_pending() {
454 Ok(pending) => self.health.drained(pending, Utc::now()),
455 Err(e) => tracing::warn!("counting the index outbox failed: {e:#}"),
456 }
457 Some(n)
458 }
459 Err(e) => {
460 tracing::warn!("index drain failed: {e:#}");
461 self.health.drain_failed(&format!("{e:#}"), Utc::now());
462 None
463 }
464 }
465 }
466
467 fn repair(&mut self, source: &str, dry_run: bool) -> Result<MaintenanceSummary> {
468 let started = Instant::now();
469 let store = &self.store;
470 let embedder = self.embedder.as_deref();
471 let repair = |w: &mut Indexes| run_repair(store, Some(w), embedder, dry_run);
472 let result = match self.writer.open().map(|_| ()) {
473 Ok(()) => self.writer.with(repair),
474 Err(e) => {
475 tracing::warn!(
476 "index writer unavailable ({e:#}); repairing the store without the indexes"
477 );
478 match run_repair(store, None, embedder, dry_run) {
479 Ok(summary) => log_stats(source, &summary.stats),
480 Err(store_error) => {
481 tracing::warn!("{source} store repair failed: {store_error:#}")
482 }
483 }
484 Err(e
485 .context("the index writer could not reopen, so the indexes were not repaired"))
486 }
487 };
488 match &result {
489 Ok(summary) => {
490 self.health.repaired(Utc::now());
491 log_stats(source, &summary.stats);
492 tracing::info!(
493 "{source} store repair finished in {:.1}s; drift remaining: {:?}",
494 started.elapsed().as_secs_f64(),
495 summary.report_after.drift()
496 );
497 if !dry_run {
498 self.reload_reader();
499 let mut guard = match self.summary.write() {
500 Ok(g) => g,
501 Err(poisoned) => poisoned.into_inner(),
502 };
503 *guard = Some(summary.clone());
504 drop(guard);
505 let threshold = self.near_duplicate_threshold;
506 let near_duplicates = &self.near_duplicates;
507 let counted = self.writer.with(|w| {
508 refresh_count(store, w, threshold, near_duplicates);
509 Ok(())
510 });
511 if let Err(e) = counted {
512 tracing::warn!("near-duplicate count failed: {e:#}");
513 }
514 if summary.stats.index_rebuilt {
515 residency::release_free_heap();
516 }
517 }
518 }
519 Err(e) => {
520 tracing::warn!("{source} store repair failed: {e:#}");
521 self.health.repair_failed(&format!("{e:#}"), Utc::now());
522 }
523 }
524 result
525 }
526
527 fn purge_applied_deletes(&mut self) -> bool {
531 let token = match self.store.delete_purge_requested() {
532 Ok(Some(token)) => token,
533 Ok(None) => return false,
534 Err(e) => {
535 tracing::warn!("reading the delete purge request failed: {e:#}");
536 return false;
537 }
538 };
539 let result = self.compact_indexes().and_then(|()| {
540 let cold = self.store.purge_cold_segments()?;
541 if cold != Default::default() {
542 tracing::info!(
543 "deleted records purged from the cold tier: {} retired, {} segments rewritten, {} files removed",
544 cold.retired,
545 cold.rewritten,
546 cold.removed_files
547 );
548 }
549 self.store.delete_purge_done(&token)
550 });
551 if let Err(e) = result {
552 tracing::warn!("purging a peer's deletes failed and will retry: {e:#}");
553 }
554 true
555 }
556
557 fn compact_indexes(&mut self) -> Result<()> {
558 let store = &self.store;
559 let took = self.writer.with(|w| {
560 while w.drain(store)? > 0 {}
561 let started = Instant::now();
562 w.compact()?;
563 Ok(started.elapsed())
564 })?;
565 self.reload_reader();
566 residency::release_free_heap();
567 tracing::info!("lexical index compacted in {:.1}s", took.as_secs_f64());
568 Ok(())
569 }
570
571 fn reload_reader(&self) {
572 let mut guard = match self.reader.write() {
573 Ok(g) => g,
574 Err(poisoned) => poisoned.into_inner(),
575 };
576 if let Err(e) = guard.reload_view() {
577 tracing::warn!("index reader reload failed: {e:#}");
578 }
579 }
580}
581
582#[cfg(test)]
583mod tests {
584 use std::fs::Permissions;
585 use std::os::unix::fs::PermissionsExt;
586
587 use super::*;
588 use second_brain_core::schema::{Memory, MemoryType};
589 use second_brain_core::store::Store;
590
591 #[test]
592 fn a_fresh_throttle_scans_on_the_first_idle_tick() {
593 let now = Instant::now();
594 assert!(ChunkThrottle::new(now).due(now));
595 }
596
597 #[test]
598 fn a_scan_that_finds_nothing_waits_out_the_idle_interval() {
599 let now = Instant::now();
600 let mut t = ChunkThrottle::new(now);
601 t.after_backfill(now, false);
602 assert!(!t.due(now + CHUNK_IDLE_RESCAN - Duration::from_secs(1)));
603 assert!(t.due(now + CHUNK_IDLE_RESCAN));
604 }
605
606 #[test]
607 fn progress_keeps_the_next_idle_tick_scanning() {
608 let now = Instant::now();
609 let mut t = ChunkThrottle::new(now);
610 t.after_backfill(now, true);
611 assert!(t.due(now));
612 }
613
614 #[test]
615 fn an_applied_drain_resets_the_idle_wait() {
616 let now = Instant::now();
617 let mut t = ChunkThrottle::new(now);
618 t.after_backfill(now, false);
619 t.after_drain(now, 0);
620 assert!(!t.due(now));
621 t.after_drain(now, 3);
622 assert!(t.due(now));
623 }
624
625 fn remember(store: &SqliteStore, content: &str) -> uuid::Uuid {
626 let m = Memory::new(
627 content.to_string(),
628 MemoryType::Semantic,
629 "test".to_string(),
630 String::new(),
631 );
632 store.store_memory(&m).unwrap();
633 m.id
634 }
635
636 fn drain_loop(
637 store: Arc<SqliteStore>,
638 reader: Arc<RwLock<Indexes>>,
639 writer: Indexes,
640 ) -> DrainLoop {
641 let (_tx, requests) = std::sync::mpsc::channel();
642 let claim = IndexClaim::acquire(writer.dir()).unwrap();
643 DrainLoop {
644 store,
645 reader,
646 writer: WriterSlot::new(writer, claim),
647 embedder: None,
648 summary: Arc::new(RwLock::new(None)),
649 near_duplicates: Arc::new(RwLock::new(None)),
650 near_duplicate_threshold: 0.97,
651 config: MaintenanceConfig {
652 enabled: false,
653 interval_secs: 3600,
654 },
655 requests,
656 health: Arc::new(Health::new(Utc::now(), None)),
657 }
658 }
659
660 fn idle_loop(dir: &std::path::Path) -> DrainLoop {
661 let store = Arc::new(SqliteStore::in_memory("m".to_string()).unwrap());
662 let mut writer = Indexes::open(dir).unwrap();
663 writer.persist().unwrap();
664 let reader = Arc::new(RwLock::new(Indexes::open_readonly(dir).unwrap()));
665 drain_loop(store, reader, writer)
666 }
667
668 #[test]
669 fn idleness_counts_from_when_an_operation_finishes() {
670 let dir = tempfile::tempdir().unwrap();
671 let mut lp = idle_loop(dir.path());
672 let mut finishing = None;
673 lp.writer
674 .with(|_| {
675 finishing = Some(Instant::now());
676 Ok(())
677 })
678 .unwrap();
679 let finishing = finishing.unwrap();
680 assert!(lp.writer.last_used >= finishing);
681 assert!(
682 !lp.writer
683 .release_if_idle(finishing + WRITER_IDLE - Duration::from_secs(1))
684 );
685 }
686
687 #[test]
688 fn a_writer_holding_changes_its_files_lack_is_never_released() {
689 let dir = tempfile::tempdir().unwrap();
690 let mut lp = idle_loop(dir.path());
691 remember(&lp.store, "the zanzibar ferry leaves at noon");
692 let tantivy = dir.path().join("tantivy");
693 let set_mode =
694 |mode| std::fs::set_permissions(&tantivy, Permissions::from_mode(mode)).unwrap();
695 set_mode(0o500);
696 let drained = lp.drain();
697 set_mode(0o700);
698 assert_eq!(
699 drained, None,
700 "the drain should fail to write the lexical index"
701 );
702 assert!(
703 !lp.writer
704 .release_if_idle(lp.writer.last_used + WRITER_IDLE * 10)
705 );
706 lp.repair("test", true).unwrap();
707 assert!(!lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
708
709 assert_eq!(lp.drain(), Some(1));
710 assert!(lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
711 }
712
713 #[test]
714 fn a_failure_that_left_the_files_current_still_releases() {
715 let dir = tempfile::tempdir().unwrap();
716 let mut lp = idle_loop(dir.path());
717 let failed: Result<()> = lp.writer.with(|_| anyhow::bail!("a store step failed"));
718 assert!(failed.is_err());
719 assert!(lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
720 }
721
722 #[test]
723 fn a_writer_that_cannot_reopen_fails_the_repair() {
724 let dir = tempfile::tempdir().unwrap();
725 let mut lp = idle_loop(dir.path());
726 assert!(lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
727 let blocker = dir.path().join("not-a-dir");
728 std::fs::write(&blocker, b"x").unwrap();
729 lp.writer.dir = blocker.join("indexes");
730
731 let err = lp.repair("test", false).unwrap_err();
732 assert!(
733 format!("{err:#}").contains("indexes were not repaired"),
734 "{err:#}"
735 );
736 assert!(lp.writer.memory().is_none());
737 }
738
739 #[test]
740 fn an_idle_writer_is_released_and_reopened_by_the_next_write() {
741 let dir = tempfile::tempdir().unwrap();
742 let store = Arc::new(SqliteStore::in_memory("m".to_string()).unwrap());
743 let mut writer = Indexes::open(dir.path()).unwrap();
744 writer.persist().unwrap();
745 let reader = Arc::new(RwLock::new(Indexes::open_readonly(dir.path()).unwrap()));
746 let mut lp = drain_loop(store.clone(), reader.clone(), writer);
747
748 let first = remember(&store, "the zanzibar ferry leaves at noon");
749 assert_eq!(lp.drain(), Some(1));
750 assert!(lp.writer.memory().is_some());
751
752 let used = lp.writer.last_used;
753 assert!(
754 !lp.writer
755 .release_if_idle(used + WRITER_IDLE - Duration::from_secs(1))
756 );
757 assert!(lp.writer.memory().is_some());
758 assert!(lp.writer.release_if_idle(used + WRITER_IDLE));
759 assert!(lp.writer.memory().is_none());
760
761 assert_eq!(lp.drain(), Some(0));
762 assert!(
763 lp.writer.memory().is_none(),
764 "an empty tick reopened the writer"
765 );
766
767 let second = remember(&store, "the kilimanjaro trail opens in june");
768 assert_eq!(lp.drain(), Some(1));
769 assert!(lp.writer.memory().is_some());
770 assert_eq!(lp.writer.open().unwrap().lexical_len(), 2);
771
772 let guard = reader.read().unwrap();
773 let hits = |q: &str| -> Vec<uuid::Uuid> {
774 guard
775 .bm25_search(q, 5)
776 .unwrap()
777 .into_iter()
778 .map(|(id, _)| id)
779 .collect()
780 };
781 assert_eq!(hits("kilimanjaro"), vec![second]);
782 assert_eq!(hits("zanzibar"), vec![first]);
783 }
784
785 fn keyed_store(dir: &std::path::Path) -> Arc<SqliteStore> {
786 let key = second_brain_core::keys::StoreKey::generate().unwrap();
787 Arc::new(SqliteStore::open_with_key(&dir.join("brain.db"), "m".to_string(), &key).unwrap())
788 }
789
790 fn index_files(dir: &std::path::Path) -> Vec<std::path::PathBuf> {
791 let mut out = Vec::new();
792 let mut stack = vec![dir.to_path_buf()];
793 while let Some(d) = stack.pop() {
794 for entry in std::fs::read_dir(d).unwrap() {
795 let path = entry.unwrap().path();
796 let name = path.file_name().unwrap().to_string_lossy().to_string();
797 if path.is_dir() {
798 stack.push(path);
799 } else if !name.ends_with(".lock") && name != second_brain_core::index::CLAIM_FILE {
800 out.push(path);
801 }
802 }
803 }
804 out
805 }
806
807 fn assert_all_sealed_under(dir: &std::path::Path, key: SealKeyId) {
808 let files = index_files(dir);
809 assert!(!files.is_empty());
810 for path in files {
811 assert_eq!(
812 second_brain_core::sealed::sealed_key_id(&path).unwrap(),
813 Some(key),
814 "{}",
815 path.display()
816 );
817 }
818 }
819
820 #[test]
821 fn indexes_opened_before_the_store_was_encrypted_are_sealed_and_recall_moves_to_a_shared_reader()
822 {
823 let data = tempfile::tempdir().unwrap();
824 let store = keyed_store(data.path());
825 let dir = data.path().join("brain.indexes");
826 let mut writer = Indexes::open(&dir).unwrap();
827 writer.persist().unwrap();
828 let reader = Arc::new(RwLock::new(Indexes::open_readonly(&dir).unwrap()));
829 let mut lp = drain_loop(store.clone(), reader.clone(), writer);
830 let first = remember(&store, "the zanzibar ferry leaves at noon");
831 assert_eq!(lp.drain(), Some(1));
832
833 lp.seal_if_due();
834 let keys = store.at_rest_keys(AtRestPurpose::Indexes).unwrap().unwrap();
835 assert!(keys.settled);
836 assert_all_sealed_under(&dir, keys.keys.current_id());
837 assert!(Indexes::open_readonly(&dir).is_err());
838
839 let second = remember(&store, "the kilimanjaro trail opens in june");
840 assert_eq!(lp.drain(), Some(1));
841 let guard = reader.read().unwrap();
842 assert_eq!(guard.bm25_search("zanzibar", 5).unwrap()[0].0, first);
843 assert_eq!(guard.bm25_search("kilimanjaro", 5).unwrap()[0].0, second);
844 assert_eq!(guard.vector_len(), lp.writer.open().unwrap().vector_len());
845 }
846
847 #[test]
848 fn a_store_key_rotation_reseals_the_indexes_under_a_new_key() {
849 let data = tempfile::tempdir().unwrap();
850 let store = keyed_store(data.path());
851 let dir = data.path().join("brain.indexes");
852 let keys = store
853 .at_rest_keys(AtRestPurpose::Indexes)
854 .unwrap()
855 .unwrap()
856 .keys;
857 let handles = crate::models::open_index_handles(&dir, &store, Some(&keys), |_| {}).unwrap();
858 let crate::models::IndexHandles {
859 reader,
860 writer,
861 claim,
862 } = handles;
863 drop(claim);
864 let mut lp = drain_loop(store.clone(), reader.clone(), writer);
865 let id = remember(&store, "the zanzibar ferry leaves at noon");
866 assert_eq!(lp.drain(), Some(1));
867 lp.seal_if_due();
868 let before = store.at_rest_keys(AtRestPurpose::Indexes).unwrap().unwrap();
869 assert!(before.settled);
870
871 let rotated = second_brain_core::keys::StoreKey::generate().unwrap();
872 store
873 .rewrite_under_key(
874 second_brain_core::sqlite_store::encrypt_migration::RewriteTo::Key(&rotated),
875 )
876 .unwrap();
877 lp.seal_if_due();
878 let after = store.at_rest_keys(AtRestPurpose::Indexes).unwrap().unwrap();
879 assert_ne!(after.keys.current_id(), before.keys.current_id());
880 assert!(after.settled);
881 assert!(after.keys.retired_ids().is_empty());
882 assert_all_sealed_under(&dir, after.keys.current_id());
883 assert_eq!(
884 reader.read().unwrap().bm25_search("zanzibar", 5).unwrap()[0].0,
885 id
886 );
887 }
888
889 #[test]
890 fn an_idle_sealed_writer_keeps_its_vectors_for_recall() {
891 let data = tempfile::tempdir().unwrap();
892 let store = keyed_store(data.path());
893 let dir = data.path().join("brain.indexes");
894 let keys = store
895 .at_rest_keys(AtRestPurpose::Indexes)
896 .unwrap()
897 .unwrap()
898 .keys;
899 let handles = crate::models::open_index_handles(&dir, &store, Some(&keys), |_| {}).unwrap();
900 let crate::models::IndexHandles {
901 reader,
902 writer,
903 claim,
904 } = handles;
905 drop(claim);
906 let mut lp = drain_loop(store.clone(), reader.clone(), writer);
907 let first = remember(&store, "the zanzibar ferry leaves at noon");
908 assert_eq!(lp.drain(), Some(1));
909 let used = lp.writer.last_used;
910 assert!(lp.writer.release_if_idle(used + WRITER_IDLE));
911 assert!(!lp.writer.release_if_idle(used + WRITER_IDLE * 2));
912 assert!(lp.writer.memory().is_some());
913
914 let second = remember(&store, "the kilimanjaro trail opens in june");
915 assert_eq!(lp.drain(), Some(1));
916 let guard = reader.read().unwrap();
917 assert_eq!(guard.bm25_search("zanzibar", 5).unwrap()[0].0, first);
918 assert_eq!(guard.bm25_search("kilimanjaro", 5).unwrap()[0].0, second);
919 }
920
921 #[test]
922 fn indexes_that_open_under_no_held_key_are_rebuilt_from_the_store() {
923 let data = tempfile::tempdir().unwrap();
924 let store = keyed_store(data.path());
925 let dir = data.path().join("brain.indexes");
926 let id = remember(&store, "the zanzibar ferry leaves at noon");
927 let foreign = SealKeys::single(second_brain_core::sealed::SealKey::generate().unwrap());
928 let handles =
929 crate::models::open_index_handles(&dir, &store, Some(&foreign), |_| {}).unwrap();
930 drop(handles);
931
932 let keys = store
933 .at_rest_keys(AtRestPurpose::Indexes)
934 .unwrap()
935 .unwrap()
936 .keys;
937 let handles = crate::models::open_index_handles(&dir, &store, Some(&keys), |_| {}).unwrap();
938 assert_eq!(
939 handles
940 .reader
941 .read()
942 .unwrap()
943 .bm25_search("zanzibar", 5)
944 .unwrap()[0]
945 .0,
946 id
947 );
948 }
949}