use std::path::PathBuf;
use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender};
use std::sync::{Arc, PoisonError, RwLock};
use std::time::{Duration, Instant};
use anyhow::{Result, anyhow};
use chrono::Utc;
use second_brain_core::config::MaintenanceConfig;
use second_brain_core::embedding::Embedder;
use second_brain_core::index::{IndexClaim, Indexes};
use second_brain_core::maintenance::{MaintenanceSummary, RepairStats};
use second_brain_core::residency::{self, IndexMemory};
use second_brain_core::sealed::{SealKeyId, SealKeys};
use second_brain_core::sqlite_store::{AtRestPurpose, SqliteStore};
use crate::health::Health;
use crate::near_duplicates::{NearDupCount, refresh_count};
const DRAIN_INTERVAL: Duration = Duration::from_secs(5);
const DELETE_PURGE_GAP: Duration = Duration::from_secs(600);
const WRITER_IDLE: Duration = Duration::from_secs(60);
const SEAL_CHECK_INTERVAL: Duration = Duration::from_secs(60);
const CHUNK_IDLE_RESCAN: Duration = Duration::from_secs(600);
const CHUNK_TICK_BUDGET: Duration = Duration::from_secs(3);
const CHUNK_MEMORIES_PER_TICK: usize = 32;
struct ChunkThrottle {
next_scan: Instant,
}
impl ChunkThrottle {
fn new(now: Instant) -> Self {
Self { next_scan: now }
}
fn due(&self, now: Instant) -> bool {
now >= self.next_scan
}
fn after_backfill(&mut self, now: Instant, progressed: bool) {
self.next_scan = if progressed {
now
} else {
now + CHUNK_IDLE_RESCAN
};
}
fn after_drain(&mut self, now: Instant, applied: usize) {
if applied > 0 {
self.next_scan = now;
}
}
}
pub struct RepairRequest {
pub dry_run: bool,
pub reply: Sender<Result<MaintenanceSummary>>,
}
pub enum DrainRequest {
Repair(RepairRequest),
CompactIndexes(Sender<Result<()>>),
Pause(Receiver<PauseCommand>),
}
pub enum PauseCommand {
Settle(Sender<Result<()>>),
}
pub fn run_repair(
store: &SqliteStore,
mut indexes: Option<&mut Indexes>,
embedder: Option<&Embedder>,
dry_run: bool,
) -> Result<MaintenanceSummary> {
let embed = embedder.map(|e| move |text: &str| e.embed(text));
let embed_ref = embed
.as_ref()
.map(|f| f as &dyn Fn(&str) -> Result<Vec<f32>>);
let stats = store.repair(indexes.as_deref_mut(), embed_ref, dry_run)?;
let report_after = store.integrity_report(indexes.as_deref())?;
Ok(MaintenanceSummary {
ran_at: Utc::now(),
stats,
report_after,
})
}
fn log_stats(source: &str, stats: &RepairStats) {
if stats.is_clean() {
tracing::info!("{source} store repair: nothing to do");
} else {
tracing::info!("{source} store repair: {}", summarize(stats));
}
}
fn summarize(s: &RepairStats) -> String {
let mut parts = Vec::new();
let mut push = |name: &str, n: u64| {
if n > 0 {
parts.push(format!("{name}={n}"));
}
};
push("duplicates_superseded", s.duplicates_superseded);
push("markers_inherited", s.distillation_markers_inherited);
push("orphaned_markers_dropped", s.orphaned_markers_dropped);
push("raws_retired", s.raws_retired);
push("relations_pruned", s.relations_pruned);
push("vec_keys_deleted", s.vec_keys_deleted);
push("sync_log_rows_deleted", s.sync_log_rows_deleted);
push("stale_cursors_dropped", s.stale_cursors_dropped);
push("embeddings_backfilled", s.embeddings_backfilled);
push("embedding_failures", s.embedding_failures);
if s.vec_keys_gc_skipped {
parts.push("vec_keys_gc=skipped".to_string());
}
if s.index_rebuilt {
parts.push("index=rebuilt".to_string());
}
if s.dry_run {
parts.push("dry_run".to_string());
}
parts.join(" ")
}
pub struct WriterSlot {
dir: PathBuf,
open: Option<Indexes>,
last_used: Instant,
shared: bool,
idle_released: bool,
_claim: IndexClaim,
}
impl WriterSlot {
pub fn new(writer: Indexes, claim: IndexClaim) -> Self {
crate::heap::record_index_writer(Some(writer.memory()));
Self {
dir: writer.dir().to_path_buf(),
shared: writer.seal_keys().is_some(),
open: Some(writer),
last_used: Instant::now(),
idle_released: false,
_claim: claim,
}
}
fn open(&mut self) -> Result<&mut Indexes> {
if self.open.is_none() {
if self.shared {
return Err(anyhow!(
"the sealed index writer at {} is gone; the daemon must restart to reopen it",
self.dir.display()
));
}
let writer = Indexes::open(&self.dir)?;
tracing::info!("index writer reopened at {}", self.dir.display());
crate::heap::record_index_writer(Some(writer.memory()));
self.open = Some(writer);
}
Ok(self.open.as_mut().expect("writer opened above"))
}
fn with<R>(&mut self, op: impl FnOnce(&mut Indexes) -> Result<R>) -> Result<R> {
let result = op(self.open()?);
self.last_used = Instant::now();
self.idle_released = false;
crate::heap::record_index_writer(self.memory());
result
}
fn release_if_idle(&mut self, now: Instant) -> bool {
let current = self
.open
.as_ref()
.is_some_and(|w| !w.has_unpersisted_changes());
if !current || now.saturating_duration_since(self.last_used) < WRITER_IDLE {
return false;
}
if self.shared {
return self.release_lexical_writer();
}
if let Some(writer) = self.open.take()
&& let Err(e) = writer.wait_for_merges()
{
tracing::warn!("index writer released without its merges: {e:#}");
}
crate::heap::record_index_writer(None);
residency::release_free_heap();
tracing::info!(
"index writer released after {}s idle",
WRITER_IDLE.as_secs()
);
true
}
fn release_lexical_writer(&mut self) -> bool {
if self.idle_released {
return false;
}
if let Some(writer) = self.open.as_mut()
&& let Err(e) = writer.close_lexical_writer()
{
tracing::warn!("lexical writer released without its merges: {e:#}");
}
self.idle_released = true;
residency::release_free_heap();
tracing::info!(
"lexical index writer released after {}s idle",
WRITER_IDLE.as_secs()
);
true
}
fn memory(&self) -> Option<IndexMemory> {
self.open.as_ref().map(Indexes::memory)
}
fn sealed_under(&self) -> Option<SealKeyId> {
self.open
.as_ref()
.and_then(Indexes::seal_keys)
.map(|keys| keys.current_id())
}
fn seal_under(&mut self, keys: SealKeys, reader: &RwLock<Indexes>) -> Result<()> {
if self.shared {
let writer = self
.open
.as_mut()
.ok_or_else(|| anyhow!("the sealed index writer is gone"))?;
writer.set_seal_keys(keys)?;
return writer.reseal();
}
if let Some(writer) = self.open.as_mut()
&& writer.has_unpersisted_changes()
{
writer.persist()?;
}
if let Some(writer) = self.open.take() {
crate::heap::record_index_writer(None);
writer.wait_for_merges()?;
}
let mut writer = Indexes::open_keyed(&self.dir, &keys)?;
writer.reseal()?;
let shared = writer.shared_reader()?;
*reader.write().unwrap_or_else(PoisonError::into_inner) = shared;
crate::heap::record_index_writer(Some(writer.memory()));
self.open = Some(writer);
self.shared = true;
self.idle_released = false;
self.last_used = Instant::now();
Ok(())
}
}
pub struct DrainLoop {
pub store: Arc<SqliteStore>,
pub reader: Arc<RwLock<Indexes>>,
pub writer: WriterSlot,
pub embedder: Option<Arc<Embedder>>,
pub summary: Arc<RwLock<Option<MaintenanceSummary>>>,
pub near_duplicates: Arc<RwLock<Option<NearDupCount>>>,
pub near_duplicate_threshold: f32,
pub config: MaintenanceConfig,
pub requests: Receiver<DrainRequest>,
pub health: Arc<Health>,
}
impl DrainLoop {
pub fn spawn(self) {
std::thread::Builder::new()
.name("index-drain".to_string())
.spawn(move || self.run())
.expect("spawning the index drain thread");
}
fn run(mut self) {
let mut last_repair: Option<Instant> = None;
let mut chunks = ChunkThrottle::new(Instant::now());
let mut last_purge: Option<Instant> = None;
let mut last_seal_check: Option<Instant> = None;
loop {
match self.requests.recv_timeout(DRAIN_INTERVAL) {
Ok(DrainRequest::Repair(request)) => {
let result = self.repair("requested", request.dry_run);
if result.is_ok() && !request.dry_run {
last_repair = Some(Instant::now());
}
let _ = request.reply.send(result);
}
Ok(DrainRequest::CompactIndexes(reply)) => {
let _ = reply.send(self.compact_indexes());
}
Ok(DrainRequest::Pause(commands)) => {
for PauseCommand::Settle(reply) in commands {
let _ = reply.send(self.settle());
}
}
Err(RecvTimeoutError::Timeout) | Err(RecvTimeoutError::Disconnected) => {
let applied = self.drain();
chunks.after_drain(Instant::now(), applied.unwrap_or(0));
let due = last_repair.is_none_or(|t| {
t.elapsed() >= Duration::from_secs(self.config.interval_secs)
});
if applied == Some(0) && self.config.enabled && due {
if self.repair("scheduled", false).is_ok() {
last_repair = Some(Instant::now());
} else {
last_repair = Some(
Instant::now()
- Duration::from_secs(
self.config.interval_secs.saturating_sub(600),
),
);
}
}
if applied == Some(0)
&& last_purge.is_none_or(|t| t.elapsed() >= DELETE_PURGE_GAP)
&& self.purge_applied_deletes()
{
last_purge = Some(Instant::now());
}
if applied == Some(0) && chunks.due(Instant::now()) {
let progressed = self.backfill_chunks();
chunks.after_backfill(Instant::now(), progressed);
}
}
}
if last_seal_check.is_none_or(|t| t.elapsed() >= SEAL_CHECK_INTERVAL) {
self.seal_if_due();
last_seal_check = Some(Instant::now());
}
self.writer.release_if_idle(Instant::now());
}
}
fn seal_if_due(&mut self) {
let keys = match self.store.at_rest_keys(AtRestPurpose::Indexes) {
Ok(Some(keys)) => keys,
Ok(None) => return,
Err(e) => {
tracing::warn!("reading the search indexes' keys failed: {e:#}");
return;
}
};
let current = keys.keys.current_id();
if keys.settled && self.writer.sealed_under() == Some(current) {
return;
}
let started = Instant::now();
if let Err(e) = self.writer.seal_under(keys.keys, &self.reader) {
tracing::warn!("sealing the search indexes failed and will retry: {e:#}");
return;
}
match self
.store
.settle_at_rest_keys(AtRestPurpose::Indexes, current)
{
Ok(()) => tracing::info!(
"search indexes sealed under key {current} in {:.1}s",
started.elapsed().as_secs_f64()
),
Err(e) => tracing::warn!("recording the sealed search indexes failed: {e:#}"),
}
}
fn backfill_chunks(&self) -> bool {
let Some(embedder) = self.embedder.as_deref() else {
return false;
};
let embed = |texts: &[&str]| embedder.embed_batch(texts);
match self.store.backfill_chunk_embeddings(
&embed,
CHUNK_MEMORIES_PER_TICK,
CHUNK_TICK_BUDGET,
) {
Ok(r) => {
if r.memories > 0 || r.failed > 0 {
tracing::info!(
"chunk backfill: memories={} chunks={} failed={} pending={}",
r.memories,
r.chunks,
r.failed,
r.pending
);
}
r.memories > 0
}
Err(e) => {
tracing::warn!("chunk backfill failed: {e:#}");
false
}
}
}
fn drain(&mut self) -> Option<usize> {
let store = &self.store;
let writer = &mut self.writer;
let result = store.outbox_pending().and_then(|pending| {
if pending == 0 {
Ok(0)
} else {
writer.with(|w| w.drain(store))
}
});
match result {
Ok(0) => {
self.health.drained(0, Utc::now());
Some(0)
}
Ok(n) => {
tracing::info!("index drain applied {n} outbox rows");
self.reload_reader();
match self.store.outbox_pending() {
Ok(pending) => self.health.drained(pending, Utc::now()),
Err(e) => tracing::warn!("counting the index outbox failed: {e:#}"),
}
Some(n)
}
Err(e) => {
tracing::warn!("index drain failed: {e:#}");
self.health.drain_failed(&format!("{e:#}"), Utc::now());
None
}
}
}
fn repair(&mut self, source: &str, dry_run: bool) -> Result<MaintenanceSummary> {
let started = Instant::now();
let store = &self.store;
let embedder = self.embedder.as_deref();
let repair = |w: &mut Indexes| run_repair(store, Some(w), embedder, dry_run);
let result = match self.writer.open().map(|_| ()) {
Ok(()) => self.writer.with(repair),
Err(e) => {
tracing::warn!(
"index writer unavailable ({e:#}); repairing the store without the indexes"
);
match run_repair(store, None, embedder, dry_run) {
Ok(summary) => log_stats(source, &summary.stats),
Err(store_error) => {
tracing::warn!("{source} store repair failed: {store_error:#}")
}
}
Err(e
.context("the index writer could not reopen, so the indexes were not repaired"))
}
};
match &result {
Ok(summary) => {
self.health.repaired(Utc::now());
log_stats(source, &summary.stats);
tracing::info!(
"{source} store repair finished in {:.1}s; drift remaining: {:?}",
started.elapsed().as_secs_f64(),
summary.report_after.drift()
);
if !dry_run {
self.reload_reader();
let mut guard = match self.summary.write() {
Ok(g) => g,
Err(poisoned) => poisoned.into_inner(),
};
*guard = Some(summary.clone());
drop(guard);
let threshold = self.near_duplicate_threshold;
let near_duplicates = &self.near_duplicates;
let counted = self.writer.with(|w| {
refresh_count(store, w, threshold, near_duplicates);
Ok(())
});
if let Err(e) = counted {
tracing::warn!("near-duplicate count failed: {e:#}");
}
if summary.stats.index_rebuilt {
residency::release_free_heap();
}
}
}
Err(e) => {
tracing::warn!("{source} store repair failed: {e:#}");
self.health.repair_failed(&format!("{e:#}"), Utc::now());
}
}
result
}
fn purge_applied_deletes(&mut self) -> bool {
let token = match self.store.delete_purge_requested() {
Ok(Some(token)) => token,
Ok(None) => return false,
Err(e) => {
tracing::warn!("reading the delete purge request failed: {e:#}");
return false;
}
};
let result = self.compact_indexes().and_then(|()| {
let cold = self.store.purge_cold_segments()?;
if cold != Default::default() {
tracing::info!(
"deleted records purged from the cold tier: {} retired, {} segments rewritten, {} files removed",
cold.retired,
cold.rewritten,
cold.removed_files
);
}
self.store.delete_purge_done(&token)
});
if let Err(e) = result {
tracing::warn!("purging a peer's deletes failed and will retry: {e:#}");
}
true
}
fn compact_indexes(&mut self) -> Result<()> {
let store = &self.store;
let took = self.writer.with(|w| {
while w.drain(store)? > 0 {}
let started = Instant::now();
w.compact()?;
Ok(started.elapsed())
})?;
self.reload_reader();
residency::release_free_heap();
tracing::info!("lexical index compacted in {:.1}s", took.as_secs_f64());
Ok(())
}
fn settle(&mut self) -> Result<()> {
let store = &self.store;
self.writer.with(|w| {
while w.drain(store)? > 0 {}
if w.has_unpersisted_changes() {
w.persist()?;
}
w.close_lexical_writer()
})?;
self.reload_reader();
Ok(())
}
fn reload_reader(&self) {
let mut guard = match self.reader.write() {
Ok(g) => g,
Err(poisoned) => poisoned.into_inner(),
};
if let Err(e) = guard.reload_view() {
tracing::warn!("index reader reload failed: {e:#}");
}
}
}
#[cfg(test)]
mod tests {
use std::fs::Permissions;
use std::os::unix::fs::PermissionsExt;
use super::*;
use second_brain_core::schema::{Memory, MemoryType};
use second_brain_core::store::Store;
#[test]
fn a_fresh_throttle_scans_on_the_first_idle_tick() {
let now = Instant::now();
assert!(ChunkThrottle::new(now).due(now));
}
#[test]
fn a_scan_that_finds_nothing_waits_out_the_idle_interval() {
let now = Instant::now();
let mut t = ChunkThrottle::new(now);
t.after_backfill(now, false);
assert!(!t.due(now + CHUNK_IDLE_RESCAN - Duration::from_secs(1)));
assert!(t.due(now + CHUNK_IDLE_RESCAN));
}
#[test]
fn progress_keeps_the_next_idle_tick_scanning() {
let now = Instant::now();
let mut t = ChunkThrottle::new(now);
t.after_backfill(now, true);
assert!(t.due(now));
}
#[test]
fn an_applied_drain_resets_the_idle_wait() {
let now = Instant::now();
let mut t = ChunkThrottle::new(now);
t.after_backfill(now, false);
t.after_drain(now, 0);
assert!(!t.due(now));
t.after_drain(now, 3);
assert!(t.due(now));
}
fn remember(store: &SqliteStore, content: &str) -> uuid::Uuid {
let m = Memory::new(
content.to_string(),
MemoryType::Semantic,
"test".to_string(),
String::new(),
);
store.store_memory(&m).unwrap();
m.id
}
fn drain_loop(
store: Arc<SqliteStore>,
reader: Arc<RwLock<Indexes>>,
writer: Indexes,
) -> DrainLoop {
let (_tx, requests) = std::sync::mpsc::channel();
let claim = IndexClaim::acquire(writer.dir()).unwrap();
DrainLoop {
store,
reader,
writer: WriterSlot::new(writer, claim),
embedder: None,
summary: Arc::new(RwLock::new(None)),
near_duplicates: Arc::new(RwLock::new(None)),
near_duplicate_threshold: 0.97,
config: MaintenanceConfig {
enabled: false,
interval_secs: 3600,
},
requests,
health: Arc::new(Health::new(Utc::now(), None)),
}
}
fn idle_loop(dir: &std::path::Path) -> DrainLoop {
let store = Arc::new(SqliteStore::in_memory("m".to_string()).unwrap());
let mut writer = Indexes::open(dir).unwrap();
writer.persist().unwrap();
let reader = Arc::new(RwLock::new(Indexes::open_readonly(dir).unwrap()));
drain_loop(store, reader, writer)
}
#[test]
fn idleness_counts_from_when_an_operation_finishes() {
let dir = tempfile::tempdir().unwrap();
let mut lp = idle_loop(dir.path());
let mut finishing = None;
lp.writer
.with(|_| {
finishing = Some(Instant::now());
Ok(())
})
.unwrap();
let finishing = finishing.unwrap();
assert!(lp.writer.last_used >= finishing);
assert!(
!lp.writer
.release_if_idle(finishing + WRITER_IDLE - Duration::from_secs(1))
);
}
#[test]
fn a_writer_holding_changes_its_files_lack_is_never_released() {
let dir = tempfile::tempdir().unwrap();
let mut lp = idle_loop(dir.path());
remember(&lp.store, "the zanzibar ferry leaves at noon");
let tantivy = dir.path().join("tantivy");
let set_mode =
|mode| std::fs::set_permissions(&tantivy, Permissions::from_mode(mode)).unwrap();
set_mode(0o500);
let drained = lp.drain();
set_mode(0o700);
assert_eq!(
drained, None,
"the drain should fail to write the lexical index"
);
assert!(
!lp.writer
.release_if_idle(lp.writer.last_used + WRITER_IDLE * 10)
);
lp.repair("test", true).unwrap();
assert!(!lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
assert_eq!(lp.drain(), Some(1));
assert!(lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
}
#[test]
fn a_failure_that_left_the_files_current_still_releases() {
let dir = tempfile::tempdir().unwrap();
let mut lp = idle_loop(dir.path());
let failed: Result<()> = lp.writer.with(|_| anyhow::bail!("a store step failed"));
assert!(failed.is_err());
assert!(lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
}
#[test]
fn a_writer_that_cannot_reopen_fails_the_repair() {
let dir = tempfile::tempdir().unwrap();
let mut lp = idle_loop(dir.path());
assert!(lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
let blocker = dir.path().join("not-a-dir");
std::fs::write(&blocker, b"x").unwrap();
lp.writer.dir = blocker.join("indexes");
let err = lp.repair("test", false).unwrap_err();
assert!(
format!("{err:#}").contains("indexes were not repaired"),
"{err:#}"
);
assert!(lp.writer.memory().is_none());
}
#[test]
fn an_idle_writer_is_released_and_reopened_by_the_next_write() {
let dir = tempfile::tempdir().unwrap();
let store = Arc::new(SqliteStore::in_memory("m".to_string()).unwrap());
let mut writer = Indexes::open(dir.path()).unwrap();
writer.persist().unwrap();
let reader = Arc::new(RwLock::new(Indexes::open_readonly(dir.path()).unwrap()));
let mut lp = drain_loop(store.clone(), reader.clone(), writer);
let first = remember(&store, "the zanzibar ferry leaves at noon");
assert_eq!(lp.drain(), Some(1));
assert!(lp.writer.memory().is_some());
let used = lp.writer.last_used;
assert!(
!lp.writer
.release_if_idle(used + WRITER_IDLE - Duration::from_secs(1))
);
assert!(lp.writer.memory().is_some());
assert!(lp.writer.release_if_idle(used + WRITER_IDLE));
assert!(lp.writer.memory().is_none());
assert_eq!(lp.drain(), Some(0));
assert!(
lp.writer.memory().is_none(),
"an empty tick reopened the writer"
);
let second = remember(&store, "the kilimanjaro trail opens in june");
assert_eq!(lp.drain(), Some(1));
assert!(lp.writer.memory().is_some());
assert_eq!(lp.writer.open().unwrap().lexical_len(), 2);
let guard = reader.read().unwrap();
let hits = |q: &str| -> Vec<uuid::Uuid> {
guard
.bm25_search(q, 5)
.unwrap()
.into_iter()
.map(|(id, _)| id)
.collect()
};
assert_eq!(hits("kilimanjaro"), vec![second]);
assert_eq!(hits("zanzibar"), vec![first]);
}
fn keyed_store(dir: &std::path::Path) -> Arc<SqliteStore> {
let key = second_brain_core::keys::StoreKey::generate().unwrap();
Arc::new(SqliteStore::open_with_key(&dir.join("brain.db"), "m".to_string(), &key).unwrap())
}
fn index_files(dir: &std::path::Path) -> Vec<std::path::PathBuf> {
let mut out = Vec::new();
let mut stack = vec![dir.to_path_buf()];
while let Some(d) = stack.pop() {
for entry in std::fs::read_dir(d).unwrap() {
let path = entry.unwrap().path();
let name = path.file_name().unwrap().to_string_lossy().to_string();
if path.is_dir() {
stack.push(path);
} else if !name.ends_with(".lock") && name != second_brain_core::index::CLAIM_FILE {
out.push(path);
}
}
}
out
}
fn assert_all_sealed_under(dir: &std::path::Path, key: SealKeyId) {
let files = index_files(dir);
assert!(!files.is_empty());
for path in files {
assert_eq!(
second_brain_core::sealed::sealed_key_id(&path).unwrap(),
Some(key),
"{}",
path.display()
);
}
}
#[test]
fn indexes_opened_before_the_store_was_encrypted_are_sealed_and_recall_moves_to_a_shared_reader()
{
let data = tempfile::tempdir().unwrap();
let store = keyed_store(data.path());
let dir = data.path().join("brain.indexes");
let mut writer = Indexes::open(&dir).unwrap();
writer.persist().unwrap();
let reader = Arc::new(RwLock::new(Indexes::open_readonly(&dir).unwrap()));
let mut lp = drain_loop(store.clone(), reader.clone(), writer);
let first = remember(&store, "the zanzibar ferry leaves at noon");
assert_eq!(lp.drain(), Some(1));
lp.seal_if_due();
let keys = store.at_rest_keys(AtRestPurpose::Indexes).unwrap().unwrap();
assert!(keys.settled);
assert_all_sealed_under(&dir, keys.keys.current_id());
assert!(Indexes::open_readonly(&dir).is_err());
let second = remember(&store, "the kilimanjaro trail opens in june");
assert_eq!(lp.drain(), Some(1));
let guard = reader.read().unwrap();
assert_eq!(guard.bm25_search("zanzibar", 5).unwrap()[0].0, first);
assert_eq!(guard.bm25_search("kilimanjaro", 5).unwrap()[0].0, second);
assert_eq!(guard.vector_len(), lp.writer.open().unwrap().vector_len());
}
#[test]
fn a_store_key_rotation_reseals_the_indexes_under_a_new_key() {
let data = tempfile::tempdir().unwrap();
let store = keyed_store(data.path());
let dir = data.path().join("brain.indexes");
let keys = store
.at_rest_keys(AtRestPurpose::Indexes)
.unwrap()
.unwrap()
.keys;
let handles = crate::models::open_index_handles(&dir, &store, Some(&keys), |_| {}).unwrap();
let crate::models::IndexHandles {
reader,
writer,
claim,
} = handles;
drop(claim);
let mut lp = drain_loop(store.clone(), reader.clone(), writer);
let id = remember(&store, "the zanzibar ferry leaves at noon");
assert_eq!(lp.drain(), Some(1));
lp.seal_if_due();
let before = store.at_rest_keys(AtRestPurpose::Indexes).unwrap().unwrap();
assert!(before.settled);
let rotated = second_brain_core::keys::StoreKey::generate().unwrap();
store
.rewrite_under_key(
second_brain_core::sqlite_store::encrypt_migration::RewriteTo::Key(&rotated),
)
.unwrap();
lp.seal_if_due();
let after = store.at_rest_keys(AtRestPurpose::Indexes).unwrap().unwrap();
assert_ne!(after.keys.current_id(), before.keys.current_id());
assert!(after.settled);
assert!(after.keys.retired_ids().is_empty());
assert_all_sealed_under(&dir, after.keys.current_id());
assert_eq!(
reader.read().unwrap().bm25_search("zanzibar", 5).unwrap()[0].0,
id
);
}
#[test]
fn an_idle_sealed_writer_keeps_its_vectors_for_recall() {
let data = tempfile::tempdir().unwrap();
let store = keyed_store(data.path());
let dir = data.path().join("brain.indexes");
let keys = store
.at_rest_keys(AtRestPurpose::Indexes)
.unwrap()
.unwrap()
.keys;
let handles = crate::models::open_index_handles(&dir, &store, Some(&keys), |_| {}).unwrap();
let crate::models::IndexHandles {
reader,
writer,
claim,
} = handles;
drop(claim);
let mut lp = drain_loop(store.clone(), reader.clone(), writer);
let first = remember(&store, "the zanzibar ferry leaves at noon");
assert_eq!(lp.drain(), Some(1));
let used = lp.writer.last_used;
assert!(lp.writer.release_if_idle(used + WRITER_IDLE));
assert!(!lp.writer.release_if_idle(used + WRITER_IDLE * 2));
assert!(lp.writer.memory().is_some());
let second = remember(&store, "the kilimanjaro trail opens in june");
assert_eq!(lp.drain(), Some(1));
let guard = reader.read().unwrap();
assert_eq!(guard.bm25_search("zanzibar", 5).unwrap()[0].0, first);
assert_eq!(guard.bm25_search("kilimanjaro", 5).unwrap()[0].0, second);
}
#[test]
fn indexes_that_open_under_no_held_key_are_rebuilt_from_the_store() {
let data = tempfile::tempdir().unwrap();
let store = keyed_store(data.path());
let dir = data.path().join("brain.indexes");
let id = remember(&store, "the zanzibar ferry leaves at noon");
let foreign = SealKeys::single(second_brain_core::sealed::SealKey::generate().unwrap());
let handles =
crate::models::open_index_handles(&dir, &store, Some(&foreign), |_| {}).unwrap();
drop(handles);
let keys = store
.at_rest_keys(AtRestPurpose::Indexes)
.unwrap()
.unwrap()
.keys;
let handles = crate::models::open_index_handles(&dir, &store, Some(&keys), |_| {}).unwrap();
assert_eq!(
handles
.reader
.read()
.unwrap()
.bm25_search("zanzibar", 5)
.unwrap()[0]
.0,
id
);
}
}