use std::path::PathBuf;
use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender};
use std::sync::{Arc, RwLock};
use std::time::{Duration, Instant};
use anyhow::Result;
use chrono::Utc;
use second_brain_core::config::MaintenanceConfig;
use second_brain_core::embedding::Embedder;
use second_brain_core::index::{IndexClaim, Indexes};
use second_brain_core::maintenance::{MaintenanceSummary, RepairStats};
use second_brain_core::residency::{self, IndexMemory};
use second_brain_core::sqlite_store::SqliteStore;
use crate::health::Health;
use crate::near_duplicates::{NearDupCount, refresh_count};
const DRAIN_INTERVAL: Duration = Duration::from_secs(5);
const WRITER_IDLE: Duration = Duration::from_secs(60);
const CHUNK_IDLE_RESCAN: Duration = Duration::from_secs(600);
const CHUNK_TICK_BUDGET: Duration = Duration::from_secs(3);
const CHUNK_MEMORIES_PER_TICK: usize = 32;
struct ChunkThrottle {
next_scan: Instant,
}
impl ChunkThrottle {
fn new(now: Instant) -> Self {
Self { next_scan: now }
}
fn due(&self, now: Instant) -> bool {
now >= self.next_scan
}
fn after_backfill(&mut self, now: Instant, progressed: bool) {
self.next_scan = if progressed {
now
} else {
now + CHUNK_IDLE_RESCAN
};
}
fn after_drain(&mut self, now: Instant, applied: usize) {
if applied > 0 {
self.next_scan = now;
}
}
}
pub struct RepairRequest {
pub dry_run: bool,
pub reply: Sender<Result<MaintenanceSummary>>,
}
pub enum DrainRequest {
Repair(RepairRequest),
CompactIndexes(Sender<Result<()>>),
}
pub fn run_repair(
store: &SqliteStore,
mut indexes: Option<&mut Indexes>,
embedder: Option<&Embedder>,
dry_run: bool,
) -> Result<MaintenanceSummary> {
let embed = embedder.map(|e| move |text: &str| e.embed(text));
let embed_ref = embed
.as_ref()
.map(|f| f as &dyn Fn(&str) -> Result<Vec<f32>>);
let stats = store.repair(indexes.as_deref_mut(), embed_ref, dry_run)?;
let report_after = store.integrity_report(indexes.as_deref())?;
Ok(MaintenanceSummary {
ran_at: Utc::now(),
stats,
report_after,
})
}
fn log_stats(source: &str, stats: &RepairStats) {
if stats.is_clean() {
tracing::info!("{source} store repair: nothing to do");
} else {
tracing::info!("{source} store repair: {}", summarize(stats));
}
}
fn summarize(s: &RepairStats) -> String {
let mut parts = Vec::new();
let mut push = |name: &str, n: u64| {
if n > 0 {
parts.push(format!("{name}={n}"));
}
};
push("duplicates_superseded", s.duplicates_superseded);
push("markers_inherited", s.distillation_markers_inherited);
push("orphaned_markers_dropped", s.orphaned_markers_dropped);
push("raws_retired", s.raws_retired);
push("relations_pruned", s.relations_pruned);
push("vec_keys_deleted", s.vec_keys_deleted);
push("sync_log_rows_deleted", s.sync_log_rows_deleted);
push("stale_cursors_dropped", s.stale_cursors_dropped);
push("embeddings_backfilled", s.embeddings_backfilled);
push("embedding_failures", s.embedding_failures);
if s.vec_keys_gc_skipped {
parts.push("vec_keys_gc=skipped".to_string());
}
if s.index_rebuilt {
parts.push("index=rebuilt".to_string());
}
if s.dry_run {
parts.push("dry_run".to_string());
}
parts.join(" ")
}
pub struct WriterSlot {
dir: PathBuf,
open: Option<Indexes>,
last_used: Instant,
_claim: IndexClaim,
}
impl WriterSlot {
pub fn new(writer: Indexes, claim: IndexClaim) -> Self {
crate::heap::record_index_writer(Some(writer.memory()));
Self {
dir: writer.dir().to_path_buf(),
open: Some(writer),
last_used: Instant::now(),
_claim: claim,
}
}
fn open(&mut self) -> Result<&mut Indexes> {
if self.open.is_none() {
let writer = Indexes::open(&self.dir)?;
tracing::info!("index writer reopened at {}", self.dir.display());
crate::heap::record_index_writer(Some(writer.memory()));
self.open = Some(writer);
}
Ok(self.open.as_mut().expect("writer opened above"))
}
fn with<R>(&mut self, op: impl FnOnce(&mut Indexes) -> Result<R>) -> Result<R> {
let result = op(self.open()?);
self.last_used = Instant::now();
crate::heap::record_index_writer(self.memory());
result
}
fn release_if_idle(&mut self, now: Instant) -> bool {
let current = self
.open
.as_ref()
.is_some_and(|w| !w.has_unpersisted_changes());
if !current || now.saturating_duration_since(self.last_used) < WRITER_IDLE {
return false;
}
if let Some(writer) = self.open.take()
&& let Err(e) = writer.wait_for_merges()
{
tracing::warn!("index writer released without its merges: {e:#}");
}
crate::heap::record_index_writer(None);
residency::release_free_heap();
tracing::info!(
"index writer released after {}s idle",
WRITER_IDLE.as_secs()
);
true
}
fn memory(&self) -> Option<IndexMemory> {
self.open.as_ref().map(Indexes::memory)
}
}
pub struct DrainLoop {
pub store: Arc<SqliteStore>,
pub reader: Arc<RwLock<Indexes>>,
pub writer: WriterSlot,
pub embedder: Option<Arc<Embedder>>,
pub summary: Arc<RwLock<Option<MaintenanceSummary>>>,
pub near_duplicates: Arc<RwLock<Option<NearDupCount>>>,
pub near_duplicate_threshold: f32,
pub config: MaintenanceConfig,
pub requests: Receiver<DrainRequest>,
pub health: Arc<Health>,
}
impl DrainLoop {
pub fn spawn(self) {
std::thread::Builder::new()
.name("index-drain".to_string())
.spawn(move || self.run())
.expect("spawning the index drain thread");
}
fn run(mut self) {
let mut last_repair: Option<Instant> = None;
let mut chunks = ChunkThrottle::new(Instant::now());
loop {
match self.requests.recv_timeout(DRAIN_INTERVAL) {
Ok(DrainRequest::Repair(request)) => {
let result = self.repair("requested", request.dry_run);
if result.is_ok() && !request.dry_run {
last_repair = Some(Instant::now());
}
let _ = request.reply.send(result);
}
Ok(DrainRequest::CompactIndexes(reply)) => {
let _ = reply.send(self.compact_indexes());
}
Err(RecvTimeoutError::Timeout) | Err(RecvTimeoutError::Disconnected) => {
let applied = self.drain();
chunks.after_drain(Instant::now(), applied.unwrap_or(0));
let due = last_repair.is_none_or(|t| {
t.elapsed() >= Duration::from_secs(self.config.interval_secs)
});
if applied == Some(0) && self.config.enabled && due {
if self.repair("scheduled", false).is_ok() {
last_repair = Some(Instant::now());
} else {
last_repair = Some(
Instant::now()
- Duration::from_secs(
self.config.interval_secs.saturating_sub(600),
),
);
}
}
if applied == Some(0) && chunks.due(Instant::now()) {
let progressed = self.backfill_chunks();
chunks.after_backfill(Instant::now(), progressed);
}
}
}
self.writer.release_if_idle(Instant::now());
}
}
fn backfill_chunks(&self) -> bool {
let Some(embedder) = self.embedder.as_deref() else {
return false;
};
let embed = |texts: &[&str]| embedder.embed_batch(texts);
match self.store.backfill_chunk_embeddings(
&embed,
CHUNK_MEMORIES_PER_TICK,
CHUNK_TICK_BUDGET,
) {
Ok(r) => {
if r.memories > 0 || r.failed > 0 {
tracing::info!(
"chunk backfill: memories={} chunks={} failed={} pending={}",
r.memories,
r.chunks,
r.failed,
r.pending
);
}
r.memories > 0
}
Err(e) => {
tracing::warn!("chunk backfill failed: {e:#}");
false
}
}
}
fn drain(&mut self) -> Option<usize> {
let store = &self.store;
let writer = &mut self.writer;
let result = store.outbox_pending().and_then(|pending| {
if pending == 0 {
Ok(0)
} else {
writer.with(|w| w.drain(store))
}
});
match result {
Ok(0) => {
self.health.drained(0, Utc::now());
Some(0)
}
Ok(n) => {
tracing::info!("index drain applied {n} outbox rows");
self.reload_reader();
match self.store.outbox_pending() {
Ok(pending) => self.health.drained(pending, Utc::now()),
Err(e) => tracing::warn!("counting the index outbox failed: {e:#}"),
}
Some(n)
}
Err(e) => {
tracing::warn!("index drain failed: {e:#}");
self.health.drain_failed(&format!("{e:#}"), Utc::now());
None
}
}
}
fn repair(&mut self, source: &str, dry_run: bool) -> Result<MaintenanceSummary> {
let started = Instant::now();
let store = &self.store;
let embedder = self.embedder.as_deref();
let repair = |w: &mut Indexes| run_repair(store, Some(w), embedder, dry_run);
let result = match self.writer.open().map(|_| ()) {
Ok(()) => self.writer.with(repair),
Err(e) => {
tracing::warn!(
"index writer unavailable ({e:#}); repairing the store without the indexes"
);
match run_repair(store, None, embedder, dry_run) {
Ok(summary) => log_stats(source, &summary.stats),
Err(store_error) => {
tracing::warn!("{source} store repair failed: {store_error:#}")
}
}
Err(e
.context("the index writer could not reopen, so the indexes were not repaired"))
}
};
match &result {
Ok(summary) => {
self.health.repaired(Utc::now());
log_stats(source, &summary.stats);
tracing::info!(
"{source} store repair finished in {:.1}s; drift remaining: {:?}",
started.elapsed().as_secs_f64(),
summary.report_after.drift()
);
if !dry_run {
self.reload_reader();
let mut guard = match self.summary.write() {
Ok(g) => g,
Err(poisoned) => poisoned.into_inner(),
};
*guard = Some(summary.clone());
drop(guard);
let threshold = self.near_duplicate_threshold;
let near_duplicates = &self.near_duplicates;
let counted = self.writer.with(|w| {
refresh_count(store, w, threshold, near_duplicates);
Ok(())
});
if let Err(e) = counted {
tracing::warn!("near-duplicate count failed: {e:#}");
}
if summary.stats.index_rebuilt {
residency::release_free_heap();
}
}
}
Err(e) => {
tracing::warn!("{source} store repair failed: {e:#}");
self.health.repair_failed(&format!("{e:#}"), Utc::now());
}
}
result
}
fn compact_indexes(&mut self) -> Result<()> {
let store = &self.store;
let took = self.writer.with(|w| {
while w.drain(store)? > 0 {}
let started = Instant::now();
w.compact()?;
Ok(started.elapsed())
})?;
self.reload_reader();
residency::release_free_heap();
tracing::info!("lexical index compacted in {:.1}s", took.as_secs_f64());
Ok(())
}
fn reload_reader(&self) {
let mut guard = match self.reader.write() {
Ok(g) => g,
Err(poisoned) => poisoned.into_inner(),
};
if let Err(e) = guard.reload_view() {
tracing::warn!("index reader reload failed: {e:#}");
}
}
}
#[cfg(test)]
mod tests {
use std::fs::Permissions;
use std::os::unix::fs::PermissionsExt;
use super::*;
use second_brain_core::schema::{Memory, MemoryType};
use second_brain_core::store::Store;
#[test]
fn a_fresh_throttle_scans_on_the_first_idle_tick() {
let now = Instant::now();
assert!(ChunkThrottle::new(now).due(now));
}
#[test]
fn a_scan_that_finds_nothing_waits_out_the_idle_interval() {
let now = Instant::now();
let mut t = ChunkThrottle::new(now);
t.after_backfill(now, false);
assert!(!t.due(now + CHUNK_IDLE_RESCAN - Duration::from_secs(1)));
assert!(t.due(now + CHUNK_IDLE_RESCAN));
}
#[test]
fn progress_keeps_the_next_idle_tick_scanning() {
let now = Instant::now();
let mut t = ChunkThrottle::new(now);
t.after_backfill(now, true);
assert!(t.due(now));
}
#[test]
fn an_applied_drain_resets_the_idle_wait() {
let now = Instant::now();
let mut t = ChunkThrottle::new(now);
t.after_backfill(now, false);
t.after_drain(now, 0);
assert!(!t.due(now));
t.after_drain(now, 3);
assert!(t.due(now));
}
fn remember(store: &SqliteStore, content: &str) -> uuid::Uuid {
let m = Memory::new(
content.to_string(),
MemoryType::Semantic,
"test".to_string(),
String::new(),
);
store.store_memory(&m).unwrap();
m.id
}
fn drain_loop(
store: Arc<SqliteStore>,
reader: Arc<RwLock<Indexes>>,
writer: Indexes,
) -> DrainLoop {
let (_tx, requests) = std::sync::mpsc::channel();
let claim = IndexClaim::acquire(writer.dir()).unwrap();
DrainLoop {
store,
reader,
writer: WriterSlot::new(writer, claim),
embedder: None,
summary: Arc::new(RwLock::new(None)),
near_duplicates: Arc::new(RwLock::new(None)),
near_duplicate_threshold: 0.97,
config: MaintenanceConfig {
enabled: false,
interval_secs: 3600,
},
requests,
health: Arc::new(Health::new(Utc::now(), None)),
}
}
fn idle_loop(dir: &std::path::Path) -> DrainLoop {
let store = Arc::new(SqliteStore::in_memory("m".to_string()).unwrap());
let mut writer = Indexes::open(dir).unwrap();
writer.persist().unwrap();
let reader = Arc::new(RwLock::new(Indexes::open_readonly(dir).unwrap()));
drain_loop(store, reader, writer)
}
#[test]
fn idleness_counts_from_when_an_operation_finishes() {
let dir = tempfile::tempdir().unwrap();
let mut lp = idle_loop(dir.path());
let mut finishing = None;
lp.writer
.with(|_| {
finishing = Some(Instant::now());
Ok(())
})
.unwrap();
let finishing = finishing.unwrap();
assert!(lp.writer.last_used >= finishing);
assert!(
!lp.writer
.release_if_idle(finishing + WRITER_IDLE - Duration::from_secs(1))
);
}
#[test]
fn a_writer_holding_changes_its_files_lack_is_never_released() {
let dir = tempfile::tempdir().unwrap();
let mut lp = idle_loop(dir.path());
remember(&lp.store, "the zanzibar ferry leaves at noon");
let tantivy = dir.path().join("tantivy");
let set_mode =
|mode| std::fs::set_permissions(&tantivy, Permissions::from_mode(mode)).unwrap();
set_mode(0o500);
let drained = lp.drain();
set_mode(0o700);
assert_eq!(
drained, None,
"the drain should fail to write the lexical index"
);
assert!(
!lp.writer
.release_if_idle(lp.writer.last_used + WRITER_IDLE * 10)
);
lp.repair("test", true).unwrap();
assert!(!lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
assert_eq!(lp.drain(), Some(1));
assert!(lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
}
#[test]
fn a_failure_that_left_the_files_current_still_releases() {
let dir = tempfile::tempdir().unwrap();
let mut lp = idle_loop(dir.path());
let failed: Result<()> = lp.writer.with(|_| anyhow::bail!("a store step failed"));
assert!(failed.is_err());
assert!(lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
}
#[test]
fn a_writer_that_cannot_reopen_fails_the_repair() {
let dir = tempfile::tempdir().unwrap();
let mut lp = idle_loop(dir.path());
assert!(lp.writer.release_if_idle(lp.writer.last_used + WRITER_IDLE));
let blocker = dir.path().join("not-a-dir");
std::fs::write(&blocker, b"x").unwrap();
lp.writer.dir = blocker.join("indexes");
let err = lp.repair("test", false).unwrap_err();
assert!(
format!("{err:#}").contains("indexes were not repaired"),
"{err:#}"
);
assert!(lp.writer.memory().is_none());
}
#[test]
fn an_idle_writer_is_released_and_reopened_by_the_next_write() {
let dir = tempfile::tempdir().unwrap();
let store = Arc::new(SqliteStore::in_memory("m".to_string()).unwrap());
let mut writer = Indexes::open(dir.path()).unwrap();
writer.persist().unwrap();
let reader = Arc::new(RwLock::new(Indexes::open_readonly(dir.path()).unwrap()));
let mut lp = drain_loop(store.clone(), reader.clone(), writer);
let first = remember(&store, "the zanzibar ferry leaves at noon");
assert_eq!(lp.drain(), Some(1));
assert!(lp.writer.memory().is_some());
let used = lp.writer.last_used;
assert!(
!lp.writer
.release_if_idle(used + WRITER_IDLE - Duration::from_secs(1))
);
assert!(lp.writer.memory().is_some());
assert!(lp.writer.release_if_idle(used + WRITER_IDLE));
assert!(lp.writer.memory().is_none());
assert_eq!(lp.drain(), Some(0));
assert!(
lp.writer.memory().is_none(),
"an empty tick reopened the writer"
);
let second = remember(&store, "the kilimanjaro trail opens in june");
assert_eq!(lp.drain(), Some(1));
assert!(lp.writer.memory().is_some());
assert_eq!(lp.writer.open().unwrap().lexical_len(), 2);
let guard = reader.read().unwrap();
let hits = |q: &str| -> Vec<uuid::Uuid> {
guard
.bm25_search(q, 5)
.unwrap()
.into_iter()
.map(|(id, _)| id)
.collect()
};
assert_eq!(hits("kilimanjaro"), vec![second]);
assert_eq!(hits("zanzibar"), vec![first]);
}
}