use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender};
use std::sync::{Arc, RwLock};
use std::time::{Duration, Instant};
use anyhow::Result;
use chrono::Utc;
use second_brain_core::config::MaintenanceConfig;
use second_brain_core::embedding::Embedder;
use second_brain_core::index::Indexes;
use second_brain_core::maintenance::{MaintenanceSummary, RepairStats};
use second_brain_core::sqlite_store::SqliteStore;
use crate::near_duplicates::{NearDupCount, refresh_count};
const DRAIN_INTERVAL: Duration = Duration::from_secs(5);
const CHUNK_IDLE_RESCAN: Duration = Duration::from_secs(600);
const CHUNK_TICK_BUDGET: Duration = Duration::from_secs(3);
const CHUNK_MEMORIES_PER_TICK: usize = 32;
struct ChunkThrottle {
next_scan: Instant,
}
impl ChunkThrottle {
fn new(now: Instant) -> Self {
Self { next_scan: now }
}
fn due(&self, now: Instant) -> bool {
now >= self.next_scan
}
fn after_backfill(&mut self, now: Instant, progressed: bool) {
self.next_scan = if progressed {
now
} else {
now + CHUNK_IDLE_RESCAN
};
}
fn after_drain(&mut self, now: Instant, applied: usize) {
if applied > 0 {
self.next_scan = now;
}
}
}
pub struct RepairRequest {
pub dry_run: bool,
pub reply: Sender<Result<MaintenanceSummary>>,
}
pub fn run_repair(
store: &SqliteStore,
mut indexes: Option<&mut Indexes>,
embedder: Option<&Embedder>,
dry_run: bool,
) -> Result<MaintenanceSummary> {
let embed = embedder.map(|e| move |text: &str| e.embed(text));
let embed_ref = embed
.as_ref()
.map(|f| f as &dyn Fn(&str) -> Result<Vec<f32>>);
let stats = store.repair(indexes.as_deref_mut(), embed_ref, dry_run)?;
let report_after = store.integrity_report(indexes.as_deref())?;
Ok(MaintenanceSummary {
ran_at: Utc::now(),
stats,
report_after,
})
}
fn log_stats(source: &str, stats: &RepairStats) {
if stats.is_clean() {
tracing::info!("{source} store repair: nothing to do");
} else {
tracing::info!("{source} store repair: {}", summarize(stats));
}
}
fn summarize(s: &RepairStats) -> String {
let mut parts = Vec::new();
let mut push = |name: &str, n: u64| {
if n > 0 {
parts.push(format!("{name}={n}"));
}
};
push("duplicates_superseded", s.duplicates_superseded);
push("markers_inherited", s.distillation_markers_inherited);
push("orphaned_markers_dropped", s.orphaned_markers_dropped);
push("raws_retired", s.raws_retired);
push("relations_pruned", s.relations_pruned);
push("vec_keys_deleted", s.vec_keys_deleted);
push("sync_log_rows_deleted", s.sync_log_rows_deleted);
push("stale_cursors_dropped", s.stale_cursors_dropped);
push("embeddings_backfilled", s.embeddings_backfilled);
push("embedding_failures", s.embedding_failures);
if s.vec_keys_gc_skipped {
parts.push("vec_keys_gc=skipped".to_string());
}
if s.index_rebuilt {
parts.push("index=rebuilt".to_string());
}
if s.dry_run {
parts.push("dry_run".to_string());
}
parts.join(" ")
}
pub struct DrainLoop {
pub store: Arc<SqliteStore>,
pub reader: Arc<RwLock<Indexes>>,
pub writer: Indexes,
pub embedder: Option<Arc<Embedder>>,
pub summary: Arc<RwLock<Option<MaintenanceSummary>>>,
pub near_duplicates: Arc<RwLock<Option<NearDupCount>>>,
pub near_duplicate_threshold: f32,
pub config: MaintenanceConfig,
pub requests: Receiver<RepairRequest>,
}
impl DrainLoop {
pub fn spawn(self) {
std::thread::Builder::new()
.name("index-drain".to_string())
.spawn(move || self.run())
.expect("spawning the index drain thread");
}
fn run(mut self) {
let mut last_repair: Option<Instant> = None;
let mut chunks = ChunkThrottle::new(Instant::now());
loop {
match self.requests.recv_timeout(DRAIN_INTERVAL) {
Ok(request) => {
let result = self.repair("requested", request.dry_run);
if result.is_ok() && !request.dry_run {
last_repair = Some(Instant::now());
}
let _ = request.reply.send(result);
}
Err(RecvTimeoutError::Timeout) | Err(RecvTimeoutError::Disconnected) => {
let applied = self.drain();
chunks.after_drain(Instant::now(), applied.unwrap_or(0));
let due = last_repair.is_none_or(|t| {
t.elapsed() >= Duration::from_secs(self.config.interval_secs)
});
if applied == Some(0) && self.config.enabled && due {
if self.repair("scheduled", false).is_ok() {
last_repair = Some(Instant::now());
} else {
last_repair = Some(
Instant::now()
- Duration::from_secs(
self.config.interval_secs.saturating_sub(600),
),
);
}
}
if applied == Some(0) && chunks.due(Instant::now()) {
let progressed = self.backfill_chunks();
chunks.after_backfill(Instant::now(), progressed);
}
}
}
}
}
fn backfill_chunks(&self) -> bool {
let Some(embedder) = self.embedder.as_deref() else {
return false;
};
let embed = |texts: &[&str]| embedder.embed_batch(texts);
match self.store.backfill_chunk_embeddings(
&embed,
CHUNK_MEMORIES_PER_TICK,
CHUNK_TICK_BUDGET,
) {
Ok(r) => {
if r.memories > 0 || r.failed > 0 {
tracing::info!(
"chunk backfill: memories={} chunks={} failed={} pending={}",
r.memories,
r.chunks,
r.failed,
r.pending
);
}
r.memories > 0
}
Err(e) => {
tracing::warn!("chunk backfill failed: {e:#}");
false
}
}
}
fn drain(&mut self) -> Option<usize> {
match self.writer.drain(&self.store) {
Ok(0) => Some(0),
Ok(n) => {
tracing::info!("index drain applied {n} outbox rows");
self.reload_reader();
Some(n)
}
Err(e) => {
tracing::warn!("index drain failed: {e:#}");
None
}
}
}
fn repair(&mut self, source: &str, dry_run: bool) -> Result<MaintenanceSummary> {
let started = Instant::now();
let result = run_repair(
&self.store,
Some(&mut self.writer),
self.embedder.as_deref(),
dry_run,
);
match &result {
Ok(summary) => {
log_stats(source, &summary.stats);
tracing::info!(
"{source} store repair finished in {:.1}s; drift remaining: {:?}",
started.elapsed().as_secs_f64(),
summary.report_after.drift()
);
if !dry_run {
self.reload_reader();
let mut guard = match self.summary.write() {
Ok(g) => g,
Err(poisoned) => poisoned.into_inner(),
};
*guard = Some(summary.clone());
drop(guard);
refresh_count(
&self.store,
&self.writer,
self.near_duplicate_threshold,
&self.near_duplicates,
);
}
}
Err(e) => tracing::warn!("{source} store repair failed: {e:#}"),
}
result
}
fn reload_reader(&self) {
let mut guard = match self.reader.write() {
Ok(g) => g,
Err(poisoned) => poisoned.into_inner(),
};
if let Err(e) = guard.reload_view() {
tracing::warn!("index reader reload failed: {e:#}");
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_fresh_throttle_scans_on_the_first_idle_tick() {
let now = Instant::now();
assert!(ChunkThrottle::new(now).due(now));
}
#[test]
fn a_scan_that_finds_nothing_waits_out_the_idle_interval() {
let now = Instant::now();
let mut t = ChunkThrottle::new(now);
t.after_backfill(now, false);
assert!(!t.due(now + CHUNK_IDLE_RESCAN - Duration::from_secs(1)));
assert!(t.due(now + CHUNK_IDLE_RESCAN));
}
#[test]
fn progress_keeps_the_next_idle_tick_scanning() {
let now = Instant::now();
let mut t = ChunkThrottle::new(now);
t.after_backfill(now, true);
assert!(t.due(now));
}
#[test]
fn an_applied_drain_resets_the_idle_wait() {
let now = Instant::now();
let mut t = ChunkThrottle::new(now);
t.after_backfill(now, false);
t.after_drain(now, 0);
assert!(!t.due(now));
t.after_drain(now, 3);
assert!(t.due(now));
}
}