second-brain-api 0.11.0

HTTP API server for second-brain: REST endpoints for recall, remember, and ingest
Documentation
//! The drain thread: applies the index outbox every few seconds and, when
//! the outbox is quiet and the interval has passed, repairs the store with
//! the writable indexes it owns. On-demand repairs arrive over a channel so
//! the same thread services them; two writers never exist.

use std::sync::mpsc::{Receiver, RecvTimeoutError, Sender};
use std::sync::{Arc, RwLock};
use std::time::{Duration, Instant};

use anyhow::Result;
use chrono::Utc;
use second_brain_core::config::MaintenanceConfig;
use second_brain_core::embedding::Embedder;
use second_brain_core::index::Indexes;
use second_brain_core::maintenance::{MaintenanceSummary, RepairStats};
use second_brain_core::sqlite_store::SqliteStore;

use crate::health::Health;
use crate::near_duplicates::{NearDupCount, refresh_count};

const DRAIN_INTERVAL: Duration = Duration::from_secs(5);

/// Finding chunk work splits every live row, so once a scan comes back
/// empty-handed the next one waits this long unless a drain applies writes.
const CHUNK_IDLE_RESCAN: Duration = Duration::from_secs(600);
/// Per idle tick, so the drain and repair behind it are never held up long.
const CHUNK_TICK_BUDGET: Duration = Duration::from_secs(3);
/// Bounds what one tick holds in memory: at most 16 chunks per memory.
const CHUNK_MEMORIES_PER_TICK: usize = 32;

/// When the drain thread may next look for memories needing chunks.
struct ChunkThrottle {
    next_scan: Instant,
}

impl ChunkThrottle {
    fn new(now: Instant) -> Self {
        Self { next_scan: now }
    }

    fn due(&self, now: Instant) -> bool {
        now >= self.next_scan
    }

    fn after_backfill(&mut self, now: Instant, progressed: bool) {
        self.next_scan = if progressed {
            now
        } else {
            now + CHUNK_IDLE_RESCAN
        };
    }

    /// Applied outbox rows may carry content changes that need re-chunking.
    fn after_drain(&mut self, now: Instant, applied: usize) {
        if applied > 0 {
            self.next_scan = now;
        }
    }
}

pub struct RepairRequest {
    pub dry_run: bool,
    pub reply: Sender<Result<MaintenanceSummary>>,
}

/// Work only the drain thread may do, because it owns the writable indexes.
pub enum DrainRequest {
    Repair(RepairRequest),
    /// Applies the whole outbox, then rewrites the lexical index into fresh
    /// segments (see `Indexes::compact`).
    CompactIndexes(Sender<Result<()>>),
}

/// Runs a repair and re-measures the store afterwards, on whatever index
/// handle the caller owns (the writer on the drain thread, none when the
/// indexes never opened).
pub fn run_repair(
    store: &SqliteStore,
    mut indexes: Option<&mut Indexes>,
    embedder: Option<&Embedder>,
    dry_run: bool,
) -> Result<MaintenanceSummary> {
    let embed = embedder.map(|e| move |text: &str| e.embed(text));
    let embed_ref = embed
        .as_ref()
        .map(|f| f as &dyn Fn(&str) -> Result<Vec<f32>>);
    let stats = store.repair(indexes.as_deref_mut(), embed_ref, dry_run)?;
    let report_after = store.integrity_report(indexes.as_deref())?;
    Ok(MaintenanceSummary {
        ran_at: Utc::now(),
        stats,
        report_after,
    })
}

fn log_stats(source: &str, stats: &RepairStats) {
    if stats.is_clean() {
        tracing::info!("{source} store repair: nothing to do");
    } else {
        tracing::info!("{source} store repair: {}", summarize(stats));
    }
}

fn summarize(s: &RepairStats) -> String {
    let mut parts = Vec::new();
    let mut push = |name: &str, n: u64| {
        if n > 0 {
            parts.push(format!("{name}={n}"));
        }
    };
    push("duplicates_superseded", s.duplicates_superseded);
    push("markers_inherited", s.distillation_markers_inherited);
    push("orphaned_markers_dropped", s.orphaned_markers_dropped);
    push("raws_retired", s.raws_retired);
    push("relations_pruned", s.relations_pruned);
    push("vec_keys_deleted", s.vec_keys_deleted);
    push("sync_log_rows_deleted", s.sync_log_rows_deleted);
    push("stale_cursors_dropped", s.stale_cursors_dropped);
    push("embeddings_backfilled", s.embeddings_backfilled);
    push("embedding_failures", s.embedding_failures);
    if s.vec_keys_gc_skipped {
        parts.push("vec_keys_gc=skipped".to_string());
    }
    if s.index_rebuilt {
        parts.push("index=rebuilt".to_string());
    }
    if s.dry_run {
        parts.push("dry_run".to_string());
    }
    parts.join(" ")
}

pub struct DrainLoop {
    pub store: Arc<SqliteStore>,
    pub reader: Arc<RwLock<Indexes>>,
    pub writer: Indexes,
    pub embedder: Option<Arc<Embedder>>,
    pub summary: Arc<RwLock<Option<MaintenanceSummary>>>,
    pub near_duplicates: Arc<RwLock<Option<NearDupCount>>>,
    pub near_duplicate_threshold: f32,
    pub config: MaintenanceConfig,
    pub requests: Receiver<DrainRequest>,
    pub health: Arc<Health>,
}

impl DrainLoop {
    pub fn spawn(self) {
        std::thread::Builder::new()
            .name("index-drain".to_string())
            .spawn(move || self.run())
            .expect("spawning the index drain thread");
    }

    fn run(mut self) {
        let mut last_repair: Option<Instant> = None;
        let mut chunks = ChunkThrottle::new(Instant::now());
        loop {
            match self.requests.recv_timeout(DRAIN_INTERVAL) {
                Ok(DrainRequest::Repair(request)) => {
                    let result = self.repair("requested", request.dry_run);
                    if result.is_ok() && !request.dry_run {
                        last_repair = Some(Instant::now());
                    }
                    // A caller that gave up waiting is not an error here.
                    let _ = request.reply.send(result);
                }
                Ok(DrainRequest::CompactIndexes(reply)) => {
                    let _ = reply.send(self.compact_indexes());
                }
                Err(RecvTimeoutError::Timeout) | Err(RecvTimeoutError::Disconnected) => {
                    let applied = self.drain();
                    chunks.after_drain(Instant::now(), applied.unwrap_or(0));
                    let due = last_repair.is_none_or(|t| {
                        t.elapsed() >= Duration::from_secs(self.config.interval_secs)
                    });
                    if applied == Some(0) && self.config.enabled && due {
                        if self.repair("scheduled", false).is_ok() {
                            last_repair = Some(Instant::now());
                        } else {
                            // Retry sooner than the full interval, but not
                            // every tick against a store that keeps failing.
                            last_repair = Some(
                                Instant::now()
                                    - Duration::from_secs(
                                        self.config.interval_secs.saturating_sub(600),
                                    ),
                            );
                        }
                    }
                    if applied == Some(0) && chunks.due(Instant::now()) {
                        let progressed = self.backfill_chunks();
                        chunks.after_backfill(Instant::now(), progressed);
                    }
                }
            }
        }
    }

    /// Embeds pending chunk sets within one tick's budget; true when any set
    /// was stored, so the next idle tick keeps going.
    fn backfill_chunks(&self) -> bool {
        let Some(embedder) = self.embedder.as_deref() else {
            return false;
        };
        let embed = |texts: &[&str]| embedder.embed_batch(texts);
        match self.store.backfill_chunk_embeddings(
            &embed,
            CHUNK_MEMORIES_PER_TICK,
            CHUNK_TICK_BUDGET,
        ) {
            Ok(r) => {
                if r.memories > 0 || r.failed > 0 {
                    tracing::info!(
                        "chunk backfill: memories={} chunks={} failed={} pending={}",
                        r.memories,
                        r.chunks,
                        r.failed,
                        r.pending
                    );
                }
                r.memories > 0
            }
            Err(e) => {
                tracing::warn!("chunk backfill failed: {e:#}");
                false
            }
        }
    }

    /// Applies pending outbox rows; None when the drain itself failed.
    fn drain(&mut self) -> Option<usize> {
        match self.writer.drain(&self.store) {
            Ok(0) => {
                self.health.drained(0, Utc::now());
                Some(0)
            }
            Ok(n) => {
                tracing::info!("index drain applied {n} outbox rows");
                self.reload_reader();
                match self.store.outbox_pending() {
                    Ok(pending) => self.health.drained(pending, Utc::now()),
                    Err(e) => tracing::warn!("counting the index outbox failed: {e:#}"),
                }
                Some(n)
            }
            Err(e) => {
                tracing::warn!("index drain failed: {e:#}");
                self.health.drain_failed(&format!("{e:#}"), Utc::now());
                None
            }
        }
    }

    fn repair(&mut self, source: &str, dry_run: bool) -> Result<MaintenanceSummary> {
        let started = Instant::now();
        let result = run_repair(
            &self.store,
            Some(&mut self.writer),
            self.embedder.as_deref(),
            dry_run,
        );
        match &result {
            Ok(summary) => {
                self.health.repaired(Utc::now());
                log_stats(source, &summary.stats);
                tracing::info!(
                    "{source} store repair finished in {:.1}s; drift remaining: {:?}",
                    started.elapsed().as_secs_f64(),
                    summary.report_after.drift()
                );
                if !dry_run {
                    self.reload_reader();
                    let mut guard = match self.summary.write() {
                        Ok(g) => g,
                        Err(poisoned) => poisoned.into_inner(),
                    };
                    *guard = Some(summary.clone());
                    drop(guard);
                    refresh_count(
                        &self.store,
                        &self.writer,
                        self.near_duplicate_threshold,
                        &self.near_duplicates,
                    );
                }
            }
            Err(e) => {
                tracing::warn!("{source} store repair failed: {e:#}");
                self.health.repair_failed(&format!("{e:#}"), Utc::now());
            }
        }
        result
    }

    fn compact_indexes(&mut self) -> Result<()> {
        while self.writer.drain(&self.store)? > 0 {}
        let started = Instant::now();
        self.writer.compact()?;
        self.reload_reader();
        tracing::info!(
            "lexical index compacted in {:.1}s",
            started.elapsed().as_secs_f64()
        );
        Ok(())
    }

    fn reload_reader(&self) {
        let mut guard = match self.reader.write() {
            Ok(g) => g,
            Err(poisoned) => poisoned.into_inner(),
        };
        if let Err(e) = guard.reload_view() {
            tracing::warn!("index reader reload failed: {e:#}");
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn a_fresh_throttle_scans_on_the_first_idle_tick() {
        let now = Instant::now();
        assert!(ChunkThrottle::new(now).due(now));
    }

    #[test]
    fn a_scan_that_finds_nothing_waits_out_the_idle_interval() {
        let now = Instant::now();
        let mut t = ChunkThrottle::new(now);
        t.after_backfill(now, false);
        assert!(!t.due(now + CHUNK_IDLE_RESCAN - Duration::from_secs(1)));
        assert!(t.due(now + CHUNK_IDLE_RESCAN));
    }

    #[test]
    fn progress_keeps_the_next_idle_tick_scanning() {
        let now = Instant::now();
        let mut t = ChunkThrottle::new(now);
        t.after_backfill(now, true);
        assert!(t.due(now));
    }

    #[test]
    fn an_applied_drain_resets_the_idle_wait() {
        let now = Instant::now();
        let mut t = ChunkThrottle::new(now);
        t.after_backfill(now, false);
        t.after_drain(now, 0);
        assert!(!t.due(now));
        t.after_drain(now, 3);
        assert!(t.due(now));
    }
}