second-brain-api 0.13.4

HTTP API server for second-brain: REST endpoints for recall, remember, and ingest
Documentation
//! The one-time compaction after the stored-content redaction pass. The pass
//! rewrites rows, but SQLite keeps their old bytes in free and split pages
//! and tantivy keeps deleted terms until a segment merges away; a VACUUM and
//! a forced merge are what take that plaintext off the disk.

use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc::Sender;
use std::time::Duration;

use anyhow::{Result, anyhow};
use chrono::Utc;
use second_brain_core::sqlite_store::encrypt_migration::NotReopened;
use second_brain_core::sqlite_store::{SqliteStore, VacuumSpace};

use crate::health::{Health, MaintenanceState};
use crate::maintenance::DrainRequest;
use crate::write_gate::WriteGate;

pub const SPACE_RETRY: Duration = Duration::from_secs(30 * 60);
/// How long the compaction waits for a pending encryption migration, across
/// restarts, before it vacuums the plaintext store itself; past a failed
/// migration's daily retries.
const AWAIT_ENCRYPTION_MICROS: i64 = 3 * 24 * 3600 * 1_000_000;
const FIRST_RETRY: Duration = Duration::from_secs(30);
const MAX_RETRY: Duration = Duration::from_secs(3600);

pub struct Compaction {
    pub store: Arc<SqliteStore>,
    pub gate: Arc<WriteGate>,
    /// The drain thread, which owns the writable indexes; None when the
    /// indexes never opened.
    pub indexes: Option<Sender<DrainRequest>>,
    pub health: Arc<Health>,
    pub space: fn(&SqliteStore) -> Result<VacuumSpace>,
    /// The store waits for the one-time encryption migration, whose rewrite
    /// copies only live pages and so does this compaction's VACUUM; the
    /// compaction waits for it instead of rewriting the store twice.
    pub awaits_encryption: bool,
    /// Set once the store was seen plaintext after the redaction pass, so a
    /// migration seen afterwards ran on the redacted rows.
    saw_plaintext: AtomicBool,
}

#[derive(Debug, PartialEq, Eq)]
enum Step {
    Done,
    NoSpace,
    AwaitingEncryption,
    /// Vacuumed, but with no drain thread the lexical index could not be
    /// merged, so the compaction stays unrecorded and runs at the next start.
    IndexesUnavailable,
}

impl Compaction {
    pub fn new(
        store: Arc<SqliteStore>,
        gate: Arc<WriteGate>,
        indexes: Option<Sender<DrainRequest>>,
        health: Arc<Health>,
    ) -> Self {
        Self {
            store,
            gate,
            indexes,
            health,
            space: SqliteStore::vacuum_space,
            awaits_encryption: false,
            saw_plaintext: AtomicBool::new(false),
        }
    }

    /// Returns once the store records the compaction as done, which may be
    /// at once; a disk without room is checked again every `SPACE_RETRY`
    /// and a failure is retried with capped backoff.
    pub fn run_until_done(&self, mut sleep: impl FnMut(Duration)) {
        let mut delay = FIRST_RETRY;
        loop {
            match self.attempt() {
                Ok(Step::Done | Step::IndexesUnavailable) => return,
                Ok(Step::NoSpace | Step::AwaitingEncryption) => sleep(SPACE_RETRY),
                Err(e) => {
                    tracing::error!(
                        "store compaction failed, retrying in {}s: {e:#}",
                        delay.as_secs()
                    );
                    let now = Utc::now();
                    self.health
                        .maintenance(MaintenanceState::Pending, None, now);
                    self.health.maintenance_failed(&format!("{e:#}"), now);
                    sleep(delay);
                    delay = (delay * 2).min(MAX_RETRY);
                }
            }
        }
    }

    fn attempt(&self) -> Result<Step> {
        if self.store.stored_redaction_compacted()? {
            self.health
                .maintenance(MaintenanceState::Done, None, Utc::now());
            return Ok(Step::Done);
        }
        if self.awaits_encryption {
            let encrypted = self.store.is_encrypted();
            if !encrypted {
                let now = Utc::now().timestamp_micros();
                let since = self.store.encryption_awaited_since(now)?;
                if now - since < AWAIT_ENCRYPTION_MICROS {
                    self.saw_plaintext.store(true, Ordering::SeqCst);
                    return Ok(Step::AwaitingEncryption);
                }
            }
            if encrypted && self.saw_plaintext.load(Ordering::SeqCst) {
                return self.merge_indexes_after_migration();
            }
        }
        let space = (self.space)(&self.store)?;
        if !space.fits() {
            tracing::warn!(
                "store compaction waits for disk space: needs {} bytes free at {}, has {}",
                space.needed_bytes,
                space.path.display(),
                space.free_bytes
            );
            self.health
                .maintenance(MaintenanceState::WaitingForSpace, Some(space), Utc::now());
            return Ok(Step::NoSpace);
        }
        self.merge_indexes(Some(space))?;
        let (before, after) = vacuum(&self.store, &self.gate, &self.health)?;
        let step = self.record()?;
        if step == Step::Done {
            tracing::info!("store compacted after redaction: {before} bytes before, {after} after");
        }
        Ok(step)
    }

    fn merge_indexes_after_migration(&self) -> Result<Step> {
        self.merge_indexes(None)?;
        let step = self.record()?;
        if step == Step::Done {
            tracing::info!("store compacted after redaction by the encryption migration");
        }
        Ok(step)
    }

    fn merge_indexes(&self, space: Option<VacuumSpace>) -> Result<()> {
        let Some(drain) = &self.indexes else {
            return Ok(());
        };
        self.health
            .maintenance(MaintenanceState::MergingIndex, space, Utc::now());
        let (reply, done) = std::sync::mpsc::channel();
        drain
            .send(DrainRequest::CompactIndexes(reply))
            .map_err(|_| anyhow!("the index drain thread is not running"))?;
        done.recv()
            .map_err(|_| anyhow!("the index drain thread stopped"))?
    }

    fn record(&self) -> Result<Step> {
        if self.indexes.is_none() {
            self.health.maintenance_failed(
                "the search indexes are unavailable, so their old segments were not merged; \
                 compaction runs again at the next start",
                Utc::now(),
            );
            return Ok(Step::IndexesUnavailable);
        }
        self.store.mark_stored_redaction_compacted()?;
        self.health
            .maintenance(MaintenanceState::Done, None, Utc::now());
        Ok(Step::Done)
    }
}

/// Runs a VACUUM with agent writes held at the gate, reporting it to health
/// for its length.
pub fn vacuum(store: &SqliteStore, gate: &WriteGate, health: &Health) -> Result<(u64, u64)> {
    let _closed = gate.close();
    let previous = health.report().maintenance.state;
    health.maintenance(MaintenanceState::Vacuuming, None, Utc::now());
    let result = store.vacuum_clearing_wal();
    health.maintenance(previous, None, Utc::now());
    if let Err(e) = &result
        && e.downcast_ref::<NotReopened>().is_some()
    {
        tracing::error!("{e:#}");
        std::process::exit(1);
    }
    result
}

#[cfg(test)]
mod tests {
    use second_brain_core::keys::{KeyGuard, StoreKey};
    use second_brain_core::sqlite_store::encrypt_migration::RewriteTo;

    use super::*;

    /// Answers index compaction requests the way the drain thread does when
    /// the merge succeeds.
    fn drain_stub() -> Sender<DrainRequest> {
        let (tx, requests) = std::sync::mpsc::channel();
        std::thread::spawn(move || {
            for request in requests {
                if let DrainRequest::CompactIndexes(reply) = request {
                    let _ = reply.send(Ok(()));
                }
            }
        });
        tx
    }

    fn compaction(dir: &std::path::Path) -> Compaction {
        let store = SqliteStore::open(&dir.join("brain.db"), "m".to_string()).unwrap();
        Compaction::new(
            Arc::new(store),
            Arc::new(WriteGate::default()),
            Some(drain_stub()),
            Arc::new(Health::new(Utc::now(), None)),
        )
    }

    static SPACE_CHECKS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);

    fn short_of_space_once(store: &SqliteStore) -> Result<VacuumSpace> {
        if SPACE_CHECKS.fetch_add(1, std::sync::atomic::Ordering::SeqCst) > 0 {
            return store.vacuum_space();
        }
        Ok(VacuumSpace {
            path: std::path::PathBuf::from("/data"),
            needed_bytes: 2_000_000,
            free_bytes: 1_000_000,
        })
    }

    #[test]
    fn a_compaction_runs_once_and_is_recorded() {
        let dir = tempfile::tempdir().unwrap();
        let c = compaction(dir.path());
        c.run_until_done(|_| panic!("nothing to wait for"));
        assert!(c.store.stored_redaction_compacted().unwrap());
        assert_eq!(c.health.report().maintenance.state, MaintenanceState::Done);
        assert!(!c.gate.is_closed());
    }

    #[test]
    fn a_store_awaiting_encryption_waits_for_the_migration_and_skips_its_own_vacuum() {
        use std::os::unix::fs::MetadataExt;

        let dir = tempfile::tempdir().unwrap();
        let db = dir.path().join("brain.db");
        let mut c = compaction(dir.path());
        c.awaits_encryption = true;
        assert_eq!(c.attempt().unwrap(), Step::AwaitingEncryption);
        assert!(!c.store.stored_redaction_compacted().unwrap());

        let key = StoreKey::from_bytes(KeyGuard::new([4; 32]));
        c.store.rewrite_under_key(RewriteTo::Key(&key)).unwrap();
        let migrated = std::fs::metadata(&db).unwrap().ino();
        c.run_until_done(|_| panic!("the migration has run"));
        assert!(c.store.stored_redaction_compacted().unwrap());
        assert_eq!(
            std::fs::metadata(&db).unwrap().ino(),
            migrated,
            "the store was not rewritten a second time"
        );
    }

    #[test]
    fn a_migration_that_never_lands_gives_way_to_the_vacuum_across_restarts() {
        let dir = tempfile::tempdir().unwrap();
        let c = compaction(dir.path());
        let started = Utc::now().timestamp_micros() - AWAIT_ENCRYPTION_MICROS - 1;
        c.store.encryption_awaited_since(started).unwrap();
        drop(c);

        let mut restarted = compaction(dir.path());
        restarted.awaits_encryption = true;
        restarted.run_until_done(|_| panic!("the wait began over three days ago"));
        assert!(restarted.store.stored_redaction_compacted().unwrap());
        assert!(!restarted.store.is_encrypted());
    }

    #[test]
    fn a_store_encrypted_before_the_compaction_looked_is_still_vacuumed() {
        use std::os::unix::fs::MetadataExt;

        let dir = tempfile::tempdir().unwrap();
        let db = dir.path().join("brain.db");
        let mut c = compaction(dir.path());
        c.awaits_encryption = true;
        let key = StoreKey::from_bytes(KeyGuard::new([5; 32]));
        c.store.rewrite_under_key(RewriteTo::Key(&key)).unwrap();
        let migrated = std::fs::metadata(&db).unwrap().ino();
        c.run_until_done(|_| panic!("nothing to wait for"));
        assert!(c.store.stored_redaction_compacted().unwrap());
        assert_ne!(std::fs::metadata(&db).unwrap().ino(), migrated);
    }

    #[test]
    fn without_indexes_it_vacuums_but_leaves_the_compaction_unrecorded() {
        let dir = tempfile::tempdir().unwrap();
        let mut c = compaction(dir.path());
        c.indexes = None;
        c.run_until_done(|_| panic!("retrying cannot bring the indexes back"));
        assert!(!c.store.stored_redaction_compacted().unwrap());
        let report = c.health.report();
        assert!(report.maintenance.last_error.is_some());
        assert!(!c.gate.is_closed());
    }

    #[test]
    fn a_failed_attempt_waits_for_its_retry_as_pending_not_merging() {
        let dir = tempfile::tempdir().unwrap();
        let mut c = compaction(dir.path());
        let (tx, requests) = std::sync::mpsc::channel();
        std::thread::spawn(move || {
            for (n, request) in requests.into_iter().enumerate() {
                if let DrainRequest::CompactIndexes(reply) = request {
                    let outcome = if n == 0 {
                        Err(anyhow!("merge failed"))
                    } else {
                        Ok(())
                    };
                    let _ = reply.send(outcome);
                }
            }
        });
        c.indexes = Some(tx);
        let health = c.health.clone();
        let mut states = Vec::new();
        c.run_until_done(|_| states.push(health.report().maintenance.state));
        assert_eq!(states, [MaintenanceState::Pending]);
        assert_eq!(c.health.report().maintenance.state, MaintenanceState::Done);
    }

    #[test]
    fn without_room_it_reports_and_waits_instead_of_failing() {
        let dir = tempfile::tempdir().unwrap();
        let mut c = compaction(dir.path());
        c.space = short_of_space_once;
        let health = c.health.clone();
        let store = c.store.clone();
        let mut waits = Vec::new();
        c.run_until_done(|d| {
            let report = health.report();
            assert_eq!(report.maintenance.state, MaintenanceState::WaitingForSpace);
            assert!(
                report
                    .reason
                    .as_deref()
                    .is_some_and(|r| r.contains("waits for disk space")),
                "{report:?}"
            );
            assert!(!store.stored_redaction_compacted().unwrap());
            waits.push(d);
        });
        assert_eq!(waits, [SPACE_RETRY]);
        assert!(c.store.stored_redaction_compacted().unwrap());
        assert_eq!(c.health.report().maintenance.state, MaintenanceState::Done);
        assert!(c.health.report().reason.is_none());
    }
}