second-brain-api 0.13.1

HTTP API server for second-brain: REST endpoints for recall, remember, and ingest
Documentation
use std::sync::Arc;
use std::time::Duration;

use axum::Json;
use axum::extract::State;
use axum::response::IntoResponse;

use second_brain_core::embedding::Embedder;
use second_brain_core::residency;
use second_brain_core::sqlite_store::{RedactionRun, SqliteStore};

use crate::AppState;
use crate::routes::internal_error;
use crate::store_compaction::Compaction;

const FIRST_RETRY: Duration = Duration::from_secs(30);
const MAX_RETRY: Duration = Duration::from_secs(3600);

/// Runs the one-time pass on its own thread, retrying a failed run with
/// capped backoff, then the compaction that clears what the pass left on
/// disk. The store records progress of both, so a restart resumes them.
pub fn spawn_pass(
    store: Arc<SqliteStore>,
    embedder: Option<Arc<Embedder>>,
    compaction: Compaction,
) -> std::io::Result<std::thread::JoinHandle<()>> {
    std::thread::Builder::new()
        .name("redaction-pass".to_string())
        .spawn(move || {
            let embed = embedder
                .as_ref()
                .map(|e| move |texts: &[&str]| e.embed_batch(texts));
            let embed_ref = embed
                .as_ref()
                .map(|f| f as &dyn Fn(&[&str]) -> anyhow::Result<Vec<Vec<f32>>>);
            let run = run_until_done(|| store.run_stored_redaction(embed_ref), std::thread::sleep);
            tracing::info!(?run, "stored-content redaction pass finished");
            compaction.run_until_done(std::thread::sleep);
            residency::release_free_heap();
        })
}

fn run_until_done(
    mut run: impl FnMut() -> anyhow::Result<RedactionRun>,
    mut sleep: impl FnMut(Duration),
) -> RedactionRun {
    let mut delay = FIRST_RETRY;
    loop {
        match run() {
            Ok(done) => return done,
            Err(e) => {
                tracing::error!(
                    "stored-content redaction pass failed, retrying in {}s: {e:#}",
                    delay.as_secs()
                );
                sleep(delay);
                delay = (delay * 2).min(MAX_RETRY);
            }
        }
    }
}

pub async fn report(State(state): State<Arc<AppState>>) -> impl IntoResponse {
    let sqlite = state.sqlite.clone();
    match tokio::task::spawn_blocking(move || sqlite.stored_redaction_report()).await {
        Ok(Ok(report)) => Json(report).into_response(),
        Ok(Err(e)) => internal_error(format!("{e:#}")),
        Err(e) => internal_error(e),
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn a_failed_run_is_retried_with_capped_backoff() {
        let mut attempts = 0;
        let mut slept = Vec::new();
        let run = run_until_done(
            || {
                attempts += 1;
                if attempts < 9 {
                    anyhow::bail!("database is locked");
                }
                Ok(RedactionRun {
                    memories_rewritten: 1,
                    ..RedactionRun::default()
                })
            },
            |d| slept.push(d.as_secs()),
        );
        assert_eq!(run.memories_rewritten, 1);
        assert_eq!(slept, [30, 60, 120, 240, 480, 960, 1920, 3600]);
    }
}