second-brain-api 0.13.5

HTTP API server for second-brain: REST endpoints for recall, remember, and ingest
Documentation
use std::collections::BTreeSet;
use std::sync::Arc;
use std::time::Duration;

use axum::Json;
use axum::extract::{Path, State};
use axum::http::StatusCode;
use axum::response::{IntoResponse, Response};
use second_brain_core::project_path;
use second_brain_core::sqlite_store::{DeleteReport, DeleteScope, session_of};
use second_brain_core::transcript_archive::TranscriptArchive;
use serde::{Deserialize, Serialize};
use uuid::Uuid;

use super::project_names::refuse_unknown_project;
use super::{bad_request, error_response, internal_error, retry_write};
use crate::AppState;
use crate::maintenance::DrainRequest;
use crate::raw_index::purge_deleted;

/// A drain plus a lexical merge of a large index takes minutes on a slow
/// disk; past this the reply says the removal is still running.
const DRAIN_WAIT: Duration = Duration::from_secs(600);

#[derive(Debug, Deserialize)]
pub struct DeleteRequest {
    memory: Option<String>,
    session: Option<String>,
    project: Option<String>,
    #[serde(default)]
    dry_run: bool,
}

#[derive(Debug, Serialize)]
struct Removed {
    memories: u64,
    distillates: u64,
    chunks: u64,
    relations: u64,
    markers: u64,
    conversations: u64,
    restored: u64,
}

/// `raw_index` and `sync_log` are null in a dry run, which counts neither.
#[derive(Debug, Serialize)]
struct Layers {
    vector: u64,
    chunk_vectors: u64,
    bm25: u64,
    raw_index: Option<u64>,
    cold_records: u64,
    archive: u64,
    sync_log: Option<u64>,
}

#[derive(Debug, Serialize)]
struct DeleteResponse {
    scope: &'static str,
    target: String,
    dry_run: bool,
    removed: Removed,
    layers: Layers,
    drained: bool,
    warnings: Vec<String>,
    /// Null in a dry run; `GET /v1/delete/{id}` shows which peers hold it.
    delete_id: Option<Uuid>,
    seq: Option<u64>,
}

enum Outcome {
    Done(DeleteResponse),
    NothingMatched(String),
}

/// Deletes a memory, a session or a project everywhere: the store rows
/// (tombstoned, so the delete reaches every peer and wins there), the hot
/// indexes after a drain and merge, the raw index, the archived transcripts
/// and the cold segments. Replies once every layer is done, with what each
/// one removed.
pub async fn delete_scope(
    State(state): State<Arc<AppState>>,
    Json(req): Json<DeleteRequest>,
) -> Response {
    let (scope, name, target) = match parse(&req) {
        Ok(parsed) => parsed,
        Err(message) => return bad_request(message),
    };
    let dry_run = req.dry_run;
    let outcome = tokio::task::spawn_blocking(move || {
        run(&state, &scope, dry_run).map(|outcome| match outcome {
            Outcome::Done(body) => Outcome::Done(DeleteResponse {
                scope: name,
                target,
                ..body
            }),
            nothing => nothing,
        })
    })
    .await;
    match outcome {
        Ok(Ok(Outcome::Done(body))) => Json(body).into_response(),
        Ok(Ok(Outcome::NothingMatched(message))) => error_response(StatusCode::NOT_FOUND, message),
        Ok(Err(e)) => internal_error(e),
        Err(e) => internal_error(e),
    }
}

/// The stored spelling of a delete id or target, as `parse` stores it.
fn status_key(key: &str) -> String {
    let key = key.trim();
    if let Ok(id) = Uuid::parse_str(key) {
        return id.hyphenated().to_string();
    }
    project_slug(key).unwrap_or_else(|_| key.to_string())
}

/// A project directory as its stored slug; anything else is a slug already.
fn project_slug(value: &str) -> Result<String, String> {
    if !value.starts_with('/') {
        return Ok(value.to_string());
    }
    let path = value.trim_end_matches('/');
    if path.is_empty() {
        return Err(format!("project {value} is not a project directory"));
    }
    Ok(project_path::encode(path))
}

fn parse(req: &DeleteRequest) -> Result<(DeleteScope, &'static str, String), String> {
    let given: Vec<(&'static str, &str)> = [
        ("memory", req.memory.as_deref()),
        ("session", req.session.as_deref()),
        ("project", req.project.as_deref()),
    ]
    .into_iter()
    .filter_map(|(name, value)| value.map(|v| (name, v.trim())))
    .collect();
    let [(name, value)] = given.as_slice() else {
        return Err("pass exactly one of memory, session or project".to_string());
    };
    if value.is_empty() {
        return Err(format!("{name} is empty"));
    }
    let scope = match *name {
        "memory" => DeleteScope::Memory(
            Uuid::parse_str(value).map_err(|_| format!("memory {value} is not a memory id"))?,
        ),
        "session" => DeleteScope::Session(
            Uuid::parse_str(value)
                .map_err(|_| format!("session {value} is not a Claude Code session id"))?
                .hyphenated()
                .to_string(),
        ),
        _ => DeleteScope::Project(project_slug(value)?),
    };
    Ok((scope, name, value.to_string()))
}

fn run(state: &AppState, scope: &DeleteScope, dry_run: bool) -> anyhow::Result<Outcome> {
    let archived = archived_sessions(state.archive.as_deref(), scope);
    let report = if dry_run {
        state.sqlite.preview_delete(scope, &archived)?
    } else {
        retry_write(|| state.sqlite.delete_scope(scope, &archived))?
    };
    if report.is_empty() {
        return Ok(Outcome::NothingMatched(nothing_matched(state, scope)?));
    }
    if dry_run {
        let archive = archive_entries_of(state.archive.as_deref(), &report.sessions);
        return Ok(Outcome::Done(response(
            &report,
            true,
            Done {
                raw_index: None,
                archive,
                drained: false,
                warnings: Vec::new(),
            },
        )));
    }

    let mut warnings = Vec::new();
    let drained = drain(state, &mut warnings);
    if drained {
        let mut left = 0;
        for id in &report.memory_ids {
            left += usize::from(state.sqlite.existing_vec_key(*id)?.is_some());
        }
        if left > 0 {
            warnings.push(format!(
                "{left} deleted memories still hold a vector key after the drain"
            ));
        }
    }
    if state.raw_index.is_none() {
        warnings.push(
            "the raw transcript index is not open here, so its passages were not checked"
                .to_string(),
        );
    }
    let (raw_index, archive) = match purge_deleted(
        state.raw_index.as_deref(),
        state.archive.as_deref(),
        &state.sqlite,
        &report.memory_ids,
    ) {
        Ok(purge) => (
            state.raw_index.as_ref().map(|_| purge.passages as u64),
            purge.archive_entries as u64,
        ),
        Err(e) => {
            warnings.push(format!(
                "the raw index and transcript archive were not purged; the raw index retries at its next pass: {e:#}"
            ));
            (None, 0)
        }
    };
    if report.counts.cold_records > 0 {
        match state.sqlite.purge_cold_segments() {
            Ok(purge) if purge.retired + purge.rewritten == 0 => warnings.push(
                "no cold segment could be rewritten without the deleted records; the cold pass retries"
                    .to_string(),
            ),
            Ok(_) => {}
            Err(e) => warnings.push(format!(
                "the cold segments were not purged; the cold pass retries: {e:#}"
            )),
        }
    }
    Ok(Outcome::Done(response(
        &report,
        false,
        Done {
            raw_index,
            archive,
            drained,
            warnings,
        },
    )))
}

/// Sessions with a transcript in the archive under the scope, so a session
/// the store holds nothing of is still deleted.
fn archived_sessions(archive: Option<&TranscriptArchive>, scope: &DeleteScope) -> Vec<String> {
    let Some(archive) = archive else {
        return Vec::new();
    };
    let mut sessions = BTreeSet::new();
    for entry in archive.entries() {
        let Some(session) = session_of(&entry.source) else {
            continue;
        };
        let wanted = match scope {
            DeleteScope::Memory(_) => false,
            DeleteScope::Session(key) => &session == key,
            DeleteScope::Project(path) => {
                project_path::from_transcript(&entry.source).as_deref() == Some(path.as_str())
            }
        };
        if wanted {
            sessions.insert(session);
        }
    }
    sessions.into_iter().collect()
}

fn archive_entries_of(archive: Option<&TranscriptArchive>, sessions: &[String]) -> u64 {
    let Some(archive) = archive else {
        return 0;
    };
    archive
        .entries()
        .iter()
        .filter(|e| session_of(&e.source).is_some_and(|s| sessions.contains(&s)))
        .count() as u64
}

fn nothing_matched(state: &AppState, scope: &DeleteScope) -> anyhow::Result<String> {
    Ok(match scope {
        DeleteScope::Memory(id) => format!("memory {id} not found"),
        DeleteScope::Session(key) => format!("nothing is stored or archived from session {key}"),
        DeleteScope::Project(path) => refuse_unknown_project(&state.sqlite, path)?
            .unwrap_or_else(|| format!("nothing is stored or archived under project {path}")),
    })
}

/// Hands the drain thread a drain-and-merge and waits for it, so the hot
/// indexes hold neither the deleted entries nor their text when this
/// returns true.
fn drain(state: &AppState, warnings: &mut Vec<String>) -> bool {
    let Some(tx) = &state.repair_tx else {
        warnings.push(
            "the search indexes are not open here; the deleted entries leave them at their next drain"
                .to_string(),
        );
        return false;
    };
    let (reply, done) = std::sync::mpsc::channel();
    if tx.send(DrainRequest::CompactIndexes(reply)).is_err() {
        warnings.push("the index drain thread is not running".to_string());
        return false;
    }
    match done.recv_timeout(DRAIN_WAIT) {
        Ok(Ok(())) => true,
        Ok(Err(e)) => {
            warnings.push(format!("the index drain failed and will retry: {e:#}"));
            false
        }
        Err(_) => {
            warnings.push(
                "the index drain is still running; the entries leave when it finishes".to_string(),
            );
            false
        }
    }
}

struct Done {
    raw_index: Option<u64>,
    archive: u64,
    drained: bool,
    warnings: Vec<String>,
}

fn response(report: &DeleteReport, dry_run: bool, done: Done) -> DeleteResponse {
    let c = &report.counts;
    DeleteResponse {
        scope: "",
        target: String::new(),
        dry_run,
        removed: Removed {
            memories: c.memories,
            distillates: c.distillates,
            chunks: c.chunks,
            relations: c.relations,
            markers: c.markers,
            conversations: c.conversations,
            restored: c.restored,
        },
        layers: Layers {
            vector: c.vector_keys,
            chunk_vectors: c.chunk_vector_keys,
            bm25: c.bm25,
            raw_index: done.raw_index,
            cold_records: c.cold_records,
            archive: done.archive,
            sync_log: (!dry_run).then_some(c.log_records),
        },
        drained: done.drained,
        warnings: done.warnings,
        delete_id: report.delete_id,
        seq: report.seq,
    }
}

/// A delete run on this machine, found by its id or its target, with each
/// floor neighbour marked acked once it acknowledged this store's log past
/// the delete's tombstones.
pub async fn delete_status(
    State(state): State<Arc<AppState>>,
    Path(key): Path<String>,
) -> Response {
    let lookup = status_key(&key);
    match tokio::task::spawn_blocking(move || state.sqlite.delete_status(&lookup)).await {
        Ok(Ok(Some(status))) => Json(status).into_response(),
        Ok(Ok(None)) => error_response(
            StatusCode::NOT_FOUND,
            format!("no delete of {key} was run on this machine"),
        ),
        Ok(Err(e)) => internal_error(e),
        Err(e) => internal_error(e),
    }
}