second-brain-api 0.11.0

HTTP API server for second-brain: REST endpoints for recall, remember, and ingest
Documentation
use std::collections::HashMap;
use std::sync::Arc;

use axum::Json;
use axum::extract::{Query, State};
use axum::http::StatusCode;
use axum::response::{IntoResponse, Response};
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use uuid::Uuid;

use second_brain_core::index::Indexes;
use second_brain_core::near_duplicates::{MergeRefusal, ReviewVerdict, check_member_ids};
use second_brain_core::redact::redact_for_store;
use second_brain_core::sqlite_store::SqliteStore;

use crate::AppState;
use crate::routes::{internal_error, retry_write};

const DEFAULT_LIMIT: usize = 20;
const MAX_LIMIT: usize = 200;

fn refuse(status: StatusCode, message: impl Into<String>) -> Response {
    (status, Json(serde_json::json!({"error": message.into()}))).into_response()
}

fn refusal(r: MergeRefusal) -> Response {
    match r {
        MergeRefusal::Stale(id) => refuse(
            StatusCode::CONFLICT,
            format!("{id} is no longer a live distillate"),
        ),
        MergeRefusal::Invalid(message) => refuse(StatusCode::BAD_REQUEST, message),
    }
}

fn parse_ids(raw: &[String]) -> Result<Vec<Uuid>, MergeRefusal> {
    let ids = raw
        .iter()
        .map(|s| Uuid::parse_str(s).map_err(|_| MergeRefusal::Invalid(format!("invalid id {s:?}"))))
        .collect::<Result<Vec<_>, _>>()?;
    check_member_ids(&ids)?;
    Ok(ids)
}

fn read_indexes(indexes: &std::sync::RwLock<Indexes>) -> std::sync::RwLockReadGuard<'_, Indexes> {
    match indexes.read() {
        Ok(g) => g,
        Err(poisoned) => poisoned.into_inner(),
    }
}

#[derive(Debug, Clone, Serialize)]
pub struct NearDupCount {
    pub groups: u64,
    pub threshold: f32,
    pub counted_at: DateTime<Utc>,
}

pub fn record_count(
    cache: &std::sync::RwLock<Option<NearDupCount>>,
    groups: usize,
    threshold: f32,
) {
    let mut guard = match cache.write() {
        Ok(g) => g,
        Err(poisoned) => poisoned.into_inner(),
    };
    *guard = Some(NearDupCount {
        groups: groups as u64,
        threshold,
        counted_at: Utc::now(),
    });
}

pub fn cached_count(cache: &std::sync::RwLock<Option<NearDupCount>>) -> Option<NearDupCount> {
    match cache.read() {
        Ok(g) => g.clone(),
        Err(poisoned) => poisoned.into_inner().clone(),
    }
}

/// Recounts at `threshold` on an index handle the caller already holds; a
/// failure only leaves the previous count in place.
pub fn refresh_count(
    sqlite: &SqliteStore,
    indexes: &Indexes,
    threshold: f32,
    cache: &std::sync::RwLock<Option<NearDupCount>>,
) {
    match sqlite.near_duplicate_group_count(indexes, threshold) {
        Ok(n) => record_count(cache, n, threshold),
        Err(e) => tracing::warn!("near-duplicate count failed: {e:#}"),
    }
}

pub async fn list(
    State(state): State<Arc<AppState>>,
    Query(params): Query<HashMap<String, String>>,
) -> Response {
    let limit = match params.get("limit").map(|s| s.parse::<usize>()) {
        None => DEFAULT_LIMIT,
        Some(Ok(n)) if (1..=MAX_LIMIT).contains(&n) => n,
        Some(_) => {
            return refuse(
                StatusCode::BAD_REQUEST,
                format!("limit must be between 1 and {MAX_LIMIT}"),
            );
        }
    };
    let threshold = match params.get("threshold").map(|s| s.parse::<f32>()) {
        None => state.near_duplicate_threshold,
        Some(Ok(t)) if t > 0.0 && t <= 1.0 => t,
        Some(_) => {
            return refuse(
                StatusCode::BAD_REQUEST,
                "threshold must be above 0 and at most 1",
            );
        }
    };
    let Some(indexes) = state.indexes.clone() else {
        return refuse(
            StatusCode::SERVICE_UNAVAILABLE,
            "the vector index is unavailable",
        );
    };
    let sqlite = state.sqlite.clone();
    let result = tokio::task::spawn_blocking(move || {
        let guard = read_indexes(&indexes);
        sqlite.near_duplicate_groups(&guard, threshold, limit)
    })
    .await;
    match result {
        Ok(Ok(listing)) => {
            if threshold == state.near_duplicate_threshold {
                record_count(&state.near_duplicates, listing.total, threshold);
            }
            Json(serde_json::json!({
                "threshold": threshold,
                "groups": listing.groups,
            }))
            .into_response()
        }
        Ok(Err(e)) => internal_error(e),
        Err(e) => internal_error(e),
    }
}

#[derive(Deserialize)]
pub struct MergeRequest {
    member_ids: Vec<String>,
    content: String,
    model: String,
}

pub async fn merge(State(state): State<Arc<AppState>>, Json(req): Json<MergeRequest>) -> Response {
    let ids = match parse_ids(&req.member_ids) {
        Ok(ids) => ids,
        Err(r) => return refusal(r),
    };
    if req.content.trim().is_empty() {
        return refuse(StatusCode::BAD_REQUEST, "content is empty");
    }
    let sqlite = state.sqlite.clone();
    let check_ids = ids.clone();
    match tokio::task::spawn_blocking(move || sqlite.check_merge(&check_ids)).await {
        Ok(Ok(Ok(()))) => {}
        Ok(Ok(Err(r))) => return refusal(r),
        Ok(Err(e)) => return internal_error(e),
        Err(e) => return internal_error(e),
    }
    let Some(embedder) = state.embedder.clone() else {
        return refuse(
            StatusCode::SERVICE_UNAVAILABLE,
            "the embedder is unavailable",
        );
    };
    let sqlite = state.sqlite.clone();
    let result = tokio::task::spawn_blocking(move || -> anyhow::Result<_> {
        // A merged distillate has no context, so its body is its content,
        // the same text consolidation embeds for a new distillate.
        let content = redact_for_store(&req.content);
        let embedding = embedder.embed(&content)?;
        retry_write(|| sqlite.merge_distillates(&ids, &content, embedding.clone(), &req.model))
    })
    .await;
    match result {
        Ok(Ok(Ok(report))) => Json(report).into_response(),
        Ok(Ok(Err(r))) => refusal(r),
        Ok(Err(e)) => internal_error(e),
        Err(e) => internal_error(e),
    }
}

fn parse_verdict(raw: &str) -> Result<ReviewVerdict, String> {
    match raw {
        "separate" => Ok(ReviewVerdict::Separate),
        "conflict" => Ok(ReviewVerdict::Conflict),
        "unmerged" => Ok(ReviewVerdict::Unmerged),
        other => Err(format!(
            "unknown verdict {other:?}; valid values: separate, conflict, unmerged"
        )),
    }
}

#[derive(Deserialize)]
pub struct ReviewRequest {
    member_ids: Vec<String>,
    verdict: String,
}

pub async fn review(
    State(state): State<Arc<AppState>>,
    Json(req): Json<ReviewRequest>,
) -> Response {
    let verdict = match parse_verdict(&req.verdict) {
        Ok(v) => v,
        Err(message) => return refuse(StatusCode::BAD_REQUEST, message),
    };
    let ids = match parse_ids(&req.member_ids) {
        Ok(ids) => ids,
        Err(r) => return refusal(r),
    };
    let sqlite = state.sqlite.clone();
    let result = tokio::task::spawn_blocking(move || {
        retry_write(|| sqlite.record_near_duplicate_review(&ids, verdict))
    })
    .await;
    match result {
        Ok(Ok(Ok(recorded))) => {
            Json(serde_json::json!({"relations_recorded": recorded})).into_response()
        }
        Ok(Ok(Err(r))) => refusal(r),
        Ok(Err(e)) => internal_error(e),
        Err(e) => internal_error(e),
    }
}

/// Takes the body as a bare value so every malformed request is a JSON 400,
/// not the extractor's plain-text 422.
pub async fn reopen(
    State(state): State<Arc<AppState>>,
    Json(body): Json<serde_json::Value>,
) -> Response {
    let Some(raw) = body.get("verdicts").and_then(|v| v.as_array()) else {
        return refuse(
            StatusCode::BAD_REQUEST,
            "verdicts must be a list of separate, conflict or unmerged",
        );
    };
    if raw.is_empty() {
        return refuse(StatusCode::BAD_REQUEST, "verdicts is empty");
    }
    let mut verdicts = Vec::with_capacity(raw.len());
    for value in raw {
        match parse_verdict(value.as_str().unwrap_or_default()) {
            Ok(v) if !verdicts.contains(&v) => verdicts.push(v),
            Ok(_) => {}
            Err(message) => return refuse(StatusCode::BAD_REQUEST, message),
        }
    }
    let sqlite = state.sqlite.clone();
    let result = tokio::task::spawn_blocking(move || {
        retry_write(|| sqlite.reopen_near_duplicate_reviews(&verdicts))
    })
    .await;
    match result {
        Ok(Ok(removed)) => Json(serde_json::json!({"reviews_removed": removed})).into_response(),
        Ok(Err(e)) => internal_error(e),
        Err(e) => internal_error(e),
    }
}