second-brain-api 0.13.5

HTTP API server for second-brain: REST endpoints for recall, remember, and ingest
Documentation
use std::sync::Arc;

use axum::Json;
use axum::extract::State;
use axum::http::StatusCode;
use axum::response::IntoResponse;
use serde::{Deserialize, Serialize};

use super::bad_request;
use crate::AppState;
use crate::ingest::{IngestStats, run_ingest};

#[derive(Deserialize, Default)]
pub(super) struct IngestRequest {
    source: Option<String>,
}

#[derive(Serialize, Default)]
struct IngestResponse {
    new_memories: usize,
    new_entities: usize,
    new_conversations: usize,
    skipped_dupes: usize,
    unchanged: usize,
    errors: usize,
    files_processed: usize,
    /// True while a pairing bootstrap holds ingest; nothing ran.
    #[serde(skip_serializing_if = "std::ops::Not::not")]
    deferred: bool,
}

impl IngestResponse {
    fn deferred() -> Self {
        Self {
            deferred: true,
            ..Self::default()
        }
    }
}

impl From<&IngestStats> for IngestResponse {
    fn from(stats: &IngestStats) -> Self {
        Self {
            new_memories: stats.new_memories,
            new_entities: stats.new_entities,
            new_conversations: stats.new_conversations,
            skipped_dupes: stats.skipped_dupes,
            unchanged: stats.unchanged,
            errors: stats.errors,
            files_processed: stats.files_processed,
            deferred: false,
        }
    }
}

#[derive(Serialize)]
struct ReingestResponse {
    cleared_log: usize,
    #[serde(flatten)]
    stats: IngestResponse,
}

pub(super) async fn ingest(
    State(state): State<Arc<AppState>>,
    body: Option<Json<IngestRequest>>,
) -> impl IntoResponse {
    let source = body.and_then(|Json(req)| req.source);
    if let Err(message) = crate::ingest::adapters_for(source.as_deref(), &state.machine.id) {
        return bad_request(message);
    }
    if state.bootstrap.is_pending() {
        return Json(IngestResponse::deferred()).into_response();
    }
    let store = state.store.clone();
    let embedder = state.embedder.clone();

    let machine_id = state.machine.id.clone();
    let result = tokio::task::spawn_blocking(move || {
        run_ingest(&store, embedder.as_deref(), &machine_id, source.as_deref())
    })
    .await;

    match result {
        Ok(Ok(stats)) => Json(IngestResponse::from(&stats)).into_response(),
        Ok(Err(e)) => (
            StatusCode::INTERNAL_SERVER_ERROR,
            Json(serde_json::json!({"error": e.to_string()})),
        )
            .into_response(),
        Err(e) => (
            StatusCode::INTERNAL_SERVER_ERROR,
            Json(serde_json::json!({"error": e.to_string()})),
        )
            .into_response(),
    }
}

#[derive(Deserialize)]
pub(super) struct ReingestRequest {
    #[serde(default)]
    clean: bool,
}

pub(super) async fn reingest(
    State(state): State<Arc<AppState>>,
    Json(req): Json<ReingestRequest>,
) -> impl IntoResponse {
    if req.clean {
        return bad_request(
            "clean is no longer supported: raw ids are derived from the transcript, so \
             deleting raws and re-ingesting would skip every chunk and leave the raws \
             deleted on every peer"
                .to_string(),
        );
    }
    if state.bootstrap.is_pending() {
        return Json(ReingestResponse {
            cleared_log: 0,
            stats: IngestResponse::deferred(),
        })
        .into_response();
    }
    let store = state.store.clone();
    let embedder = state.embedder.clone();

    let machine_id = state.machine.id.clone();
    let result = tokio::task::spawn_blocking(move || -> anyhow::Result<ReingestResponse> {
        let (cleared_log, stats) =
            crate::ingest::reingest(&store, embedder.as_deref(), &machine_id)?;
        Ok(ReingestResponse {
            cleared_log,
            stats: IngestResponse::from(&stats),
        })
    })
    .await;

    match result {
        Ok(Ok(response)) => Json(response).into_response(),
        Ok(Err(e)) => (
            StatusCode::INTERNAL_SERVER_ERROR,
            Json(serde_json::json!({"error": e.to_string()})),
        )
            .into_response(),
        Err(e) => (
            StatusCode::INTERNAL_SERVER_ERROR,
            Json(serde_json::json!({"error": e.to_string()})),
        )
            .into_response(),
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn an_answer_that_ran_carries_no_deferred_field() {
        let ran = serde_json::to_value(IngestResponse::from(&IngestStats {
            new_memories: 2,
            ..IngestStats::default()
        }))
        .unwrap();
        assert!(ran.get("deferred").is_none(), "{ran}");
        assert_eq!(ran["new_memories"], 2);
        let deferred = serde_json::to_value(IngestResponse::deferred()).unwrap();
        assert_eq!(deferred["deferred"], true);
    }
}