second-brain-api 0.12.0

HTTP API server for second-brain: REST endpoints for recall, remember, and ingest
Documentation
use std::path::PathBuf;
use std::sync::{Arc, RwLock};

use tokio::net::TcpListener;

use chrono::Utc;
use second_brain_api::backfill::{Backfill, LiveIngest, PAUSE_BETWEEN_FILES};
use second_brain_api::footprint::{FootprintCheck, StorePaths};
use second_brain_api::health::{Health, Windows};
use second_brain_api::maintenance::{DrainLoop, WriterSlot};
use second_brain_api::models::{self, IndexHandles};
use second_brain_api::note_cover::NoteCover;
use second_brain_api::progress::ProgressCheck;
use second_brain_api::raw_index::{PAUSE_AFTER_EMBED, RawCoverage, RawIndexer, open_or_rebuild};
use second_brain_api::sources::{GitHub, PullRequests};
use second_brain_api::startup::Startup;
use second_brain_api::store_compaction::Compaction;
use second_brain_api::sync_loop::SyncLoop;
use second_brain_api::write_gate::WriteGate;
use second_brain_api::write_queue::{self, WriteQueue};
use second_brain_api::{AppState, alert, file_modes, sources, stored_redaction, vaults, watcher};
use second_brain_core::api_token;
use second_brain_core::config::Config;
use second_brain_core::embedding::DocumentEmbedder;
use second_brain_core::index::raw;
use second_brain_core::machine::{self, MachineIdentity};
use second_brain_core::sqlite_store::SqliteStore;
use second_brain_core::store::StoreAdmin;
use second_brain_core::transcript_archive::TranscriptArchive;

#[tokio::main]
async fn main() {
    unsafe { libc::umask(0o077) };

    second_brain_api::logging::init();

    let bind = std::env::var("SECOND_BRAIN_BIND").unwrap_or_else(|_| "127.0.0.1:7200".to_string());
    let listener = match TcpListener::bind(&bind).await {
        Ok(l) => l,
        Err(e) => {
            tracing::error!(
                "cannot listen on {bind}: {e}; another second-brain daemon probably holds it"
            );
            std::process::exit(1);
        }
    };
    let startup = Startup::new();
    let server = tokio::spawn(startup.clone().serve(listener));
    tracing::info!("listening on {bind}");

    let db_path = std::env::var("SECOND_BRAIN_DB")
        .map(PathBuf::from)
        .unwrap_or_else(|_| {
            let home = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string());
            PathBuf::from(home).join(".second-brain").join("brain.db")
        });

    if let Some(parent) = db_path.parent() {
        std::fs::create_dir_all(parent).ok();
    }

    let data_dir = machine::data_dir().expect("cannot determine home directory");
    let api_token = match api_token::ensure(&data_dir) {
        Ok(token) => token,
        Err(e) => {
            tracing::error!(
                "cannot create the API token in {}: {e:#}",
                data_dir.display()
            );
            std::process::exit(1);
        }
    };

    let config_dir = MachineIdentity::config_dir().expect("cannot determine home directory");
    let machine = MachineIdentity::load_or_create(&config_dir)
        .expect("failed to load or create machine identity");

    tracing::info!("machine identity: {} ({})", machine.name, machine.id);

    startup.phase("opening the database");
    let sqlite = match SqliteStore::open(&db_path, machine.id.clone()) {
        Ok(s) => Arc::new(s),
        Err(e) => {
            tracing::error!("failed to open database at {}: {e}", db_path.display());
            std::process::exit(1);
        }
    };
    let store: Arc<dyn StoreAdmin> = sqlite.clone();
    store
        .register_machine(&machine)
        .expect("failed to register machine identity");

    tracing::info!("database opened at {}", db_path.display());

    let (config, config_loaded) = match Config::load_default() {
        Ok(c) => (c, true),
        Err(e) => {
            tracing::warn!("config.toml unusable, running with defaults: {e:#}");
            (Config::default(), false)
        }
    };
    // A config that failed to load may still hold an [llm] section, so it
    // counts as configured and never puts the daemon in no-GPU mode.
    let llm_configured = !config_loaded || config.llm.is_some();

    let health = Arc::new(
        Health::new(
            Utc::now(),
            alert::start(&config.alert, machine.name.clone()),
        )
        .with_windows(Windows::from_config(&config.alert)),
    );

    startup.phase("loading the embedding model");
    let embedder = models::load_embedder();

    startup.phase("opening the search indexes");
    let index_dir = db_path.with_extension("indexes");
    let index_handles = match models::open_index_handles(&index_dir, &sqlite, |n| {
        startup.phase(format!("rebuilding search indexes for {n} memories"))
    }) {
        Ok(h) => Some(h),
        Err(e) => {
            tracing::warn!(
                "search indexes unavailable at {}: {e}; recall falls back to the legacy engine",
                index_dir.display()
            );
            health.indexes_unavailable(&format!("{e:#}"), Utc::now());
            None
        }
    };
    let indexes = index_handles.as_ref().map(|h| h.reader.clone());

    let archive_dir = data_dir.join("transcripts");
    let archive = match TranscriptArchive::open(&archive_dir) {
        Ok(a) => Some(Arc::new(a)),
        Err(e) => {
            tracing::error!(
                "cannot open the transcript archive at {}: {e:#}",
                archive_dir.display()
            );
            None
        }
    };
    startup.phase("opening the raw transcript index");
    let raw_dir = raw::dir_for_db(&db_path);
    let raw_index = match open_or_rebuild(&raw_dir) {
        Ok(index) => Some(Arc::new(RwLock::new(index))),
        Err(e) => {
            tracing::warn!(
                "raw transcript index unavailable at {}: {e:#}",
                raw_dir.display()
            );
            None
        }
    };
    for (path, e) in file_modes::make_private(&data_dir, &db_path, &index_dir) {
        tracing::warn!("could not make {} private: {e}", path.display());
    }

    startup.phase("loading the reranker");
    let reranker = models::load_reranker(indexes.is_some());
    startup.phase("starting background tasks");

    let maintenance = Arc::new(RwLock::new(None));
    let near_duplicates = Arc::new(RwLock::new(None));
    let mut repair_tx = None;
    if let Some(IndexHandles {
        reader,
        writer,
        claim,
    }) = index_handles
    {
        let (tx, requests) = std::sync::mpsc::channel();
        repair_tx = Some(tx);
        DrainLoop {
            store: sqlite.clone(),
            reader,
            writer: WriterSlot::new(writer, claim),
            embedder: embedder.clone(),
            summary: maintenance.clone(),
            near_duplicates: near_duplicates.clone(),
            near_duplicate_threshold: config.distill.near_duplicate_threshold,
            config: config.maintenance.clone(),
            requests,
            health: health.clone(),
        }
        .spawn();
    }

    let raw_coverage = raw_index
        .as_ref()
        .map(|_| Arc::new(RawCoverage::new(config.raw_index.max_age_days)));
    if let (Some(index), Some(coverage), Some(embedder)) = (&raw_index, &raw_coverage, &embedder) {
        let embedder = embedder.clone();
        RawIndexer {
            index: index.clone(),
            archive: archive.clone(),
            store: sqlite.clone(),
            embed: Arc::new(move |texts: &[&str]| embedder.embed_batch(texts)),
            pause: PAUSE_AFTER_EMBED,
            coverage: coverage.clone(),
        }
        .spawn();
    }
    let live = Arc::new(LiveIngest::default());
    watcher::spawn(
        store.clone(),
        embedder.clone(),
        machine.id.clone(),
        health.clone(),
        archive.clone(),
        live.clone(),
    );
    vaults::spawn(
        sqlite.clone(),
        embedder.clone(),
        config.sources.obsidian_vaults.clone(),
        machine.id.clone(),
    );
    Backfill {
        store: sqlite.clone(),
        embedder: embedder.clone(),
        machine_id: machine.id.clone(),
        projects_dir: watcher::projects_dir(),
        archive,
        live,
        health: health.clone(),
        pause: PAUSE_BETWEEN_FILES,
    }
    .spawn();

    sources::Sources {
        store: sqlite.clone(),
        embedder: embedder.clone().map(|e| e as Arc<dyn DocumentEmbedder>),
        machine_id: machine.id.clone(),
        pull_requests: GitHub::from_env(tokio::runtime::Handle::current())
            .map(|g| Arc::new(g) as Arc<dyn PullRequests>),
    }
    .spawn();

    if !llm_configured {
        NoteCover {
            store: sqlite.clone(),
        }
        .spawn();
    }

    let gate = Arc::new(WriteGate::new(data_dir.join(write_queue::DIR_NAME)));
    let compaction = Compaction::new(
        sqlite.clone(),
        gate.clone(),
        repair_tx.clone(),
        health.clone(),
    );
    if let Err(e) = stored_redaction::spawn_pass(sqlite.clone(), embedder.clone(), compaction) {
        tracing::error!("cannot start the stored-content redaction pass: {e}");
    }

    models::spawn_idle_unload(embedder.clone(), reranker.clone());

    let progress = ProgressCheck {
        store: sqlite.clone(),
        data_dir: data_dir.clone(),
        health: health.clone(),
        llm_configured,
    };

    SyncLoop::over_ssh(data_dir.clone(), sqlite.clone(), gate.clone())
        .with_health(health.clone())
        .spawn();

    let state = Arc::new(AppState {
        store,
        sqlite,
        indexes,
        raw_index,
        raw_coverage,
        reranker,
        // Gates and fuse_top come from [retrieval] in config.toml. The
        // 2026-09 sweep on the distilled corpus (evals/fuse-top-distilled.md)
        // moved the fuse_top default from 10 to 20.
        tunables: second_brain_api::daemon_tunables(config.retrieval.tunables()),
        embedder,
        machine,
        maintenance,
        repair_tx,
        near_duplicate_threshold: config.distill.near_duplicate_threshold,
        near_duplicates,
        health: health.clone(),
        gate: gate.clone(),
    });

    FootprintCheck {
        paths: StorePaths {
            raw_index: raw_dir.clone(),
            db: db_path,
            indexes: index_dir,
            archive: archive_dir,
        },
        state: state.clone(),
        health: health.clone(),
    }
    .spawn();

    let app = second_brain_api::app(state, api_token.clone());

    let announced = health.clone();
    tokio::spawn(async move {
        progress.start().await;
        announced.announce(Utc::now());
    });

    WriteQueue::new(
        data_dir.join(write_queue::DIR_NAME),
        app.clone(),
        api_token,
        health,
        gate,
    )
    .spawn();

    second_brain_api::self_update::spawn();

    startup.ready(app);
    tracing::info!("startup finished; serving every route on {bind}");

    match server.await {
        Ok(Ok(())) => {}
        Ok(Err(e)) => {
            tracing::error!("serving on {bind} stopped: {e}");
            std::process::exit(1);
        }
        Err(e) => {
            tracing::error!("the server task on {bind} ended abnormally: {e}");
            std::process::exit(1);
        }
    }
}