second-brain-core 0.13.5

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
use std::path::{Path, PathBuf};

use anyhow::{Context, Result, bail};
use serde::Deserialize;

use crate::retrieval::{RetrievalTunables, blend};

pub const CONFIG_FILE: &str = "config.toml";

#[derive(Debug, Clone, Default, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Config {
    pub llm: Option<LlmConfig>,
    #[serde(default)]
    pub cloud: CloudConfig,
    #[serde(default)]
    pub distill: DistillConfig,
    #[serde(default)]
    pub retrieval: RetrievalConfig,
    #[serde(default)]
    pub sources: SourcesConfig,
    #[serde(default)]
    pub maintenance: MaintenanceConfig,
    #[serde(default)]
    pub alert: AlertConfig,
    #[serde(default)]
    pub raw_index: RawIndexConfig,
}

/// How far back the raw fallback index reaches. Transcripts and archived
/// raws whose newest session time is older than `max_age_days` are left out
/// until the bound is raised, so a first pass over years of history stays
/// bounded and the newest work is indexed first.
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct RawIndexConfig {
    #[serde(default = "default_raw_max_age_days")]
    pub max_age_days: u32,
}

impl Default for RawIndexConfig {
    fn default() -> Self {
        Self {
            max_age_days: default_raw_max_age_days(),
        }
    }
}

fn default_raw_max_age_days() -> u32 {
    30
}

/// Where the daemon POSTs its health state when it changes. Any service
/// that accepts a JSON POST works; nothing is sent when unset.
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct AlertConfig {
    pub webhook_url: Option<String>,
    /// Health degrades when no sync has succeeded for this long on a device
    /// that lists peers in `sync-peers` or has synced before; 0 turns the
    /// check off.
    #[serde(default = "default_overdue_hours")]
    pub sync_overdue_hours: u64,
    /// Health degrades when a raw has waited this long to be distilled; 0
    /// turns the check off.
    #[serde(default = "default_overdue_hours")]
    pub undistilled_overdue_hours: u64,
}

impl Default for AlertConfig {
    fn default() -> Self {
        Self {
            webhook_url: None,
            sync_overdue_hours: default_overdue_hours(),
            undistilled_overdue_hours: default_overdue_hours(),
        }
    }
}

fn default_overdue_hours() -> u64 {
    24
}

/// Scheduled store repair (see `maintenance::repair`). The daemon runs it
/// once at startup and again whenever `interval_secs` have passed since the
/// last run, from the thread that owns the writable indexes.
#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct MaintenanceConfig {
    #[serde(default = "default_true")]
    pub enabled: bool,
    #[serde(default = "default_maintenance_interval")]
    pub interval_secs: u64,
}

#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct LlmConfig {
    // No default endpoint on purpose: the v0.5 hardcoded localhost:8081 default
    // silently pointed every LLM stage at a dead machine for weeks.
    pub endpoint: String,
    pub model: String,
    pub api_key_env: Option<String>,
    #[serde(default = "default_context_budget")]
    pub context_budget_tokens: usize,
}

#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct CloudConfig {
    pub vault_id: Option<String>,
    #[serde(default)]
    pub providers: Vec<ProviderConfig>,
    #[serde(default = "default_data_shards")]
    pub data_shards: usize,
    #[serde(default = "default_total_shards")]
    pub total_shards: usize,
}

#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ProviderConfig {
    pub name: String,
    pub endpoint: String,
    pub bucket: String,
    #[serde(default)]
    pub region: Option<String>,
    pub access_key_id: String,
    pub secret_key_env: String,
}

#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct DistillConfig {
    #[serde(default = "default_min_idle_minutes")]
    pub min_idle_minutes: u64,
    #[serde(default = "default_archive_after_days")]
    pub archive_after_days: u64,
    /// Cosine at which two live distillates of one project and type are
    /// offered for merging. On the live store (2026-09-23) pairs at 0.97 and
    /// above were paraphrases of one fact, while 0.95 to 0.97 still held
    /// pairs that disagree on a detail.
    #[serde(default = "default_near_duplicate_threshold")]
    pub near_duplicate_threshold: f32,
}

#[derive(Debug, Clone, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct RetrievalConfig {
    #[serde(default = "default_rerank_gate")]
    pub rerank_gate: f32,
    #[serde(default = "default_cosine_gate")]
    pub cosine_gate: f32,
    #[serde(default = "default_rerank_model")]
    pub rerank_model: String,
    #[serde(default = "default_true")]
    pub rerank_enabled: bool,
    /// How many fused candidates reach the reranker. Measured on the
    /// distilled corpus (evals/fuse-top-distilled.md): 20 beats 10 on
    /// recall@3 and MRR with the confidence interval clear of zero.
    #[serde(default = "default_fuse_top")]
    pub fuse_top: usize,
    /// Cosine floor for a recall whose scope was scanned whole; see
    /// `RetrievalTunables::exhausted_cosine_gate`.
    #[serde(default = "default_exhausted_cosine_gate")]
    pub exhausted_cosine_gate: f32,
    /// See `RetrievalTunables::fusion_floor`.
    #[serde(default = "default_fusion_floor")]
    pub fusion_floor: usize,
    /// See `RetrievalTunables::chunk_rerank`.
    #[serde(default = "default_true")]
    pub chunk_rerank: bool,
    /// See `RetrievalTunables::rerank_whole_chunked`.
    #[serde(default = "default_true")]
    pub rerank_whole_chunked: bool,
}

impl RetrievalConfig {
    /// The gates and fan-out the daemon ranks with. Candidate depths are
    /// not configurable and keep their built-in values.
    pub fn tunables(&self) -> RetrievalTunables {
        RetrievalTunables {
            fuse_top: self.fuse_top,
            rerank_gate: self.rerank_gate,
            cosine_gate: self.cosine_gate,
            exhausted_cosine_gate: self.exhausted_cosine_gate,
            fusion_floor: self.fusion_floor,
            chunk_rerank: self.chunk_rerank,
            rerank_whole_chunked: self.rerank_whole_chunked,
            ..RetrievalTunables::default()
        }
    }
}

#[derive(Debug, Clone, Default, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct SourcesConfig {
    #[serde(default)]
    pub obsidian_vaults: Vec<PathBuf>,
    #[serde(default)]
    pub codex_sessions: Option<PathBuf>,
}

fn default_context_budget() -> usize {
    16000
}
fn default_data_shards() -> usize {
    2
}
fn default_total_shards() -> usize {
    3
}
fn default_min_idle_minutes() -> u64 {
    60
}
fn default_archive_after_days() -> u64 {
    7
}
fn default_near_duplicate_threshold() -> f32 {
    0.97
}
fn default_rerank_gate() -> f32 {
    0.30
}
fn default_cosine_gate() -> f32 {
    blend::MIN_SIMILARITY
}
fn default_exhausted_cosine_gate() -> f32 {
    blend::EXHAUSTED_MIN_SIMILARITY
}
fn default_fusion_floor() -> usize {
    RetrievalTunables::default().fusion_floor
}
fn default_fuse_top() -> usize {
    20
}
fn default_rerank_model() -> String {
    "bge-reranker-v2-m3".to_string()
}
fn default_true() -> bool {
    true
}
fn default_maintenance_interval() -> u64 {
    21600
}

impl Default for CloudConfig {
    fn default() -> Self {
        Self {
            vault_id: None,
            providers: Vec::new(),
            data_shards: default_data_shards(),
            total_shards: default_total_shards(),
        }
    }
}

impl Default for DistillConfig {
    fn default() -> Self {
        Self {
            min_idle_minutes: default_min_idle_minutes(),
            archive_after_days: default_archive_after_days(),
            near_duplicate_threshold: default_near_duplicate_threshold(),
        }
    }
}

impl Default for MaintenanceConfig {
    fn default() -> Self {
        Self {
            enabled: true,
            interval_secs: default_maintenance_interval(),
        }
    }
}

impl Default for RetrievalConfig {
    fn default() -> Self {
        Self {
            rerank_gate: default_rerank_gate(),
            cosine_gate: default_cosine_gate(),
            rerank_model: default_rerank_model(),
            rerank_enabled: true,
            fuse_top: default_fuse_top(),
            exhausted_cosine_gate: default_exhausted_cosine_gate(),
            fusion_floor: default_fusion_floor(),
            chunk_rerank: true,
            rerank_whole_chunked: true,
        }
    }
}

impl Config {
    pub fn parse(text: &str) -> Result<Self> {
        let config: Config = toml::from_str(text).context("invalid config.toml")?;
        config.validate()?;
        Ok(config)
    }

    pub fn load(path: &Path) -> Result<Self> {
        let text = std::fs::read_to_string(path)
            .with_context(|| format!("cannot read config at {}", path.display()))?;
        Self::parse(&text)
    }

    /// Loads ~/.second-brain/config.toml, falling back to all-default config
    /// (no llm, no cloud) when the file does not exist so read-only commands
    /// work on an unconfigured machine.
    pub fn load_default() -> Result<Self> {
        let path = crate::machine::data_dir()?.join(CONFIG_FILE);
        if path.exists() {
            Self::load(&path)
        } else {
            Ok(Self::default())
        }
    }

    fn validate(&self) -> Result<()> {
        if let Some(llm) = &self.llm {
            if llm.endpoint.trim().is_empty() {
                bail!("llm.endpoint must not be empty");
            }
            if llm.model.trim().is_empty() {
                bail!("llm.model must not be empty");
            }
        }
        if self.retrieval.fuse_top == 0 {
            bail!("retrieval.fuse_top must be at least 1");
        }
        for (name, gate) in [
            ("cosine_gate", self.retrieval.cosine_gate),
            (
                "exhausted_cosine_gate",
                self.retrieval.exhausted_cosine_gate,
            ),
        ] {
            if !(0.0..=1.0).contains(&gate) {
                bail!("retrieval.{name} must be between 0 and 1 (got {gate})");
            }
        }
        if let Some(url) = &self.alert.webhook_url
            && !(url.starts_with("http://") || url.starts_with("https://"))
        {
            bail!("alert.webhook_url must be an http:// or https:// URL (got {url})");
        }
        for (key, hours) in [
            ("sync_overdue_hours", self.alert.sync_overdue_hours),
            (
                "undistilled_overdue_hours",
                self.alert.undistilled_overdue_hours,
            ),
        ] {
            if i64::try_from(hours)
                .ok()
                .and_then(chrono::Duration::try_hours)
                .is_none()
            {
                bail!("alert.{key} is too large (got {hours})");
            }
        }
        let dup = self.distill.near_duplicate_threshold;
        if !(dup > 0.0 && dup <= 1.0) {
            bail!("distill.near_duplicate_threshold must be above 0 and at most 1 (got {dup})");
        }
        let c = &self.cloud;
        if c.data_shards == 0 || c.total_shards <= c.data_shards {
            bail!(
                "cloud erasure config requires 0 < data_shards < total_shards (got {}/{})",
                c.data_shards,
                c.total_shards
            );
        }
        if !c.providers.is_empty() && c.providers.len() < c.total_shards {
            bail!(
                "{} cloud providers configured but total_shards = {}; one shard per provider is required",
                c.providers.len(),
                c.total_shards
            );
        }
        Ok(())
    }

    pub fn llm(&self) -> Result<&LlmConfig> {
        self.llm.as_ref().context(
            "no LLM endpoint configured; add an [llm] section with endpoint and model \
             to ~/.second-brain/config.toml (there is deliberately no default)",
        )
    }

    pub fn cloud(&self) -> Result<&CloudConfig> {
        if self.cloud.providers.is_empty() {
            bail!(
                "no cloud providers configured; add [[cloud.providers]] entries \
                 to ~/.second-brain/config.toml"
            );
        }
        Ok(&self.cloud)
    }
}