second-brain-api 0.12.0

HTTP API server for second-brain: REST endpoints for recall, remember, and ingest
Documentation
use std::collections::{HashMap, HashSet};
use std::path::Path;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex, RwLock};
use std::time::Duration;

use anyhow::Result;
use chrono::{DateTime, Utc};
use second_brain_adapters::raw_text::raw_passages;
use second_brain_core::index::EMBEDDING_DIM;
use second_brain_core::index::raw::{RawIndex, RawOrigin, RawPassage};
use second_brain_core::project_path;
use second_brain_core::residency;
use second_brain_core::sqlite_store::SqliteStore;
use second_brain_core::store::Store;
use second_brain_core::transcript_archive::{ArchiveEntry, TranscriptArchive};
use uuid::Uuid;

const EMBED_BATCH: usize = 8;
const COMMIT_EVERY: usize = 20;
/// Embedding sessions are handed out without fairness, so without this gap
/// the indexer takes the session straight back and an ingest or recall
/// waiting for it times out.
pub const PAUSE_AFTER_EMBED: Duration = Duration::from_millis(100);
const PASS_INTERVAL: Duration = Duration::from_secs(600);

/// Opens the raw index, or rebuilds it from scratch when its files cannot
/// be read as an index, for example after a schema change: everything in it
/// is derived from the transcript archive and the store, so the next pass
/// refills it. An I/O error opening the directories themselves is returned
/// instead; tantivy and usearch report their own I/O faults without an I/O
/// cause, so those still rebuild.
pub fn open_or_rebuild(dir: &Path) -> Result<RawIndex> {
    match RawIndex::open(dir) {
        Ok(index) => Ok(index),
        Err(e)
            if e.chain()
                .any(|c| c.downcast_ref::<std::io::Error>().is_some()) =>
        {
            Err(e)
        }
        Err(e) => {
            tracing::warn!(
                "raw index at {} cannot be opened, rebuilding it: {e:#}",
                dir.display()
            );
            std::fs::remove_dir_all(dir)?;
            RawIndex::open(dir)
        }
    }
}

pub type EmbedBatch = dyn Fn(&[&str]) -> Result<Vec<Vec<f32>>> + Send + Sync;

#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub struct PassSummary {
    pub transcripts: usize,
    pub memories: usize,
    pub passages: usize,
    pub restored: usize,
    /// Sources that could not be read or embedded; they are retried next pass.
    pub failed: usize,
    /// Passages left out because their source is older than the age bound.
    pub skipped_by_age: usize,
}

/// The age bound an indexer works to and what its last pass left out,
/// shared with status. It also remembers when each transcript left out
/// was last active, so later passes skip it without reading it again.
pub struct RawCoverage {
    max_age_days: u32,
    skipped_by_age: AtomicUsize,
    aged: Mutex<HashMap<String, Dated>>,
}

#[derive(Clone, Copy)]
struct Dated {
    newest: DateTime<Utc>,
    passages: usize,
}

impl RawCoverage {
    pub fn new(max_age_days: u32) -> Self {
        Self {
            max_age_days,
            skipped_by_age: AtomicUsize::new(0),
            aged: Mutex::new(HashMap::new()),
        }
    }

    pub fn max_age_days(&self) -> u32 {
        self.max_age_days
    }

    /// Passages the last finished pass left out for age.
    pub fn skipped_by_age(&self) -> usize {
        self.skipped_by_age.load(Ordering::Relaxed)
    }

    fn cutoff(&self) -> DateTime<Utc> {
        Utc::now()
            .checked_sub_signed(chrono::Duration::days(i64::from(self.max_age_days)))
            .unwrap_or(DateTime::<Utc>::MIN_UTC)
    }

    fn aged(&self) -> std::sync::MutexGuard<'_, HashMap<String, Dated>> {
        self.aged.lock().unwrap_or_else(|p| p.into_inner())
    }
}

/// Feeds the raw fallback index from the transcript archive and from raw
/// memories consolidation archived, embedding outside the index lock so
/// recall is never held behind the model.
pub struct RawIndexer {
    pub index: Arc<RwLock<RawIndex>>,
    pub archive: Option<Arc<TranscriptArchive>>,
    pub store: Arc<SqliteStore>,
    pub embed: Arc<EmbedBatch>,
    pub pause: Duration,
    pub coverage: Arc<RawCoverage>,
}

impl RawIndexer {
    pub fn spawn(self) {
        std::thread::Builder::new()
            .name("raw-index".to_string())
            .spawn(move || {
                loop {
                    match self.pass() {
                        Ok(s)
                            if PassSummary {
                                skipped_by_age: 0,
                                ..s
                            } != PassSummary::default() =>
                        {
                            tracing::info!(
                                "raw index: {} transcripts, {} archived memories, {} passages, {} vectors restored, {} sources failed, {} passages older than {} days skipped",
                                s.transcripts,
                                s.memories,
                                s.passages,
                                s.restored,
                                s.failed,
                                s.skipped_by_age,
                                self.coverage.max_age_days()
                            );
                            residency::release_free_heap();
                        }
                        Ok(_) => {}
                        Err(e) => {
                            tracing::warn!("raw index pass failed: {e:#}");
                            residency::release_free_heap();
                        }
                    }
                    std::thread::sleep(PASS_INTERVAL);
                }
            })
            .expect("spawning the raw index thread");
    }

    /// Indexes every source not indexed yet that is within the age bound,
    /// newest first. A source that fails is counted, skipped and retried on
    /// the next pass, so one bad blob or a passing embedder error never stops
    /// the rest.
    pub fn pass(&self) -> Result<PassSummary> {
        let mut summary = PassSummary::default();
        let mut first_error = None;
        let mut pending = 0;
        let cutoff = self.coverage.cutoff();
        if let Some(archive) = &self.archive {
            let mut queue = Vec::new();
            let mut aged = HashMap::new();
            for entry in archive.entries() {
                let origin = RawOrigin::Transcript {
                    sha256: entry.sha256.clone(),
                };
                if self.indexed(&origin) {
                    continue;
                }
                match self.date_transcript(archive, &entry) {
                    Ok(dated) if dated.newest < cutoff => {
                        summary.skipped_by_age += dated.passages;
                        aged.insert(entry.sha256.clone(), dated);
                    }
                    Ok(dated) => queue.push((dated.newest, entry, origin)),
                    Err(e) => {
                        summary.failed += 1;
                        first_error.get_or_insert(e);
                    }
                }
            }
            *self.coverage.aged() = aged;
            queue.sort_by_key(|q| std::cmp::Reverse(q.0));
            for (_, entry, origin) in queue {
                match self.add_transcript(archive, &entry, &origin) {
                    Ok(added) => {
                        summary.passages += added;
                        summary.transcripts += 1;
                        pending = self.commit_every(pending + 1)?;
                    }
                    Err(e) => {
                        summary.failed += 1;
                        first_error.get_or_insert(e);
                    }
                }
            }
        }

        let raws = self.store.archived_raws_since(cutoff)?;
        summary.skipped_by_age += raws
            .older
            .into_iter()
            .filter(|&id| !self.indexed(&RawOrigin::Memory { id }))
            .count();
        for id in raws.recent {
            let origin = RawOrigin::Memory { id };
            if self.indexed(&origin) {
                continue;
            }
            match self.add_memory(id, &origin) {
                Ok((added, restored)) => {
                    summary.passages += added;
                    summary.restored += usize::from(restored);
                    summary.memories += 1;
                    pending = self.commit_every(pending + 1)?;
                }
                Err(e) => {
                    summary.failed += 1;
                    first_error.get_or_insert(e);
                }
            }
        }
        if pending > 0 {
            self.write().commit()?;
        }
        self.coverage
            .skipped_by_age
            .store(summary.skipped_by_age, Ordering::Relaxed);
        if let Some(e) = first_error {
            tracing::warn!(
                "raw index: {} sources failed and will be retried, first: {e:#}",
                summary.failed
            );
        }
        Ok(summary)
    }

    /// When a transcript was last active: its newest passage time, or when
    /// it was archived if no line carries one. A transcript left out for age
    /// is dated from memory rather than read again.
    fn date_transcript(&self, archive: &TranscriptArchive, entry: &ArchiveEntry) -> Result<Dated> {
        if let Some(dated) = self.coverage.aged().get(&entry.sha256) {
            return Ok(*dated);
        }
        let source = Path::new(&entry.source);
        let passages = match archive.read(source)? {
            Some(content) => raw_passages(source, &content),
            None => Vec::new(),
        };
        Ok(Dated {
            newest: passages
                .iter()
                .filter_map(|p| p.at)
                .max()
                .unwrap_or(entry.archived_at),
            passages: passages.len(),
        })
    }

    fn add_transcript(
        &self,
        archive: &TranscriptArchive,
        entry: &ArchiveEntry,
        origin: &RawOrigin,
    ) -> Result<usize> {
        let source = Path::new(&entry.source);
        let Some(content) = archive.read(source)? else {
            return self.add(origin, Vec::new());
        };
        let project = project_path::from_transcript(&entry.source);
        let passages = raw_passages(source, &content)
            .into_iter()
            .map(|p| RawPassage {
                text: p.text,
                at: p.at,
                project: project.clone(),
                memory: p.memory,
            })
            .collect();
        self.add(origin, passages)
    }

    fn add_memory(&self, id: Uuid, origin: &RawOrigin) -> Result<(usize, bool)> {
        let Some(memory) = self.store.get_memory_with_embedding(id)? else {
            return Ok((0, false));
        };
        let mut restored = false;
        let embedding = if memory.embedding.len() == EMBEDDING_DIM {
            memory.embedding.clone()
        } else {
            let embedding = self
                .embed_and_yield(&[memory.content.as_str()])?
                .into_iter()
                .next()
                .unwrap_or_default();
            if embedding.len() == EMBEDDING_DIM {
                restored = self.store.restore_archived_embedding(id, &embedding)?;
            }
            embedding
        };
        let passage = RawPassage {
            text: memory.content.clone(),
            at: memory.session_at,
            project: memory.project_path.clone(),
            memory: Some(id),
        };
        let added = self.write().add_source(origin, &[(passage, embedding)])?;
        Ok((added, restored))
    }

    /// Embeds only passages whose text is not indexed yet, so a transcript
    /// archived again as it grows costs only its new text.
    fn add(&self, origin: &RawOrigin, passages: Vec<RawPassage>) -> Result<usize> {
        let passages: Vec<RawPassage> = {
            let index = self.read();
            let mut seen = HashSet::new();
            passages
                .into_iter()
                .filter(|p| seen.insert(p.text.trim().to_string()) && !index.has_text(&p.text))
                .collect()
        };
        let mut embedded = Vec::with_capacity(passages.len());
        for batch in passages.chunks(EMBED_BATCH) {
            let texts: Vec<&str> = batch.iter().map(|p| p.text.as_str()).collect();
            let vectors = self.embed_and_yield(&texts)?;
            embedded.extend(batch.iter().cloned().zip(vectors));
        }
        self.write().add_source(origin, &embedded)
    }

    fn embed_and_yield(&self, texts: &[&str]) -> Result<Vec<Vec<f32>>> {
        let vectors = (self.embed)(texts);
        std::thread::sleep(self.pause);
        vectors
    }

    fn commit_every(&self, pending: usize) -> Result<usize> {
        if pending < COMMIT_EVERY {
            return Ok(pending);
        }
        self.write().commit()?;
        Ok(0)
    }

    fn indexed(&self, origin: &RawOrigin) -> bool {
        self.read().has_source(origin)
    }

    fn read(&self) -> std::sync::RwLockReadGuard<'_, RawIndex> {
        self.index.read().unwrap_or_else(|p| p.into_inner())
    }

    fn write(&self) -> std::sync::RwLockWriteGuard<'_, RawIndex> {
        self.index.write().unwrap_or_else(|p| p.into_inner())
    }
}