second-brain-core 0.11.0

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
use std::ffi::CString;
use std::os::unix::ffi::OsStrExt;
use std::os::unix::fs::MetadataExt;
use std::path::{Path, PathBuf};

use anyhow::{Context, Result, bail};
use rusqlite::{OptionalExtension, params};
use serde::Serialize;

use super::SqliteStore;
use super::pool::BUSY_TIMEOUT;

const COMPACTED_KEY: &str = "stored_redaction_v1_compacted";
const VACUUM_BUSY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(600);
const SPACE_MARGIN: u64 = 16 * 1024 * 1024;

/// Free space against what a VACUUM of this store needs, on the filesystem
/// with the least room to spare.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct VacuumSpace {
    pub path: PathBuf,
    pub needed_bytes: u64,
    pub free_bytes: u64,
}

impl VacuumSpace {
    pub fn fits(&self) -> bool {
        self.free_bytes >= self.needed_bytes
    }
}

impl SqliteStore {
    /// `VACUUM` cannot run inside a transaction and needs the writer alone;
    /// returns (bytes before, bytes after).
    pub fn vacuum(&self) -> Result<(u64, u64)> {
        self.vacuum_then(false)
    }

    /// `vacuum`, then a checkpoint that truncates the WAL, because the frames
    /// VACUUM appended, and older ones past them, would otherwise keep every
    /// page's previous contents on disk. Errors when readers keep the
    /// checkpoint from finishing, though the VACUUM itself went through.
    pub fn vacuum_clearing_wal(&self) -> Result<(u64, u64)> {
        self.vacuum_then(true)
    }

    fn vacuum_then(&self, truncate_wal: bool) -> Result<(u64, u64)> {
        let conn = self.conn();
        let size = |sql: &str| -> Result<u64> {
            let page_size: i64 = conn.query_row("PRAGMA page_size", [], |r| r.get(0))?;
            let pages: i64 = conn.query_row(sql, [], |r| r.get(0))?;
            Ok((page_size * pages) as u64)
        };
        let before = size("PRAGMA page_count")?;
        conn.busy_timeout(VACUUM_BUSY_TIMEOUT)?;
        let vacuumed = conn.execute_batch("VACUUM;").and_then(|()| {
            if !truncate_wal {
                return Ok(0);
            }
            conn.query_row("PRAGMA wal_checkpoint(TRUNCATE)", [], |r| {
                r.get::<_, i64>(0)
            })
        });
        conn.busy_timeout(BUSY_TIMEOUT)?;
        if vacuumed.context("vacuuming the store")? != 0 {
            bail!("the WAL checkpoint after VACUUM was blocked by readers");
        }
        let after = size("PRAGMA page_count")?;
        Ok((before, after))
    }

    /// VACUUM writes a full copy of the live pages to SQLite's temp
    /// directory and then the same again into the WAL, so both need room.
    pub fn vacuum_space(&self) -> Result<VacuumSpace> {
        let Some(db) = self.db_path()? else {
            return Ok(VacuumSpace {
                path: PathBuf::new(),
                needed_bytes: 0,
                free_bytes: 0,
            });
        };
        let live = self.live_bytes()?;
        let copy = live + live / 16 + SPACE_MARGIN;
        let db_dir = db
            .parent()
            .map_or_else(|| PathBuf::from("."), Path::to_path_buf);
        let temp_dir = sqlite_temp_dir();
        let mut needs = vec![(db_dir.clone(), copy)];
        if same_filesystem(&db_dir, &temp_dir)? {
            needs[0].1 += copy;
        } else {
            needs.push((temp_dir, copy));
        }
        let mut tightest: Option<VacuumSpace> = None;
        for (path, needed_bytes) in needs {
            let free_bytes = fs4::available_space(&path)
                .with_context(|| format!("reading free space at {}", path.display()))?;
            let space = VacuumSpace {
                path,
                needed_bytes,
                free_bytes,
            };
            let headroom = |s: &VacuumSpace| s.free_bytes as i128 - s.needed_bytes as i128;
            if tightest
                .as_ref()
                .is_none_or(|t| headroom(&space) < headroom(t))
            {
                tightest = Some(space);
            }
        }
        Ok(tightest.expect("the database directory is always checked"))
    }

    pub fn stored_redaction_compacted(&self) -> Result<bool> {
        let value: Option<String> = self
            .reader()
            .query_row(
                "SELECT value FROM index_meta WHERE key = ?1",
                params![COMPACTED_KEY],
                |r| r.get(0),
            )
            .optional()?;
        Ok(value.as_deref() == Some("done"))
    }

    pub fn mark_stored_redaction_compacted(&self) -> Result<()> {
        self.conn().execute(
            "INSERT OR REPLACE INTO index_meta (key, value) VALUES (?1, 'done')",
            params![COMPACTED_KEY],
        )?;
        Ok(())
    }

    fn db_path(&self) -> Result<Option<PathBuf>> {
        let file: String = self
            .reader()
            .query_row("PRAGMA database_list", [], |r| r.get(2))?;
        Ok((!file.is_empty()).then(|| PathBuf::from(file)))
    }

    fn live_bytes(&self) -> Result<u64> {
        let conn = self.reader();
        let page_size: i64 = conn.query_row("PRAGMA page_size", [], |r| r.get(0))?;
        let pages: i64 = conn.query_row("PRAGMA page_count", [], |r| r.get(0))?;
        let free: i64 = conn.query_row("PRAGMA freelist_count", [], |r| r.get(0))?;
        Ok((pages - free).max(0) as u64 * page_size as u64)
    }
}

/// The directory SQLite's unix VFS picks for temp files, in its own order:
/// the first candidate that is a directory this process can write and enter.
fn sqlite_temp_dir() -> PathBuf {
    let from_env = ["SQLITE_TMPDIR", "TMPDIR"]
        .into_iter()
        .filter_map(|var| std::env::var_os(var).map(PathBuf::from));
    let fixed = ["/var/tmp", "/usr/tmp", "/tmp", "."]
        .into_iter()
        .map(PathBuf::from);
    from_env
        .chain(fixed)
        .find(|dir| usable_temp_dir(dir))
        .unwrap_or_else(|| PathBuf::from("."))
}

fn usable_temp_dir(dir: &Path) -> bool {
    if !dir.is_dir() {
        return false;
    }
    let Ok(path) = CString::new(dir.as_os_str().as_bytes()) else {
        return false;
    };
    unsafe { libc::access(path.as_ptr(), libc::W_OK | libc::X_OK) == 0 }
}

fn same_filesystem(a: &Path, b: &Path) -> Result<bool> {
    let dev = |p: &Path| {
        std::fs::metadata(p)
            .map(|m| m.dev())
            .with_context(|| format!("reading {}", p.display()))
    };
    Ok(dev(a)? == dev(b)?)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn an_in_memory_store_needs_no_space() {
        let store = SqliteStore::in_memory("m".to_string()).unwrap();
        assert!(store.vacuum_space().unwrap().fits());
    }

    #[test]
    fn a_shortfall_does_not_fit() {
        let space = VacuumSpace {
            path: PathBuf::from("/"),
            needed_bytes: 10,
            free_bytes: 9,
        };
        assert!(!space.fits());
    }

    #[test]
    fn the_temp_dir_is_one_sqlite_can_write() {
        assert!(usable_temp_dir(&sqlite_temp_dir()));
    }
}