second-brain-core 0.13.5

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
use std::io::{self, ErrorKind};
use std::path::{Path, PathBuf};

pub const PRIVATE_DIR_MODE: u32 = 0o700;
pub const PRIVATE_FILE_MODE: u32 = 0o600;

/// Creates one directory, failing if it exists, with `PRIVATE_DIR_MODE`
/// from the start rather than tightened after.
pub fn create_private_dir(dir: &Path) -> anyhow::Result<()> {
    let mut builder = std::fs::DirBuilder::new();
    #[cfg(unix)]
    std::os::unix::fs::DirBuilderExt::mode(&mut builder, PRIVATE_DIR_MODE);
    anyhow::Context::with_context(builder.create(dir), || {
        format!("creating {}", dir.display())
    })
}

/// Calls `visit` with every existing path the daemon keeps private and the
/// mode it must have: the data dir, the API token, the sync key, the device
/// key in either its plain or sealed file, the database with its WAL and SHM
/// files, and every directory and file in the index tree, in the raw
/// transcript index tree and in the cold tier beside the database.
/// A directory is visited before its entries are read, so a visitor may make
/// it readable first; symlinks inside the index tree are skipped. Returns the
/// paths that could not be inspected.
pub fn visit_private_paths(
    data_dir: &Path,
    db_path: &Path,
    index_dir: &Path,
    mut visit: impl FnMut(&Path, u32),
) -> Vec<(PathBuf, io::Error)> {
    let mut errors = Vec::new();
    visit_if_present(data_dir, PRIVATE_DIR_MODE, &mut visit, &mut errors);
    for name in [
        "api.token",
        "sync.key",
        crate::keys::DEVICE_KEY_FILE,
        crate::keys::SEALED_DEVICE_KEY_FILE,
    ] {
        visit_if_present(
            &data_dir.join(name),
            PRIVATE_FILE_MODE,
            &mut visit,
            &mut errors,
        );
    }
    for suffix in ["", "-wal", "-shm"] {
        let mut path = db_path.as_os_str().to_owned();
        path.push(suffix);
        visit_if_present(Path::new(&path), PRIVATE_FILE_MODE, &mut visit, &mut errors);
    }
    visit_tree(index_dir, &mut visit, &mut errors);
    visit_tree(
        &crate::index::raw::dir_for_db(db_path),
        &mut visit,
        &mut errors,
    );
    visit_tree(&crate::cold::dir_for_db(db_path), &mut visit, &mut errors);
    errors
}

fn visit_if_present(
    path: &Path,
    mode: u32,
    visit: &mut impl FnMut(&Path, u32),
    errors: &mut Vec<(PathBuf, io::Error)>,
) {
    match std::fs::metadata(path) {
        Ok(_) => visit(path, mode),
        Err(e) if e.kind() == ErrorKind::NotFound => {}
        Err(e) => errors.push((path.to_path_buf(), e)),
    }
}

fn visit_tree(
    dir: &Path,
    visit: &mut impl FnMut(&Path, u32),
    errors: &mut Vec<(PathBuf, io::Error)>,
) {
    match std::fs::metadata(dir) {
        Ok(meta) if meta.is_dir() => {}
        Ok(_) => return,
        Err(e) if e.kind() == ErrorKind::NotFound => return,
        Err(e) => return errors.push((dir.to_path_buf(), e)),
    }
    visit(dir, PRIVATE_DIR_MODE);
    let entries = match std::fs::read_dir(dir) {
        Ok(entries) => entries,
        Err(e) => return errors.push((dir.to_path_buf(), e)),
    };
    for entry in entries {
        let entry = match entry {
            Ok(entry) => entry,
            Err(e) => {
                errors.push((dir.to_path_buf(), e));
                continue;
            }
        };
        let path = entry.path();
        match entry.file_type() {
            Ok(t) if t.is_dir() => visit_tree(&path, visit, errors),
            Ok(t) if t.is_file() => visit(&path, PRIVATE_FILE_MODE),
            Ok(_) => {}
            Err(e) => errors.push((path, e)),
        }
    }
}

#[cfg(test)]
mod tests {
    use std::collections::BTreeSet;
    use std::os::unix::fs::PermissionsExt;

    use super::*;

    fn touch(p: &Path) {
        std::fs::write(p, "x").unwrap();
    }

    fn set(p: &Path, mode: u32) {
        std::fs::set_permissions(p, std::fs::Permissions::from_mode(mode)).unwrap();
    }

    fn collect(data: &Path, db: &Path, index: &Path) -> BTreeSet<(PathBuf, u32)> {
        let mut seen = BTreeSet::new();
        let errors = visit_private_paths(data, db, index, |path, mode| {
            seen.insert((path.to_path_buf(), mode));
        });
        assert!(errors.is_empty(), "{errors:?}");
        seen
    }

    fn running_as_root() -> bool {
        unsafe { libc::getuid() == 0 }
    }

    #[test]
    fn visits_every_existing_private_path_with_its_mode() {
        let root = tempfile::tempdir().unwrap();
        let data = root.path().join(".second-brain");
        let index = data.join("brain.indexes");
        let nested = index.join("tantivy");
        std::fs::create_dir_all(&nested).unwrap();
        for name in [
            "api.token",
            "device.key",
            "device.key.cred",
            "brain.db",
            "brain.db-wal",
            "config.toml",
        ] {
            touch(&data.join(name));
        }
        touch(&index.join("vectors.usearch"));
        touch(&nested.join("meta.json"));

        let seen = collect(&data, &data.join("brain.db"), &index);

        let expected: BTreeSet<(PathBuf, u32)> = [
            (data.clone(), PRIVATE_DIR_MODE),
            (data.join("api.token"), PRIVATE_FILE_MODE),
            (data.join("device.key"), PRIVATE_FILE_MODE),
            (data.join("device.key.cred"), PRIVATE_FILE_MODE),
            (data.join("brain.db"), PRIVATE_FILE_MODE),
            (data.join("brain.db-wal"), PRIVATE_FILE_MODE),
            (index.clone(), PRIVATE_DIR_MODE),
            (index.join("vectors.usearch"), PRIVATE_FILE_MODE),
            (nested.clone(), PRIVATE_DIR_MODE),
            (nested.join("meta.json"), PRIVATE_FILE_MODE),
        ]
        .into_iter()
        .collect();
        assert_eq!(seen, expected);
    }

    #[test]
    fn a_db_and_indexes_outside_the_data_dir_are_visited() {
        let root = tempfile::tempdir().unwrap();
        let data = root.path().join("data");
        let elsewhere = root.path().join("elsewhere");
        std::fs::create_dir_all(&data).unwrap();
        std::fs::create_dir_all(&elsewhere).unwrap();
        let db = elsewhere.join("custom.db");
        touch(&db);
        touch(&elsewhere.join("custom.db-shm"));
        let index = elsewhere.join("custom.indexes");
        std::fs::create_dir(&index).unwrap();
        touch(&index.join("f"));
        let raw = elsewhere.join("custom.raw-index");
        let raw_text = raw.join("text");
        std::fs::create_dir_all(&raw_text).unwrap();
        touch(&raw.join("segment-0.usearch"));
        touch(&raw_text.join("meta.json"));
        let cold = elsewhere.join("custom.cold");
        std::fs::create_dir(&cold).unwrap();
        touch(&cold.join("segment.seg"));

        let seen = collect(&data, &db, &index);

        let expected: BTreeSet<(PathBuf, u32)> = [
            (data.clone(), PRIVATE_DIR_MODE),
            (db.clone(), PRIVATE_FILE_MODE),
            (elsewhere.join("custom.db-shm"), PRIVATE_FILE_MODE),
            (index.clone(), PRIVATE_DIR_MODE),
            (index.join("f"), PRIVATE_FILE_MODE),
            (raw.clone(), PRIVATE_DIR_MODE),
            (raw.join("segment-0.usearch"), PRIVATE_FILE_MODE),
            (raw_text.clone(), PRIVATE_DIR_MODE),
            (raw_text.join("meta.json"), PRIVATE_FILE_MODE),
            (cold.clone(), PRIVATE_DIR_MODE),
            (cold.join("segment.seg"), PRIVATE_FILE_MODE),
        ]
        .into_iter()
        .collect();
        assert_eq!(seen, expected);
    }

    #[test]
    fn nothing_is_visited_when_the_install_is_absent() {
        let root = tempfile::tempdir().unwrap();
        let data = root.path().join("absent");

        let seen = collect(&data, &data.join("brain.db"), &data.join("brain.indexes"));

        assert!(seen.is_empty(), "{seen:?}");
    }

    #[test]
    fn a_fresh_data_dir_is_the_only_path_visited() {
        let root = tempfile::tempdir().unwrap();
        let data = root.path().join(".second-brain");
        std::fs::create_dir(&data).unwrap();

        let seen = collect(&data, &data.join("brain.db"), &data.join("brain.indexes"));

        assert_eq!(seen, BTreeSet::from([(data.clone(), PRIVATE_DIR_MODE)]));
    }

    #[test]
    fn symlinks_in_the_index_are_not_visited() {
        let root = tempfile::tempdir().unwrap();
        let outside = root.path().join("outside.txt");
        touch(&outside);
        let index = root.path().join("brain.indexes");
        std::fs::create_dir(&index).unwrap();
        std::os::unix::fs::symlink(&outside, index.join("link")).unwrap();

        let seen = collect(root.path(), &root.path().join("brain.db"), &index);

        assert!(
            seen.contains(&(index.clone(), PRIVATE_DIR_MODE)),
            "{seen:?}"
        );
        assert!(
            !seen
                .iter()
                .any(|(p, _)| p == &index.join("link") || p == &outside),
            "{seen:?}"
        );
    }

    #[test]
    fn a_directory_is_visited_before_its_entries_are_read() {
        if running_as_root() {
            return;
        }
        let root = tempfile::tempdir().unwrap();
        let index = root.path().join("brain.indexes");
        let locked = index.join("tantivy");
        std::fs::create_dir_all(&locked).unwrap();
        touch(&locked.join("meta.json"));
        set(&locked, 0o300);

        let mut seen = BTreeSet::new();
        let errors = visit_private_paths(
            root.path(),
            &root.path().join("brain.db"),
            &index,
            |path, mode| {
                if mode == PRIVATE_DIR_MODE {
                    set(path, PRIVATE_DIR_MODE);
                }
                seen.insert(path.to_path_buf());
            },
        );

        set(&locked, 0o700);
        assert!(errors.is_empty(), "{errors:?}");
        assert!(seen.contains(&locked.join("meta.json")), "{seen:?}");
    }

    #[test]
    fn an_unreadable_index_dir_is_reported_and_still_visited() {
        if running_as_root() {
            return;
        }
        let root = tempfile::tempdir().unwrap();
        let index = root.path().join("brain.indexes");
        let locked = index.join("tantivy");
        std::fs::create_dir_all(&locked).unwrap();
        set(&locked, 0o000);

        let mut seen = BTreeSet::new();
        let errors = visit_private_paths(
            root.path(),
            &root.path().join("brain.db"),
            &index,
            |path, _| {
                seen.insert(path.to_path_buf());
            },
        );

        set(&locked, 0o700);
        assert!(seen.contains(&locked), "{seen:?}");
        assert_eq!(
            errors.iter().map(|(p, _)| p.clone()).collect::<Vec<_>>(),
            vec![locked.clone()],
            "{errors:?}"
        );
    }
}