second-brain-core 0.13.0

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
use std::collections::BTreeSet;
use std::fs;
use std::path::Path;

use anyhow::{Context, Result, anyhow, ensure};

use super::device::sync_dir;
use super::store_key::{dir_of, remove_if_present, same_dir, store_file_in};
use super::{
    DeviceKeys, KeyStorage, NEXT_STORE_KEY_FILE, STORE_KEY_FILE, StoreKey, WRAPPED_STORE_KEY_FILE,
    WrapRefresh, create_device_keys, load_device_keys_in, read_store_key,
    refresh_wrapped_store_key, store_key_for, write_store_key,
};
use crate::sqlite_store::SqliteStore;
use crate::sqlite_store::encrypt_migration::{
    FileState, NotReopened, Rewrite, RewriteTo, db_file_state, remove_interrupted_rewrite,
};

#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum StoreEncryption {
    Encrypted,
    /// Plaintext, waiting for the one-time migration.
    Pending,
    /// Plaintext, and never migrated: the store is outside the data dir
    /// (a tool's copy, whose key files would sit in someone else's data dir),
    /// or the build has no SQLCipher.
    Plaintext,
}

#[non_exhaustive]
pub struct DeviceStore {
    pub store: SqliteStore,
    pub encryption: StoreEncryption,
}

/// Opens the daemon's store. A store in `data_dir` that does not exist yet
/// is created encrypted, generating the device keys and the store key
/// first; an encrypted store opens under its key from `data_dir`. A copy
/// left by an interrupted rewrite is dropped first.
pub fn open_device_store(
    db: &Path,
    data_dir: &Path,
    machine_id: String,
    storage: &KeyStorage,
) -> Result<DeviceStore> {
    remove_interrupted_rewrite(db)?;
    let in_data_dir = same_dir(dir_of(db), data_dir);
    let can_encrypt = in_data_dir && cfg!(feature = "encryption");
    match db_file_state(db)? {
        FileState::Encrypted => {
            let device = load_device_keys_in(data_dir, storage)?.ok_or_else(|| {
                anyhow!(
                    "{} is encrypted, but {} holds no device key to open its store key; \
                     restore the data dir from a backup with `sb restore`",
                    db.display(),
                    data_dir.display()
                )
            })?;
            let key = store_key_for(db, &device)?;
            Ok(DeviceStore {
                store: SqliteStore::open_with_key(db, machine_id, &key)?,
                encryption: StoreEncryption::Encrypted,
            })
        }
        FileState::Absent if can_encrypt => {
            let device = match load_device_keys_in(data_dir, storage)? {
                Some(device) => device,
                None => create_device_keys(data_dir, storage)?,
            };
            let key = pending_store_key(data_dir, &device)?;
            let store = SqliteStore::open_with_key(db, machine_id, &key)?;
            store.mark_revocations_covered(&store.rotation_causes()?)?;
            Ok(DeviceStore {
                store,
                encryption: StoreEncryption::Encrypted,
            })
        }
        _ => Ok(DeviceStore {
            store: SqliteStore::open(db, machine_id)?,
            encryption: if can_encrypt {
                StoreEncryption::Pending
            } else {
                StoreEncryption::Plaintext
            },
        }),
    }
}

/// The one-time migration of a plaintext store in `data_dir` to SQLCipher.
/// The store key is written before the rewrite, so a migration cut short
/// at any point reuses it on the next attempt.
pub fn encrypt_store(store: &SqliteStore, data_dir: &Path, device: &DeviceKeys) -> Result<Rewrite> {
    store_file_in(store, data_dir)?;
    ensure!(!store.is_encrypted(), "the store is already encrypted");
    let key = pending_store_key(data_dir, device)?;
    let covering = store.rotation_causes()?;
    let rewrite = store.rewrite_under_key(RewriteTo::Key(&key))?;
    if let Err(e) = store.record_encrypted(chrono::Utc::now().timestamp_micros(), rewrite.pause) {
        tracing::warn!("cannot record the encryption of the store: {e:#}");
    }
    after_new_key(store, data_dir, device, &key, &covering);
    Ok(rewrite)
}

/// `refresh_wrapped_store_key` with the key the store is open under.
pub fn refresh_store_key_wrap(
    store: &SqliteStore,
    data_dir: &Path,
    device: &DeviceKeys,
) -> Result<WrapRefresh> {
    store_file_in(store, data_dir)?;
    let key = store
        .store_key()
        .ok_or_else(|| anyhow!("a plaintext store has no store key to wrap"))?;
    refresh_wrapped_store_key(store, data_dir, device, &key)
}

/// Rewrites the store under a fresh store key, so a copy of the old key,
/// in a backup or a revoked device's hands, opens no later copy of it.
pub fn rotate_store_key(
    store: &SqliteStore,
    data_dir: &Path,
    device: &DeviceKeys,
) -> Result<Rewrite> {
    let db = store_file_in(store, data_dir)?;
    ensure!(
        store.is_encrypted(),
        "a plaintext store has no key to rotate"
    );
    settle_next_key(&db, data_dir, device)?;
    let key = StoreKey::generate()?;
    let covering = store.rotation_causes()?;
    write_store_key(data_dir, NEXT_STORE_KEY_FILE, &key, device)?;
    let rewrite = match store.rewrite_under_key(RewriteTo::Key(&key)) {
        Ok(rewrite) => rewrite,
        Err(e) => return Err(settle_failed_rotation(data_dir, e)),
    };
    promote_next_key(data_dir)?;
    after_new_key(store, data_dir, device, &key, &covering);
    Ok(rewrite)
}

/// A `store.key.next` left by an earlier rotation may be the only copy of
/// the key the file is under. It is settled as an open settles it, by the
/// file rather than the store's in-memory key (which keeps the old key after
/// a failed reopen), before a new rotation overwrites it.
fn settle_next_key(db: &Path, data_dir: &Path, device: &DeviceKeys) -> Result<()> {
    let next = data_dir.join(NEXT_STORE_KEY_FILE);
    if !fs::exists(&next).with_context(|| format!("checking {}", next.display()))? {
        return Ok(());
    }
    store_key_for(db, device).map(drop)
}

fn promote_next_key(data_dir: &Path) -> Result<()> {
    fs::rename(
        data_dir.join(NEXT_STORE_KEY_FILE),
        data_dir.join(STORE_KEY_FILE),
    )
    .context("promoting the new store key")?;
    sync_dir(data_dir)
}

/// A rewrite that renamed its copy into place before failing leaves the
/// store under the new key, which then lives only in `store.key.next`;
/// that file is promoted at the next open. Any other failure left the
/// store under its old key, so the new key goes.
fn settle_failed_rotation(data_dir: &Path, error: anyhow::Error) -> anyhow::Error {
    if error.downcast_ref::<NotReopened>().is_some() {
        return error;
    }
    match remove_if_present(&data_dir.join(NEXT_STORE_KEY_FILE)) {
        Ok(()) => error,
        Err(cleanup) => error.context(format!("{cleanup:#}")),
    }
}

/// `covering` is the revocation and retirement set read before the
/// rewrite, so one that lands during it still asks for another rotation.
fn after_new_key(
    store: &SqliteStore,
    data_dir: &Path,
    device: &DeviceKeys,
    key: &StoreKey,
    covering: &BTreeSet<String>,
) {
    if let Err(e) = store.mark_revocations_covered(covering) {
        tracing::warn!("cannot record which revocations the store key follows: {e:#}");
    }
    if let Err(e) = refresh_wrapped_store_key(store, data_dir, device, key) {
        tracing::warn!("cannot write {WRAPPED_STORE_KEY_FILE}: {e:#}");
    }
}

fn pending_store_key(data_dir: &Path, device: &DeviceKeys) -> Result<StoreKey> {
    if let Some(key) = read_store_key(&data_dir.join(STORE_KEY_FILE), device)? {
        return Ok(key);
    }
    let key = StoreKey::generate()?;
    write_store_key(data_dir, STORE_KEY_FILE, &key, device)?;
    Ok(key)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn a_rotation_that_failed_after_its_rename_keeps_the_new_key() {
        let dir = tempfile::tempdir().unwrap();
        let next = dir.path().join(NEXT_STORE_KEY_FILE);
        std::fs::write(&next, b"sealed").unwrap();
        let reopen = anyhow::Error::from(NotReopened(anyhow!("EMFILE")));
        let kept = settle_failed_rotation(dir.path(), reopen);
        assert!(kept.downcast_ref::<NotReopened>().is_some());
        assert!(next.exists(), "the only copy of the key the store is under");

        let before_rename = anyhow!("the copy failed its check");
        settle_failed_rotation(dir.path(), before_rename);
        assert!(!next.exists(), "the store is still under its old key");
    }

    #[cfg(feature = "encryption")]
    #[test]
    fn a_rotation_retried_after_a_failed_reopen_keeps_the_key_the_file_is_under() {
        use crate::schema::{Memory, MemoryType};
        use crate::sqlite_store::encrypt_migration::FAIL_REOPEN;
        use crate::store::Store;

        let dir = tempfile::tempdir().unwrap();
        let db = dir.path().join("brain.db");
        let open = || open_device_store(&db, dir.path(), "m".into(), &KeyStorage::File).unwrap();
        let opened = open();
        opened
            .store
            .store_memory(&Memory::new(
                "kept".to_string(),
                MemoryType::Semantic,
                "test".to_string(),
                String::new(),
            ))
            .unwrap();
        let device = load_device_keys_in(dir.path(), &KeyStorage::File)
            .unwrap()
            .unwrap();

        FAIL_REOPEN.set(true);
        let failed = rotate_store_key(&opened.store, dir.path(), &device).unwrap_err();
        FAIL_REOPEN.set(false);
        assert!(failed.downcast_ref::<NotReopened>().is_some(), "{failed:#}");
        let _retried = rotate_store_key(&opened.store, dir.path(), &device);
        drop(opened);

        assert_eq!(open().store.memory_count().unwrap(), 1);
    }
}