second-brain-core 0.13.0

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
use anyhow::{Context, Result, anyhow};
use hkdf::Hkdf;
use rusqlite::{Connection, OptionalExtension, params};
use serde::{Deserialize, Serialize};
use sha2::Sha256;

use super::SqliteStore;
use crate::keys::{KeyGuard, StoreKey, hex};
use crate::sealed::{SealKey, SealKeyId, SealKeys};

/// Every at-rest key row starts with this, so a plaintext copy of the
/// store can drop them.
pub(crate) const ROW_PREFIX: &str = "at_rest_key:";
const STORE_KEY_ID_INFO: &[u8] = b"second-brain at-rest store key id v1";

/// What a set of files beside the store is sealed for. Each purpose has
/// its own keys, so each owner reseals its files on its own schedule.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum AtRestPurpose {
    Indexes,
    RawIndex,
    TranscriptArchive,
}

impl AtRestPurpose {
    fn row(self) -> &'static str {
        match self {
            Self::Indexes => "at_rest_key:indexes",
            Self::RawIndex => "at_rest_key:raw_index",
            Self::TranscriptArchive => "at_rest_key:transcript_archive",
        }
    }
}

/// The keys for one purpose. Until `settled`, files may still be plaintext
/// or sealed under a retired key, and their owner reseals them, then calls
/// `SqliteStore::settle_at_rest_keys`.
#[derive(Clone, Debug)]
pub struct AtRestKeys {
    pub keys: SealKeys,
    pub settled: bool,
}

#[derive(Serialize, Deserialize)]
struct Stored {
    key: String,
    store_key: String,
    #[serde(default)]
    retired: Vec<String>,
    settled: bool,
}

fn store_key_id(key: &StoreKey) -> Result<String> {
    let mut id = [0u8; 8];
    Hkdf::<Sha256>::new(None, key.expose())
        .expand(STORE_KEY_ID_INFO, &mut id)
        .map_err(|e| anyhow!("deriving the store key id: {e}"))?;
    Ok(hex(&id))
}

fn seal_key(text: &str) -> Result<SealKey> {
    let bytes = crate::keys::unhex::<32>(text)
        .ok_or_else(|| anyhow!("an at-rest key is not 64 hex digits"))?;
    SealKey::from_bytes(KeyGuard::new(bytes))
}

impl Stored {
    fn keys(&self) -> Result<SealKeys> {
        let retired = self
            .retired
            .iter()
            .map(|k| seal_key(k))
            .collect::<Result<Vec<_>>>()?;
        Ok(SealKeys::new(seal_key(&self.key)?, retired))
    }
}

fn read_stored(conn: &Connection, purpose: AtRestPurpose) -> Result<Option<Stored>> {
    let stored: Option<String> = conn
        .query_row(
            "SELECT value FROM index_meta WHERE key = ?1",
            params![purpose.row()],
            |r| r.get(0),
        )
        .optional()?;
    stored
        .map(|text| serde_json::from_str(&text))
        .transpose()
        .with_context(|| format!("reading {}", purpose.row()))
}

impl SqliteStore {
    /// The keys files for `purpose` are sealed under; None for a store with
    /// no store key, whose files stay plaintext. The first call mints them.
    /// Once the store key has changed since the current key was minted (a
    /// rotation, or the store's first encryption), a new current key is
    /// minted and the old one is kept as retired until the files are
    /// resealed, so a copy of the old store key opens none of the files
    /// written after that.
    pub fn at_rest_keys(&self, purpose: AtRestPurpose) -> Result<Option<AtRestKeys>> {
        let Some(store_key) = self.store_key() else {
            return Ok(None);
        };
        let store_key = store_key_id(&store_key)?;
        if let Some(stored) = read_stored(&self.conn(), purpose)?
            && stored.store_key == store_key
        {
            return Ok(Some(AtRestKeys {
                keys: stored.keys()?,
                settled: stored.settled,
            }));
        }
        let next = self.with_durable_txn(|tx| {
            let stored = read_stored(tx, purpose)?;
            if let Some(stored) = stored.as_ref().filter(|s| s.store_key == store_key) {
                return Ok(Stored {
                    key: stored.key.clone(),
                    store_key: stored.store_key.clone(),
                    retired: stored.retired.clone(),
                    settled: stored.settled,
                });
            }
            let minted = SealKey::generate()?;
            let mut retired = Vec::new();
            if let Some(old) = stored {
                retired.push(old.key);
                retired.extend(old.retired);
            }
            let next = Stored {
                key: hex(minted.expose()),
                store_key: store_key.clone(),
                retired,
                settled: false,
            };
            tx.execute(
                "INSERT OR REPLACE INTO index_meta (key, value) VALUES (?1, ?2)",
                params![purpose.row(), serde_json::to_string(&next)?],
            )?;
            Ok(next)
        })?;
        Ok(Some(AtRestKeys {
            keys: next.keys()?,
            settled: next.settled,
        }))
    }

    /// Records that every file for `purpose` is sealed under `current`, and
    /// forgets the retired keys. Nothing changes if `current` is no longer
    /// the current key.
    pub fn settle_at_rest_keys(&self, purpose: AtRestPurpose, current: SealKeyId) -> Result<()> {
        let conn = self.conn();
        let Some(mut stored) = read_stored(&conn, purpose)? else {
            return Ok(());
        };
        if seal_key(&stored.key)?.id() != current {
            return Ok(());
        }
        stored.retired.clear();
        stored.settled = true;
        conn.execute(
            "UPDATE index_meta SET value = ?2 WHERE key = ?1",
            params![purpose.row(), serde_json::to_string(&stored)?],
        )?;
        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn a_store_without_a_store_key_seals_nothing() {
        let store = SqliteStore::in_memory("m".to_string()).unwrap();
        assert!(
            store
                .at_rest_keys(AtRestPurpose::Indexes)
                .unwrap()
                .is_none()
        );
    }

    #[cfg(feature = "encryption")]
    mod keyed {
        use super::*;
        use crate::sqlite_store::encrypt_migration::RewriteTo;

        fn keyed_store(dir: &std::path::Path) -> SqliteStore {
            let key = StoreKey::generate().unwrap();
            SqliteStore::open_with_key(&dir.join("brain.db"), "m".to_string(), &key).unwrap()
        }

        #[test]
        fn a_plaintext_copy_of_the_store_holds_no_at_rest_key() {
            let dir = tempfile::tempdir().unwrap();
            let store = keyed_store(dir.path());
            store.at_rest_keys(AtRestPurpose::Indexes).unwrap().unwrap();
            store
                .at_rest_keys(AtRestPurpose::RawIndex)
                .unwrap()
                .unwrap();
            let copy = store.write_plaintext_copy(dir.path()).unwrap();
            let conn = rusqlite::Connection::open(&copy).unwrap();
            let rows: i64 = conn
                .query_row(
                    "SELECT COUNT(*) FROM index_meta WHERE key LIKE 'at_rest_key:%'",
                    [],
                    |r| r.get(0),
                )
                .unwrap();
            assert_eq!(rows, 0);
        }

        #[test]
        fn keys_are_minted_once_and_kept_per_purpose() {
            let dir = tempfile::tempdir().unwrap();
            let store = keyed_store(dir.path());
            let first = store.at_rest_keys(AtRestPurpose::Indexes).unwrap().unwrap();
            assert!(!first.settled);
            assert!(first.keys.retired_ids().is_empty());
            let again = store.at_rest_keys(AtRestPurpose::Indexes).unwrap().unwrap();
            assert_eq!(first.keys.current_id(), again.keys.current_id());
            let other = store
                .at_rest_keys(AtRestPurpose::RawIndex)
                .unwrap()
                .unwrap();
            assert_ne!(first.keys.current_id(), other.keys.current_id());

            store
                .settle_at_rest_keys(AtRestPurpose::Indexes, first.keys.current_id())
                .unwrap();
            assert!(
                store
                    .at_rest_keys(AtRestPurpose::Indexes)
                    .unwrap()
                    .unwrap()
                    .settled
            );
            assert!(
                !store
                    .at_rest_keys(AtRestPurpose::RawIndex)
                    .unwrap()
                    .unwrap()
                    .settled
            );
        }

        #[test]
        fn a_new_store_key_mints_a_new_key_and_retires_the_old_until_settled() {
            let dir = tempfile::tempdir().unwrap();
            let store = keyed_store(dir.path());
            let before = store.at_rest_keys(AtRestPurpose::Indexes).unwrap().unwrap();
            store
                .settle_at_rest_keys(AtRestPurpose::Indexes, before.keys.current_id())
                .unwrap();

            let rotated = StoreKey::generate().unwrap();
            store.rewrite_under_key(RewriteTo::Key(&rotated)).unwrap();
            let after = store.at_rest_keys(AtRestPurpose::Indexes).unwrap().unwrap();
            assert_ne!(after.keys.current_id(), before.keys.current_id());
            assert_eq!(after.keys.retired_ids(), vec![before.keys.current_id()]);
            assert!(!after.settled);

            store
                .settle_at_rest_keys(AtRestPurpose::Indexes, before.keys.current_id())
                .unwrap();
            assert!(
                !store
                    .at_rest_keys(AtRestPurpose::Indexes)
                    .unwrap()
                    .unwrap()
                    .settled
            );
            store
                .settle_at_rest_keys(AtRestPurpose::Indexes, after.keys.current_id())
                .unwrap();
            let settled = store.at_rest_keys(AtRestPurpose::Indexes).unwrap().unwrap();
            assert!(settled.settled);
            assert!(settled.keys.retired_ids().is_empty());
        }
    }
}