second-brain-core 0.13.3

Core library for second-brain: SQLite storage with vector and lexical indexes, BGE embeddings, and hybrid retrieval
Documentation
use std::fmt;

use age::x25519::{Identity, Recipient};
use anyhow::{Result, anyhow, bail};
use hkdf::Hkdf;
use rusqlite::{OptionalExtension, params};
use sha2::{Digest, Sha256};
use zeroize::Zeroizing;

use super::tables::query_envelopes;
use super::{Envelope, Epoch, IdentityKey, IdentityTag, KeyGuard, hex, wrap};
use crate::sqlite_store::SqliteStore;

const ENTROPY_LEN: usize = 16;
const CHECKSUM_LEN: usize = 4;
const ENCODED_LEN: usize = (ENTROPY_LEN + CHECKSUM_LEN) * 8 / 5;
const GROUP_LEN: usize = 4;
const ALPHABET: &[u8; 32] = b"0123456789ABCDEFGHJKMNPQRSTVWXYZ";
const SECRET_KEY_HRP: &str = "age-secret-key-";
const IDENTITY_SALT: &[u8] = b"second-brain recovery phrase v1";
const IDENTITY_INFO: &[u8] = b"x25519 recipient";
const CHECKSUM_DOMAIN: &[u8] = b"second-brain recovery checksum v1";
const FINGERPRINT_DOMAIN: &[u8] = b"second-brain recovery fingerprint v1";
const OFFERED_KEY: &str = "recovery_offered_at";

/// A generated recovery secret of 128 bits. It only decrypts: one X25519
/// recipient derives from it, and every epoch a device holds is wrapped to
/// that recipient. It is never stored; `Debug` shows only its fingerprint.
pub struct RecoveryPhrase(KeyGuard<[u8; ENTROPY_LEN]>);

impl RecoveryPhrase {
    pub fn generate() -> Result<Self> {
        Ok(Self(KeyGuard::random()?))
    }

    /// Accepts the phrase in any case, with or without spaces and dashes,
    /// reading O as 0 and I or L as 1. A wrong checksum is reported as a
    /// typo, never as a different key.
    pub fn parse(text: &str) -> Result<Self> {
        let mut values = Zeroizing::new(Vec::with_capacity(ENCODED_LEN));
        for c in text.chars() {
            if c.is_whitespace() || c == '-' {
                continue;
            }
            let c = match c.to_ascii_uppercase() {
                'O' => '0',
                'I' | 'L' => '1',
                other => other,
            };
            let Some(value) = ALPHABET.iter().position(|&a| char::from(a) == c) else {
                bail!("a recovery phrase holds only letters and digits; {c:?} is not one");
            };
            values.push(value as u8);
        }
        if values.len() != ENCODED_LEN {
            bail!(
                "a recovery phrase has {ENCODED_LEN} letters and digits, this one has {}",
                values.len()
            );
        }
        let bytes = unpack(&values);
        let mut entropy = KeyGuard::new([0u8; ENTROPY_LEN]);
        entropy.expose_mut().copy_from_slice(&bytes[..ENTROPY_LEN]);
        if bytes[ENTROPY_LEN..] != checksum(entropy.expose()) {
            bail!("the recovery phrase has a typo: its check letters do not match");
        }
        Ok(Self(entropy))
    }

    /// The phrase as shown to the user: groups of four separated by dashes.
    pub fn expose_text(&self) -> Zeroizing<String> {
        let mut bytes = Zeroizing::new(Vec::with_capacity(ENTROPY_LEN + CHECKSUM_LEN));
        bytes.extend_from_slice(self.0.expose());
        bytes.extend_from_slice(&checksum(self.0.expose()));
        let values = pack(&bytes);
        let mut text = Zeroizing::new(String::with_capacity(ENCODED_LEN + ENCODED_LEN / GROUP_LEN));
        for (i, value) in values.iter().enumerate() {
            if i > 0 && i % GROUP_LEN == 0 {
                text.push('-');
            }
            text.push(char::from(ALPHABET[usize::from(*value)]));
        }
        text
    }

    pub fn identity(&self) -> Result<Identity> {
        let mut secret = Zeroizing::new([0u8; 32]);
        Hkdf::<Sha256>::new(Some(IDENTITY_SALT), self.0.expose())
            .expand(IDENTITY_INFO, secret.as_mut())
            .map_err(|e| anyhow!("deriving the recovery key: {e}"))?;
        let hrp =
            bech32::Hrp::parse(SECRET_KEY_HRP).map_err(|e| anyhow!("recovery key prefix: {e}"))?;
        let encoded = Zeroizing::new(
            bech32::encode_upper::<bech32::Bech32>(hrp, secret.as_ref())
                .map_err(|e| anyhow!("encoding the recovery key: {e}"))?,
        );
        encoded
            .parse::<Identity>()
            .map_err(|e| anyhow!("building the recovery key: {e}"))
    }

    pub fn recipient(&self) -> Result<Recipient> {
        Ok(self.identity()?.to_public())
    }

    pub fn fingerprint(&self) -> Result<String> {
        Ok(recovery_fingerprint(&self.recipient()?.to_string()))
    }
}

impl fmt::Debug for RecoveryPhrase {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self.fingerprint() {
            Ok(fingerprint) => write!(f, "RecoveryPhrase({fingerprint})"),
            Err(_) => f.write_str("RecoveryPhrase(..)"),
        }
    }
}

/// The 16 hex characters a device lists in its signed row for a recovery
/// recipient, so the user can tell phrases apart without seeing them.
pub fn recovery_fingerprint(recipient: &str) -> String {
    let digest = Sha256::new()
        .chain_update(FINGERPRINT_DOMAIN)
        .chain_update(recipient.as_bytes())
        .finalize();
    hex(&digest[..8])
}

fn checksum(entropy: &[u8; ENTROPY_LEN]) -> [u8; CHECKSUM_LEN] {
    let digest = Sha256::new()
        .chain_update(CHECKSUM_DOMAIN)
        .chain_update(entropy)
        .finalize();
    let mut out = [0u8; CHECKSUM_LEN];
    out.copy_from_slice(&digest[..CHECKSUM_LEN]);
    out
}

fn pack(bytes: &[u8]) -> Zeroizing<Vec<u8>> {
    let mut out = Zeroizing::new(Vec::with_capacity(bytes.len() * 8 / 5));
    let mut acc: u16 = 0;
    let mut bits = 0;
    for &byte in bytes {
        acc = (acc << 8) | u16::from(byte);
        bits += 8;
        while bits >= 5 {
            bits -= 5;
            out.push(((acc >> bits) & 0x1f) as u8);
        }
        acc &= (1 << bits) - 1;
    }
    out
}

fn unpack(values: &[u8]) -> Zeroizing<Vec<u8>> {
    let mut out = Zeroizing::new(Vec::with_capacity(values.len() * 5 / 8));
    let mut acc: u16 = 0;
    let mut bits = 0;
    for &value in values {
        acc = (acc << 5) | u16::from(value);
        bits += 5;
        if bits >= 8 {
            bits -= 8;
            out.push((acc >> bits) as u8);
        }
        acc &= (1 << bits) - 1;
    }
    out
}

/// Wraps every held epoch to each recipient that lacks an envelope for it
/// and is not retired, under `tag`, with no machine id. Returns how many
/// envelopes were added.
pub fn wrap_for_recovery(
    store: &SqliteStore,
    tag: &IdentityTag,
    held: &[(Epoch, &IdentityKey)],
    recipients: &[Recipient],
) -> Result<usize> {
    let retired = store.retired_fingerprints()?;
    let mut added = 0;
    for recipient in recipients {
        let text = recipient.to_string();
        if retired.contains(&recovery_fingerprint(&text)) {
            continue;
        }
        for (epoch, key) in held {
            if store.envelope(tag, epoch, &text)?.is_some() {
                continue;
            }
            if store.put_envelope(&wrap(key, epoch.number, tag, recipient, None)?)? {
                added += 1;
            }
        }
    }
    Ok(added)
}

impl SqliteStore {
    /// The recovery recipients this device wraps `tag`'s epochs to: the
    /// recipients of its envelopes that name no machine, less retired ones.
    /// Handshakes only store envelopes addressed to this machine, so a peer
    /// never adds one.
    pub fn recovery_recipients(&self, tag: &IdentityTag) -> Result<Vec<Recipient>> {
        let retired = self.retired_fingerprints()?;
        let conn = self.reader();
        let mut stmt = conn.prepare(
            "SELECT DISTINCT recipient FROM key_envelopes \
             WHERE identity_tag = ?1 AND machine_id IS NULL ORDER BY recipient",
        )?;
        let texts = stmt
            .query_map(params![tag.as_str()], |r| r.get::<_, String>(0))?
            .collect::<rusqlite::Result<Vec<_>>>()?;
        texts
            .iter()
            .filter(|text| !retired.contains(&recovery_fingerprint(text)))
            .map(|text| {
                text.parse::<Recipient>()
                    .map_err(|e| anyhow!("recovery recipient {text} is invalid: {e}"))
            })
            .collect()
    }

    /// Every envelope, under any identity, wrapped to `recipient` with no
    /// machine id.
    pub fn recovery_envelopes(&self, recipient: &Recipient) -> Result<Vec<Envelope>> {
        let conn = self.reader();
        query_envelopes(
            &conn,
            "recipient = ?1 AND machine_id IS NULL",
            params![recipient.to_string()],
        )
    }

    pub fn recovery_offered(&self) -> Result<bool> {
        let value: Option<String> = self
            .reader()
            .query_row(
                "SELECT value FROM index_meta WHERE key = ?1",
                params![OFFERED_KEY],
                |r| r.get(0),
            )
            .optional()?;
        Ok(value.is_some())
    }

    pub fn mark_recovery_offered(&self, now_micros: i64) -> Result<()> {
        self.conn().execute(
            "INSERT OR IGNORE INTO index_meta (key, value) VALUES (?1, ?2)",
            params![OFFERED_KEY, now_micros.to_string()],
        )?;
        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn a_phrase_reads_back_to_the_same_recipient() {
        let phrase = RecoveryPhrase::generate().unwrap();
        let text = phrase.expose_text();
        assert_eq!(text.len(), ENCODED_LEN + ENCODED_LEN / GROUP_LEN - 1);
        assert_eq!(text.split('-').count(), ENCODED_LEN / GROUP_LEN);
        let parsed = RecoveryPhrase::parse(&text).unwrap();
        assert_eq!(
            parsed.recipient().unwrap().to_string(),
            phrase.recipient().unwrap().to_string()
        );
        assert_eq!(parsed.fingerprint().unwrap(), phrase.fingerprint().unwrap());
    }

    #[test]
    fn a_phrase_is_read_in_any_case_and_spacing_with_lookalike_letters() {
        let phrase = RecoveryPhrase::generate().unwrap();
        let text = phrase.expose_text();
        let loose: String = text
            .chars()
            .filter(|c| *c != '-')
            .map(|c| match c {
                '0' => 'o',
                '1' => 'l',
                other => other.to_ascii_lowercase(),
            })
            .collect::<Vec<_>>()
            .chunks(5)
            .map(|chunk| chunk.iter().collect::<String>())
            .collect::<Vec<_>>()
            .join("  ");
        let parsed = RecoveryPhrase::parse(&loose).unwrap();
        assert_eq!(parsed.fingerprint().unwrap(), phrase.fingerprint().unwrap());
    }

    #[test]
    fn a_typo_is_reported_as_a_typo() {
        let phrase = RecoveryPhrase::generate().unwrap();
        let mut text: Vec<char> = phrase.expose_text().chars().collect();
        text[0] = if text[0] == 'A' { 'B' } else { 'A' };
        let typo: String = text.into_iter().collect();
        let err = format!("{:#}", RecoveryPhrase::parse(&typo).unwrap_err());
        assert!(err.contains("typo"), "{err}");

        let short = &phrase.expose_text()[..10];
        let err = format!("{:#}", RecoveryPhrase::parse(short).unwrap_err());
        assert!(err.contains("letters and digits"), "{err}");
        let err = format!("{:#}", RecoveryPhrase::parse("U!").unwrap_err());
        assert!(err.contains("only letters"), "{err}");
    }

    #[test]
    fn phrases_differ_and_debug_shows_only_the_fingerprint() {
        let a = RecoveryPhrase::generate().unwrap();
        let b = RecoveryPhrase::generate().unwrap();
        assert_ne!(a.fingerprint().unwrap(), b.fingerprint().unwrap());
        let shown = format!("{a:?}");
        assert!(shown.contains(&a.fingerprint().unwrap()), "{shown}");
        assert!(!shown.contains(a.expose_text().as_str()), "{shown}");
        assert_eq!(a.fingerprint().unwrap().len(), 16);
    }

    #[test]
    fn held_epochs_are_wrapped_once_to_each_recipient_and_open_with_the_phrase() {
        let store = SqliteStore::in_memory("local".to_string()).unwrap();
        let first = IdentityKey::generate().unwrap();
        let second = IdentityKey::generate().unwrap();
        let tag = IdentityTag::migrated(&first.key_id());
        let epoch = |number, key: &IdentityKey| Epoch {
            number,
            key_id: key.key_id(),
        };
        let phrase = RecoveryPhrase::generate().unwrap();
        let recipients = [phrase.recipient().unwrap()];

        let held = [(epoch(0, &first), &first)];
        assert_eq!(
            wrap_for_recovery(&store, &tag, &held, &recipients).unwrap(),
            1
        );
        let held = [(epoch(0, &first), &first), (epoch(1, &second), &second)];
        assert_eq!(
            wrap_for_recovery(&store, &tag, &held, &recipients).unwrap(),
            1
        );
        assert_eq!(
            wrap_for_recovery(&store, &tag, &held, &recipients).unwrap(),
            0
        );

        let found = store.recovery_recipients(&tag).unwrap();
        assert_eq!(found.len(), 1);
        assert_eq!(found[0].to_string(), recipients[0].to_string());
        let envelopes = store.recovery_envelopes(&recipients[0]).unwrap();
        assert_eq!(envelopes.len(), 2);
        let reparsed = RecoveryPhrase::parse(&phrase.expose_text()).unwrap();
        let identity = reparsed.identity().unwrap();
        for envelope in &envelopes {
            assert_eq!(envelope.machine_id, None);
            envelope.open(&identity).unwrap();
        }
        assert!(store.envelopes_for_machine("local").unwrap().is_empty());
    }

    #[test]
    fn envelopes_for_a_machine_are_not_recovery_recipients() {
        let store = SqliteStore::in_memory("local".to_string()).unwrap();
        let key = IdentityKey::generate().unwrap();
        let tag = IdentityTag::migrated(&key.key_id());
        let device = super::super::DeviceKeys::generate().unwrap();
        store
            .put_envelope(
                &wrap(&key, 0, &tag, &device.age_recipient(), Some("local".into())).unwrap(),
            )
            .unwrap();
        assert!(store.recovery_recipients(&tag).unwrap().is_empty());
    }

    #[test]
    fn the_offer_is_remembered() {
        let store = SqliteStore::in_memory("local".to_string()).unwrap();
        assert!(!store.recovery_offered().unwrap());
        store.mark_recovery_offered(5).unwrap();
        store.mark_recovery_offered(9).unwrap();
        assert!(store.recovery_offered().unwrap());
    }
}