second-brain-sync 0.13.3

Bidirectional sync for second-brain: SSH transport, JSONL change log, conflict resolution
Documentation
use std::fs::{self, OpenOptions};
use std::io::{ErrorKind, Write};
use std::os::unix::ffi::OsStrExt;
use std::os::unix::fs::{DirBuilderExt, MetadataExt, OpenOptionsExt};
use std::path::{Path, PathBuf};

use anyhow::{Context, Result, bail};
use base64::Engine;
use base64::engine::general_purpose::STANDARD;
use ed25519_dalek::SigningKey;
use rand_core::{OsRng, RngCore};
use sha2::{Digest, Sha256};

use crate::ssh::write_atomic;

const KEY_DIR_PREFIX: &str = "second-brain-ssh-";
const KEY_FILE: &str = "id";
const CREATE_ATTEMPTS: usize = 3;

/// This device's ssh private key in a 0600 file, for one ssh transport to
/// pass to `ssh -i`, removed with its directory on drop. Each identity gets a
/// fresh 0700 directory with a random name under `$XDG_RUNTIME_DIR` or
/// `/dev/shm` (both RAM) when one exists, else the temp dir, which is disk on
/// macOS; a directory another user made first is never used.
pub struct EphemeralIdentity {
    path: PathBuf,
}

impl EphemeralIdentity {
    pub fn create(data_dir: &Path, key: &SigningKey, comment: &str) -> Result<Self> {
        Self::create_in(&runtime_base(), data_dir, key, comment)
    }

    pub(crate) fn create_in(
        base: &Path,
        data_dir: &Path,
        key: &SigningKey,
        comment: &str,
    ) -> Result<Self> {
        let dir = create_private_dir(base, data_dir)?;
        let path = dir.join(KEY_FILE);
        let written = OpenOptions::new()
            .write(true)
            .create_new(true)
            .mode(0o600)
            .open(&path)
            .and_then(|mut file| file.write_all(identity_pem(key, comment).as_bytes()))
            .with_context(|| format!("writing {}", path.display()));
        if let Err(e) = written {
            let _ = fs::remove_file(&path);
            let _ = fs::remove_dir(&dir);
            return Err(e);
        }
        Ok(Self { path })
    }

    pub fn path(&self) -> &Path {
        &self.path
    }
}

impl Drop for EphemeralIdentity {
    fn drop(&mut self) {
        if let Err(e) = fs::remove_file(&self.path)
            && e.kind() != ErrorKind::NotFound
        {
            tracing::warn!("removing the ssh key file {}: {e}", self.path.display());
        }
        if let Some(dir) = self.path.parent()
            && let Err(e) = fs::remove_dir(dir)
            && e.kind() != ErrorKind::NotFound
        {
            tracing::warn!("removing {}: {e}", dir.display());
        }
    }
}

/// Removes the key directories a killed daemon on `data_dir` left; returns
/// how many. Only real directories owned by `data_dir`'s owner are touched.
pub fn sweep_ephemeral_identities(data_dir: &Path) -> Result<usize> {
    sweep_ephemeral_identities_in(&runtime_base(), data_dir)
}

pub(crate) fn sweep_ephemeral_identities_in(base: &Path, data_dir: &Path) -> Result<usize> {
    let owner = fs::metadata(data_dir)
        .with_context(|| format!("reading {}", data_dir.display()))?
        .uid();
    let prefix = format!("{}-", key_dir_stem(data_dir));
    let entries = match fs::read_dir(base) {
        Ok(entries) => entries,
        Err(e) if e.kind() == ErrorKind::NotFound => return Ok(0),
        Err(e) => return Err(e).with_context(|| format!("listing {}", base.display())),
    };
    let mut removed = 0;
    for entry in entries {
        let entry = entry.with_context(|| format!("listing {}", base.display()))?;
        if !entry.file_name().to_string_lossy().starts_with(&prefix) {
            continue;
        }
        let dir = entry.path();
        let meta =
            fs::symlink_metadata(&dir).with_context(|| format!("reading {}", dir.display()))?;
        if !meta.is_dir() || meta.uid() != owner {
            continue;
        }
        match fs::remove_file(dir.join(KEY_FILE)) {
            Ok(()) => {}
            Err(e) if e.kind() == ErrorKind::NotFound => {}
            Err(e) => {
                return Err(e).with_context(|| format!("removing the key in {}", dir.display()));
            }
        }
        fs::remove_dir(&dir).with_context(|| format!("removing {}", dir.display()))?;
        removed += 1;
    }
    Ok(removed)
}

fn runtime_base() -> PathBuf {
    if let Some(dir) = std::env::var_os("XDG_RUNTIME_DIR").map(PathBuf::from)
        && dir.is_absolute()
        && dir.is_dir()
    {
        return dir;
    }
    let shm = Path::new("/dev/shm");
    if shm.is_dir() {
        return shm.to_path_buf();
    }
    std::env::temp_dir()
}

fn key_dir_stem(data_dir: &Path) -> String {
    let digest = Sha256::digest(data_dir.as_os_str().as_bytes());
    format!("{KEY_DIR_PREFIX}{}", &hex(&digest)[..16])
}

/// mkdir fails on any existing name, a symlink included, so the directory
/// returned is new and this process's own.
fn create_private_dir(base: &Path, data_dir: &Path) -> Result<PathBuf> {
    let stem = key_dir_stem(data_dir);
    let mut suffix = [0u8; 16];
    for _ in 0..CREATE_ATTEMPTS {
        OsRng.fill_bytes(&mut suffix);
        let dir = base.join(format!("{stem}-{}", hex(&suffix)));
        match fs::DirBuilder::new().mode(0o700).create(&dir) {
            Ok(()) => return Ok(dir),
            Err(e) if e.kind() == ErrorKind::AlreadyExists => {}
            Err(e) => return Err(e).with_context(|| format!("creating {}", dir.display())),
        }
    }
    bail!(
        "could not create a new ssh key directory in {}",
        base.display()
    )
}

fn hex(bytes: &[u8]) -> String {
    bytes.iter().map(|b| format!("{b:02x}")).collect()
}

const PEM_LABEL: &str = concat!("OPENSSH ", "PRIVATE KEY");

const KEY_TYPE: &[u8] = b"ssh-ed25519";
const MAGIC: &[u8] = b"openssh-key-v1\0";
const BLOCK: usize = 8;

fn put_string(buf: &mut Vec<u8>, bytes: &[u8]) {
    buf.extend_from_slice(&(bytes.len() as u32).to_be_bytes());
    buf.extend_from_slice(bytes);
}

fn public_blob(key: &SigningKey) -> Vec<u8> {
    let mut blob = Vec::new();
    put_string(&mut blob, KEY_TYPE);
    put_string(&mut blob, key.verifying_key().as_bytes());
    blob
}

pub fn public_line(key: &SigningKey) -> String {
    format!("ssh-ed25519 {}", STANDARD.encode(public_blob(key)))
}

fn identity_pem(key: &SigningKey, comment: &str) -> String {
    let public = key.verifying_key();
    let digest = Sha256::digest(public.as_bytes());
    let check = &digest[..4];

    let mut private = Vec::new();
    private.extend_from_slice(check);
    private.extend_from_slice(check);
    put_string(&mut private, KEY_TYPE);
    put_string(&mut private, public.as_bytes());
    let mut pair = Vec::with_capacity(64);
    pair.extend_from_slice(&key.to_bytes());
    pair.extend_from_slice(public.as_bytes());
    put_string(&mut private, &pair);
    put_string(&mut private, comment.as_bytes());
    let mut pad = 1u8;
    while private.len() % BLOCK != 0 {
        private.push(pad);
        pad += 1;
    }

    let mut body = Vec::new();
    body.extend_from_slice(MAGIC);
    put_string(&mut body, b"none");
    put_string(&mut body, b"none");
    put_string(&mut body, b"");
    body.extend_from_slice(&1u32.to_be_bytes());
    put_string(&mut body, &public_blob(key));
    put_string(&mut body, &private);

    let encoded = STANDARD.encode(&body);
    let mut pem = format!("-----BEGIN {PEM_LABEL}-----\n");
    for chunk in encoded.as_bytes().chunks(70) {
        pem.push_str(std::str::from_utf8(chunk).expect("base64 is ascii"));
        pem.push('\n');
    }
    pem.push_str(&format!("-----END {PEM_LABEL}-----\n"));
    pem
}

/// Writes `key` as an unencrypted OpenSSH private key, byte for byte the same
/// for the same key and comment. Returns false when the file already held it.
pub fn write_identity_file(path: &Path, key: &SigningKey, comment: &str) -> Result<bool> {
    let pem = identity_pem(key, comment);
    match fs::read(path) {
        Ok(existing) if existing == pem.as_bytes() => return Ok(false),
        Ok(_) => {}
        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {}
        Err(e) => return Err(e).with_context(|| format!("reading {}", path.display())),
    }
    write_atomic(path, pem.as_bytes(), Some(0o600))?;
    Ok(true)
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::fs;
    use std::os::unix::fs::PermissionsExt;
    use std::process::Command;

    fn key() -> SigningKey {
        SigningKey::from_bytes(&[7u8; 32])
    }

    #[test]
    fn rewrite_of_same_key_writes_nothing() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("id");
        assert!(write_identity_file(&path, &key(), "sb").unwrap());
        let first = fs::read(&path).unwrap();
        assert!(!write_identity_file(&path, &key(), "sb").unwrap());
        assert_eq!(fs::read(&path).unwrap(), first);
        let other = SigningKey::from_bytes(&[8u8; 32]);
        assert!(write_identity_file(&path, &other, "sb").unwrap());
    }

    #[test]
    fn file_is_private_openssh_pem() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("keys").join("id");
        write_identity_file(&path, &key(), "sb").unwrap();
        let mode = |p: &Path| fs::metadata(p).unwrap().permissions().mode() & 0o777;
        assert_eq!(mode(&path), 0o600);
        assert_eq!(mode(path.parent().unwrap()), 0o700);
        let text = fs::read_to_string(&path).unwrap();
        assert!(text.starts_with(&format!("-----BEGIN {PEM_LABEL}-----\n")));
        assert!(text.ends_with(&format!("-----END {PEM_LABEL}-----\n")));
        assert!(
            text.lines()
                .all(|l| l.len() <= 70 || l.starts_with("-----"))
        );
    }

    #[test]
    fn public_line_shape() {
        let line = public_line(&key());
        assert!(line.starts_with("ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI"));
        assert_eq!(line.split(' ').count(), 2);
    }

    #[test]
    fn ssh_keygen_derives_the_same_public_key() {
        let dir = tempfile::tempdir().unwrap();
        let path = dir.path().join("id");
        write_identity_file(&path, &key(), "sb").unwrap();
        let out = match Command::new("ssh-keygen")
            .arg("-y")
            .arg("-f")
            .arg(&path)
            .output()
        {
            Ok(out) => out,
            Err(e) => {
                eprintln!("skipping ssh-keygen check: {e}");
                return;
            }
        };
        assert!(
            out.status.success(),
            "{}",
            String::from_utf8_lossy(&out.stderr)
        );
        let printed = String::from_utf8_lossy(&out.stdout);
        let mut fields = printed.split_whitespace();
        let got = format!("{} {}", fields.next().unwrap(), fields.next().unwrap());
        assert_eq!(got, public_line(&key()));
    }

    fn private_mode(path: &Path) -> u32 {
        fs::metadata(path).unwrap().permissions().mode() & 0o777
    }

    #[test]
    fn an_ephemeral_identity_is_a_private_file_outside_the_data_dir_removed_on_drop() {
        let base = tempfile::tempdir().unwrap();
        let data_dir = tempfile::tempdir().unwrap();
        let identity =
            EphemeralIdentity::create_in(base.path(), data_dir.path(), &key(), "sb").unwrap();
        let path = identity.path().to_path_buf();
        assert!(path.starts_with(base.path()));
        assert_eq!(private_mode(&path), 0o600);
        assert_eq!(private_mode(path.parent().unwrap()), 0o700);
        assert_eq!(
            fs::read_to_string(&path).unwrap(),
            identity_pem(&key(), "sb")
        );

        let second =
            EphemeralIdentity::create_in(base.path(), data_dir.path(), &key(), "sb").unwrap();
        assert_ne!(second.path().parent(), path.parent());
        drop(identity);
        assert!(!path.parent().unwrap().exists());
        assert!(second.path().exists());
        drop(second);
        assert_eq!(fs::read_dir(base.path()).unwrap().count(), 0);
    }

    #[test]
    fn directories_another_user_made_first_do_not_stop_a_new_identity() {
        let base = tempfile::tempdir().unwrap();
        let data_dir = tempfile::tempdir().unwrap();
        let stem = key_dir_stem(data_dir.path());
        for name in [stem.clone(), format!("{stem}-00")] {
            let squatted = base.path().join(name);
            fs::create_dir(&squatted).unwrap();
            fs::set_permissions(&squatted, fs::Permissions::from_mode(0o777)).unwrap();
        }
        let identity =
            EphemeralIdentity::create_in(base.path(), data_dir.path(), &key(), "sb").unwrap();
        assert_eq!(private_mode(identity.path().parent().unwrap()), 0o700);
    }

    #[test]
    fn identities_left_by_a_killed_process_are_swept_for_their_data_dir_only() {
        let base = tempfile::tempdir().unwrap();
        let mine = tempfile::tempdir().unwrap();
        let other = tempfile::tempdir().unwrap();
        let left = EphemeralIdentity::create_in(base.path(), mine.path(), &key(), "sb").unwrap();
        let left_dir = left.path().parent().unwrap().to_path_buf();
        std::mem::forget(left);
        let others = EphemeralIdentity::create_in(base.path(), other.path(), &key(), "sb").unwrap();

        assert_eq!(
            sweep_ephemeral_identities_in(base.path(), mine.path()).unwrap(),
            1
        );
        assert!(!left_dir.exists());
        assert!(others.path().exists());
        assert_eq!(
            sweep_ephemeral_identities_in(base.path(), mine.path()).unwrap(),
            0
        );
    }

    #[test]
    fn the_sweep_never_follows_a_symlink() {
        let base = tempfile::tempdir().unwrap();
        let data_dir = tempfile::tempdir().unwrap();
        let elsewhere = tempfile::tempdir().unwrap();
        fs::write(elsewhere.path().join(KEY_FILE), "kept").unwrap();
        std::os::unix::fs::symlink(
            elsewhere.path(),
            base.path()
                .join(format!("{}-ff", key_dir_stem(data_dir.path()))),
        )
        .unwrap();

        assert_eq!(
            sweep_ephemeral_identities_in(base.path(), data_dir.path()).unwrap(),
            0
        );
        assert!(elsewhere.path().join(KEY_FILE).exists());
    }
}