second-brain-sync 0.12.0

Bidirectional sync for second-brain: SSH transport, JSONL change log, conflict resolution
Documentation
use std::collections::BTreeMap;
use std::io::{BufRead, BufReader, BufWriter, Read, Write};
use std::path::Path;

use anyhow::{Context, Result};
use second_brain_core::store::SkipReason;
use serde_json::Value;

use crate::crypto::{SyncEncryptor, open_line, seal_line};
use crate::protocol::{SyncHeader, read_header};
use crate::ssh::untrusted_host_hint;

/// Records posted to the local store per import call while the peer's export
/// is still streaming, so a long-lagging peer never sits in memory whole.
const IMPORT_CHUNK: usize = 1000;
const EXPORT_PAGE: usize = 2000;

/// This machine's store: `sb` reaches it through the daemon's API, the daemon
/// directly.
pub trait Local {
    fn cursors(&self) -> Result<Vec<(String, u64)>>;
    fn set_cursor(&self, peer_id: &str, last_seq: u64) -> Result<()>;
    /// `header` is the stream's, sent by the peer, not this machine's.
    fn import(&self, header: &SyncHeader, records: &[Value]) -> Result<Imported>;
    fn export(&self, after_seq: u64, limit: usize) -> Result<Exported>;
}

#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub struct Imported {
    pub created: u64,
    pub updated: u64,
    pub deleted: u64,
    pub skipped: u64,
    pub dangling: u64,
    pub max_seq: u64,
}

impl Imported {
    pub fn dangling_in(skipped_by_reason: &BTreeMap<String, u64>) -> u64 {
        skipped_by_reason
            .get(SkipReason::DanglingEndpoint.as_str())
            .copied()
            .unwrap_or(0)
    }
}

pub struct Exported {
    pub records: Vec<Value>,
    pub max_seq: u64,
}

pub trait Transport {
    fn host(&self) -> &str;
    fn whoami(&self) -> Result<String>;
    fn export(&self, after_seq: u64) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)>;
    fn import(&self) -> Result<(Box<dyn Write + Send>, Box<dyn Remote>)>;
}

/// One running `sb sync-export` or `sb sync-import` on a peer.
pub trait Remote {
    fn wait(&mut self) -> Result<Exit>;
    fn stop(&mut self) -> Result<()>;
}

pub struct Exit {
    pub success: bool,
    pub status: String,
}

/// A full pull then push with one peer, recorded as a success for health
/// only once both have finished.
pub fn sync(
    local: &dyn Local,
    transport: &dyn Transport,
    encryptor: &dyn SyncEncryptor,
    data_dir: &Path,
    report: &dyn Fn(&str),
) -> Result<()> {
    let peer_host = transport.host();
    let peer_machine_id = transport
        .whoami()
        .context("resolving peer machine id (is the peer's daemon running and reachable?)")?;

    report(&format!("pulling from {peer_host}..."));
    let pull_stats = pull(local, transport, &peer_machine_id, encryptor)?;
    report(&format!(
        "  {} created, {} updated, {} deleted, {} skipped",
        pull_stats.created, pull_stats.updated, pull_stats.deleted, pull_stats.skipped
    ));
    if pull_stats.dangling > 0 {
        report(&format!(
            "  warning: {} records dropped with unresolvable endpoints; \
             run `sb prune-relations` on the peer if this persists",
            pull_stats.dangling
        ));
    }

    report(&format!("pushing to {peer_host}..."));
    let max_seq = push(local, transport, &peer_machine_id, encryptor, report)?;
    report(&format!("  pushed through seq {max_seq}"));

    crate::peers::record_success(data_dir, chrono::Utc::now())
        .context("recording that the sync succeeded")?;
    report("sync complete.");
    Ok(())
}

/// Prefers the machine-id-keyed cursor and falls back to the one keyed by the
/// `sync-peers` connection string, so a peer synced before cursors were keyed
/// by identity resumes where it was instead of re-pulling its whole history.
fn cursor(local: &dyn Local, key: &str, legacy_key: &str) -> Result<u64> {
    let cursors = local.cursors().context("getting sync status")?;
    Ok(cursors
        .iter()
        .find(|(peer_id, _)| peer_id == key)
        .or_else(|| cursors.iter().find(|(peer_id, _)| peer_id == legacy_key))
        .map(|(_, last_seq)| *last_seq)
        .unwrap_or(0))
}

pub fn pull(
    local: &dyn Local,
    transport: &dyn Transport,
    peer_machine_id: &str,
    encryptor: &dyn SyncEncryptor,
) -> Result<Imported> {
    let peer_host = transport.host();
    let after_seq = cursor(local, peer_machine_id, peer_host)?;

    let (output, mut remote) = transport.export(after_seq)?;
    let mut reader = BufReader::new(output);
    let peer_header = match read_checked_header(&mut reader, peer_host) {
        Ok(header) => header,
        Err(e) => {
            // A peer that failed before writing its header ends the stream with
            // a failing exit, and that failure, not its version, is the cause.
            if matches!(reader.fill_buf(), Ok(rest) if rest.is_empty()) {
                let exit = remote.wait()?;
                if !exit.success {
                    anyhow::bail!(
                        "sync-export on {peer_host} exited with {}; its error is printed above.\n  hint: {}",
                        exit.status,
                        untrusted_host_hint(peer_host)
                    );
                }
            } else {
                remote.stop()?;
            }
            return Err(e);
        }
    };

    let mut totals = Imported::default();
    let mut chunk = Vec::with_capacity(IMPORT_CHUNK);
    let mut post_chunk = |chunk: &[Value]| -> Result<()> {
        if chunk.is_empty() {
            return Ok(());
        }
        let resp = local.import(&peer_header, chunk)?;
        totals.created += resp.created;
        totals.updated += resp.updated;
        totals.deleted += resp.deleted;
        totals.skipped += resp.skipped;
        totals.dangling += resp.dangling;
        totals.max_seq = totals.max_seq.max(resp.max_seq);
        Ok(())
    };
    let mut import_stream = || -> Result<()> {
        for line in (&mut reader).lines() {
            let line = line?;
            if line.trim().is_empty() {
                continue;
            }
            let plaintext = open_line(encryptor, &line)?;
            chunk.push(serde_json::from_slice(&plaintext)?);
            if chunk.len() >= IMPORT_CHUNK {
                post_chunk(&chunk)?;
                chunk.clear();
            }
        }
        post_chunk(&chunk)?;
        chunk.clear();
        Ok(())
    };
    if let Err(e) = import_stream() {
        remote.stop()?;
        return Err(e);
    }

    let exit = remote.wait()?;
    if !exit.success {
        anyhow::bail!(
            "ssh pull from {peer_host} exited with {}.\n  hint: {}",
            exit.status,
            untrusted_host_hint(peer_host)
        );
    }

    if totals.max_seq > after_seq {
        local
            .set_cursor(peer_machine_id, totals.max_seq)
            .context("updating sync state")?;
    }

    Ok(totals)
}

pub fn push(
    local: &dyn Local,
    transport: &dyn Transport,
    peer_machine_id: &str,
    encryptor: &dyn SyncEncryptor,
    report: &dyn Fn(&str),
) -> Result<u64> {
    let peer_host = transport.host();
    let push_peer = format!("push:{peer_machine_id}");
    let mut cursor = cursor(local, &push_peer, &format!("push:{peer_host}"))?;

    let (input, mut remote) = transport.import()?;
    let mut total_pushed = 0u64;
    let stream = || -> Result<()> {
        let mut writer = BufWriter::new(input);
        writeln!(writer, "{}", SyncHeader::local().to_line())?;

        loop {
            let export = local.export(cursor, EXPORT_PAGE)?;
            if export.records.is_empty() {
                break;
            }
            let batch_count = export.records.len();
            for record in &export.records {
                let plaintext = serde_json::to_vec(record)?;
                let line = seal_line(encryptor, &plaintext)?;
                writer.write_all(line.as_bytes())?;
                writer.write_all(b"\n")?;
            }
            writer.flush()?;

            total_pushed += batch_count as u64;
            cursor = export.max_seq;
            report(&format!(
                "  batch: {batch_count} records (through seq {cursor})"
            ));

            if batch_count < EXPORT_PAGE {
                break;
            }
        }
        writer.flush()?;
        Ok(())
    };
    let streamed = stream();

    // A peer that refuses the header exits before reading on, which surfaces
    // here as a broken pipe; its exit status is the more useful error.
    let exit = remote.wait()?;
    if !exit.success {
        anyhow::bail!(
            "ssh push to {peer_host} exited with {}.\n  hint: {}",
            exit.status,
            untrusted_host_hint(peer_host)
        );
    }
    streamed?;

    if total_pushed > 0 {
        local
            .set_cursor(&push_peer, cursor)
            .context("updating push sync state")?;
    }

    report(&format!("  total: {total_pushed} records pushed"));
    Ok(cursor)
}

pub fn read_checked_header<R: BufRead>(reader: &mut R, sender: &str) -> Result<SyncHeader> {
    let header =
        read_header(reader).with_context(|| format!("reading the sync header from {sender}"))?;
    SyncHeader::local().ensure_matches("this machine", &header, sender)?;
    Ok(header)
}