second-brain-sync 0.11.0

Bidirectional sync for second-brain: SSH transport, JSONL change log, conflict resolution
Documentation
use std::fmt;
use std::io::BufRead;

use anyhow::{Context, Result};
use second_brain_core::sqlite_store::SCHEMA_VERSION;
use serde::{Deserialize, Serialize};

/// Bump whenever the sync wire format or the meaning of a field changes:
/// the record envelope, the header line, the line framing or the crypto frame.
/// `wire_format_matches_the_protocol_version` pins the current format.
/// Releases up to second-brain 0.10.0 sent no header and count as protocol 1.
pub const SYNC_PROTOCOL_VERSION: u32 = 2;

const UNVERSIONED_PROTOCOL: u32 = 1;

/// Makes the header line neither base64 nor JSON: binaries up to 0.10.0 pass
/// any non-base64 line through as a plaintext JSON record, and only a parse
/// failure on the first line stops them before they apply anything.
const HEADER_PREFIX: &str = "second-brain-sync";

/// First line of every sync stream: `HEADER_PREFIX`, a space, then plaintext
/// JSON, so a peer holding the wrong key still gets a version error. It carries
/// no data, is not a change-log entry and has no sequence number.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct SyncHeader {
    pub sync_protocol: u32,
    /// `PRAGMA user_version` of the store whose records the stream carries.
    pub schema_version: i64,
}

impl SyncHeader {
    pub fn current(schema_version: i64) -> Self {
        Self {
            sync_protocol: SYNC_PROTOCOL_VERSION,
            schema_version,
        }
    }

    /// The versions this binary was built with.
    pub fn local() -> Self {
        Self::current(SCHEMA_VERSION)
    }

    pub fn to_line(&self) -> String {
        let json = serde_json::to_string(self).expect("SyncHeader serializes");
        format!("{HEADER_PREFIX} {json}")
    }

    pub fn parse_line(line: &str) -> Result<Self, VersionError> {
        line.trim()
            .strip_prefix(HEADER_PREFIX)
            .and_then(|rest| serde_json::from_str(rest).ok())
            .ok_or(VersionError::MissingHeader)
    }

    /// Refuses unless both sides speak the same protocol over the same schema.
    /// `local_name` and `peer_name` name the two sides in the error message.
    pub fn ensure_matches(
        &self,
        local_name: &str,
        peer: &Self,
        peer_name: &str,
    ) -> Result<(), VersionError> {
        if self == peer {
            return Ok(());
        }
        Err(VersionError::Mismatch {
            local: *self,
            local_name: local_name.to_string(),
            peer: *peer,
            peer_name: peer_name.to_string(),
        })
    }
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub enum VersionError {
    /// A stream or request without a header: its sender predates versioning.
    MissingHeader,
    /// The local daemon has no version route: it predates versioning.
    UnversionedDaemon,
    Mismatch {
        local: SyncHeader,
        local_name: String,
        peer: SyncHeader,
        peer_name: String,
    },
}

impl fmt::Display for VersionError {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            VersionError::MissingHeader => write_unversioned(f, "the sender"),
            VersionError::UnversionedDaemon => write_unversioned(f, "the local daemon"),
            VersionError::Mismatch {
                local,
                local_name,
                peer,
                peer_name,
            } => {
                let older = if (local.sync_protocol, local.schema_version)
                    < (peer.sync_protocol, peer.schema_version)
                {
                    local_name
                } else {
                    peer_name
                };
                write!(
                    f,
                    "sync version mismatch: {local_name} has sync protocol {} and schema {}, \
                     {peer_name} has sync protocol {} and schema {}; upgrade {older}, \
                     nothing was applied",
                    local.sync_protocol,
                    local.schema_version,
                    peer.sync_protocol,
                    peer.schema_version
                )
            }
        }
    }
}

fn write_unversioned(f: &mut fmt::Formatter<'_>, side: &str) -> fmt::Result {
    write!(
        f,
        "{side} declared no sync version, so it speaks sync protocol {UNVERSIONED_PROTOCOL} \
         (second-brain 0.10.0 or older) and this side speaks protocol {SYNC_PROTOCOL_VERSION}; \
         upgrade {side}, nothing was applied"
    )
}

impl std::error::Error for VersionError {}

/// Reads the header line that must open a sync stream, skipping blank lines.
/// An empty stream is refused like a header-less one: a current sender always
/// writes the header, even when it has no records.
pub fn read_header<R: BufRead>(reader: &mut R) -> Result<SyncHeader> {
    let mut line = String::new();
    loop {
        line.clear();
        if reader
            .read_line(&mut line)
            .context("reading sync stream header")?
            == 0
        {
            return Err(VersionError::MissingHeader.into());
        }
        if !line.trim().is_empty() {
            return Ok(SyncHeader::parse_line(&line)?);
        }
    }
}

#[cfg(test)]
mod tests {
    use std::io::BufReader;

    use base64::Engine;
    use base64::engine::general_purpose::STANDARD as B64;
    use chrono::{TimeZone, Utc};
    use second_brain_core::schema::{SyncNodeType, SyncOp, SyncRecord};

    use super::*;
    use crate::crypto::{XChaChaEncryptor, seal_line};

    fn header(protocol: u32, schema: i64) -> SyncHeader {
        SyncHeader {
            sync_protocol: protocol,
            schema_version: schema,
        }
    }

    #[test]
    fn header_line_round_trips() {
        let h = SyncHeader::current(7);
        let line = h.to_line();
        assert!(!line.contains('\n'));
        assert_eq!(SyncHeader::parse_line(&line).unwrap(), h);
    }

    #[test]
    fn header_line_aborts_an_unversioned_reader() {
        let line = SyncHeader::current(7).to_line();
        assert!(
            serde_json::from_str::<serde_json::Value>(&line).is_err(),
            "{line}"
        );
        assert!(B64.decode(line.trim()).is_err(), "{line}");
    }

    #[test]
    fn read_header_consumes_only_the_header_line() {
        let enc = XChaChaEncryptor::new([3; 32]);
        let record = seal_line(&enc, b"{\"local_seq\":1}").unwrap();
        let stream = format!("\n{}\n{record}\n", SyncHeader::current(7).to_line());
        let mut reader = BufReader::new(stream.as_bytes());
        assert_eq!(read_header(&mut reader).unwrap(), SyncHeader::current(7));
        let rest: Vec<String> = reader.lines().map(|l| l.unwrap()).collect();
        assert_eq!(
            rest,
            vec![record],
            "records after the header are left unread"
        );
    }

    #[test]
    fn stream_starting_with_a_record_has_no_header() {
        let enc = XChaChaEncryptor::new([3; 32]);
        let record = seal_line(&enc, b"{\"local_seq\":1}").unwrap();
        let stream = format!("{record}\n");
        let mut reader = BufReader::new(stream.as_bytes());
        let err = read_header(&mut reader).unwrap_err();
        assert_eq!(
            err.downcast_ref::<VersionError>(),
            Some(&VersionError::MissingHeader)
        );
        assert!(err.to_string().contains("protocol 1"), "{err}");
    }

    #[test]
    fn legacy_plaintext_record_is_not_a_header() {
        let legacy = r#"{"local_seq":1,"origin_machine_id":"m","origin_seq":1,"op":"create","node_type":"memory","node_id":"x","timestamp":"2026-01-01T00:00:00Z","data":null}"#;
        assert_eq!(
            SyncHeader::parse_line(legacy),
            Err(VersionError::MissingHeader)
        );
    }

    #[test]
    fn empty_stream_has_no_header() {
        let mut reader = BufReader::new(&b""[..]);
        let err = read_header(&mut reader).unwrap_err();
        assert_eq!(
            err.downcast_ref::<VersionError>(),
            Some(&VersionError::MissingHeader)
        );
    }

    #[test]
    fn matching_headers_pass() {
        let h = SyncHeader::current(7);
        assert!(h.ensure_matches("this machine", &h, "peer").is_ok());
    }

    #[test]
    fn protocol_mismatch_names_both_versions_and_the_older_side() {
        let err = header(2, 7)
            .ensure_matches("this machine", &header(3, 7), "pi")
            .unwrap_err()
            .to_string();
        assert!(
            err.contains("this machine has sync protocol 2 and schema 7"),
            "{err}"
        );
        assert!(err.contains("pi has sync protocol 3 and schema 7"), "{err}");
        assert!(err.contains("upgrade this machine"), "{err}");
    }

    #[test]
    fn schema_mismatch_names_the_older_side() {
        let err = header(2, 8)
            .ensure_matches("this machine", &header(2, 7), "pi")
            .unwrap_err()
            .to_string();
        assert!(
            err.contains("schema 8") && err.contains("schema 7"),
            "{err}"
        );
        assert!(err.contains("upgrade pi"), "{err}");
    }

    fn every_op() -> Vec<SyncOp> {
        let ops = vec![SyncOp::Create, SyncOp::Update, SyncOp::Delete];
        for op in &ops {
            match op {
                SyncOp::Create | SyncOp::Update | SyncOp::Delete => {}
            }
        }
        ops
    }

    fn every_node_type() -> Vec<SyncNodeType> {
        let types = vec![
            SyncNodeType::Memory,
            SyncNodeType::Entity,
            SyncNodeType::Conversation,
            SyncNodeType::Relation,
            SyncNodeType::Distillation,
        ];
        for t in &types {
            match t {
                SyncNodeType::Memory
                | SyncNodeType::Entity
                | SyncNodeType::Conversation
                | SyncNodeType::Relation
                | SyncNodeType::Distillation => {}
            }
        }
        types
    }

    fn wire_format() -> String {
        let record = SyncRecord {
            local_seq: 1,
            origin_machine_id: "m".to_string(),
            origin_seq: 2,
            op: SyncOp::Create,
            node_type: SyncNodeType::Memory,
            node_id: "n".to_string(),
            timestamp: Utc.with_ymd_and_hms(2026, 1, 2, 3, 4, 5).unwrap(),
            data: Some(serde_json::json!({"k": "v"})),
        };
        let ops: Vec<String> = every_op()
            .iter()
            .map(|o| serde_json::to_string(o).unwrap())
            .collect();
        let types: Vec<String> = every_node_type()
            .iter()
            .map(|t| serde_json::to_string(t).unwrap())
            .collect();
        format!(
            "header={}\nrecord={}\nops={}\nnode_types={}\nframe={}",
            SyncHeader::current(7).to_line(),
            serde_json::to_string(&record).unwrap(),
            ops.join(","),
            types.join(","),
            crate::crypto::frame_format(),
        )
    }

    #[test]
    fn wire_format_matches_the_protocol_version() {
        const PINNED: (u32, &str) = (
            2,
            "header=second-brain-sync {\"sync_protocol\":2,\"schema_version\":7}\n\
             record={\"local_seq\":1,\"origin_machine_id\":\"m\",\"origin_seq\":2,\"op\":\"create\",\
             \"node_type\":\"memory\",\"node_id\":\"n\",\"timestamp\":\"2026-01-02T03:04:05Z\",\
             \"data\":{\"k\":\"v\"}}\n\
             ops=\"create\",\"update\",\"delete\"\n\
             node_types=\"memory\",\"entity\",\"conversation\",\"relation\",\"distillation\"\n\
             frame=base64(0x01 | nonce[24] | xchacha20poly1305 ciphertext)",
        );
        assert_eq!(
            (SYNC_PROTOCOL_VERSION, wire_format().as_str()),
            PINNED,
            "the sync wire format changed: bump SYNC_PROTOCOL_VERSION and re-pin this test"
        );
    }
}