use std::fmt;
use std::io::BufRead;
use anyhow::{Context, Result};
use base64::Engine;
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
use second_brain_core::keys::{
DeviceRoute, Envelope, Epoch, EpochCause, IdentityTag, SignedRecord, SignedRetirement,
SignedRevocation,
};
use second_brain_core::sqlite_store::{SCHEMA_VERSION, SqliteStore};
use serde::{Deserialize, Serialize};
pub const SYNC_PROTOCOL_VERSION: u32 = 9;
const UNVERSIONED_PROTOCOL: u32 = 1;
const HEADER_PREFIX: &str = "second-brain-sync";
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct SyncHeader {
pub sync_protocol: u32,
pub schema_version: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub hello: Option<Hello>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Hello {
pub machine_id: String,
pub wall_micros: i64,
pub synced: bool,
pub cursor: Option<u64>,
pub held_origin_seq: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub keys: Option<HandshakeKeys>,
#[serde(default)]
pub peer_in: Option<u64>,
#[serde(default)]
pub log_floor: u64,
#[serde(default)]
pub head: u64,
#[serde(default)]
pub applied: Option<u64>,
#[serde(default)]
pub merging: Option<Merging>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Merging {
pub snapshot_id: String,
pub exporter: String,
pub seq: u64,
}
const BELOW_FLOOR_PREFIX: &str = "second-brain-sync-below-floor";
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct BelowFloorLine {
pub after_seq: u64,
pub log_floor: u64,
}
impl BelowFloorLine {
pub fn to_line(&self) -> String {
format!(
"{BELOW_FLOOR_PREFIX} {}",
serde_json::to_string(self).expect("BelowFloorLine serializes")
)
}
pub fn parse_line(line: &str) -> Option<Self> {
let json = line.trim().strip_prefix(BELOW_FLOOR_PREFIX)?;
serde_json::from_str(json.trim()).ok()
}
}
const REFUSED_PREFIX: &str = "second-brain-sync-refused";
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RefusedLine {
pub error: String,
#[serde(default)]
pub identity: bool,
}
impl RefusedLine {
pub fn to_line(&self) -> String {
format!(
"{REFUSED_PREFIX} {}",
serde_json::to_string(self).expect("RefusedLine serializes")
)
}
pub fn parse_line(line: &str) -> Option<Self> {
let json = line.trim().strip_prefix(REFUSED_PREFIX)?;
serde_json::from_str(json.trim()).ok()
}
pub fn from_refusal_body(body: &str) -> Option<Self> {
let value: serde_json::Value = serde_json::from_str(body).ok()?;
if value["below_floor"] == true || value["merging"] == true {
return None;
}
Some(Self {
error: value["error"].as_str()?.to_string(),
identity: value["identity"] == true,
})
}
}
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct HandshakeKeys {
pub identity_tag: Option<IdentityTag>,
#[serde(default)]
pub epochs: Vec<Epoch>,
#[serde(default)]
pub devices: Vec<SignedRecord>,
#[serde(default)]
pub envelopes: Vec<Envelope>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub revocations: Vec<SignedRevocation>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub retirements: Vec<SignedRetirement>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub causes: Vec<(Epoch, EpochCause)>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub routes: Vec<DeviceRoute>,
}
impl Hello {
pub fn applied_of_receiver(&self) -> Option<u64> {
self.cursor.max(self.peer_in)
}
pub fn acked_by_receiver(&self) -> Option<u64> {
self.applied.or(self.cursor).max(self.peer_in)
}
pub fn to_arg(&self) -> String {
URL_SAFE_NO_PAD.encode(serde_json::to_vec(self).expect("Hello serializes"))
}
pub fn keys(&self) -> Result<&HandshakeKeys, VersionError> {
self.keys
.as_ref()
.ok_or_else(|| VersionError::KeylessHello {
peer: self.machine_id.clone(),
})
}
pub fn from_arg(arg: &str) -> Result<Self> {
let json = URL_SAFE_NO_PAD
.decode(arg.trim())
.context("decoding the sync hello")?;
serde_json::from_slice(&json).context("parsing the sync hello")
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub struct PushApplied {
pub applied: u64,
}
impl PushApplied {
pub fn to_line(&self) -> String {
serde_json::to_string(self).expect("PushApplied serializes")
}
pub fn parse_line(line: &str) -> Result<Self> {
serde_json::from_str(line.trim()).context("parsing the push result line")
}
}
impl SyncHeader {
pub fn current(schema_version: i64) -> Self {
Self {
sync_protocol: SYNC_PROTOCOL_VERSION,
schema_version,
hello: None,
}
}
pub fn local() -> Self {
Self::current(SCHEMA_VERSION)
}
pub fn to_line(&self) -> String {
let json = serde_json::to_string(self).expect("SyncHeader serializes");
format!("{HEADER_PREFIX} {json}")
}
pub fn parse_line(line: &str) -> Result<Self, VersionError> {
line.trim()
.strip_prefix(HEADER_PREFIX)
.and_then(|rest| serde_json::from_str(rest).ok())
.ok_or(VersionError::MissingHeader)
}
pub fn ensure_matches(
&self,
local_name: &str,
peer: &Self,
peer_name: &str,
) -> Result<(), VersionError> {
if (self.sync_protocol, self.schema_version) == (peer.sync_protocol, peer.schema_version) {
return Ok(());
}
Err(VersionError::Mismatch {
local: self.clone(),
local_name: local_name.to_string(),
peer: peer.clone(),
peer_name: peer_name.to_string(),
})
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum VersionError {
MissingHeader,
UnversionedDaemon,
KeylessHello { peer: String },
Mismatch {
local: SyncHeader,
local_name: String,
peer: SyncHeader,
peer_name: String,
},
}
impl fmt::Display for VersionError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
VersionError::MissingHeader => write_unversioned(f, "the sender"),
VersionError::UnversionedDaemon => write_unversioned(f, "the local daemon"),
VersionError::KeylessHello { peer } => write!(
f,
"{peer} sent a sync hello without a sync identity, so it speaks sync protocol 3 \
or older and this side speaks protocol {SYNC_PROTOCOL_VERSION}; upgrade {peer}, \
nothing was applied"
),
VersionError::Mismatch {
local,
local_name,
peer,
peer_name,
} => {
let older = if (local.sync_protocol, local.schema_version)
< (peer.sync_protocol, peer.schema_version)
{
local_name
} else {
peer_name
};
write!(
f,
"sync version mismatch: {local_name} has sync protocol {} and schema {}, \
{peer_name} has sync protocol {} and schema {}; upgrade {older}, \
nothing was applied",
local.sync_protocol,
local.schema_version,
peer.sync_protocol,
peer.schema_version
)
}
}
}
}
fn write_unversioned(f: &mut fmt::Formatter<'_>, side: &str) -> fmt::Result {
write!(
f,
"{side} declared no sync version, so it speaks sync protocol {UNVERSIONED_PROTOCOL} \
(second-brain 0.10.0 or older) and this side speaks protocol {SYNC_PROTOCOL_VERSION}; \
upgrade {side}, nothing was applied"
)
}
impl std::error::Error for VersionError {}
pub fn read_header<R: BufRead>(reader: &mut R) -> Result<SyncHeader> {
let mut line = String::new();
loop {
line.clear();
if reader
.read_line(&mut line)
.context("reading sync stream header")?
== 0
{
return Err(VersionError::MissingHeader.into());
}
if !line.trim().is_empty() {
return Ok(SyncHeader::parse_line(&line)?);
}
}
}
pub fn store_header(sqlite: &SqliteStore) -> Result<SyncHeader> {
let store = SyncHeader::current(sqlite.schema_version()?);
SyncHeader::current(SCHEMA_VERSION).ensure_matches(
"this daemon's binary",
&store,
"its database",
)?;
Ok(store)
}
pub fn check_store_header(
sqlite: &SqliteStore,
declared: Option<&SyncHeader>,
sender: &str,
) -> Result<()> {
let ours = store_header(sqlite)?;
let theirs = declared.ok_or(VersionError::MissingHeader)?;
ours.ensure_matches("this daemon", theirs, sender)?;
Ok(())
}
#[cfg(test)]
mod tests {
use std::io::BufReader;
use base64::Engine;
use base64::engine::general_purpose::STANDARD as B64;
use chrono::{TimeZone, Utc};
use second_brain_core::keys::DeviceRecord;
use second_brain_core::schema::{SyncNodeType, SyncOp, SyncRecord};
use super::*;
use crate::crypto::{XChaChaEncryptor, seal_line};
fn header(protocol: u32, schema: i64) -> SyncHeader {
SyncHeader {
sync_protocol: protocol,
schema_version: schema,
hello: None,
}
}
#[test]
fn header_line_round_trips() {
let h = SyncHeader::current(7);
let line = h.to_line();
assert!(!line.contains('\n'));
assert_eq!(SyncHeader::parse_line(&line).unwrap(), h);
}
#[test]
fn header_line_aborts_an_unversioned_reader() {
let line = SyncHeader::current(7).to_line();
assert!(
serde_json::from_str::<serde_json::Value>(&line).is_err(),
"{line}"
);
assert!(B64.decode(line.trim()).is_err(), "{line}");
}
#[test]
fn read_header_consumes_only_the_header_line() {
let enc = XChaChaEncryptor::new([3; 32]);
let record = seal_line(&enc, b"{\"local_seq\":1}").unwrap();
let stream = format!("\n{}\n{record}\n", SyncHeader::current(7).to_line());
let mut reader = BufReader::new(stream.as_bytes());
assert_eq!(read_header(&mut reader).unwrap(), SyncHeader::current(7));
let rest: Vec<String> = reader.lines().map(|l| l.unwrap()).collect();
assert_eq!(
rest,
vec![record],
"records after the header are left unread"
);
}
#[test]
fn stream_starting_with_a_record_has_no_header() {
let enc = XChaChaEncryptor::new([3; 32]);
let record = seal_line(&enc, b"{\"local_seq\":1}").unwrap();
let stream = format!("{record}\n");
let mut reader = BufReader::new(stream.as_bytes());
let err = read_header(&mut reader).unwrap_err();
assert_eq!(
err.downcast_ref::<VersionError>(),
Some(&VersionError::MissingHeader)
);
assert!(err.to_string().contains("protocol 1"), "{err}");
}
#[test]
fn legacy_plaintext_record_is_not_a_header() {
let legacy = r#"{"local_seq":1,"origin_machine_id":"m","origin_seq":1,"op":"create","node_type":"memory","node_id":"x","timestamp":"2026-01-01T00:00:00Z","data":null}"#;
assert_eq!(
SyncHeader::parse_line(legacy),
Err(VersionError::MissingHeader)
);
}
#[test]
fn empty_stream_has_no_header() {
let mut reader = BufReader::new(&b""[..]);
let err = read_header(&mut reader).unwrap_err();
assert_eq!(
err.downcast_ref::<VersionError>(),
Some(&VersionError::MissingHeader)
);
}
#[test]
fn matching_headers_pass() {
let h = SyncHeader::current(7);
assert!(h.ensure_matches("this machine", &h, "peer").is_ok());
}
fn sample_hello() -> Hello {
Hello {
machine_id: "m".to_string(),
wall_micros: 1_767_225_600_000_000,
synced: true,
cursor: Some(5),
held_origin_seq: Some((1 << 32) | 1),
keys: Some(HandshakeKeys {
identity_tag: Some(IdentityTag::from("m-0001020304050607".to_string())),
epochs: vec![Epoch {
number: 1,
key_id: "08090a0b0c0d0e0f".parse().unwrap(),
}],
devices: vec![sample_row()],
envelopes: vec![Envelope {
identity_tag: IdentityTag::from("m-0001020304050607".to_string()),
epoch: Epoch {
number: 1,
key_id: "08090a0b0c0d0e0f".parse().unwrap(),
},
recipient: "age1recipient".to_string(),
machine_id: Some("m".to_string()),
ciphertext: vec![1, 2, 3],
}],
revocations: Vec::new(),
retirements: Vec::new(),
causes: Vec::new(),
routes: Vec::new(),
}),
peer_in: Some(3),
log_floor: 2,
head: 9,
applied: None,
merging: None,
}
}
fn sample_row() -> SignedRecord {
let key = ed25519_dalek::SigningKey::from_bytes(&[1; 32]);
DeviceRecord {
machine_id: "m".to_string(),
identity_tag: "m-0001020304050607".to_string(),
version: 1,
name: "n".to_string(),
age_recipient: "age1recipient".to_string(),
ssh_public_key: "ssh-ed25519 AAAA".to_string(),
signing_public_key: key.verifying_key().to_bytes(),
recovery_fingerprints: vec!["fp".to_string()],
ssh_login: Some("user".to_string()),
ssh_port: Some(22),
ssh_host_key: Some("ssh-ed25519 BBBB".to_string()),
}
.sign("m", &key)
}
fn with_hello(schema: i64) -> SyncHeader {
SyncHeader {
hello: Some(sample_hello()),
..SyncHeader::current(schema)
}
}
#[test]
fn header_line_with_a_hello_round_trips() {
let h = with_hello(7);
let line = h.to_line();
assert!(!line.contains('\n'));
assert_eq!(SyncHeader::parse_line(&line).unwrap(), h);
}
#[test]
fn header_line_without_a_hello_omits_it() {
let line = SyncHeader::current(7).to_line();
assert!(!line.contains("hello"), "{line}");
assert_eq!(SyncHeader::parse_line(&line).unwrap().hello, None);
}
#[test]
fn ensure_matches_ignores_the_hello() {
let bare = SyncHeader::current(7);
let greeted = with_hello(7);
assert!(
bare.ensure_matches("this machine", &greeted, "peer")
.is_ok()
);
assert!(
greeted
.ensure_matches("this machine", &bare, "peer")
.is_ok()
);
assert!(
with_hello(8)
.ensure_matches("this machine", &greeted, "peer")
.is_err()
);
}
#[test]
fn a_hello_from_before_sync_identities_asks_for_an_upgrade_not_pairing() {
let old = r#"{"machine_id":"mac","wall_micros":1,"synced":false,"cursor":null,"held_origin_seq":null}"#;
let hello: Hello = serde_json::from_str(old).unwrap();
let err = hello.keys().unwrap_err().to_string();
assert!(err.contains("upgrade mac"), "{err}");
assert!(!err.contains("pair"), "{err}");
assert!(sample_hello().keys().is_ok());
}
#[test]
fn hello_argument_round_trips_in_one_shell_word() {
let hello = sample_hello();
let arg = hello.to_arg();
assert!(
arg.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_'),
"{arg}"
);
assert_eq!(Hello::from_arg(&arg).unwrap(), hello);
assert!(Hello::from_arg("not a hello").is_err());
}
#[test]
fn protocol_mismatch_names_both_versions_and_the_older_side() {
let err = header(2, 7)
.ensure_matches("this machine", &header(3, 7), "pi")
.unwrap_err()
.to_string();
assert!(
err.contains("this machine has sync protocol 2 and schema 7"),
"{err}"
);
assert!(err.contains("pi has sync protocol 3 and schema 7"), "{err}");
assert!(err.contains("upgrade this machine"), "{err}");
}
#[test]
fn schema_mismatch_names_the_older_side() {
let err = header(2, 8)
.ensure_matches("this machine", &header(2, 7), "pi")
.unwrap_err()
.to_string();
assert!(
err.contains("schema 8") && err.contains("schema 7"),
"{err}"
);
assert!(err.contains("upgrade pi"), "{err}");
}
fn every_op() -> Vec<SyncOp> {
let ops = vec![SyncOp::Create, SyncOp::Update, SyncOp::Delete];
for op in &ops {
match op {
SyncOp::Create | SyncOp::Update | SyncOp::Delete => {}
}
}
ops
}
fn every_node_type() -> Vec<SyncNodeType> {
let types = vec![
SyncNodeType::Memory,
SyncNodeType::Entity,
SyncNodeType::Conversation,
SyncNodeType::Relation,
SyncNodeType::Distillation,
];
for t in &types {
match t {
SyncNodeType::Memory
| SyncNodeType::Entity
| SyncNodeType::Conversation
| SyncNodeType::Relation
| SyncNodeType::Distillation => {}
}
}
types
}
fn wire_format() -> String {
let record = SyncRecord {
local_seq: 1,
origin_machine_id: "m".to_string(),
origin_seq: 2,
op: SyncOp::Create,
node_type: SyncNodeType::Memory,
node_id: "n".to_string(),
timestamp: Utc.with_ymd_and_hms(2026, 1, 2, 3, 4, 5).unwrap(),
data: Some(serde_json::json!({"k": "v"})),
};
let ops: Vec<String> = every_op()
.iter()
.map(|o| serde_json::to_string(o).unwrap())
.collect();
let types: Vec<String> = every_node_type()
.iter()
.map(|t| serde_json::to_string(t).unwrap())
.collect();
format!(
"header={}\nhello_header={}\npush_applied={}\nrecord={}\nops={}\nnode_types={}\nframe={}\nbelow_floor={}\nrefused={}\nsnapshot={}",
SyncHeader::current(7).to_line(),
with_hello(7).to_line(),
PushApplied { applied: 4 }.to_line(),
serde_json::to_string(&record).unwrap(),
ops.join(","),
types.join(","),
crate::crypto::frame_format(),
BelowFloorLine {
after_seq: 1,
log_floor: 2
}
.to_line(),
RefusedLine {
error: "e".into(),
identity: true
}
.to_line(),
crate::snapshot::snapshot_format(),
)
}
#[test]
fn refused_line_round_trips_and_is_not_a_header() {
let refused = RefusedLine {
error: "machine-b holds a different sync identity; run `sb pair`".into(),
identity: true,
};
let line = refused.to_line();
assert_eq!(RefusedLine::parse_line(&line), Some(refused));
assert!(SyncHeader::parse_line(&line).is_err());
assert_eq!(BelowFloorLine::parse_line(&line), None);
assert_eq!(
RefusedLine::parse_line(&SyncHeader::local().to_line()),
None
);
}
#[test]
fn a_refusal_body_reads_its_error_and_identity_flag() {
assert_eq!(
RefusedLine::from_refusal_body(r#"{"error":"run `sb pair`","identity":true}"#),
Some(RefusedLine {
error: "run `sb pair`".into(),
identity: true
})
);
assert_eq!(
RefusedLine::from_refusal_body(r#"{"error":"sync version mismatch"}"#),
Some(RefusedLine {
error: "sync version mismatch".into(),
identity: false
})
);
}
#[test]
fn a_refusal_body_asking_for_a_snapshot_or_a_wait_is_not_a_refusal() {
for body in [
r#"{"error":"below","below_floor":true,"after_seq":1,"log_floor":2}"#,
r#"{"error":"merging","merging":true,"snapshot":{}}"#,
r#"{"error":7}"#,
r#"{"message":"no error field"}"#,
"not json",
] {
assert_eq!(RefusedLine::from_refusal_body(body), None, "{body}");
}
}
#[test]
fn wire_format_matches_the_protocol_version() {
const PINNED: (u32, &str) = (
9,
"header=second-brain-sync {\"sync_protocol\":9,\"schema_version\":7}\n\
hello_header=second-brain-sync {\"sync_protocol\":9,\"schema_version\":7,\"hello\
\":{\"machine_id\":\"m\",\"wall_micros\":1767225600000000,\"synced\":true,\"curs\
or\":5,\"held_origin_seq\":4294967297,\"keys\":{\"identity_tag\":\"m-00010203040\
50607\",\"epochs\":[{\"number\":1,\"key_id\":\"08090a0b0c0d0e0f\"}],\"devices\":\
[{\"record\":{\"machine_id\":\"m\",\"identity_tag\":\"m-0001020304050607\",\"ver\
sion\":1,\"name\":\"n\",\"age_recipient\":\"age1recipient\",\"ssh_public_key\":\"\
ssh-ed25519 AAAA\",\"signing_public_key\":\"8a88e3dd7409f195fd52db2d3cba5d72ca67\
09bf1d94121bf3748801b40f6f5c\",\"recovery_fingerprints\":[\"fp\"],\"ssh_login\":\
\"user\",\"ssh_port\":22,\"ssh_host_key\":\"ssh-ed25519 BBBB\"},\"signatures\":[\
{\"signer\":\"m\",\"signature\":\"2db3155897e491f055ffdba51499bd9f272db9f0c4ff20\
134883805b167ca8257a583705b6613ad5649f4cc23cab8a495630b1b8e279f17967257ba14a6c2f\
0d\"}]}],\"envelopes\":[{\"identity_tag\":\"m-0001020304050607\",\"epoch\":{\"nu\
mber\":1,\"key_id\":\"08090a0b0c0d0e0f\"},\"recipient\":\"age1recipient\",\"mach\
ine_id\":\"m\",\"ciphertext\":[1,2,3]}]},\"peer_in\":3,\"log_floor\":2,\"head\":9,\"applied\":null,\"mergin\
g\":null}}\n\
push_applied={\"applied\":4}\n\
record={\"local_seq\":1,\"origin_machine_id\":\"m\",\"origin_seq\":2,\"op\":\"create\",\
\"node_type\":\"memory\",\"node_id\":\"n\",\"timestamp\":\"2026-01-02T03:04:05Z\",\
\"data\":{\"k\":\"v\"}}\n\
ops=\"create\",\"update\",\"delete\"\n\
node_types=\"memory\",\"entity\",\"conversation\",\"relation\",\"distillation\"\n\
frame=base64(0x02 | epoch u32 be | key id[8] | nonce[24] | xchacha20poly1305 \
ciphertext, aad = first 13 bytes)\n\
below_floor=second-brain-sync-below-floor {\"after_seq\":1,\"log_floor\":2}\n\
refused=second-brain-sync-refused {\"error\":\"e\",\"identity\":true}\n\
snapshot=second-brain-snapshot <json header> \\n, then frames of kind u8 (1 rows, 2 \
manifest) | len u32 be | nonce[24] | xchacha20poly1305(zstd(jsonl rows) or json \
manifest), aad = \"second-brain snapshot frame v1\" | epoch u32 be | key id[8] | \
snapshot id | kind | frame index u64 be",
);
assert_eq!(
(SYNC_PROTOCOL_VERSION, wire_format().as_str()),
PINNED,
"the sync wire format changed: bump SYNC_PROTOCOL_VERSION and re-pin this test"
);
}
}