use std::collections::BTreeMap;
use std::fmt;
use std::io::{BufRead, BufReader, BufWriter, Cursor, Read, Write};
use std::path::Path;
use crate::keyring::IdentityMismatch;
use crate::protocol::{
BelowFloorLine, Hello, Merging, PushApplied, RefusedLine, SYNC_PROTOCOL_VERSION, SyncHeader,
read_header,
};
use crate::ssh::untrusted_host_hint;
use anyhow::{Context, Result};
use second_brain_core::store::SkipReason;
const IMPORT_CHUNK: usize = 1000;
const PUSH_IMPORT_CHUNK: usize = 500;
const EXPORT_PAGE: usize = 2000;
pub trait Local {
fn cursors(&self) -> Result<Vec<(String, u64)>>;
fn set_cursor(&self, peer_id: &str, last_seq: u64) -> Result<()>;
fn import(&self, header: &SyncHeader, lines: &[String]) -> Result<Imported>;
fn export(&self, peer_id: &str, after_seq: u64, limit: usize) -> Result<Exported>;
fn hello_for(&self, peer_id: &str) -> Result<Hello>;
fn accept_hello(&self, hello: &Hello) -> Result<()>;
fn record_ack(&self, peer_id: &str, acked_seq: u64) -> Result<()>;
fn record_peer_in(&self, peer_id: &str, seq: u64) -> Result<()>;
fn push_report(&self, peer_id: &str) -> Result<Option<u64>>;
fn record_push_report(&self, peer_id: &str, reported: Option<u64>) -> Result<()>;
fn merge_snapshot(&self, exporter: &str, stream: Box<dyn Read + Send>) -> Result<Merged> {
let _ = (exporter, stream);
anyhow::bail!("this store cannot merge a snapshot")
}
fn merging(&self) -> Result<Option<Merging>> {
Ok(None)
}
fn export_snapshot(&self, peer: &Hello) -> Result<Box<dyn Read + Send>> {
let _ = peer;
anyhow::bail!("this store cannot export a snapshot")
}
fn bootstrap_pending(&self) -> Result<bool> {
Ok(false)
}
fn finish_bootstrap(&self) -> Result<()> {
Ok(())
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Merged {
Done {
seq: u64,
},
Running(Merging),
}
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub struct Imported {
pub created: u64,
pub updated: u64,
pub deleted: u64,
pub skipped: u64,
pub dangling: u64,
pub errors: u64,
pub max_seq: u64,
pub reached: u64,
}
impl Imported {
pub fn dangling_in(skipped_by_reason: &BTreeMap<String, u64>) -> u64 {
skipped_by_reason
.get(SkipReason::DanglingEndpoint.as_str())
.copied()
.unwrap_or(0)
}
}
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub struct Watermark {
reached: u64,
applied: u64,
capped: Option<u64>,
}
impl Watermark {
pub fn add(&mut self, chunk: &Imported) {
self.reached = self.reached.max(chunk.reached.max(chunk.max_seq));
self.applied = self.applied.max(chunk.max_seq);
if chunk.errors > 0 {
self.capped = Some(self.capped.map_or(chunk.max_seq, |c| c.min(chunk.max_seq)));
}
}
pub fn applied(&self) -> u64 {
self.capped
.map_or(self.applied, |cap| self.applied.min(cap))
}
pub fn reached(&self) -> u64 {
self.reached
}
}
pub struct Exported {
pub lines: Vec<String>,
pub max_seq: u64,
}
pub trait Transport {
fn host(&self) -> &str;
fn whoami(&self) -> Result<String>;
fn export(
&self,
after_seq: u64,
hello: &Hello,
) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)>;
fn import(&self) -> Result<(Box<dyn Write + Send>, Box<dyn Read + Send>, Box<dyn Remote>)>;
fn snapshot_export(&self, hello: &Hello) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)> {
let _ = hello;
anyhow::bail!("the transport to {} carries no snapshots", self.host())
}
fn snapshot_import(
&self,
) -> Result<(Box<dyn Write + Send>, Box<dyn Read + Send>, Box<dyn Remote>)> {
anyhow::bail!("the transport to {} carries no snapshots", self.host())
}
}
pub trait Remote {
fn wait(&mut self) -> Result<Exit>;
fn stop(&mut self) -> Result<()>;
}
pub struct Exit {
pub success: bool,
pub status: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct PeerRefused {
pub host: String,
pub error: String,
pub identity: bool,
}
impl PeerRefused {
fn from_line(host: &str, refused: RefusedLine) -> Self {
Self {
host: host.to_string(),
error: refused.error,
identity: refused.identity,
}
}
}
impl fmt::Display for PeerRefused {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{} refused the sync: {}", self.host, self.error)
}
}
impl std::error::Error for PeerRefused {}
pub fn sync(
local: &dyn Local,
transport: &dyn Transport,
data_dir: &Path,
report: &dyn Fn(&str),
) -> Result<()> {
let peer_machine_id = transport
.whoami()
.context("resolving peer machine id (is the peer's daemon running and reachable?)")?;
sync_with(local, transport, &peer_machine_id, data_dir, report)
}
pub fn sync_with(
local: &dyn Local,
transport: &dyn Transport,
peer_machine_id: &str,
data_dir: &Path,
report: &dyn Fn(&str),
) -> Result<()> {
let peer_host = transport.host();
report(&format!("pulling from {peer_host}..."));
let pull_stats = pull(local, transport, peer_machine_id)?;
report(&format!(
" {} created, {} updated, {} deleted, {} skipped",
pull_stats.created, pull_stats.updated, pull_stats.deleted, pull_stats.skipped
));
if pull_stats.dangling > 0 {
report(&format!(
" warning: {} records dropped with unresolvable endpoints; \
run `sb prune-relations` on the peer if this persists",
pull_stats.dangling
));
}
report(&format!("pushing to {peer_host}..."));
let max_seq = push(local, transport, peer_machine_id, report)?;
report(&format!(" pushed through seq {max_seq}"));
crate::peers::record_success(data_dir, chrono::Utc::now())
.context("recording that the sync succeeded")?;
report("sync complete.");
Ok(())
}
fn cursor(local: &dyn Local, key: &str, legacy_key: &str) -> Result<u64> {
let cursors = local.cursors().context("getting sync status")?;
Ok(cursors
.iter()
.find(|(peer_id, _)| peer_id == key)
.or_else(|| cursors.iter().find(|(peer_id, _)| peer_id == legacy_key))
.map(|(_, last_seq)| *last_seq)
.unwrap_or(0))
}
pub fn pull(
local: &dyn Local,
transport: &dyn Transport,
peer_machine_id: &str,
) -> Result<Imported> {
if let Some(merging) = local.merging()? {
anyhow::bail!(
"a snapshot merge from {} through seq {} is still running; sync again once it finishes",
merging.exporter,
merging.seq
);
}
let bootstrapping = local.bootstrap_pending()?;
if bootstrapping && cursor(local, peer_machine_id, transport.host())? == 0 {
let start = BelowFloorLine {
after_seq: 0,
log_floor: 0,
};
merge_peer_snapshot(local, transport, peer_machine_id, start)?;
}
let imported = pull_after_floor(local, transport, peer_machine_id)?;
if bootstrapping {
local.finish_bootstrap()?;
}
Ok(imported)
}
fn pull_after_floor(
local: &dyn Local,
transport: &dyn Transport,
peer_machine_id: &str,
) -> Result<Imported> {
match pull_log(local, transport, peer_machine_id)? {
Pulled::Log(imported) => Ok(imported),
Pulled::BelowFloor(below) => {
merge_peer_snapshot(local, transport, peer_machine_id, below)?;
match pull_log(local, transport, peer_machine_id)? {
Pulled::Log(imported) => Ok(imported),
Pulled::BelowFloor(again) => anyhow::bail!(
"{} truncated its log past seq {} again right after a snapshot; sync again",
transport.host(),
again.log_floor
),
}
}
}
}
enum Pulled {
Log(Imported),
BelowFloor(BelowFloorLine),
}
fn pull_hello(local: &dyn Local, peer_machine_id: &str, after_seq: u64) -> Result<Hello> {
let hello = local
.hello_for(peer_machine_id)
.context("building the sync hello")?;
Ok(Hello {
cursor: Some(after_seq),
applied: Some(hello.applied.unwrap_or(after_seq).min(after_seq)),
..hello
})
}
fn merge_peer_snapshot(
local: &dyn Local,
transport: &dyn Transport,
peer_machine_id: &str,
below: BelowFloorLine,
) -> Result<()> {
let peer_host = transport.host();
let hello = pull_hello(local, peer_machine_id, below.after_seq)?;
let (stream, mut remote) = transport.snapshot_export(&hello)?;
let merged = local.merge_snapshot(peer_machine_id, stream);
let exit = remote.wait()?;
if !exit.success {
anyhow::bail!(
"sync-snapshot on {peer_host} exited with {}; its error is printed above.\n hint: {}",
exit.status,
untrusted_host_hint(peer_host)
);
}
match merged.with_context(|| format!("merging the snapshot from {peer_host}"))? {
Merged::Done { .. } => Ok(()),
Merged::Running(merging) => anyhow::bail!(
"merging the snapshot from {peer_host} through seq {} continues in the daemon; \
sync again once it finishes",
merging.seq
),
}
}
fn pull_log(local: &dyn Local, transport: &dyn Transport, peer_machine_id: &str) -> Result<Pulled> {
let peer_host = transport.host();
let after_seq = cursor(local, peer_machine_id, peer_host)?;
let hello = pull_hello(local, peer_machine_id, after_seq)?;
let (output, mut remote) = transport.export(after_seq, &hello)?;
let mut reader = BufReader::new(output);
let mut first = String::new();
let read = reader.read_line(&mut first);
if let Some(below) = BelowFloorLine::parse_line(&first) {
let exit = remote.wait()?;
if !exit.success {
anyhow::bail!(
"sync-export on {peer_host} exited with {}.\n hint: {}",
exit.status,
untrusted_host_hint(peer_host)
);
}
return Ok(Pulled::BelowFloor(below));
}
if let Some(refused) = RefusedLine::parse_line(&first) {
if let Err(e) = remote.wait() {
tracing::warn!("sync-export on {peer_host} refused and then failed to exit: {e:#}");
}
return Err(PeerRefused::from_line(peer_host, refused).into());
}
let header = read
.map_err(anyhow::Error::from)
.and_then(|_| read_checked_header(&mut Cursor::new(first.as_bytes()), peer_host));
let peer_header = match header {
Ok(header) => header,
Err(e) => {
if first.trim().is_empty() && matches!(reader.fill_buf(), Ok(rest) if rest.is_empty()) {
let exit = remote.wait()?;
if !exit.success {
anyhow::bail!(
"sync-export on {peer_host} exited with {}; its error is printed above.\n hint: {}",
exit.status,
untrusted_host_hint(peer_host)
);
}
} else {
remote.stop()?;
}
return Err(e);
}
};
if let Err(e) = accept_peer_hello(local, &peer_header, peer_host) {
remote.stop()?;
return Err(e);
}
let (mut totals, watermark) = match import_lines(local, &peer_header, &mut reader, IMPORT_CHUNK)
{
Ok(imported) => imported,
Err(e) => {
remote.stop()?;
return Err(e);
}
};
let exit = remote.wait()?;
if !exit.success {
anyhow::bail!(
"ssh pull from {peer_host} exited with {}.\n hint: {}",
exit.status,
untrusted_host_hint(peer_host)
);
}
totals.max_seq = watermark.reached();
if totals.max_seq > after_seq {
local
.set_cursor(peer_machine_id, totals.max_seq)
.context("updating sync state")?;
}
Ok(Pulled::Log(totals))
}
pub fn push(
local: &dyn Local,
transport: &dyn Transport,
peer_machine_id: &str,
report: &dyn Fn(&str),
) -> Result<u64> {
let peer_host = transport.host();
let push_peer = format!("push:{peer_machine_id}");
let push_cursor = cursor(local, &push_peer, &format!("push:{peer_host}"))?;
let header = SyncHeader {
hello: Some(
local
.hello_for(peer_machine_id)
.context("building the sync hello")?,
),
..SyncHeader::local()
};
let (input, output, mut remote) = transport.import()?;
let mut writer = BufWriter::new(input);
let mut replies = BufReader::new(output);
let own_floor = header.hello.as_ref().map_or(0, |hello| hello.log_floor);
let mut handshake = || -> Result<(u64, Hello)> {
writeln!(writer, "{}", header.to_line())?;
writer.flush()?;
let first = first_line(&mut replies)
.with_context(|| format!("reading the sync header from {peer_host}"))?;
if let Some(refused) = RefusedLine::parse_line(&first) {
return Err(PeerRefused::from_line(peer_host, refused).into());
}
let receiver = read_checked_header(&mut Cursor::new(first.as_bytes()), peer_host)?;
accept_peer_hello(local, &receiver, peer_host)?;
let reported = receiver.hello.as_ref().and_then(Hello::applied_of_receiver);
let start = push_start(push_cursor, local.push_report(peer_machine_id)?, reported);
if start < push_cursor {
local
.set_cursor(&push_peer, start)
.context("lowering the push cursor for a restored peer")?;
}
local.record_push_report(peer_machine_id, reported)?;
if let Some(acked) = receiver.hello.as_ref().and_then(Hello::acked_by_receiver) {
local.record_ack(peer_machine_id, acked)?;
}
let hello = receiver
.hello
.clone()
.expect("accept_peer_hello refuses a header without its hello");
Ok((start, hello))
};
let (start, receiver) = match handshake() {
Ok(handshake) => handshake,
Err(e) => {
drop(writer);
let refused = e.downcast_ref::<PeerRefused>().is_some();
let exit = match remote.wait() {
Ok(exit) => exit,
Err(wait) if refused => {
tracing::warn!(
"ssh push to {peer_host} refused and then failed to exit: {wait:#}"
);
return Err(e);
}
Err(wait) => return Err(wait),
};
if !exit.success && !refused {
anyhow::bail!(
"ssh push to {peer_host} exited with {}.\n hint: {}",
exit.status,
untrusted_host_hint(peer_host)
);
}
return Err(e);
}
};
if receiver.merging.is_some() || start < own_floor {
drop(writer);
let applied = read_push_applied(&mut replies);
let exit = remote.wait()?;
if !exit.success {
anyhow::bail!(
"ssh push to {peer_host} exited with {}.\n hint: {}",
exit.status,
untrusted_host_hint(peer_host)
);
}
applied.with_context(|| format!("reading what {peer_host} applied"))?;
if let Some(merging) = receiver.merging {
report(&format!(
" {peer_host} is merging snapshot {} through seq {}; nothing pushed",
merging.snapshot_id, merging.seq
));
} else {
push_snapshot(local, transport, &receiver, start, report)?;
}
return Ok(start.min(push_cursor));
}
let mut cursor = start;
let mut total_pushed = 0u64;
let mut stream = || -> Result<()> {
loop {
let export = local.export(peer_machine_id, cursor, EXPORT_PAGE)?;
if export.lines.is_empty() {
break;
}
let batch_count = export.lines.len();
for line in &export.lines {
writer.write_all(line.as_bytes())?;
writer.write_all(b"\n")?;
}
writer.flush()?;
total_pushed += batch_count as u64;
cursor = export.max_seq;
report(&format!(
" batch: {batch_count} records (through seq {cursor})"
));
if batch_count < EXPORT_PAGE {
break;
}
}
writer.flush()?;
Ok(())
};
let streamed = stream();
drop(writer);
let applied = read_push_applied(&mut replies);
let exit = remote.wait()?;
if !exit.success {
anyhow::bail!(
"ssh push to {peer_host} exited with {}.\n hint: {}",
exit.status,
untrusted_host_hint(peer_host)
);
}
streamed?;
let applied = applied.with_context(|| format!("reading what {peer_host} applied"))?;
local
.record_ack(peer_machine_id, applied.applied)
.context("recording the peer's ack")?;
if cursor != start.min(push_cursor) {
local
.set_cursor(&push_peer, cursor)
.context("updating push sync state")?;
}
report(&format!(" total: {total_pushed} records pushed"));
Ok(cursor)
}
fn push_snapshot(
local: &dyn Local,
transport: &dyn Transport,
receiver: &Hello,
start: u64,
report: &dyn Fn(&str),
) -> Result<()> {
let peer_host = transport.host();
report(&format!(
" {peer_host} needs rows after seq {start}, below this log's floor; sending a snapshot"
));
let mut snapshot = local.export_snapshot(receiver)?;
let (input, output, mut remote) = transport.snapshot_import()?;
let mut writer = BufWriter::new(input);
let copied = std::io::copy(&mut snapshot, &mut writer).and_then(|_| writer.flush());
drop(writer);
let mut line = String::new();
let answered = BufReader::new(output).read_line(&mut line);
let exit = remote.wait()?;
if !exit.success {
anyhow::bail!(
"sync-snapshot on {peer_host} exited with {}.\n hint: {}",
exit.status,
untrusted_host_hint(peer_host)
);
}
copied.with_context(|| format!("streaming the snapshot to {peer_host}"))?;
answered?;
let merging: Merging = serde_json::from_str(line.trim())
.with_context(|| format!("reading the merge {peer_host} started"))?;
report(&format!(
" {peer_host} is merging snapshot {} through seq {}",
merging.snapshot_id, merging.seq
));
Ok(())
}
pub fn push_start(push_cursor: u64, previous: Option<u64>, reported: Option<u64>) -> u64 {
let restored = previous.is_some_and(|previous| reported.is_none_or(|r| r < previous));
if restored {
return reported.unwrap_or(0);
}
push_cursor.max(reported.unwrap_or(0))
}
fn first_line<R: BufRead>(reader: &mut R) -> std::io::Result<String> {
let mut line = String::new();
loop {
line.clear();
if reader.read_line(&mut line)? == 0 || !line.trim().is_empty() {
return Ok(line);
}
}
}
fn read_push_applied<R: BufRead>(replies: &mut R) -> Result<PushApplied> {
let mut line = String::new();
loop {
line.clear();
if replies.read_line(&mut line)? == 0 {
anyhow::bail!("the receiver closed the stream without saying what it applied");
}
if !line.trim().is_empty() {
return PushApplied::parse_line(&line);
}
}
}
pub fn serve_push<R: BufRead, W: Write>(
local: &dyn Local,
header: &SyncHeader,
mut input: R,
mut output: W,
) -> Result<Imported> {
if let Err(e) = accept_peer_hello(local, header, "the sender") {
return Err(refuse(&mut output, e));
}
let pusher = header
.hello
.as_ref()
.map(|hello| hello.machine_id.clone())
.expect("accept_peer_hello refuses a header without its hello");
let reply = SyncHeader {
hello: Some(
local
.hello_for(&pusher)
.context("building the sync hello")?,
),
..SyncHeader::local()
};
writeln!(output, "{}", reply.to_line())?;
output.flush()?;
let (mut totals, watermark) = import_lines(local, header, &mut input, PUSH_IMPORT_CHUNK)?;
totals.max_seq = watermark.applied();
if totals.max_seq > 0 {
local
.record_peer_in(&pusher, totals.max_seq)
.context("recording what the push applied")?;
}
writeln!(
output,
"{}",
PushApplied {
applied: totals.max_seq
}
.to_line()
)?;
output.flush()?;
Ok(totals)
}
fn refuse<W: Write>(output: &mut W, e: anyhow::Error) -> anyhow::Error {
let mismatch = e.downcast_ref::<IdentityMismatch>();
let refused = RefusedLine {
error: mismatch.map_or_else(|| format!("{e:#}"), ToString::to_string),
identity: mismatch.is_some(),
};
match writeln!(output, "{}", refused.to_line()).and_then(|()| output.flush()) {
Ok(()) => e,
Err(told) => e.context(format!("the sender did not get the refusal: {told}")),
}
}
fn import_lines<R: BufRead>(
local: &dyn Local,
header: &SyncHeader,
reader: &mut R,
chunk_len: usize,
) -> Result<(Imported, Watermark)> {
let mut totals = Imported::default();
let mut watermark = Watermark::default();
let mut chunk = Vec::with_capacity(chunk_len);
let mut post_chunk = |chunk: &mut Vec<String>| -> Result<()> {
if chunk.is_empty() {
return Ok(());
}
let resp = local.import(header, chunk)?;
totals.created += resp.created;
totals.updated += resp.updated;
totals.deleted += resp.deleted;
totals.skipped += resp.skipped;
totals.dangling += resp.dangling;
totals.errors += resp.errors;
watermark.add(&resp);
chunk.clear();
Ok(())
};
for line in reader.lines() {
let line = line?;
if line.trim().is_empty() {
continue;
}
chunk.push(line);
if chunk.len() >= chunk_len {
post_chunk(&mut chunk)?;
}
}
post_chunk(&mut chunk)?;
Ok((totals, watermark))
}
fn accept_peer_hello(local: &dyn Local, header: &SyncHeader, peer_host: &str) -> Result<()> {
let hello = header.hello.as_ref().ok_or_else(|| {
anyhow::anyhow!(
"{peer_host} sent a sync protocol {SYNC_PROTOCOL_VERSION} header without its hello, \
nothing was applied"
)
})?;
local
.accept_hello(hello)
.with_context(|| format!("accepting the sync hello from {peer_host}"))
}
pub fn read_checked_header<R: BufRead>(reader: &mut R, sender: &str) -> Result<SyncHeader> {
let header =
read_header(reader).with_context(|| format!("reading the sync header from {sender}"))?;
SyncHeader::local().ensure_matches("this machine", &header, sender)?;
Ok(header)
}
#[cfg(test)]
mod tests {
use std::sync::{Arc, Mutex};
use super::*;
const PEER: &str = "peer-machine";
fn hello(machine_id: &str, cursor: Option<u64>) -> Hello {
Hello {
machine_id: machine_id.to_string(),
wall_micros: 1_767_225_600_000_000,
synced: true,
cursor,
held_origin_seq: Some(3),
keys: None,
peer_in: None,
log_floor: 0,
head: 0,
applied: None,
merging: None,
}
}
#[derive(Default)]
struct FakeLocal {
cursors: Vec<(String, u64)>,
events: Mutex<Vec<String>>,
accepted: Mutex<Vec<Hello>>,
imported: Mutex<Vec<String>>,
exports: Vec<String>,
export_starts: Mutex<Vec<u64>>,
set_cursors: Mutex<Vec<(String, u64)>>,
acks: Mutex<Vec<(String, u64)>>,
peer_in: Mutex<Vec<(String, u64)>>,
last_report: Mutex<Option<u64>>,
report_fails: bool,
import_results: Mutex<Vec<Imported>>,
bootstrapping: Mutex<bool>,
refuses_hello: Option<String>,
}
impl Local for FakeLocal {
fn cursors(&self) -> Result<Vec<(String, u64)>> {
Ok(self.cursors.clone())
}
fn set_cursor(&self, peer_id: &str, last_seq: u64) -> Result<()> {
self.set_cursors
.lock()
.unwrap()
.push((peer_id.to_string(), last_seq));
Ok(())
}
fn import(&self, _header: &SyncHeader, lines: &[String]) -> Result<Imported> {
self.events.lock().unwrap().push("import".to_string());
self.imported.lock().unwrap().extend_from_slice(lines);
let mut scripted = self.import_results.lock().unwrap();
if !scripted.is_empty() {
return Ok(scripted.remove(0));
}
Ok(Imported {
created: lines.len() as u64,
max_seq: 8,
..Imported::default()
})
}
fn export(&self, peer_id: &str, after_seq: u64, _limit: usize) -> Result<Exported> {
assert_eq!(peer_id, PEER);
let mut starts = self.export_starts.lock().unwrap();
let lines = if starts.is_empty() {
self.exports.clone()
} else {
Vec::new()
};
starts.push(after_seq);
Ok(Exported {
max_seq: after_seq + lines.len() as u64,
lines,
})
}
fn hello_for(&self, peer_id: &str) -> Result<Hello> {
assert_eq!(peer_id, PEER);
Ok(hello("local-machine", Some(99)))
}
fn accept_hello(&self, hello: &Hello) -> Result<()> {
if let Some(message) = &self.refuses_hello {
return Err(anyhow::Error::new(IdentityMismatch(message.clone()))
.context("sync/accept-hello returned error"));
}
self.events.lock().unwrap().push("accept".to_string());
self.accepted.lock().unwrap().push(hello.clone());
Ok(())
}
fn record_ack(&self, peer_id: &str, acked_seq: u64) -> Result<()> {
self.acks
.lock()
.unwrap()
.push((peer_id.to_string(), acked_seq));
Ok(())
}
fn record_peer_in(&self, peer_id: &str, seq: u64) -> Result<()> {
self.peer_in
.lock()
.unwrap()
.push((peer_id.to_string(), seq));
Ok(())
}
fn push_report(&self, peer_id: &str) -> Result<Option<u64>> {
assert_eq!(peer_id, PEER);
Ok(*self.last_report.lock().unwrap())
}
fn record_push_report(&self, peer_id: &str, reported: Option<u64>) -> Result<()> {
assert_eq!(peer_id, PEER);
if self.report_fails {
anyhow::bail!("the store is busy");
}
*self.last_report.lock().unwrap() = reported;
Ok(())
}
fn merge_snapshot(&self, exporter: &str, _stream: Box<dyn Read + Send>) -> Result<Merged> {
assert_eq!(exporter, PEER);
self.events.lock().unwrap().push("merge".to_string());
Ok(Merged::Done { seq: 5 })
}
fn bootstrap_pending(&self) -> Result<bool> {
Ok(*self.bootstrapping.lock().unwrap())
}
fn finish_bootstrap(&self) -> Result<()> {
self.events.lock().unwrap().push("finish".to_string());
*self.bootstrapping.lock().unwrap() = false;
Ok(())
}
}
struct Finished;
impl Remote for Finished {
fn wait(&mut self) -> Result<Exit> {
Ok(Exit {
success: true,
status: "exit status: 0".to_string(),
})
}
fn stop(&mut self) -> Result<()> {
Ok(())
}
}
struct Lost;
impl Remote for Lost {
fn wait(&mut self) -> Result<Exit> {
anyhow::bail!("ssh vanished")
}
fn stop(&mut self) -> Result<()> {
Ok(())
}
}
struct Failed;
impl Remote for Failed {
fn wait(&mut self) -> Result<Exit> {
Ok(Exit {
success: false,
status: "exit status: 1".to_string(),
})
}
fn stop(&mut self) -> Result<()> {
Ok(())
}
}
#[derive(Clone, Default)]
struct Shared(Arc<Mutex<Vec<u8>>>);
impl Write for Shared {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.0.lock().unwrap().extend_from_slice(buf);
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
struct FakePeer {
export_stream: String,
exports: Mutex<Vec<(u64, Hello)>>,
snapshots: Mutex<Vec<Hello>>,
pushed: Shared,
push_replies: String,
remote_fails: bool,
remote_lost: bool,
}
impl FakePeer {
fn sending(header: SyncHeader) -> Self {
Self {
export_stream: format!("{}\n{SEALED}\n", header.to_line()),
exports: Mutex::default(),
snapshots: Mutex::default(),
pushed: Shared::default(),
push_replies: receiving(hello(PEER, None), 0),
remote_fails: false,
remote_lost: false,
}
}
fn remote(&self) -> Box<dyn Remote> {
if self.remote_lost {
Box::new(Lost)
} else if self.remote_fails {
Box::new(Failed)
} else {
Box::new(Finished)
}
}
fn taking_pushes(replies: String) -> Self {
Self {
push_replies: replies,
..Self::sending(SyncHeader::local())
}
}
}
fn receiving(peer_hello: Hello, applied: u64) -> String {
let header = SyncHeader {
hello: Some(peer_hello),
..SyncHeader::local()
};
format!(
"{}\n{}\n",
header.to_line(),
PushApplied { applied }.to_line()
)
}
impl Transport for FakePeer {
fn host(&self) -> &str {
"user@peer"
}
fn whoami(&self) -> Result<String> {
Ok(PEER.to_string())
}
fn export(
&self,
after_seq: u64,
hello: &Hello,
) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)> {
self.exports
.lock()
.unwrap()
.push((after_seq, hello.clone()));
Ok((
Box::new(Cursor::new(self.export_stream.clone().into_bytes())),
self.remote(),
))
}
fn snapshot_export(
&self,
hello: &Hello,
) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)> {
self.snapshots.lock().unwrap().push(hello.clone());
Ok((Box::new(Cursor::new(Vec::new())), Box::new(Finished)))
}
fn import(&self) -> Result<(Box<dyn Write + Send>, Box<dyn Read + Send>, Box<dyn Remote>)> {
Ok((
Box::new(self.pushed.clone()),
Box::new(Cursor::new(self.push_replies.clone().into_bytes())),
self.remote(),
))
}
}
const SEALED: &str = "AgAAAAEAAQIDBAUGBw";
#[test]
fn a_pull_sends_a_hello_at_its_cursor_and_accepts_the_peers_before_importing() {
let local = FakeLocal {
cursors: vec![(PEER.to_string(), 7)],
..FakeLocal::default()
};
let peer_hello = hello(PEER, Some(4));
let peer = FakePeer::sending(SyncHeader {
hello: Some(peer_hello.clone()),
..SyncHeader::local()
});
let imported = pull(&local, &peer, PEER).unwrap();
assert_eq!(imported.created, 1);
assert_eq!(
*peer.exports.lock().unwrap(),
vec![(
7,
Hello {
applied: Some(7),
..hello("local-machine", Some(7))
}
)],
"the hello's cursor is the cursor the pull resumes from, and nothing failed"
);
assert_eq!(*local.accepted.lock().unwrap(), vec![peer_hello]);
assert_eq!(*local.events.lock().unwrap(), vec!["accept", "import"]);
assert_eq!(
*local.imported.lock().unwrap(),
vec![SEALED.to_string()],
"the peer's sealed line reaches the store unopened"
);
}
#[test]
fn a_bootstrapping_device_merges_a_snapshot_before_its_first_pull_and_then_finishes() {
let local = FakeLocal {
bootstrapping: Mutex::new(true),
..FakeLocal::default()
};
let peer = FakePeer::sending(SyncHeader {
hello: Some(hello(PEER, None)),
..SyncHeader::local()
});
pull(&local, &peer, PEER).unwrap();
let snapshots = peer.snapshots.lock().unwrap();
assert_eq!(
snapshots.len(),
1,
"a peer whose log starts at 0 still sends a snapshot"
);
assert_eq!(snapshots[0].cursor, Some(0));
assert_eq!(
*local.events.lock().unwrap(),
vec!["merge", "accept", "import", "finish"]
);
assert!(!*local.bootstrapping.lock().unwrap());
}
#[test]
fn a_bootstrap_whose_snapshot_already_merged_only_pulls_and_one_that_fails_stays_pending() {
let merged = FakeLocal {
cursors: vec![(PEER.to_string(), 7)],
bootstrapping: Mutex::new(true),
..FakeLocal::default()
};
let peer = FakePeer::sending(SyncHeader {
hello: Some(hello(PEER, None)),
..SyncHeader::local()
});
pull(&merged, &peer, PEER).unwrap();
assert!(peer.snapshots.lock().unwrap().is_empty());
assert_eq!(
*merged.events.lock().unwrap(),
vec!["accept", "import", "finish"]
);
let failing = FakeLocal {
bootstrapping: Mutex::new(true),
..FakeLocal::default()
};
let no_hello = FakePeer::sending(SyncHeader::local());
assert!(pull(&failing, &no_hello, PEER).is_err());
assert!(*failing.bootstrapping.lock().unwrap());
assert!(
!failing
.events
.lock()
.unwrap()
.contains(&"finish".to_string())
);
}
#[test]
fn a_pull_whose_header_has_no_hello_imports_nothing() {
let local = FakeLocal::default();
let peer = FakePeer::sending(SyncHeader::local());
let error = pull(&local, &peer, PEER).unwrap_err();
assert!(format!("{error:#}").contains("hello"), "{error:#}");
assert!(local.events.lock().unwrap().is_empty());
}
#[test]
fn a_push_header_carries_the_hello() {
let local = FakeLocal {
exports: vec![SEALED.to_string()],
..FakeLocal::default()
};
let peer = FakePeer::sending(SyncHeader::local());
push(&local, &peer, PEER, &|_| {}).unwrap();
let pushed = peer.pushed.0.lock().unwrap().clone();
let mut reader = BufReader::new(pushed.as_slice());
let header = read_header(&mut reader).unwrap();
let rest: Vec<String> = reader.lines().map(|l| l.unwrap()).collect();
assert_eq!(
rest,
vec![SEALED.to_string()],
"sealed lines go out as exported"
);
assert_eq!(header.hello, Some(hello("local-machine", Some(99))));
assert_eq!(
(header.sync_protocol, header.schema_version),
(
SyncHeader::local().sync_protocol,
SyncHeader::local().schema_version
)
);
}
#[test]
fn a_failed_chunk_caps_the_watermark_whatever_later_chunks_apply() {
let mut watermark = Watermark::default();
let chunk = |max_seq, errors| Imported {
max_seq,
errors,
..Imported::default()
};
watermark.add(&chunk(10, 0));
watermark.add(&chunk(15, 1));
watermark.add(&chunk(30, 0));
assert_eq!(watermark.applied(), 15);
assert_eq!(watermark.reached(), 30);
}
#[test]
fn a_pull_cursor_moves_past_a_failed_chunk() {
let local = FakeLocal {
import_results: Mutex::new(vec![
Imported {
max_seq: 5,
errors: 1,
..Imported::default()
},
Imported {
max_seq: 20,
..Imported::default()
},
]),
..FakeLocal::default()
};
let peer = FakePeer {
export_stream: format!(
"{}\n{}",
SyncHeader {
hello: Some(hello(PEER, None)),
..SyncHeader::local()
}
.to_line(),
format!("{SEALED}\n").repeat(IMPORT_CHUNK + 1)
),
..FakePeer::sending(SyncHeader::local())
};
let imported = pull(&local, &peer, PEER).unwrap();
assert_eq!(imported.max_seq, 20);
assert_eq!(
*local.set_cursors.lock().unwrap(),
vec![(PEER.to_string(), 20)]
);
}
#[test]
fn the_receivers_report_sets_where_a_push_starts() {
assert_eq!(push_start(5, None, None), 5, "no report: the cursor");
assert_eq!(push_start(0, None, Some(9)), 9, "no cursor: the report");
assert_eq!(push_start(5, None, Some(9)), 9);
assert_eq!(push_start(9, Some(4), Some(6)), 9);
assert_eq!(
push_start(9, Some(9), Some(4)),
4,
"a lower report: restored"
);
assert_eq!(
push_start(9, Some(9), None),
0,
"no report after one: restored"
);
}
#[test]
fn a_push_resumes_from_the_receivers_report_and_records_its_acks() {
let local = FakeLocal {
cursors: vec![(format!("push:{PEER}"), 3)],
exports: vec![SEALED.to_string()],
..FakeLocal::default()
};
let receiver = Hello {
peer_in: Some(6),
..hello(PEER, Some(2))
};
let peer = FakePeer::taking_pushes(receiving(receiver.clone(), 7));
push(&local, &peer, PEER, &|_| {}).unwrap();
assert_eq!(*local.accepted.lock().unwrap(), vec![receiver]);
assert_eq!(local.export_starts.lock().unwrap()[0], 6);
assert_eq!(*local.last_report.lock().unwrap(), Some(6));
assert_eq!(
*local.acks.lock().unwrap(),
vec![(PEER.to_string(), 6), (PEER.to_string(), 7)],
"the report and then the final watermark are acks"
);
}
#[test]
fn a_receiver_holding_a_failed_record_acks_only_what_it_applied_without_a_gap() {
let local = FakeLocal {
cursors: vec![(format!("push:{PEER}"), 9)],
exports: vec![SEALED.to_string()],
..FakeLocal::default()
};
let receiver = Hello {
applied: Some(4),
..hello(PEER, Some(9))
};
let peer = FakePeer::taking_pushes(receiving(receiver, 4));
push(&local, &peer, PEER, &|_| {}).unwrap();
assert_eq!(
local.export_starts.lock().unwrap()[0],
9,
"the push resumes from the receiver's cursor, not from the failed record"
);
assert_eq!(local.acks.lock().unwrap()[0], (PEER.to_string(), 4));
}
#[test]
fn a_push_to_a_receiver_that_closes_without_a_result_fails() {
let local = FakeLocal {
exports: vec![SEALED.to_string()],
..FakeLocal::default()
};
let header = SyncHeader {
hello: Some(hello(PEER, None)),
..SyncHeader::local()
};
let peer = FakePeer::taking_pushes(format!("{}\n", header.to_line()));
let error = push(&local, &peer, PEER, &|_| {}).unwrap_err();
assert!(format!("{error:#}").contains("applied"), "{error:#}");
assert!(local.set_cursors.lock().unwrap().is_empty());
}
#[test]
fn serving_a_push_answers_with_this_hello_and_reports_what_it_applied() {
let local = FakeLocal::default();
let pusher = hello(PEER, Some(1));
let input = format!(
"{}\n{SEALED}\n{SEALED}\n",
SyncHeader {
hello: Some(pusher.clone()),
..SyncHeader::local()
}
.to_line()
);
let mut output = Vec::new();
let mut input = input.as_bytes();
let header = read_checked_header(&mut input, "the sender").unwrap();
let imported = serve_push(&local, &header, input, &mut output).unwrap();
assert_eq!(imported.created, 2);
assert_eq!(*local.accepted.lock().unwrap(), vec![pusher]);
assert_eq!(*local.peer_in.lock().unwrap(), vec![(PEER.to_string(), 8)]);
let mut replies = BufReader::new(output.as_slice());
let reply = read_header(&mut replies).unwrap();
assert_eq!(reply.hello, Some(hello("local-machine", Some(99))));
assert_eq!(
read_push_applied(&mut replies).unwrap(),
PushApplied { applied: 8 }
);
}
#[test]
fn a_restored_receiver_is_pushed_from_its_report_again_after_a_failed_push() {
let local = FakeLocal {
cursors: vec![(format!("push:{PEER}"), 9)],
exports: vec![SEALED.to_string()],
last_report: Mutex::new(Some(9)),
..FakeLocal::default()
};
let header = SyncHeader {
hello: Some(hello(PEER, Some(4))),
..SyncHeader::local()
};
let peer = FakePeer::taking_pushes(format!("{}\n", header.to_line()));
push(&local, &peer, PEER, &|_| {}).unwrap_err();
assert_eq!(local.export_starts.lock().unwrap()[0], 4);
assert_eq!(
*local.set_cursors.lock().unwrap(),
vec![(format!("push:{PEER}"), 4)],
"the lowered start outlives the failed push"
);
}
#[test]
fn a_restore_seen_by_a_push_that_fails_before_streaming_is_seen_again() {
let local = FakeLocal {
cursors: vec![(format!("push:{PEER}"), 9)],
exports: vec![SEALED.to_string()],
last_report: Mutex::new(Some(9)),
report_fails: true,
..FakeLocal::default()
};
let peer = FakePeer::taking_pushes(receiving(hello(PEER, Some(4)), 4));
push(&local, &peer, PEER, &|_| {}).unwrap_err();
assert_eq!(*local.last_report.lock().unwrap(), Some(9));
assert_eq!(
*local.set_cursors.lock().unwrap(),
vec![(format!("push:{PEER}"), 4)]
);
assert!(local.export_starts.lock().unwrap().is_empty());
}
const MISMATCH: &str = "peer-machine holds a different sync identity; pair it with `sb pair`";
fn refused_identity() -> RefusedLine {
RefusedLine {
error: MISMATCH.to_string(),
identity: true,
}
}
fn expect_refused(error: &anyhow::Error) -> &PeerRefused {
error
.downcast_ref::<PeerRefused>()
.unwrap_or_else(|| panic!("not a PeerRefused: {error:#}"))
}
#[test]
fn a_pull_the_peer_refuses_returns_its_named_error_not_its_exit_status() {
let local = FakeLocal::default();
let peer = FakePeer {
export_stream: format!("{}\n", refused_identity().to_line()),
remote_fails: true,
..FakePeer::sending(SyncHeader::local())
};
let error = pull(&local, &peer, PEER).unwrap_err();
assert_eq!(
expect_refused(&error),
&PeerRefused {
host: "user@peer".to_string(),
error: MISMATCH.to_string(),
identity: true,
}
);
assert!(format!("{error}").contains("sb pair"), "{error}");
assert!(local.events.lock().unwrap().is_empty());
assert!(local.set_cursors.lock().unwrap().is_empty());
}
#[test]
fn a_push_the_receiver_refuses_returns_its_named_error_and_streams_nothing() {
let local = FakeLocal {
exports: vec![SEALED.to_string()],
..FakeLocal::default()
};
let peer = FakePeer {
remote_fails: true,
..FakePeer::taking_pushes(format!("\n{}\n", refused_identity().to_line()))
};
let error = push(&local, &peer, PEER, &|_| {}).unwrap_err();
let refusal = expect_refused(&error);
assert!(refusal.identity);
assert_eq!(refusal.error, MISMATCH);
assert!(local.export_starts.lock().unwrap().is_empty());
assert!(local.set_cursors.lock().unwrap().is_empty());
assert!(local.acks.lock().unwrap().is_empty());
assert_eq!(*local.last_report.lock().unwrap(), None);
}
#[test]
fn a_refusal_survives_a_remote_that_cannot_be_waited_on() {
let local = FakeLocal::default();
let puller = FakePeer {
export_stream: format!("{}\n", refused_identity().to_line()),
remote_lost: true,
..FakePeer::sending(SyncHeader::local())
};
let error = pull(&local, &puller, PEER).unwrap_err();
assert!(expect_refused(&error).identity, "{error:#}");
let pusher = FakePeer {
remote_lost: true,
..FakePeer::taking_pushes(format!("\n{}\n", refused_identity().to_line()))
};
let error = push(&local, &pusher, PEER, &|_| {}).unwrap_err();
assert!(expect_refused(&error).identity, "{error:#}");
}
#[test]
fn a_push_whose_receiver_fails_without_a_refusal_still_reports_its_exit() {
let local = FakeLocal::default();
let peer = FakePeer {
remote_fails: true,
..FakePeer::taking_pushes(String::new())
};
let error = push(&local, &peer, PEER, &|_| {}).unwrap_err();
assert!(error.downcast_ref::<PeerRefused>().is_none(), "{error:#}");
assert!(format!("{error:#}").contains("exited with"), "{error:#}");
}
#[test]
fn serving_a_push_whose_hello_is_refused_tells_the_pusher_the_mismatch() {
let receiver = FakeLocal {
refuses_hello: Some(MISMATCH.to_string()),
..FakeLocal::default()
};
let header = SyncHeader {
hello: Some(hello(PEER, Some(1))),
..SyncHeader::local()
};
let input = format!("{SEALED}\n");
let mut output = Vec::new();
let error = serve_push(&receiver, &header, input.as_bytes(), &mut output).unwrap_err();
assert!(
error.downcast_ref::<IdentityMismatch>().is_some(),
"{error:#}"
);
assert!(receiver.imported.lock().unwrap().is_empty());
assert!(receiver.peer_in.lock().unwrap().is_empty());
let replies = String::from_utf8(output).unwrap();
assert_eq!(
RefusedLine::parse_line(&replies),
Some(refused_identity()),
"{replies}"
);
let pusher = FakeLocal::default();
let peer = FakePeer {
remote_fails: true,
..FakePeer::taking_pushes(replies)
};
let error = push(&pusher, &peer, PEER, &|_| {}).unwrap_err();
assert!(expect_refused(&error).identity);
}
#[test]
fn serving_a_push_refused_for_another_reason_sends_the_whole_error() {
let header = SyncHeader {
hello: None,
..SyncHeader::local()
};
let mut output = Vec::new();
serve_push(&FakeLocal::default(), &header, &b""[..], &mut output).unwrap_err();
let refused = RefusedLine::parse_line(&String::from_utf8(output).unwrap()).unwrap();
assert!(!refused.identity);
assert!(
refused.error.contains("without its hello"),
"{}",
refused.error
);
}
}