1use std::collections::BTreeMap;
2use std::fmt;
3use std::io::{BufRead, BufReader, BufWriter, Cursor, Read, Write};
4use std::path::Path;
5
6use crate::keyring::IdentityMismatch;
7use crate::protocol::{
8 BelowFloorLine, Hello, Merging, PushApplied, RefusedLine, SYNC_PROTOCOL_VERSION, SyncHeader,
9 read_header,
10};
11use crate::ssh::untrusted_host_hint;
12use anyhow::{Context, Result};
13use second_brain_core::store::SkipReason;
14
15const IMPORT_CHUNK: usize = 1000;
18const PUSH_IMPORT_CHUNK: usize = 500;
19const EXPORT_PAGE: usize = 2000;
20
21pub trait Local {
25 fn cursors(&self) -> Result<Vec<(String, u64)>>;
26 fn set_cursor(&self, peer_id: &str, last_seq: u64) -> Result<()>;
27 fn import(&self, header: &SyncHeader, lines: &[String]) -> Result<Imported>;
29 fn export(&self, peer_id: &str, after_seq: u64, limit: usize) -> Result<Exported>;
31 fn hello_for(&self, peer_id: &str) -> Result<Hello>;
32 fn accept_hello(&self, hello: &Hello) -> Result<()>;
35 fn record_ack(&self, peer_id: &str, acked_seq: u64) -> Result<()>;
37 fn record_peer_in(&self, peer_id: &str, seq: u64) -> Result<()>;
39 fn push_report(&self, peer_id: &str) -> Result<Option<u64>>;
42 fn record_push_report(&self, peer_id: &str, reported: Option<u64>) -> Result<()>;
43 fn merge_snapshot(&self, exporter: &str, stream: Box<dyn Read + Send>) -> Result<Merged> {
46 let _ = (exporter, stream);
47 anyhow::bail!("this store cannot merge a snapshot")
48 }
49 fn merging(&self) -> Result<Option<Merging>> {
51 Ok(None)
52 }
53 fn export_snapshot(&self, peer: &Hello) -> Result<Box<dyn Read + Send>> {
56 let _ = peer;
57 anyhow::bail!("this store cannot export a snapshot")
58 }
59 fn bootstrap_pending(&self) -> Result<bool> {
62 Ok(false)
63 }
64 fn finish_bootstrap(&self) -> Result<()> {
66 Ok(())
67 }
68}
69
70#[derive(Debug, Clone, PartialEq, Eq)]
71pub enum Merged {
72 Done {
74 seq: u64,
75 },
76 Running(Merging),
77}
78
79#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
80pub struct Imported {
81 pub created: u64,
82 pub updated: u64,
83 pub deleted: u64,
84 pub skipped: u64,
85 pub dangling: u64,
86 pub errors: u64,
87 pub max_seq: u64,
89 pub reached: u64,
91}
92
93impl Imported {
94 pub fn dangling_in(skipped_by_reason: &BTreeMap<String, u64>) -> u64 {
95 skipped_by_reason
96 .get(SkipReason::DanglingEndpoint.as_str())
97 .copied()
98 .unwrap_or(0)
99 }
100}
101
102#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
105pub struct Watermark {
106 reached: u64,
107 applied: u64,
108 capped: Option<u64>,
109}
110
111impl Watermark {
112 pub fn add(&mut self, chunk: &Imported) {
113 self.reached = self.reached.max(chunk.reached.max(chunk.max_seq));
114 self.applied = self.applied.max(chunk.max_seq);
115 if chunk.errors > 0 {
116 self.capped = Some(self.capped.map_or(chunk.max_seq, |c| c.min(chunk.max_seq)));
117 }
118 }
119
120 pub fn applied(&self) -> u64 {
124 self.capped
125 .map_or(self.applied, |cap| self.applied.min(cap))
126 }
127
128 pub fn reached(&self) -> u64 {
130 self.reached
131 }
132}
133
134pub struct Exported {
135 pub lines: Vec<String>,
136 pub max_seq: u64,
137}
138
139pub trait Transport {
140 fn host(&self) -> &str;
141 fn whoami(&self) -> Result<String>;
142 fn export(
143 &self,
144 after_seq: u64,
145 hello: &Hello,
146 ) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)>;
147 fn import(&self) -> Result<(Box<dyn Write + Send>, Box<dyn Read + Send>, Box<dyn Remote>)>;
150 fn snapshot_export(&self, hello: &Hello) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)> {
152 let _ = hello;
153 anyhow::bail!("the transport to {} carries no snapshots", self.host())
154 }
155 fn snapshot_import(
158 &self,
159 ) -> Result<(Box<dyn Write + Send>, Box<dyn Read + Send>, Box<dyn Remote>)> {
160 anyhow::bail!("the transport to {} carries no snapshots", self.host())
161 }
162}
163
164pub trait Remote {
166 fn wait(&mut self) -> Result<Exit>;
167 fn stop(&mut self) -> Result<()>;
168}
169
170pub struct Exit {
171 pub success: bool,
172 pub status: String,
173}
174
175#[derive(Debug, Clone, PartialEq, Eq)]
177pub struct PeerRefused {
178 pub host: String,
179 pub error: String,
180 pub identity: bool,
182}
183
184impl PeerRefused {
185 fn from_line(host: &str, refused: RefusedLine) -> Self {
186 Self {
187 host: host.to_string(),
188 error: refused.error,
189 identity: refused.identity,
190 }
191 }
192}
193
194impl fmt::Display for PeerRefused {
195 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
196 write!(f, "{} refused the sync: {}", self.host, self.error)
197 }
198}
199
200impl std::error::Error for PeerRefused {}
201
202pub fn sync(
205 local: &dyn Local,
206 transport: &dyn Transport,
207 data_dir: &Path,
208 report: &dyn Fn(&str),
209) -> Result<()> {
210 let peer_machine_id = transport
211 .whoami()
212 .context("resolving peer machine id (is the peer's daemon running and reachable?)")?;
213 sync_with(local, transport, &peer_machine_id, data_dir, report)
214}
215
216pub fn sync_with(
218 local: &dyn Local,
219 transport: &dyn Transport,
220 peer_machine_id: &str,
221 data_dir: &Path,
222 report: &dyn Fn(&str),
223) -> Result<()> {
224 let peer_host = transport.host();
225 report(&format!("pulling from {peer_host}..."));
226 let pull_stats = pull(local, transport, peer_machine_id)?;
227 report(&format!(
228 " {} created, {} updated, {} deleted, {} skipped",
229 pull_stats.created, pull_stats.updated, pull_stats.deleted, pull_stats.skipped
230 ));
231 if pull_stats.dangling > 0 {
232 report(&format!(
233 " warning: {} records dropped with unresolvable endpoints; \
234 run `sb prune-relations` on the peer if this persists",
235 pull_stats.dangling
236 ));
237 }
238
239 report(&format!("pushing to {peer_host}..."));
240 let max_seq = push(local, transport, peer_machine_id, report)?;
241 report(&format!(" pushed through seq {max_seq}"));
242
243 crate::peers::record_success(data_dir, chrono::Utc::now())
244 .context("recording that the sync succeeded")?;
245 report("sync complete.");
246 Ok(())
247}
248
249fn cursor(local: &dyn Local, key: &str, legacy_key: &str) -> Result<u64> {
253 let cursors = local.cursors().context("getting sync status")?;
254 Ok(cursors
255 .iter()
256 .find(|(peer_id, _)| peer_id == key)
257 .or_else(|| cursors.iter().find(|(peer_id, _)| peer_id == legacy_key))
258 .map(|(_, last_seq)| *last_seq)
259 .unwrap_or(0))
260}
261
262pub fn pull(
263 local: &dyn Local,
264 transport: &dyn Transport,
265 peer_machine_id: &str,
266) -> Result<Imported> {
267 if let Some(merging) = local.merging()? {
268 anyhow::bail!(
269 "a snapshot merge from {} through seq {} is still running; sync again once it finishes",
270 merging.exporter,
271 merging.seq
272 );
273 }
274 let bootstrapping = local.bootstrap_pending()?;
275 if bootstrapping && cursor(local, peer_machine_id, transport.host())? == 0 {
276 let start = BelowFloorLine {
277 after_seq: 0,
278 log_floor: 0,
279 };
280 merge_peer_snapshot(local, transport, peer_machine_id, start)?;
281 }
282 let imported = pull_after_floor(local, transport, peer_machine_id)?;
283 if bootstrapping {
284 local.finish_bootstrap()?;
285 }
286 Ok(imported)
287}
288
289fn pull_after_floor(
290 local: &dyn Local,
291 transport: &dyn Transport,
292 peer_machine_id: &str,
293) -> Result<Imported> {
294 match pull_log(local, transport, peer_machine_id)? {
295 Pulled::Log(imported) => Ok(imported),
296 Pulled::BelowFloor(below) => {
297 merge_peer_snapshot(local, transport, peer_machine_id, below)?;
298 match pull_log(local, transport, peer_machine_id)? {
299 Pulled::Log(imported) => Ok(imported),
300 Pulled::BelowFloor(again) => anyhow::bail!(
301 "{} truncated its log past seq {} again right after a snapshot; sync again",
302 transport.host(),
303 again.log_floor
304 ),
305 }
306 }
307 }
308}
309
310enum Pulled {
311 Log(Imported),
312 BelowFloor(BelowFloorLine),
313}
314
315fn pull_hello(local: &dyn Local, peer_machine_id: &str, after_seq: u64) -> Result<Hello> {
316 let hello = local
317 .hello_for(peer_machine_id)
318 .context("building the sync hello")?;
319 Ok(Hello {
320 cursor: Some(after_seq),
321 applied: Some(hello.applied.unwrap_or(after_seq).min(after_seq)),
322 ..hello
323 })
324}
325
326fn merge_peer_snapshot(
330 local: &dyn Local,
331 transport: &dyn Transport,
332 peer_machine_id: &str,
333 below: BelowFloorLine,
334) -> Result<()> {
335 let peer_host = transport.host();
336 let hello = pull_hello(local, peer_machine_id, below.after_seq)?;
337 let (stream, mut remote) = transport.snapshot_export(&hello)?;
338 let merged = local.merge_snapshot(peer_machine_id, stream);
339 let exit = remote.wait()?;
340 if !exit.success {
341 anyhow::bail!(
342 "sync-snapshot on {peer_host} exited with {}; its error is printed above.\n hint: {}",
343 exit.status,
344 untrusted_host_hint(peer_host)
345 );
346 }
347 match merged.with_context(|| format!("merging the snapshot from {peer_host}"))? {
348 Merged::Done { .. } => Ok(()),
349 Merged::Running(merging) => anyhow::bail!(
350 "merging the snapshot from {peer_host} through seq {} continues in the daemon; \
351 sync again once it finishes",
352 merging.seq
353 ),
354 }
355}
356
357fn pull_log(local: &dyn Local, transport: &dyn Transport, peer_machine_id: &str) -> Result<Pulled> {
358 let peer_host = transport.host();
359 let after_seq = cursor(local, peer_machine_id, peer_host)?;
360 let hello = pull_hello(local, peer_machine_id, after_seq)?;
361
362 let (output, mut remote) = transport.export(after_seq, &hello)?;
363 let mut reader = BufReader::new(output);
364 let mut first = String::new();
365 let read = reader.read_line(&mut first);
366 if let Some(below) = BelowFloorLine::parse_line(&first) {
367 let exit = remote.wait()?;
368 if !exit.success {
369 anyhow::bail!(
370 "sync-export on {peer_host} exited with {}.\n hint: {}",
371 exit.status,
372 untrusted_host_hint(peer_host)
373 );
374 }
375 return Ok(Pulled::BelowFloor(below));
376 }
377 if let Some(refused) = RefusedLine::parse_line(&first) {
378 if let Err(e) = remote.wait() {
379 tracing::warn!("sync-export on {peer_host} refused and then failed to exit: {e:#}");
380 }
381 return Err(PeerRefused::from_line(peer_host, refused).into());
382 }
383 let header = read
384 .map_err(anyhow::Error::from)
385 .and_then(|_| read_checked_header(&mut Cursor::new(first.as_bytes()), peer_host));
386 let peer_header = match header {
387 Ok(header) => header,
388 Err(e) => {
389 if first.trim().is_empty() && matches!(reader.fill_buf(), Ok(rest) if rest.is_empty()) {
392 let exit = remote.wait()?;
393 if !exit.success {
394 anyhow::bail!(
395 "sync-export on {peer_host} exited with {}; its error is printed above.\n hint: {}",
396 exit.status,
397 untrusted_host_hint(peer_host)
398 );
399 }
400 } else {
401 remote.stop()?;
402 }
403 return Err(e);
404 }
405 };
406 if let Err(e) = accept_peer_hello(local, &peer_header, peer_host) {
407 remote.stop()?;
408 return Err(e);
409 }
410
411 let (mut totals, watermark) = match import_lines(local, &peer_header, &mut reader, IMPORT_CHUNK)
412 {
413 Ok(imported) => imported,
414 Err(e) => {
415 remote.stop()?;
416 return Err(e);
417 }
418 };
419
420 let exit = remote.wait()?;
421 if !exit.success {
422 anyhow::bail!(
423 "ssh pull from {peer_host} exited with {}.\n hint: {}",
424 exit.status,
425 untrusted_host_hint(peer_host)
426 );
427 }
428
429 totals.max_seq = watermark.reached();
433 if totals.max_seq > after_seq {
434 local
435 .set_cursor(peer_machine_id, totals.max_seq)
436 .context("updating sync state")?;
437 }
438
439 Ok(Pulled::Log(totals))
440}
441
442pub fn push(
443 local: &dyn Local,
444 transport: &dyn Transport,
445 peer_machine_id: &str,
446 report: &dyn Fn(&str),
447) -> Result<u64> {
448 let peer_host = transport.host();
449 let push_peer = format!("push:{peer_machine_id}");
450 let push_cursor = cursor(local, &push_peer, &format!("push:{peer_host}"))?;
451 let header = SyncHeader {
452 hello: Some(
453 local
454 .hello_for(peer_machine_id)
455 .context("building the sync hello")?,
456 ),
457 ..SyncHeader::local()
458 };
459
460 let (input, output, mut remote) = transport.import()?;
461 let mut writer = BufWriter::new(input);
462 let mut replies = BufReader::new(output);
463 let own_floor = header.hello.as_ref().map_or(0, |hello| hello.log_floor);
464 let mut handshake = || -> Result<(u64, Hello)> {
465 writeln!(writer, "{}", header.to_line())?;
466 writer.flush()?;
467 let first = first_line(&mut replies)
468 .with_context(|| format!("reading the sync header from {peer_host}"))?;
469 if let Some(refused) = RefusedLine::parse_line(&first) {
470 return Err(PeerRefused::from_line(peer_host, refused).into());
471 }
472 let receiver = read_checked_header(&mut Cursor::new(first.as_bytes()), peer_host)?;
473 accept_peer_hello(local, &receiver, peer_host)?;
474 let reported = receiver.hello.as_ref().and_then(Hello::applied_of_receiver);
475 let start = push_start(push_cursor, local.push_report(peer_machine_id)?, reported);
476 if start < push_cursor {
480 local
481 .set_cursor(&push_peer, start)
482 .context("lowering the push cursor for a restored peer")?;
483 }
484 local.record_push_report(peer_machine_id, reported)?;
485 if let Some(acked) = receiver.hello.as_ref().and_then(Hello::acked_by_receiver) {
486 local.record_ack(peer_machine_id, acked)?;
487 }
488 let hello = receiver
489 .hello
490 .clone()
491 .expect("accept_peer_hello refuses a header without its hello");
492 Ok((start, hello))
493 };
494 let (start, receiver) = match handshake() {
495 Ok(handshake) => handshake,
496 Err(e) => {
497 drop(writer);
498 let refused = e.downcast_ref::<PeerRefused>().is_some();
499 let exit = match remote.wait() {
500 Ok(exit) => exit,
501 Err(wait) if refused => {
502 tracing::warn!(
503 "ssh push to {peer_host} refused and then failed to exit: {wait:#}"
504 );
505 return Err(e);
506 }
507 Err(wait) => return Err(wait),
508 };
509 if !exit.success && !refused {
510 anyhow::bail!(
511 "ssh push to {peer_host} exited with {}.\n hint: {}",
512 exit.status,
513 untrusted_host_hint(peer_host)
514 );
515 }
516 return Err(e);
517 }
518 };
519
520 if receiver.merging.is_some() || start < own_floor {
521 drop(writer);
522 let applied = read_push_applied(&mut replies);
523 let exit = remote.wait()?;
524 if !exit.success {
525 anyhow::bail!(
526 "ssh push to {peer_host} exited with {}.\n hint: {}",
527 exit.status,
528 untrusted_host_hint(peer_host)
529 );
530 }
531 applied.with_context(|| format!("reading what {peer_host} applied"))?;
532 if let Some(merging) = receiver.merging {
533 report(&format!(
534 " {peer_host} is merging snapshot {} through seq {}; nothing pushed",
535 merging.snapshot_id, merging.seq
536 ));
537 } else {
538 push_snapshot(local, transport, &receiver, start, report)?;
539 }
540 return Ok(start.min(push_cursor));
541 }
542
543 let mut cursor = start;
544 let mut total_pushed = 0u64;
545 let mut stream = || -> Result<()> {
546 loop {
547 let export = local.export(peer_machine_id, cursor, EXPORT_PAGE)?;
548 if export.lines.is_empty() {
549 break;
550 }
551 let batch_count = export.lines.len();
552 for line in &export.lines {
553 writer.write_all(line.as_bytes())?;
554 writer.write_all(b"\n")?;
555 }
556 writer.flush()?;
557
558 total_pushed += batch_count as u64;
559 cursor = export.max_seq;
560 report(&format!(
561 " batch: {batch_count} records (through seq {cursor})"
562 ));
563
564 if batch_count < EXPORT_PAGE {
565 break;
566 }
567 }
568 writer.flush()?;
569 Ok(())
570 };
571 let streamed = stream();
572 drop(writer);
573 let applied = read_push_applied(&mut replies);
574
575 let exit = remote.wait()?;
578 if !exit.success {
579 anyhow::bail!(
580 "ssh push to {peer_host} exited with {}.\n hint: {}",
581 exit.status,
582 untrusted_host_hint(peer_host)
583 );
584 }
585 streamed?;
586 let applied = applied.with_context(|| format!("reading what {peer_host} applied"))?;
587 local
588 .record_ack(peer_machine_id, applied.applied)
589 .context("recording the peer's ack")?;
590
591 if cursor != start.min(push_cursor) {
592 local
593 .set_cursor(&push_peer, cursor)
594 .context("updating push sync state")?;
595 }
596
597 report(&format!(" total: {total_pushed} records pushed"));
598 Ok(cursor)
599}
600
601fn push_snapshot(
605 local: &dyn Local,
606 transport: &dyn Transport,
607 receiver: &Hello,
608 start: u64,
609 report: &dyn Fn(&str),
610) -> Result<()> {
611 let peer_host = transport.host();
612 report(&format!(
613 " {peer_host} needs rows after seq {start}, below this log's floor; sending a snapshot"
614 ));
615 let mut snapshot = local.export_snapshot(receiver)?;
616 let (input, output, mut remote) = transport.snapshot_import()?;
617 let mut writer = BufWriter::new(input);
618 let copied = std::io::copy(&mut snapshot, &mut writer).and_then(|_| writer.flush());
619 drop(writer);
620 let mut line = String::new();
621 let answered = BufReader::new(output).read_line(&mut line);
622 let exit = remote.wait()?;
623 if !exit.success {
624 anyhow::bail!(
625 "sync-snapshot on {peer_host} exited with {}.\n hint: {}",
626 exit.status,
627 untrusted_host_hint(peer_host)
628 );
629 }
630 copied.with_context(|| format!("streaming the snapshot to {peer_host}"))?;
631 answered?;
632 let merging: Merging = serde_json::from_str(line.trim())
633 .with_context(|| format!("reading the merge {peer_host} started"))?;
634 report(&format!(
635 " {peer_host} is merging snapshot {} through seq {}",
636 merging.snapshot_id, merging.seq
637 ));
638 Ok(())
639}
640
641pub fn push_start(push_cursor: u64, previous: Option<u64>, reported: Option<u64>) -> u64 {
646 let restored = previous.is_some_and(|previous| reported.is_none_or(|r| r < previous));
647 if restored {
648 return reported.unwrap_or(0);
649 }
650 push_cursor.max(reported.unwrap_or(0))
651}
652
653fn first_line<R: BufRead>(reader: &mut R) -> std::io::Result<String> {
655 let mut line = String::new();
656 loop {
657 line.clear();
658 if reader.read_line(&mut line)? == 0 || !line.trim().is_empty() {
659 return Ok(line);
660 }
661 }
662}
663
664fn read_push_applied<R: BufRead>(replies: &mut R) -> Result<PushApplied> {
665 let mut line = String::new();
666 loop {
667 line.clear();
668 if replies.read_line(&mut line)? == 0 {
669 anyhow::bail!("the receiver closed the stream without saying what it applied");
670 }
671 if !line.trim().is_empty() {
672 return PushApplied::parse_line(&line);
673 }
674 }
675}
676
677pub fn serve_push<R: BufRead, W: Write>(
683 local: &dyn Local,
684 header: &SyncHeader,
685 mut input: R,
686 mut output: W,
687) -> Result<Imported> {
688 if let Err(e) = accept_peer_hello(local, header, "the sender") {
689 return Err(refuse(&mut output, e));
690 }
691 let pusher = header
692 .hello
693 .as_ref()
694 .map(|hello| hello.machine_id.clone())
695 .expect("accept_peer_hello refuses a header without its hello");
696 let reply = SyncHeader {
697 hello: Some(
698 local
699 .hello_for(&pusher)
700 .context("building the sync hello")?,
701 ),
702 ..SyncHeader::local()
703 };
704 writeln!(output, "{}", reply.to_line())?;
705 output.flush()?;
706
707 let (mut totals, watermark) = import_lines(local, header, &mut input, PUSH_IMPORT_CHUNK)?;
708 totals.max_seq = watermark.applied();
709 if totals.max_seq > 0 {
710 local
711 .record_peer_in(&pusher, totals.max_seq)
712 .context("recording what the push applied")?;
713 }
714 writeln!(
715 output,
716 "{}",
717 PushApplied {
718 applied: totals.max_seq
719 }
720 .to_line()
721 )?;
722 output.flush()?;
723 Ok(totals)
724}
725
726fn refuse<W: Write>(output: &mut W, e: anyhow::Error) -> anyhow::Error {
729 let mismatch = e.downcast_ref::<IdentityMismatch>();
730 let refused = RefusedLine {
731 error: mismatch.map_or_else(|| format!("{e:#}"), ToString::to_string),
732 identity: mismatch.is_some(),
733 };
734 match writeln!(output, "{}", refused.to_line()).and_then(|()| output.flush()) {
735 Ok(()) => e,
736 Err(told) => e.context(format!("the sender did not get the refusal: {told}")),
737 }
738}
739
740fn import_lines<R: BufRead>(
743 local: &dyn Local,
744 header: &SyncHeader,
745 reader: &mut R,
746 chunk_len: usize,
747) -> Result<(Imported, Watermark)> {
748 let mut totals = Imported::default();
749 let mut watermark = Watermark::default();
750 let mut chunk = Vec::with_capacity(chunk_len);
751 let mut post_chunk = |chunk: &mut Vec<String>| -> Result<()> {
752 if chunk.is_empty() {
753 return Ok(());
754 }
755 let resp = local.import(header, chunk)?;
756 totals.created += resp.created;
757 totals.updated += resp.updated;
758 totals.deleted += resp.deleted;
759 totals.skipped += resp.skipped;
760 totals.dangling += resp.dangling;
761 totals.errors += resp.errors;
762 watermark.add(&resp);
763 chunk.clear();
764 Ok(())
765 };
766 for line in reader.lines() {
767 let line = line?;
768 if line.trim().is_empty() {
769 continue;
770 }
771 chunk.push(line);
772 if chunk.len() >= chunk_len {
773 post_chunk(&mut chunk)?;
774 }
775 }
776 post_chunk(&mut chunk)?;
777 Ok((totals, watermark))
778}
779
780fn accept_peer_hello(local: &dyn Local, header: &SyncHeader, peer_host: &str) -> Result<()> {
781 let hello = header.hello.as_ref().ok_or_else(|| {
782 anyhow::anyhow!(
783 "{peer_host} sent a sync protocol {SYNC_PROTOCOL_VERSION} header without its hello, \
784 nothing was applied"
785 )
786 })?;
787 local
788 .accept_hello(hello)
789 .with_context(|| format!("accepting the sync hello from {peer_host}"))
790}
791
792pub fn read_checked_header<R: BufRead>(reader: &mut R, sender: &str) -> Result<SyncHeader> {
793 let header =
794 read_header(reader).with_context(|| format!("reading the sync header from {sender}"))?;
795 SyncHeader::local().ensure_matches("this machine", &header, sender)?;
796 Ok(header)
797}
798
799#[cfg(test)]
800mod tests {
801 use std::sync::{Arc, Mutex};
802
803 use super::*;
804
805 const PEER: &str = "peer-machine";
806
807 fn hello(machine_id: &str, cursor: Option<u64>) -> Hello {
808 Hello {
809 machine_id: machine_id.to_string(),
810 wall_micros: 1_767_225_600_000_000,
811 synced: true,
812 cursor,
813 held_origin_seq: Some(3),
814 keys: None,
815 peer_in: None,
816 log_floor: 0,
817 head: 0,
818 applied: None,
819 merging: None,
820 }
821 }
822
823 #[derive(Default)]
824 struct FakeLocal {
825 cursors: Vec<(String, u64)>,
826 events: Mutex<Vec<String>>,
827 accepted: Mutex<Vec<Hello>>,
828 imported: Mutex<Vec<String>>,
829 exports: Vec<String>,
830 export_starts: Mutex<Vec<u64>>,
831 set_cursors: Mutex<Vec<(String, u64)>>,
832 acks: Mutex<Vec<(String, u64)>>,
833 peer_in: Mutex<Vec<(String, u64)>>,
834 last_report: Mutex<Option<u64>>,
835 report_fails: bool,
836 import_results: Mutex<Vec<Imported>>,
837 bootstrapping: Mutex<bool>,
838 refuses_hello: Option<String>,
839 }
840
841 impl Local for FakeLocal {
842 fn cursors(&self) -> Result<Vec<(String, u64)>> {
843 Ok(self.cursors.clone())
844 }
845
846 fn set_cursor(&self, peer_id: &str, last_seq: u64) -> Result<()> {
847 self.set_cursors
848 .lock()
849 .unwrap()
850 .push((peer_id.to_string(), last_seq));
851 Ok(())
852 }
853
854 fn import(&self, _header: &SyncHeader, lines: &[String]) -> Result<Imported> {
855 self.events.lock().unwrap().push("import".to_string());
856 self.imported.lock().unwrap().extend_from_slice(lines);
857 let mut scripted = self.import_results.lock().unwrap();
858 if !scripted.is_empty() {
859 return Ok(scripted.remove(0));
860 }
861 Ok(Imported {
862 created: lines.len() as u64,
863 max_seq: 8,
864 ..Imported::default()
865 })
866 }
867
868 fn export(&self, peer_id: &str, after_seq: u64, _limit: usize) -> Result<Exported> {
869 assert_eq!(peer_id, PEER);
870 let mut starts = self.export_starts.lock().unwrap();
871 let lines = if starts.is_empty() {
872 self.exports.clone()
873 } else {
874 Vec::new()
875 };
876 starts.push(after_seq);
877 Ok(Exported {
878 max_seq: after_seq + lines.len() as u64,
879 lines,
880 })
881 }
882
883 fn hello_for(&self, peer_id: &str) -> Result<Hello> {
884 assert_eq!(peer_id, PEER);
885 Ok(hello("local-machine", Some(99)))
886 }
887
888 fn accept_hello(&self, hello: &Hello) -> Result<()> {
889 if let Some(message) = &self.refuses_hello {
890 return Err(anyhow::Error::new(IdentityMismatch(message.clone()))
891 .context("sync/accept-hello returned error"));
892 }
893 self.events.lock().unwrap().push("accept".to_string());
894 self.accepted.lock().unwrap().push(hello.clone());
895 Ok(())
896 }
897
898 fn record_ack(&self, peer_id: &str, acked_seq: u64) -> Result<()> {
899 self.acks
900 .lock()
901 .unwrap()
902 .push((peer_id.to_string(), acked_seq));
903 Ok(())
904 }
905
906 fn record_peer_in(&self, peer_id: &str, seq: u64) -> Result<()> {
907 self.peer_in
908 .lock()
909 .unwrap()
910 .push((peer_id.to_string(), seq));
911 Ok(())
912 }
913
914 fn push_report(&self, peer_id: &str) -> Result<Option<u64>> {
915 assert_eq!(peer_id, PEER);
916 Ok(*self.last_report.lock().unwrap())
917 }
918
919 fn record_push_report(&self, peer_id: &str, reported: Option<u64>) -> Result<()> {
920 assert_eq!(peer_id, PEER);
921 if self.report_fails {
922 anyhow::bail!("the store is busy");
923 }
924 *self.last_report.lock().unwrap() = reported;
925 Ok(())
926 }
927
928 fn merge_snapshot(&self, exporter: &str, _stream: Box<dyn Read + Send>) -> Result<Merged> {
929 assert_eq!(exporter, PEER);
930 self.events.lock().unwrap().push("merge".to_string());
931 Ok(Merged::Done { seq: 5 })
932 }
933
934 fn bootstrap_pending(&self) -> Result<bool> {
935 Ok(*self.bootstrapping.lock().unwrap())
936 }
937
938 fn finish_bootstrap(&self) -> Result<()> {
939 self.events.lock().unwrap().push("finish".to_string());
940 *self.bootstrapping.lock().unwrap() = false;
941 Ok(())
942 }
943 }
944
945 struct Finished;
946
947 impl Remote for Finished {
948 fn wait(&mut self) -> Result<Exit> {
949 Ok(Exit {
950 success: true,
951 status: "exit status: 0".to_string(),
952 })
953 }
954
955 fn stop(&mut self) -> Result<()> {
956 Ok(())
957 }
958 }
959
960 struct Lost;
961
962 impl Remote for Lost {
963 fn wait(&mut self) -> Result<Exit> {
964 anyhow::bail!("ssh vanished")
965 }
966
967 fn stop(&mut self) -> Result<()> {
968 Ok(())
969 }
970 }
971
972 struct Failed;
973
974 impl Remote for Failed {
975 fn wait(&mut self) -> Result<Exit> {
976 Ok(Exit {
977 success: false,
978 status: "exit status: 1".to_string(),
979 })
980 }
981
982 fn stop(&mut self) -> Result<()> {
983 Ok(())
984 }
985 }
986
987 #[derive(Clone, Default)]
988 struct Shared(Arc<Mutex<Vec<u8>>>);
989
990 impl Write for Shared {
991 fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
992 self.0.lock().unwrap().extend_from_slice(buf);
993 Ok(buf.len())
994 }
995
996 fn flush(&mut self) -> std::io::Result<()> {
997 Ok(())
998 }
999 }
1000
1001 struct FakePeer {
1002 export_stream: String,
1003 exports: Mutex<Vec<(u64, Hello)>>,
1004 snapshots: Mutex<Vec<Hello>>,
1005 pushed: Shared,
1006 push_replies: String,
1007 remote_fails: bool,
1008 remote_lost: bool,
1009 }
1010
1011 impl FakePeer {
1012 fn sending(header: SyncHeader) -> Self {
1013 Self {
1014 export_stream: format!("{}\n{SEALED}\n", header.to_line()),
1015 exports: Mutex::default(),
1016 snapshots: Mutex::default(),
1017 pushed: Shared::default(),
1018 push_replies: receiving(hello(PEER, None), 0),
1019 remote_fails: false,
1020 remote_lost: false,
1021 }
1022 }
1023
1024 fn remote(&self) -> Box<dyn Remote> {
1025 if self.remote_lost {
1026 Box::new(Lost)
1027 } else if self.remote_fails {
1028 Box::new(Failed)
1029 } else {
1030 Box::new(Finished)
1031 }
1032 }
1033
1034 fn taking_pushes(replies: String) -> Self {
1035 Self {
1036 push_replies: replies,
1037 ..Self::sending(SyncHeader::local())
1038 }
1039 }
1040 }
1041
1042 fn receiving(peer_hello: Hello, applied: u64) -> String {
1045 let header = SyncHeader {
1046 hello: Some(peer_hello),
1047 ..SyncHeader::local()
1048 };
1049 format!(
1050 "{}\n{}\n",
1051 header.to_line(),
1052 PushApplied { applied }.to_line()
1053 )
1054 }
1055
1056 impl Transport for FakePeer {
1057 fn host(&self) -> &str {
1058 "user@peer"
1059 }
1060
1061 fn whoami(&self) -> Result<String> {
1062 Ok(PEER.to_string())
1063 }
1064
1065 fn export(
1066 &self,
1067 after_seq: u64,
1068 hello: &Hello,
1069 ) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)> {
1070 self.exports
1071 .lock()
1072 .unwrap()
1073 .push((after_seq, hello.clone()));
1074 Ok((
1075 Box::new(Cursor::new(self.export_stream.clone().into_bytes())),
1076 self.remote(),
1077 ))
1078 }
1079
1080 fn snapshot_export(
1081 &self,
1082 hello: &Hello,
1083 ) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)> {
1084 self.snapshots.lock().unwrap().push(hello.clone());
1085 Ok((Box::new(Cursor::new(Vec::new())), Box::new(Finished)))
1086 }
1087
1088 fn import(&self) -> Result<(Box<dyn Write + Send>, Box<dyn Read + Send>, Box<dyn Remote>)> {
1089 Ok((
1090 Box::new(self.pushed.clone()),
1091 Box::new(Cursor::new(self.push_replies.clone().into_bytes())),
1092 self.remote(),
1093 ))
1094 }
1095 }
1096
1097 const SEALED: &str = "AgAAAAEAAQIDBAUGBw";
1098
1099 #[test]
1100 fn a_pull_sends_a_hello_at_its_cursor_and_accepts_the_peers_before_importing() {
1101 let local = FakeLocal {
1102 cursors: vec![(PEER.to_string(), 7)],
1103 ..FakeLocal::default()
1104 };
1105 let peer_hello = hello(PEER, Some(4));
1106 let peer = FakePeer::sending(SyncHeader {
1107 hello: Some(peer_hello.clone()),
1108 ..SyncHeader::local()
1109 });
1110
1111 let imported = pull(&local, &peer, PEER).unwrap();
1112
1113 assert_eq!(imported.created, 1);
1114 assert_eq!(
1115 *peer.exports.lock().unwrap(),
1116 vec![(
1117 7,
1118 Hello {
1119 applied: Some(7),
1120 ..hello("local-machine", Some(7))
1121 }
1122 )],
1123 "the hello's cursor is the cursor the pull resumes from, and nothing failed"
1124 );
1125 assert_eq!(*local.accepted.lock().unwrap(), vec![peer_hello]);
1126 assert_eq!(*local.events.lock().unwrap(), vec!["accept", "import"]);
1127 assert_eq!(
1128 *local.imported.lock().unwrap(),
1129 vec![SEALED.to_string()],
1130 "the peer's sealed line reaches the store unopened"
1131 );
1132 }
1133
1134 #[test]
1135 fn a_bootstrapping_device_merges_a_snapshot_before_its_first_pull_and_then_finishes() {
1136 let local = FakeLocal {
1137 bootstrapping: Mutex::new(true),
1138 ..FakeLocal::default()
1139 };
1140 let peer = FakePeer::sending(SyncHeader {
1141 hello: Some(hello(PEER, None)),
1142 ..SyncHeader::local()
1143 });
1144
1145 pull(&local, &peer, PEER).unwrap();
1146
1147 let snapshots = peer.snapshots.lock().unwrap();
1148 assert_eq!(
1149 snapshots.len(),
1150 1,
1151 "a peer whose log starts at 0 still sends a snapshot"
1152 );
1153 assert_eq!(snapshots[0].cursor, Some(0));
1154 assert_eq!(
1155 *local.events.lock().unwrap(),
1156 vec!["merge", "accept", "import", "finish"]
1157 );
1158 assert!(!*local.bootstrapping.lock().unwrap());
1159 }
1160
1161 #[test]
1162 fn a_bootstrap_whose_snapshot_already_merged_only_pulls_and_one_that_fails_stays_pending() {
1163 let merged = FakeLocal {
1164 cursors: vec![(PEER.to_string(), 7)],
1165 bootstrapping: Mutex::new(true),
1166 ..FakeLocal::default()
1167 };
1168 let peer = FakePeer::sending(SyncHeader {
1169 hello: Some(hello(PEER, None)),
1170 ..SyncHeader::local()
1171 });
1172 pull(&merged, &peer, PEER).unwrap();
1173 assert!(peer.snapshots.lock().unwrap().is_empty());
1174 assert_eq!(
1175 *merged.events.lock().unwrap(),
1176 vec!["accept", "import", "finish"]
1177 );
1178
1179 let failing = FakeLocal {
1180 bootstrapping: Mutex::new(true),
1181 ..FakeLocal::default()
1182 };
1183 let no_hello = FakePeer::sending(SyncHeader::local());
1184 assert!(pull(&failing, &no_hello, PEER).is_err());
1185 assert!(*failing.bootstrapping.lock().unwrap());
1186 assert!(
1187 !failing
1188 .events
1189 .lock()
1190 .unwrap()
1191 .contains(&"finish".to_string())
1192 );
1193 }
1194
1195 #[test]
1196 fn a_pull_whose_header_has_no_hello_imports_nothing() {
1197 let local = FakeLocal::default();
1198 let peer = FakePeer::sending(SyncHeader::local());
1199
1200 let error = pull(&local, &peer, PEER).unwrap_err();
1201
1202 assert!(format!("{error:#}").contains("hello"), "{error:#}");
1203 assert!(local.events.lock().unwrap().is_empty());
1204 }
1205
1206 #[test]
1207 fn a_push_header_carries_the_hello() {
1208 let local = FakeLocal {
1209 exports: vec![SEALED.to_string()],
1210 ..FakeLocal::default()
1211 };
1212 let peer = FakePeer::sending(SyncHeader::local());
1213
1214 push(&local, &peer, PEER, &|_| {}).unwrap();
1215
1216 let pushed = peer.pushed.0.lock().unwrap().clone();
1217 let mut reader = BufReader::new(pushed.as_slice());
1218 let header = read_header(&mut reader).unwrap();
1219 let rest: Vec<String> = reader.lines().map(|l| l.unwrap()).collect();
1220 assert_eq!(
1221 rest,
1222 vec![SEALED.to_string()],
1223 "sealed lines go out as exported"
1224 );
1225 assert_eq!(header.hello, Some(hello("local-machine", Some(99))));
1226 assert_eq!(
1227 (header.sync_protocol, header.schema_version),
1228 (
1229 SyncHeader::local().sync_protocol,
1230 SyncHeader::local().schema_version
1231 )
1232 );
1233 }
1234
1235 #[test]
1236 fn a_failed_chunk_caps_the_watermark_whatever_later_chunks_apply() {
1237 let mut watermark = Watermark::default();
1238 let chunk = |max_seq, errors| Imported {
1239 max_seq,
1240 errors,
1241 ..Imported::default()
1242 };
1243 watermark.add(&chunk(10, 0));
1244 watermark.add(&chunk(15, 1));
1245 watermark.add(&chunk(30, 0));
1246 assert_eq!(watermark.applied(), 15);
1247 assert_eq!(watermark.reached(), 30);
1248 }
1249
1250 #[test]
1251 fn a_pull_cursor_moves_past_a_failed_chunk() {
1252 let local = FakeLocal {
1253 import_results: Mutex::new(vec![
1254 Imported {
1255 max_seq: 5,
1256 errors: 1,
1257 ..Imported::default()
1258 },
1259 Imported {
1260 max_seq: 20,
1261 ..Imported::default()
1262 },
1263 ]),
1264 ..FakeLocal::default()
1265 };
1266 let peer = FakePeer {
1267 export_stream: format!(
1268 "{}\n{}",
1269 SyncHeader {
1270 hello: Some(hello(PEER, None)),
1271 ..SyncHeader::local()
1272 }
1273 .to_line(),
1274 format!("{SEALED}\n").repeat(IMPORT_CHUNK + 1)
1275 ),
1276 ..FakePeer::sending(SyncHeader::local())
1277 };
1278
1279 let imported = pull(&local, &peer, PEER).unwrap();
1280
1281 assert_eq!(imported.max_seq, 20);
1282 assert_eq!(
1283 *local.set_cursors.lock().unwrap(),
1284 vec![(PEER.to_string(), 20)]
1285 );
1286 }
1287
1288 #[test]
1289 fn the_receivers_report_sets_where_a_push_starts() {
1290 assert_eq!(push_start(5, None, None), 5, "no report: the cursor");
1291 assert_eq!(push_start(0, None, Some(9)), 9, "no cursor: the report");
1292 assert_eq!(push_start(5, None, Some(9)), 9);
1293 assert_eq!(push_start(9, Some(4), Some(6)), 9);
1294 assert_eq!(
1295 push_start(9, Some(9), Some(4)),
1296 4,
1297 "a lower report: restored"
1298 );
1299 assert_eq!(
1300 push_start(9, Some(9), None),
1301 0,
1302 "no report after one: restored"
1303 );
1304 }
1305
1306 #[test]
1307 fn a_push_resumes_from_the_receivers_report_and_records_its_acks() {
1308 let local = FakeLocal {
1309 cursors: vec![(format!("push:{PEER}"), 3)],
1310 exports: vec![SEALED.to_string()],
1311 ..FakeLocal::default()
1312 };
1313 let receiver = Hello {
1314 peer_in: Some(6),
1315 ..hello(PEER, Some(2))
1316 };
1317 let peer = FakePeer::taking_pushes(receiving(receiver.clone(), 7));
1318
1319 push(&local, &peer, PEER, &|_| {}).unwrap();
1320
1321 assert_eq!(*local.accepted.lock().unwrap(), vec![receiver]);
1322 assert_eq!(local.export_starts.lock().unwrap()[0], 6);
1323 assert_eq!(*local.last_report.lock().unwrap(), Some(6));
1324 assert_eq!(
1325 *local.acks.lock().unwrap(),
1326 vec![(PEER.to_string(), 6), (PEER.to_string(), 7)],
1327 "the report and then the final watermark are acks"
1328 );
1329 }
1330
1331 #[test]
1332 fn a_receiver_holding_a_failed_record_acks_only_what_it_applied_without_a_gap() {
1333 let local = FakeLocal {
1334 cursors: vec![(format!("push:{PEER}"), 9)],
1335 exports: vec![SEALED.to_string()],
1336 ..FakeLocal::default()
1337 };
1338 let receiver = Hello {
1339 applied: Some(4),
1340 ..hello(PEER, Some(9))
1341 };
1342 let peer = FakePeer::taking_pushes(receiving(receiver, 4));
1343
1344 push(&local, &peer, PEER, &|_| {}).unwrap();
1345
1346 assert_eq!(
1347 local.export_starts.lock().unwrap()[0],
1348 9,
1349 "the push resumes from the receiver's cursor, not from the failed record"
1350 );
1351 assert_eq!(local.acks.lock().unwrap()[0], (PEER.to_string(), 4));
1352 }
1353
1354 #[test]
1355 fn a_push_to_a_receiver_that_closes_without_a_result_fails() {
1356 let local = FakeLocal {
1357 exports: vec![SEALED.to_string()],
1358 ..FakeLocal::default()
1359 };
1360 let header = SyncHeader {
1361 hello: Some(hello(PEER, None)),
1362 ..SyncHeader::local()
1363 };
1364 let peer = FakePeer::taking_pushes(format!("{}\n", header.to_line()));
1365
1366 let error = push(&local, &peer, PEER, &|_| {}).unwrap_err();
1367
1368 assert!(format!("{error:#}").contains("applied"), "{error:#}");
1369 assert!(local.set_cursors.lock().unwrap().is_empty());
1370 }
1371
1372 #[test]
1373 fn serving_a_push_answers_with_this_hello_and_reports_what_it_applied() {
1374 let local = FakeLocal::default();
1375 let pusher = hello(PEER, Some(1));
1376 let input = format!(
1377 "{}\n{SEALED}\n{SEALED}\n",
1378 SyncHeader {
1379 hello: Some(pusher.clone()),
1380 ..SyncHeader::local()
1381 }
1382 .to_line()
1383 );
1384 let mut output = Vec::new();
1385 let mut input = input.as_bytes();
1386 let header = read_checked_header(&mut input, "the sender").unwrap();
1387
1388 let imported = serve_push(&local, &header, input, &mut output).unwrap();
1389
1390 assert_eq!(imported.created, 2);
1391 assert_eq!(*local.accepted.lock().unwrap(), vec![pusher]);
1392 assert_eq!(*local.peer_in.lock().unwrap(), vec![(PEER.to_string(), 8)]);
1393 let mut replies = BufReader::new(output.as_slice());
1394 let reply = read_header(&mut replies).unwrap();
1395 assert_eq!(reply.hello, Some(hello("local-machine", Some(99))));
1396 assert_eq!(
1397 read_push_applied(&mut replies).unwrap(),
1398 PushApplied { applied: 8 }
1399 );
1400 }
1401
1402 #[test]
1403 fn a_restored_receiver_is_pushed_from_its_report_again_after_a_failed_push() {
1404 let local = FakeLocal {
1405 cursors: vec![(format!("push:{PEER}"), 9)],
1406 exports: vec![SEALED.to_string()],
1407 last_report: Mutex::new(Some(9)),
1408 ..FakeLocal::default()
1409 };
1410 let header = SyncHeader {
1411 hello: Some(hello(PEER, Some(4))),
1412 ..SyncHeader::local()
1413 };
1414 let peer = FakePeer::taking_pushes(format!("{}\n", header.to_line()));
1415
1416 push(&local, &peer, PEER, &|_| {}).unwrap_err();
1417
1418 assert_eq!(local.export_starts.lock().unwrap()[0], 4);
1419 assert_eq!(
1420 *local.set_cursors.lock().unwrap(),
1421 vec![(format!("push:{PEER}"), 4)],
1422 "the lowered start outlives the failed push"
1423 );
1424 }
1425
1426 #[test]
1427 fn a_restore_seen_by_a_push_that_fails_before_streaming_is_seen_again() {
1428 let local = FakeLocal {
1429 cursors: vec![(format!("push:{PEER}"), 9)],
1430 exports: vec![SEALED.to_string()],
1431 last_report: Mutex::new(Some(9)),
1432 report_fails: true,
1433 ..FakeLocal::default()
1434 };
1435 let peer = FakePeer::taking_pushes(receiving(hello(PEER, Some(4)), 4));
1436
1437 push(&local, &peer, PEER, &|_| {}).unwrap_err();
1438
1439 assert_eq!(*local.last_report.lock().unwrap(), Some(9));
1440 assert_eq!(
1441 *local.set_cursors.lock().unwrap(),
1442 vec![(format!("push:{PEER}"), 4)]
1443 );
1444 assert!(local.export_starts.lock().unwrap().is_empty());
1445 }
1446
1447 const MISMATCH: &str = "peer-machine holds a different sync identity; pair it with `sb pair`";
1448
1449 fn refused_identity() -> RefusedLine {
1450 RefusedLine {
1451 error: MISMATCH.to_string(),
1452 identity: true,
1453 }
1454 }
1455
1456 fn expect_refused(error: &anyhow::Error) -> &PeerRefused {
1457 error
1458 .downcast_ref::<PeerRefused>()
1459 .unwrap_or_else(|| panic!("not a PeerRefused: {error:#}"))
1460 }
1461
1462 #[test]
1463 fn a_pull_the_peer_refuses_returns_its_named_error_not_its_exit_status() {
1464 let local = FakeLocal::default();
1465 let peer = FakePeer {
1466 export_stream: format!("{}\n", refused_identity().to_line()),
1467 remote_fails: true,
1468 ..FakePeer::sending(SyncHeader::local())
1469 };
1470
1471 let error = pull(&local, &peer, PEER).unwrap_err();
1472
1473 assert_eq!(
1474 expect_refused(&error),
1475 &PeerRefused {
1476 host: "user@peer".to_string(),
1477 error: MISMATCH.to_string(),
1478 identity: true,
1479 }
1480 );
1481 assert!(format!("{error}").contains("sb pair"), "{error}");
1482 assert!(local.events.lock().unwrap().is_empty());
1483 assert!(local.set_cursors.lock().unwrap().is_empty());
1484 }
1485
1486 #[test]
1487 fn a_push_the_receiver_refuses_returns_its_named_error_and_streams_nothing() {
1488 let local = FakeLocal {
1489 exports: vec![SEALED.to_string()],
1490 ..FakeLocal::default()
1491 };
1492 let peer = FakePeer {
1493 remote_fails: true,
1494 ..FakePeer::taking_pushes(format!("\n{}\n", refused_identity().to_line()))
1495 };
1496
1497 let error = push(&local, &peer, PEER, &|_| {}).unwrap_err();
1498
1499 let refusal = expect_refused(&error);
1500 assert!(refusal.identity);
1501 assert_eq!(refusal.error, MISMATCH);
1502 assert!(local.export_starts.lock().unwrap().is_empty());
1503 assert!(local.set_cursors.lock().unwrap().is_empty());
1504 assert!(local.acks.lock().unwrap().is_empty());
1505 assert_eq!(*local.last_report.lock().unwrap(), None);
1506 }
1507
1508 #[test]
1509 fn a_refusal_survives_a_remote_that_cannot_be_waited_on() {
1510 let local = FakeLocal::default();
1511 let puller = FakePeer {
1512 export_stream: format!("{}\n", refused_identity().to_line()),
1513 remote_lost: true,
1514 ..FakePeer::sending(SyncHeader::local())
1515 };
1516 let error = pull(&local, &puller, PEER).unwrap_err();
1517 assert!(expect_refused(&error).identity, "{error:#}");
1518
1519 let pusher = FakePeer {
1520 remote_lost: true,
1521 ..FakePeer::taking_pushes(format!("\n{}\n", refused_identity().to_line()))
1522 };
1523 let error = push(&local, &pusher, PEER, &|_| {}).unwrap_err();
1524 assert!(expect_refused(&error).identity, "{error:#}");
1525 }
1526
1527 #[test]
1528 fn a_push_whose_receiver_fails_without_a_refusal_still_reports_its_exit() {
1529 let local = FakeLocal::default();
1530 let peer = FakePeer {
1531 remote_fails: true,
1532 ..FakePeer::taking_pushes(String::new())
1533 };
1534
1535 let error = push(&local, &peer, PEER, &|_| {}).unwrap_err();
1536
1537 assert!(error.downcast_ref::<PeerRefused>().is_none(), "{error:#}");
1538 assert!(format!("{error:#}").contains("exited with"), "{error:#}");
1539 }
1540
1541 #[test]
1542 fn serving_a_push_whose_hello_is_refused_tells_the_pusher_the_mismatch() {
1543 let receiver = FakeLocal {
1544 refuses_hello: Some(MISMATCH.to_string()),
1545 ..FakeLocal::default()
1546 };
1547 let header = SyncHeader {
1548 hello: Some(hello(PEER, Some(1))),
1549 ..SyncHeader::local()
1550 };
1551 let input = format!("{SEALED}\n");
1552 let mut output = Vec::new();
1553
1554 let error = serve_push(&receiver, &header, input.as_bytes(), &mut output).unwrap_err();
1555
1556 assert!(
1557 error.downcast_ref::<IdentityMismatch>().is_some(),
1558 "{error:#}"
1559 );
1560 assert!(receiver.imported.lock().unwrap().is_empty());
1561 assert!(receiver.peer_in.lock().unwrap().is_empty());
1562 let replies = String::from_utf8(output).unwrap();
1563 assert_eq!(
1564 RefusedLine::parse_line(&replies),
1565 Some(refused_identity()),
1566 "{replies}"
1567 );
1568
1569 let pusher = FakeLocal::default();
1570 let peer = FakePeer {
1571 remote_fails: true,
1572 ..FakePeer::taking_pushes(replies)
1573 };
1574 let error = push(&pusher, &peer, PEER, &|_| {}).unwrap_err();
1575 assert!(expect_refused(&error).identity);
1576 }
1577
1578 #[test]
1579 fn serving_a_push_refused_for_another_reason_sends_the_whole_error() {
1580 let header = SyncHeader {
1581 hello: None,
1582 ..SyncHeader::local()
1583 };
1584 let mut output = Vec::new();
1585
1586 serve_push(&FakeLocal::default(), &header, &b""[..], &mut output).unwrap_err();
1587
1588 let refused = RefusedLine::parse_line(&String::from_utf8(output).unwrap()).unwrap();
1589 assert!(!refused.identity);
1590 assert!(
1591 refused.error.contains("without its hello"),
1592 "{}",
1593 refused.error
1594 );
1595 }
1596}