Skip to main content

second_brain_sync/
session.rs

1use std::collections::BTreeMap;
2use std::fmt;
3use std::io::{BufRead, BufReader, BufWriter, Cursor, Read, Write};
4use std::path::Path;
5
6use crate::keyring::IdentityMismatch;
7use crate::protocol::{
8    BelowFloorLine, Hello, Merging, PushApplied, RefusedLine, SYNC_PROTOCOL_VERSION, SyncHeader,
9    read_header,
10};
11use crate::ssh::untrusted_host_hint;
12use anyhow::{Context, Result};
13use second_brain_core::store::SkipReason;
14
15/// Records posted to the local store per import call while the peer's export
16/// is still streaming, so a long-lagging peer never sits in memory whole.
17const IMPORT_CHUNK: usize = 1000;
18const PUSH_IMPORT_CHUNK: usize = 500;
19const EXPORT_PAGE: usize = 2000;
20
21/// This machine's store: `sb` reaches it through the daemon's API, the daemon
22/// directly. Records cross it as sealed wire lines, so only the daemon holds
23/// a key and the session only carries lines between it and the peer.
24pub trait Local {
25    fn cursors(&self) -> Result<Vec<(String, u64)>>;
26    fn set_cursor(&self, peer_id: &str, last_seq: u64) -> Result<()>;
27    /// `header` is the stream's, sent by the peer, not this machine's.
28    fn import(&self, header: &SyncHeader, lines: &[String]) -> Result<Imported>;
29    /// Records after `after_seq`, sealed for `peer_id`.
30    fn export(&self, peer_id: &str, after_seq: u64, limit: usize) -> Result<Exported>;
31    fn hello_for(&self, peer_id: &str) -> Result<Hello>;
32    /// Raises this store's clock floors from a peer's hello; fails the sync
33    /// when the peer's clock is out of bounds.
34    fn accept_hello(&self, hello: &Hello) -> Result<()>;
35    /// `peer_id` has applied this store's log through `acked_seq`.
36    fn record_ack(&self, peer_id: &str, acked_seq: u64) -> Result<()>;
37    /// `peer_id`'s push applied its log through `seq` here.
38    fn record_peer_in(&self, peer_id: &str, seq: u64) -> Result<()>;
39    /// How far into this store's log `peer_id` last reported it had applied
40    /// when it took a push.
41    fn push_report(&self, peer_id: &str) -> Result<Option<u64>>;
42    fn record_push_report(&self, peer_id: &str, reported: Option<u64>) -> Result<()>;
43    /// Spools a snapshot stream from `exporter` and merges it, or leaves the
44    /// merge running where the store lives and says so.
45    fn merge_snapshot(&self, exporter: &str, stream: Box<dyn Read + Send>) -> Result<Merged> {
46        let _ = (exporter, stream);
47        anyhow::bail!("this store cannot merge a snapshot")
48    }
49    /// The snapshot merge this store has not finished, if any.
50    fn merging(&self) -> Result<Option<Merging>> {
51        Ok(None)
52    }
53    /// A snapshot of this store sealed for the peer whose hello this is,
54    /// which is pinned at the store's head before the first row is read.
55    fn export_snapshot(&self, peer: &Hello) -> Result<Box<dyn Read + Send>> {
56        let _ = peer;
57        anyhow::bail!("this store cannot export a snapshot")
58    }
59    /// This device was paired with an empty store and has not yet merged a
60    /// snapshot and pulled after it, so its first pull takes a snapshot.
61    fn bootstrap_pending(&self) -> Result<bool> {
62        Ok(false)
63    }
64    /// The first pull after a bootstrap snapshot succeeded.
65    fn finish_bootstrap(&self) -> Result<()> {
66        Ok(())
67    }
68}
69
70#[derive(Debug, Clone, PartialEq, Eq)]
71pub enum Merged {
72    /// Merged; the cursor for the exporter is now at `seq`.
73    Done {
74        seq: u64,
75    },
76    Running(Merging),
77}
78
79#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
80pub struct Imported {
81    pub created: u64,
82    pub updated: u64,
83    pub deleted: u64,
84    pub skipped: u64,
85    pub dangling: u64,
86    pub errors: u64,
87    /// The highest seq applied without a gap.
88    pub max_seq: u64,
89    /// The highest seq the chunk carried, past failed records.
90    pub reached: u64,
91}
92
93impl Imported {
94    pub fn dangling_in(skipped_by_reason: &BTreeMap<String, u64>) -> u64 {
95        skipped_by_reason
96            .get(SkipReason::DanglingEndpoint.as_str())
97            .copied()
98            .unwrap_or(0)
99    }
100}
101
102/// How far a run of import chunks got. Each chunk's `max_seq` is already
103/// its own watermark: the seq before its first failed record, if any.
104#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
105pub struct Watermark {
106    reached: u64,
107    applied: u64,
108    capped: Option<u64>,
109}
110
111impl Watermark {
112    pub fn add(&mut self, chunk: &Imported) {
113        self.reached = self.reached.max(chunk.reached.max(chunk.max_seq));
114        self.applied = self.applied.max(chunk.max_seq);
115        if chunk.errors > 0 {
116            self.capped = Some(self.capped.map_or(chunk.max_seq, |c| c.min(chunk.max_seq)));
117        }
118    }
119
120    /// The highest seq applied without a gap: a failed chunk caps it
121    /// whatever later chunks apply, so an ack built from it never claims a
122    /// failed record.
123    pub fn applied(&self) -> u64 {
124        self.capped
125            .map_or(self.applied, |cap| self.applied.min(cap))
126    }
127
128    /// The highest seq any chunk reached, past failed records.
129    pub fn reached(&self) -> u64 {
130        self.reached
131    }
132}
133
134pub struct Exported {
135    pub lines: Vec<String>,
136    pub max_seq: u64,
137}
138
139pub trait Transport {
140    fn host(&self) -> &str;
141    fn whoami(&self) -> Result<String>;
142    fn export(
143        &self,
144        after_seq: u64,
145        hello: &Hello,
146    ) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)>;
147    /// The pushed stream goes to the peer's `sb sync-import` on the
148    /// writer; its handshake and result come back on the reader.
149    fn import(&self) -> Result<(Box<dyn Write + Send>, Box<dyn Read + Send>, Box<dyn Remote>)>;
150    /// The peer's snapshot, sealed for the puller named in `hello`.
151    fn snapshot_export(&self, hello: &Hello) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)> {
152        let _ = hello;
153        anyhow::bail!("the transport to {} carries no snapshots", self.host())
154    }
155    /// A snapshot streamed to the peer, which spools it and answers with the
156    /// merge it started.
157    fn snapshot_import(
158        &self,
159    ) -> Result<(Box<dyn Write + Send>, Box<dyn Read + Send>, Box<dyn Remote>)> {
160        anyhow::bail!("the transport to {} carries no snapshots", self.host())
161    }
162}
163
164/// One running `sb sync-export` or `sb sync-import` on a peer.
165pub trait Remote {
166    fn wait(&mut self) -> Result<Exit>;
167    fn stop(&mut self) -> Result<()>;
168}
169
170pub struct Exit {
171    pub success: bool,
172    pub status: String,
173}
174
175/// The peer's daemon refused the sync and the peer said why on the stream.
176#[derive(Debug, Clone, PartialEq, Eq)]
177pub struct PeerRefused {
178    pub host: String,
179    pub error: String,
180    /// The peer holds another sync identity, so pairing is the fix.
181    pub identity: bool,
182}
183
184impl PeerRefused {
185    fn from_line(host: &str, refused: RefusedLine) -> Self {
186        Self {
187            host: host.to_string(),
188            error: refused.error,
189            identity: refused.identity,
190        }
191    }
192}
193
194impl fmt::Display for PeerRefused {
195    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
196        write!(f, "{} refused the sync: {}", self.host, self.error)
197    }
198}
199
200impl std::error::Error for PeerRefused {}
201
202/// A full pull then push with one peer, recorded as a success for health
203/// only once both have finished.
204pub fn sync(
205    local: &dyn Local,
206    transport: &dyn Transport,
207    data_dir: &Path,
208    report: &dyn Fn(&str),
209) -> Result<()> {
210    let peer_machine_id = transport
211        .whoami()
212        .context("resolving peer machine id (is the peer's daemon running and reachable?)")?;
213    sync_with(local, transport, &peer_machine_id, data_dir, report)
214}
215
216/// `sync` with a peer whose machine id the caller already asked for.
217pub fn sync_with(
218    local: &dyn Local,
219    transport: &dyn Transport,
220    peer_machine_id: &str,
221    data_dir: &Path,
222    report: &dyn Fn(&str),
223) -> Result<()> {
224    let peer_host = transport.host();
225    report(&format!("pulling from {peer_host}..."));
226    let pull_stats = pull(local, transport, peer_machine_id)?;
227    report(&format!(
228        "  {} created, {} updated, {} deleted, {} skipped",
229        pull_stats.created, pull_stats.updated, pull_stats.deleted, pull_stats.skipped
230    ));
231    if pull_stats.dangling > 0 {
232        report(&format!(
233            "  warning: {} records dropped with unresolvable endpoints; \
234             run `sb prune-relations` on the peer if this persists",
235            pull_stats.dangling
236        ));
237    }
238
239    report(&format!("pushing to {peer_host}..."));
240    let max_seq = push(local, transport, peer_machine_id, report)?;
241    report(&format!("  pushed through seq {max_seq}"));
242
243    crate::peers::record_success(data_dir, chrono::Utc::now())
244        .context("recording that the sync succeeded")?;
245    report("sync complete.");
246    Ok(())
247}
248
249/// Prefers the machine-id-keyed cursor and falls back to the one keyed by the
250/// `sync-peers` connection string, so a peer synced before cursors were keyed
251/// by identity resumes where it was instead of re-pulling its whole history.
252fn cursor(local: &dyn Local, key: &str, legacy_key: &str) -> Result<u64> {
253    let cursors = local.cursors().context("getting sync status")?;
254    Ok(cursors
255        .iter()
256        .find(|(peer_id, _)| peer_id == key)
257        .or_else(|| cursors.iter().find(|(peer_id, _)| peer_id == legacy_key))
258        .map(|(_, last_seq)| *last_seq)
259        .unwrap_or(0))
260}
261
262pub fn pull(
263    local: &dyn Local,
264    transport: &dyn Transport,
265    peer_machine_id: &str,
266) -> Result<Imported> {
267    if let Some(merging) = local.merging()? {
268        anyhow::bail!(
269            "a snapshot merge from {} through seq {} is still running; sync again once it finishes",
270            merging.exporter,
271            merging.seq
272        );
273    }
274    let bootstrapping = local.bootstrap_pending()?;
275    if bootstrapping && cursor(local, peer_machine_id, transport.host())? == 0 {
276        let start = BelowFloorLine {
277            after_seq: 0,
278            log_floor: 0,
279        };
280        merge_peer_snapshot(local, transport, peer_machine_id, start)?;
281    }
282    let imported = pull_after_floor(local, transport, peer_machine_id)?;
283    if bootstrapping {
284        local.finish_bootstrap()?;
285    }
286    Ok(imported)
287}
288
289fn pull_after_floor(
290    local: &dyn Local,
291    transport: &dyn Transport,
292    peer_machine_id: &str,
293) -> Result<Imported> {
294    match pull_log(local, transport, peer_machine_id)? {
295        Pulled::Log(imported) => Ok(imported),
296        Pulled::BelowFloor(below) => {
297            merge_peer_snapshot(local, transport, peer_machine_id, below)?;
298            match pull_log(local, transport, peer_machine_id)? {
299                Pulled::Log(imported) => Ok(imported),
300                Pulled::BelowFloor(again) => anyhow::bail!(
301                    "{} truncated its log past seq {} again right after a snapshot; sync again",
302                    transport.host(),
303                    again.log_floor
304                ),
305            }
306        }
307    }
308}
309
310enum Pulled {
311    Log(Imported),
312    BelowFloor(BelowFloorLine),
313}
314
315fn pull_hello(local: &dyn Local, peer_machine_id: &str, after_seq: u64) -> Result<Hello> {
316    let hello = local
317        .hello_for(peer_machine_id)
318        .context("building the sync hello")?;
319    Ok(Hello {
320        cursor: Some(after_seq),
321        applied: Some(hello.applied.unwrap_or(after_seq).min(after_seq)),
322        ..hello
323    })
324}
325
326/// The peer's log no longer holds the rows after this store's cursor, so
327/// its snapshot is merged instead; the cursor then sits at the seq the
328/// snapshot is consistent at.
329fn merge_peer_snapshot(
330    local: &dyn Local,
331    transport: &dyn Transport,
332    peer_machine_id: &str,
333    below: BelowFloorLine,
334) -> Result<()> {
335    let peer_host = transport.host();
336    let hello = pull_hello(local, peer_machine_id, below.after_seq)?;
337    let (stream, mut remote) = transport.snapshot_export(&hello)?;
338    let merged = local.merge_snapshot(peer_machine_id, stream);
339    let exit = remote.wait()?;
340    if !exit.success {
341        anyhow::bail!(
342            "sync-snapshot on {peer_host} exited with {}; its error is printed above.\n  hint: {}",
343            exit.status,
344            untrusted_host_hint(peer_host)
345        );
346    }
347    match merged.with_context(|| format!("merging the snapshot from {peer_host}"))? {
348        Merged::Done { .. } => Ok(()),
349        Merged::Running(merging) => anyhow::bail!(
350            "merging the snapshot from {peer_host} through seq {} continues in the daemon; \
351             sync again once it finishes",
352            merging.seq
353        ),
354    }
355}
356
357fn pull_log(local: &dyn Local, transport: &dyn Transport, peer_machine_id: &str) -> Result<Pulled> {
358    let peer_host = transport.host();
359    let after_seq = cursor(local, peer_machine_id, peer_host)?;
360    let hello = pull_hello(local, peer_machine_id, after_seq)?;
361
362    let (output, mut remote) = transport.export(after_seq, &hello)?;
363    let mut reader = BufReader::new(output);
364    let mut first = String::new();
365    let read = reader.read_line(&mut first);
366    if let Some(below) = BelowFloorLine::parse_line(&first) {
367        let exit = remote.wait()?;
368        if !exit.success {
369            anyhow::bail!(
370                "sync-export on {peer_host} exited with {}.\n  hint: {}",
371                exit.status,
372                untrusted_host_hint(peer_host)
373            );
374        }
375        return Ok(Pulled::BelowFloor(below));
376    }
377    if let Some(refused) = RefusedLine::parse_line(&first) {
378        if let Err(e) = remote.wait() {
379            tracing::warn!("sync-export on {peer_host} refused and then failed to exit: {e:#}");
380        }
381        return Err(PeerRefused::from_line(peer_host, refused).into());
382    }
383    let header = read
384        .map_err(anyhow::Error::from)
385        .and_then(|_| read_checked_header(&mut Cursor::new(first.as_bytes()), peer_host));
386    let peer_header = match header {
387        Ok(header) => header,
388        Err(e) => {
389            // A peer that failed before writing its header ends the stream with
390            // a failing exit, and that failure, not its version, is the cause.
391            if first.trim().is_empty() && matches!(reader.fill_buf(), Ok(rest) if rest.is_empty()) {
392                let exit = remote.wait()?;
393                if !exit.success {
394                    anyhow::bail!(
395                        "sync-export on {peer_host} exited with {}; its error is printed above.\n  hint: {}",
396                        exit.status,
397                        untrusted_host_hint(peer_host)
398                    );
399                }
400            } else {
401                remote.stop()?;
402            }
403            return Err(e);
404        }
405    };
406    if let Err(e) = accept_peer_hello(local, &peer_header, peer_host) {
407        remote.stop()?;
408        return Err(e);
409    }
410
411    let (mut totals, watermark) = match import_lines(local, &peer_header, &mut reader, IMPORT_CHUNK)
412    {
413        Ok(imported) => imported,
414        Err(e) => {
415            remote.stop()?;
416            return Err(e);
417        }
418    };
419
420    let exit = remote.wait()?;
421    if !exit.success {
422        anyhow::bail!(
423            "ssh pull from {peer_host} exited with {}.\n  hint: {}",
424            exit.status,
425            untrusted_host_hint(peer_host)
426        );
427    }
428
429    // A record that fails is kept and retried by the store, and holds the
430    // ack (`Hello.applied`) back until it settles, so the cursor moves past
431    // it instead of pulling it again.
432    totals.max_seq = watermark.reached();
433    if totals.max_seq > after_seq {
434        local
435            .set_cursor(peer_machine_id, totals.max_seq)
436            .context("updating sync state")?;
437    }
438
439    Ok(Pulled::Log(totals))
440}
441
442pub fn push(
443    local: &dyn Local,
444    transport: &dyn Transport,
445    peer_machine_id: &str,
446    report: &dyn Fn(&str),
447) -> Result<u64> {
448    let peer_host = transport.host();
449    let push_peer = format!("push:{peer_machine_id}");
450    let push_cursor = cursor(local, &push_peer, &format!("push:{peer_host}"))?;
451    let header = SyncHeader {
452        hello: Some(
453            local
454                .hello_for(peer_machine_id)
455                .context("building the sync hello")?,
456        ),
457        ..SyncHeader::local()
458    };
459
460    let (input, output, mut remote) = transport.import()?;
461    let mut writer = BufWriter::new(input);
462    let mut replies = BufReader::new(output);
463    let own_floor = header.hello.as_ref().map_or(0, |hello| hello.log_floor);
464    let mut handshake = || -> Result<(u64, Hello)> {
465        writeln!(writer, "{}", header.to_line())?;
466        writer.flush()?;
467        let first = first_line(&mut replies)
468            .with_context(|| format!("reading the sync header from {peer_host}"))?;
469        if let Some(refused) = RefusedLine::parse_line(&first) {
470            return Err(PeerRefused::from_line(peer_host, refused).into());
471        }
472        let receiver = read_checked_header(&mut Cursor::new(first.as_bytes()), peer_host)?;
473        accept_peer_hello(local, &receiver, peer_host)?;
474        let reported = receiver.hello.as_ref().and_then(Hello::applied_of_receiver);
475        let start = push_start(push_cursor, local.push_report(peer_machine_id)?, reported);
476        // The cursor is lowered before the report that revealed the restore
477        // replaces the last one, so a push that fails here is still seen
478        // as a restore next time.
479        if start < push_cursor {
480            local
481                .set_cursor(&push_peer, start)
482                .context("lowering the push cursor for a restored peer")?;
483        }
484        local.record_push_report(peer_machine_id, reported)?;
485        if let Some(acked) = receiver.hello.as_ref().and_then(Hello::acked_by_receiver) {
486            local.record_ack(peer_machine_id, acked)?;
487        }
488        let hello = receiver
489            .hello
490            .clone()
491            .expect("accept_peer_hello refuses a header without its hello");
492        Ok((start, hello))
493    };
494    let (start, receiver) = match handshake() {
495        Ok(handshake) => handshake,
496        Err(e) => {
497            drop(writer);
498            let refused = e.downcast_ref::<PeerRefused>().is_some();
499            let exit = match remote.wait() {
500                Ok(exit) => exit,
501                Err(wait) if refused => {
502                    tracing::warn!(
503                        "ssh push to {peer_host} refused and then failed to exit: {wait:#}"
504                    );
505                    return Err(e);
506                }
507                Err(wait) => return Err(wait),
508            };
509            if !exit.success && !refused {
510                anyhow::bail!(
511                    "ssh push to {peer_host} exited with {}.\n  hint: {}",
512                    exit.status,
513                    untrusted_host_hint(peer_host)
514                );
515            }
516            return Err(e);
517        }
518    };
519
520    if receiver.merging.is_some() || start < own_floor {
521        drop(writer);
522        let applied = read_push_applied(&mut replies);
523        let exit = remote.wait()?;
524        if !exit.success {
525            anyhow::bail!(
526                "ssh push to {peer_host} exited with {}.\n  hint: {}",
527                exit.status,
528                untrusted_host_hint(peer_host)
529            );
530        }
531        applied.with_context(|| format!("reading what {peer_host} applied"))?;
532        if let Some(merging) = receiver.merging {
533            report(&format!(
534                "  {peer_host} is merging snapshot {} through seq {}; nothing pushed",
535                merging.snapshot_id, merging.seq
536            ));
537        } else {
538            push_snapshot(local, transport, &receiver, start, report)?;
539        }
540        return Ok(start.min(push_cursor));
541    }
542
543    let mut cursor = start;
544    let mut total_pushed = 0u64;
545    let mut stream = || -> Result<()> {
546        loop {
547            let export = local.export(peer_machine_id, cursor, EXPORT_PAGE)?;
548            if export.lines.is_empty() {
549                break;
550            }
551            let batch_count = export.lines.len();
552            for line in &export.lines {
553                writer.write_all(line.as_bytes())?;
554                writer.write_all(b"\n")?;
555            }
556            writer.flush()?;
557
558            total_pushed += batch_count as u64;
559            cursor = export.max_seq;
560            report(&format!(
561                "  batch: {batch_count} records (through seq {cursor})"
562            ));
563
564            if batch_count < EXPORT_PAGE {
565                break;
566            }
567        }
568        writer.flush()?;
569        Ok(())
570    };
571    let streamed = stream();
572    drop(writer);
573    let applied = read_push_applied(&mut replies);
574
575    // A peer that refuses the stream exits before reading on, which surfaces
576    // here as a broken pipe; its exit status is the more useful error.
577    let exit = remote.wait()?;
578    if !exit.success {
579        anyhow::bail!(
580            "ssh push to {peer_host} exited with {}.\n  hint: {}",
581            exit.status,
582            untrusted_host_hint(peer_host)
583        );
584    }
585    streamed?;
586    let applied = applied.with_context(|| format!("reading what {peer_host} applied"))?;
587    local
588        .record_ack(peer_machine_id, applied.applied)
589        .context("recording the peer's ack")?;
590
591    if cursor != start.min(push_cursor) {
592        local
593            .set_cursor(&push_peer, cursor)
594            .context("updating push sync state")?;
595    }
596
597    report(&format!("  total: {total_pushed} records pushed"));
598    Ok(cursor)
599}
600
601/// The receiver needs rows this store's log no longer holds, so it gets a
602/// snapshot and merges it; the push cursor moves only once the receiver
603/// reports a `peer_in` at or past the snapshot's seq.
604fn push_snapshot(
605    local: &dyn Local,
606    transport: &dyn Transport,
607    receiver: &Hello,
608    start: u64,
609    report: &dyn Fn(&str),
610) -> Result<()> {
611    let peer_host = transport.host();
612    report(&format!(
613        "  {peer_host} needs rows after seq {start}, below this log's floor; sending a snapshot"
614    ));
615    let mut snapshot = local.export_snapshot(receiver)?;
616    let (input, output, mut remote) = transport.snapshot_import()?;
617    let mut writer = BufWriter::new(input);
618    let copied = std::io::copy(&mut snapshot, &mut writer).and_then(|_| writer.flush());
619    drop(writer);
620    let mut line = String::new();
621    let answered = BufReader::new(output).read_line(&mut line);
622    let exit = remote.wait()?;
623    if !exit.success {
624        anyhow::bail!(
625            "sync-snapshot on {peer_host} exited with {}.\n  hint: {}",
626            exit.status,
627            untrusted_host_hint(peer_host)
628        );
629    }
630    copied.with_context(|| format!("streaming the snapshot to {peer_host}"))?;
631    answered?;
632    let merging: Merging = serde_json::from_str(line.trim())
633        .with_context(|| format!("reading the merge {peer_host} started"))?;
634    report(&format!(
635        "  {peer_host} is merging snapshot {} through seq {}",
636        merging.snapshot_id, merging.seq
637    ));
638    Ok(())
639}
640
641/// The receiver's report is authoritative: the pusher resumes from the
642/// higher of its cursor and the report, unless the report fell below the
643/// last one, which means the receiver was restored and gets everything
644/// after what it still holds again.
645pub fn push_start(push_cursor: u64, previous: Option<u64>, reported: Option<u64>) -> u64 {
646    let restored = previous.is_some_and(|previous| reported.is_none_or(|r| r < previous));
647    if restored {
648        return reported.unwrap_or(0);
649    }
650    push_cursor.max(reported.unwrap_or(0))
651}
652
653/// The first line that is not blank, or an empty string at the end.
654fn first_line<R: BufRead>(reader: &mut R) -> std::io::Result<String> {
655    let mut line = String::new();
656    loop {
657        line.clear();
658        if reader.read_line(&mut line)? == 0 || !line.trim().is_empty() {
659            return Ok(line);
660        }
661    }
662}
663
664fn read_push_applied<R: BufRead>(replies: &mut R) -> Result<PushApplied> {
665    let mut line = String::new();
666    loop {
667        line.clear();
668        if replies.read_line(&mut line)? == 0 {
669            anyhow::bail!("the receiver closed the stream without saying what it applied");
670        }
671        if !line.trim().is_empty() {
672            return PushApplied::parse_line(&line);
673        }
674    }
675}
676
677/// The receiving end of `push`, run by `sb sync-import`: answers the
678/// pusher's hello with this store's, imports the rest of `input`, then
679/// reports the highest seq it applied without a gap. `header` is the
680/// stream's, which the caller reads with `read_checked_header` before it
681/// reaches the store, so a refused stream never does.
682pub fn serve_push<R: BufRead, W: Write>(
683    local: &dyn Local,
684    header: &SyncHeader,
685    mut input: R,
686    mut output: W,
687) -> Result<Imported> {
688    if let Err(e) = accept_peer_hello(local, header, "the sender") {
689        return Err(refuse(&mut output, e));
690    }
691    let pusher = header
692        .hello
693        .as_ref()
694        .map(|hello| hello.machine_id.clone())
695        .expect("accept_peer_hello refuses a header without its hello");
696    let reply = SyncHeader {
697        hello: Some(
698            local
699                .hello_for(&pusher)
700                .context("building the sync hello")?,
701        ),
702        ..SyncHeader::local()
703    };
704    writeln!(output, "{}", reply.to_line())?;
705    output.flush()?;
706
707    let (mut totals, watermark) = import_lines(local, header, &mut input, PUSH_IMPORT_CHUNK)?;
708    totals.max_seq = watermark.applied();
709    if totals.max_seq > 0 {
710        local
711            .record_peer_in(&pusher, totals.max_seq)
712            .context("recording what the push applied")?;
713    }
714    writeln!(
715        output,
716        "{}",
717        PushApplied {
718            applied: totals.max_seq
719        }
720        .to_line()
721    )?;
722    output.flush()?;
723    Ok(totals)
724}
725
726/// Tells the pusher why its hello was refused: the identity mismatch's own
727/// text when there is one, since that names the fix.
728fn refuse<W: Write>(output: &mut W, e: anyhow::Error) -> anyhow::Error {
729    let mismatch = e.downcast_ref::<IdentityMismatch>();
730    let refused = RefusedLine {
731        error: mismatch.map_or_else(|| format!("{e:#}"), ToString::to_string),
732        identity: mismatch.is_some(),
733    };
734    match writeln!(output, "{}", refused.to_line()).and_then(|()| output.flush()) {
735        Ok(()) => e,
736        Err(told) => e.context(format!("the sender did not get the refusal: {told}")),
737    }
738}
739
740/// Imports a stream's records in chunks; the caller picks the totals'
741/// `max_seq` from the watermark.
742fn import_lines<R: BufRead>(
743    local: &dyn Local,
744    header: &SyncHeader,
745    reader: &mut R,
746    chunk_len: usize,
747) -> Result<(Imported, Watermark)> {
748    let mut totals = Imported::default();
749    let mut watermark = Watermark::default();
750    let mut chunk = Vec::with_capacity(chunk_len);
751    let mut post_chunk = |chunk: &mut Vec<String>| -> Result<()> {
752        if chunk.is_empty() {
753            return Ok(());
754        }
755        let resp = local.import(header, chunk)?;
756        totals.created += resp.created;
757        totals.updated += resp.updated;
758        totals.deleted += resp.deleted;
759        totals.skipped += resp.skipped;
760        totals.dangling += resp.dangling;
761        totals.errors += resp.errors;
762        watermark.add(&resp);
763        chunk.clear();
764        Ok(())
765    };
766    for line in reader.lines() {
767        let line = line?;
768        if line.trim().is_empty() {
769            continue;
770        }
771        chunk.push(line);
772        if chunk.len() >= chunk_len {
773            post_chunk(&mut chunk)?;
774        }
775    }
776    post_chunk(&mut chunk)?;
777    Ok((totals, watermark))
778}
779
780fn accept_peer_hello(local: &dyn Local, header: &SyncHeader, peer_host: &str) -> Result<()> {
781    let hello = header.hello.as_ref().ok_or_else(|| {
782        anyhow::anyhow!(
783            "{peer_host} sent a sync protocol {SYNC_PROTOCOL_VERSION} header without its hello, \
784             nothing was applied"
785        )
786    })?;
787    local
788        .accept_hello(hello)
789        .with_context(|| format!("accepting the sync hello from {peer_host}"))
790}
791
792pub fn read_checked_header<R: BufRead>(reader: &mut R, sender: &str) -> Result<SyncHeader> {
793    let header =
794        read_header(reader).with_context(|| format!("reading the sync header from {sender}"))?;
795    SyncHeader::local().ensure_matches("this machine", &header, sender)?;
796    Ok(header)
797}
798
799#[cfg(test)]
800mod tests {
801    use std::sync::{Arc, Mutex};
802
803    use super::*;
804
805    const PEER: &str = "peer-machine";
806
807    fn hello(machine_id: &str, cursor: Option<u64>) -> Hello {
808        Hello {
809            machine_id: machine_id.to_string(),
810            wall_micros: 1_767_225_600_000_000,
811            synced: true,
812            cursor,
813            held_origin_seq: Some(3),
814            keys: None,
815            peer_in: None,
816            log_floor: 0,
817            head: 0,
818            applied: None,
819            merging: None,
820        }
821    }
822
823    #[derive(Default)]
824    struct FakeLocal {
825        cursors: Vec<(String, u64)>,
826        events: Mutex<Vec<String>>,
827        accepted: Mutex<Vec<Hello>>,
828        imported: Mutex<Vec<String>>,
829        exports: Vec<String>,
830        export_starts: Mutex<Vec<u64>>,
831        set_cursors: Mutex<Vec<(String, u64)>>,
832        acks: Mutex<Vec<(String, u64)>>,
833        peer_in: Mutex<Vec<(String, u64)>>,
834        last_report: Mutex<Option<u64>>,
835        report_fails: bool,
836        import_results: Mutex<Vec<Imported>>,
837        bootstrapping: Mutex<bool>,
838        refuses_hello: Option<String>,
839    }
840
841    impl Local for FakeLocal {
842        fn cursors(&self) -> Result<Vec<(String, u64)>> {
843            Ok(self.cursors.clone())
844        }
845
846        fn set_cursor(&self, peer_id: &str, last_seq: u64) -> Result<()> {
847            self.set_cursors
848                .lock()
849                .unwrap()
850                .push((peer_id.to_string(), last_seq));
851            Ok(())
852        }
853
854        fn import(&self, _header: &SyncHeader, lines: &[String]) -> Result<Imported> {
855            self.events.lock().unwrap().push("import".to_string());
856            self.imported.lock().unwrap().extend_from_slice(lines);
857            let mut scripted = self.import_results.lock().unwrap();
858            if !scripted.is_empty() {
859                return Ok(scripted.remove(0));
860            }
861            Ok(Imported {
862                created: lines.len() as u64,
863                max_seq: 8,
864                ..Imported::default()
865            })
866        }
867
868        fn export(&self, peer_id: &str, after_seq: u64, _limit: usize) -> Result<Exported> {
869            assert_eq!(peer_id, PEER);
870            let mut starts = self.export_starts.lock().unwrap();
871            let lines = if starts.is_empty() {
872                self.exports.clone()
873            } else {
874                Vec::new()
875            };
876            starts.push(after_seq);
877            Ok(Exported {
878                max_seq: after_seq + lines.len() as u64,
879                lines,
880            })
881        }
882
883        fn hello_for(&self, peer_id: &str) -> Result<Hello> {
884            assert_eq!(peer_id, PEER);
885            Ok(hello("local-machine", Some(99)))
886        }
887
888        fn accept_hello(&self, hello: &Hello) -> Result<()> {
889            if let Some(message) = &self.refuses_hello {
890                return Err(anyhow::Error::new(IdentityMismatch(message.clone()))
891                    .context("sync/accept-hello returned error"));
892            }
893            self.events.lock().unwrap().push("accept".to_string());
894            self.accepted.lock().unwrap().push(hello.clone());
895            Ok(())
896        }
897
898        fn record_ack(&self, peer_id: &str, acked_seq: u64) -> Result<()> {
899            self.acks
900                .lock()
901                .unwrap()
902                .push((peer_id.to_string(), acked_seq));
903            Ok(())
904        }
905
906        fn record_peer_in(&self, peer_id: &str, seq: u64) -> Result<()> {
907            self.peer_in
908                .lock()
909                .unwrap()
910                .push((peer_id.to_string(), seq));
911            Ok(())
912        }
913
914        fn push_report(&self, peer_id: &str) -> Result<Option<u64>> {
915            assert_eq!(peer_id, PEER);
916            Ok(*self.last_report.lock().unwrap())
917        }
918
919        fn record_push_report(&self, peer_id: &str, reported: Option<u64>) -> Result<()> {
920            assert_eq!(peer_id, PEER);
921            if self.report_fails {
922                anyhow::bail!("the store is busy");
923            }
924            *self.last_report.lock().unwrap() = reported;
925            Ok(())
926        }
927
928        fn merge_snapshot(&self, exporter: &str, _stream: Box<dyn Read + Send>) -> Result<Merged> {
929            assert_eq!(exporter, PEER);
930            self.events.lock().unwrap().push("merge".to_string());
931            Ok(Merged::Done { seq: 5 })
932        }
933
934        fn bootstrap_pending(&self) -> Result<bool> {
935            Ok(*self.bootstrapping.lock().unwrap())
936        }
937
938        fn finish_bootstrap(&self) -> Result<()> {
939            self.events.lock().unwrap().push("finish".to_string());
940            *self.bootstrapping.lock().unwrap() = false;
941            Ok(())
942        }
943    }
944
945    struct Finished;
946
947    impl Remote for Finished {
948        fn wait(&mut self) -> Result<Exit> {
949            Ok(Exit {
950                success: true,
951                status: "exit status: 0".to_string(),
952            })
953        }
954
955        fn stop(&mut self) -> Result<()> {
956            Ok(())
957        }
958    }
959
960    struct Lost;
961
962    impl Remote for Lost {
963        fn wait(&mut self) -> Result<Exit> {
964            anyhow::bail!("ssh vanished")
965        }
966
967        fn stop(&mut self) -> Result<()> {
968            Ok(())
969        }
970    }
971
972    struct Failed;
973
974    impl Remote for Failed {
975        fn wait(&mut self) -> Result<Exit> {
976            Ok(Exit {
977                success: false,
978                status: "exit status: 1".to_string(),
979            })
980        }
981
982        fn stop(&mut self) -> Result<()> {
983            Ok(())
984        }
985    }
986
987    #[derive(Clone, Default)]
988    struct Shared(Arc<Mutex<Vec<u8>>>);
989
990    impl Write for Shared {
991        fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
992            self.0.lock().unwrap().extend_from_slice(buf);
993            Ok(buf.len())
994        }
995
996        fn flush(&mut self) -> std::io::Result<()> {
997            Ok(())
998        }
999    }
1000
1001    struct FakePeer {
1002        export_stream: String,
1003        exports: Mutex<Vec<(u64, Hello)>>,
1004        snapshots: Mutex<Vec<Hello>>,
1005        pushed: Shared,
1006        push_replies: String,
1007        remote_fails: bool,
1008        remote_lost: bool,
1009    }
1010
1011    impl FakePeer {
1012        fn sending(header: SyncHeader) -> Self {
1013            Self {
1014                export_stream: format!("{}\n{SEALED}\n", header.to_line()),
1015                exports: Mutex::default(),
1016                snapshots: Mutex::default(),
1017                pushed: Shared::default(),
1018                push_replies: receiving(hello(PEER, None), 0),
1019                remote_fails: false,
1020                remote_lost: false,
1021            }
1022        }
1023
1024        fn remote(&self) -> Box<dyn Remote> {
1025            if self.remote_lost {
1026                Box::new(Lost)
1027            } else if self.remote_fails {
1028                Box::new(Failed)
1029            } else {
1030                Box::new(Finished)
1031            }
1032        }
1033
1034        fn taking_pushes(replies: String) -> Self {
1035            Self {
1036                push_replies: replies,
1037                ..Self::sending(SyncHeader::local())
1038            }
1039        }
1040    }
1041
1042    /// What a peer's `sb sync-import` writes back: its hello, then what it
1043    /// applied.
1044    fn receiving(peer_hello: Hello, applied: u64) -> String {
1045        let header = SyncHeader {
1046            hello: Some(peer_hello),
1047            ..SyncHeader::local()
1048        };
1049        format!(
1050            "{}\n{}\n",
1051            header.to_line(),
1052            PushApplied { applied }.to_line()
1053        )
1054    }
1055
1056    impl Transport for FakePeer {
1057        fn host(&self) -> &str {
1058            "user@peer"
1059        }
1060
1061        fn whoami(&self) -> Result<String> {
1062            Ok(PEER.to_string())
1063        }
1064
1065        fn export(
1066            &self,
1067            after_seq: u64,
1068            hello: &Hello,
1069        ) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)> {
1070            self.exports
1071                .lock()
1072                .unwrap()
1073                .push((after_seq, hello.clone()));
1074            Ok((
1075                Box::new(Cursor::new(self.export_stream.clone().into_bytes())),
1076                self.remote(),
1077            ))
1078        }
1079
1080        fn snapshot_export(
1081            &self,
1082            hello: &Hello,
1083        ) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)> {
1084            self.snapshots.lock().unwrap().push(hello.clone());
1085            Ok((Box::new(Cursor::new(Vec::new())), Box::new(Finished)))
1086        }
1087
1088        fn import(&self) -> Result<(Box<dyn Write + Send>, Box<dyn Read + Send>, Box<dyn Remote>)> {
1089            Ok((
1090                Box::new(self.pushed.clone()),
1091                Box::new(Cursor::new(self.push_replies.clone().into_bytes())),
1092                self.remote(),
1093            ))
1094        }
1095    }
1096
1097    const SEALED: &str = "AgAAAAEAAQIDBAUGBw";
1098
1099    #[test]
1100    fn a_pull_sends_a_hello_at_its_cursor_and_accepts_the_peers_before_importing() {
1101        let local = FakeLocal {
1102            cursors: vec![(PEER.to_string(), 7)],
1103            ..FakeLocal::default()
1104        };
1105        let peer_hello = hello(PEER, Some(4));
1106        let peer = FakePeer::sending(SyncHeader {
1107            hello: Some(peer_hello.clone()),
1108            ..SyncHeader::local()
1109        });
1110
1111        let imported = pull(&local, &peer, PEER).unwrap();
1112
1113        assert_eq!(imported.created, 1);
1114        assert_eq!(
1115            *peer.exports.lock().unwrap(),
1116            vec![(
1117                7,
1118                Hello {
1119                    applied: Some(7),
1120                    ..hello("local-machine", Some(7))
1121                }
1122            )],
1123            "the hello's cursor is the cursor the pull resumes from, and nothing failed"
1124        );
1125        assert_eq!(*local.accepted.lock().unwrap(), vec![peer_hello]);
1126        assert_eq!(*local.events.lock().unwrap(), vec!["accept", "import"]);
1127        assert_eq!(
1128            *local.imported.lock().unwrap(),
1129            vec![SEALED.to_string()],
1130            "the peer's sealed line reaches the store unopened"
1131        );
1132    }
1133
1134    #[test]
1135    fn a_bootstrapping_device_merges_a_snapshot_before_its_first_pull_and_then_finishes() {
1136        let local = FakeLocal {
1137            bootstrapping: Mutex::new(true),
1138            ..FakeLocal::default()
1139        };
1140        let peer = FakePeer::sending(SyncHeader {
1141            hello: Some(hello(PEER, None)),
1142            ..SyncHeader::local()
1143        });
1144
1145        pull(&local, &peer, PEER).unwrap();
1146
1147        let snapshots = peer.snapshots.lock().unwrap();
1148        assert_eq!(
1149            snapshots.len(),
1150            1,
1151            "a peer whose log starts at 0 still sends a snapshot"
1152        );
1153        assert_eq!(snapshots[0].cursor, Some(0));
1154        assert_eq!(
1155            *local.events.lock().unwrap(),
1156            vec!["merge", "accept", "import", "finish"]
1157        );
1158        assert!(!*local.bootstrapping.lock().unwrap());
1159    }
1160
1161    #[test]
1162    fn a_bootstrap_whose_snapshot_already_merged_only_pulls_and_one_that_fails_stays_pending() {
1163        let merged = FakeLocal {
1164            cursors: vec![(PEER.to_string(), 7)],
1165            bootstrapping: Mutex::new(true),
1166            ..FakeLocal::default()
1167        };
1168        let peer = FakePeer::sending(SyncHeader {
1169            hello: Some(hello(PEER, None)),
1170            ..SyncHeader::local()
1171        });
1172        pull(&merged, &peer, PEER).unwrap();
1173        assert!(peer.snapshots.lock().unwrap().is_empty());
1174        assert_eq!(
1175            *merged.events.lock().unwrap(),
1176            vec!["accept", "import", "finish"]
1177        );
1178
1179        let failing = FakeLocal {
1180            bootstrapping: Mutex::new(true),
1181            ..FakeLocal::default()
1182        };
1183        let no_hello = FakePeer::sending(SyncHeader::local());
1184        assert!(pull(&failing, &no_hello, PEER).is_err());
1185        assert!(*failing.bootstrapping.lock().unwrap());
1186        assert!(
1187            !failing
1188                .events
1189                .lock()
1190                .unwrap()
1191                .contains(&"finish".to_string())
1192        );
1193    }
1194
1195    #[test]
1196    fn a_pull_whose_header_has_no_hello_imports_nothing() {
1197        let local = FakeLocal::default();
1198        let peer = FakePeer::sending(SyncHeader::local());
1199
1200        let error = pull(&local, &peer, PEER).unwrap_err();
1201
1202        assert!(format!("{error:#}").contains("hello"), "{error:#}");
1203        assert!(local.events.lock().unwrap().is_empty());
1204    }
1205
1206    #[test]
1207    fn a_push_header_carries_the_hello() {
1208        let local = FakeLocal {
1209            exports: vec![SEALED.to_string()],
1210            ..FakeLocal::default()
1211        };
1212        let peer = FakePeer::sending(SyncHeader::local());
1213
1214        push(&local, &peer, PEER, &|_| {}).unwrap();
1215
1216        let pushed = peer.pushed.0.lock().unwrap().clone();
1217        let mut reader = BufReader::new(pushed.as_slice());
1218        let header = read_header(&mut reader).unwrap();
1219        let rest: Vec<String> = reader.lines().map(|l| l.unwrap()).collect();
1220        assert_eq!(
1221            rest,
1222            vec![SEALED.to_string()],
1223            "sealed lines go out as exported"
1224        );
1225        assert_eq!(header.hello, Some(hello("local-machine", Some(99))));
1226        assert_eq!(
1227            (header.sync_protocol, header.schema_version),
1228            (
1229                SyncHeader::local().sync_protocol,
1230                SyncHeader::local().schema_version
1231            )
1232        );
1233    }
1234
1235    #[test]
1236    fn a_failed_chunk_caps_the_watermark_whatever_later_chunks_apply() {
1237        let mut watermark = Watermark::default();
1238        let chunk = |max_seq, errors| Imported {
1239            max_seq,
1240            errors,
1241            ..Imported::default()
1242        };
1243        watermark.add(&chunk(10, 0));
1244        watermark.add(&chunk(15, 1));
1245        watermark.add(&chunk(30, 0));
1246        assert_eq!(watermark.applied(), 15);
1247        assert_eq!(watermark.reached(), 30);
1248    }
1249
1250    #[test]
1251    fn a_pull_cursor_moves_past_a_failed_chunk() {
1252        let local = FakeLocal {
1253            import_results: Mutex::new(vec![
1254                Imported {
1255                    max_seq: 5,
1256                    errors: 1,
1257                    ..Imported::default()
1258                },
1259                Imported {
1260                    max_seq: 20,
1261                    ..Imported::default()
1262                },
1263            ]),
1264            ..FakeLocal::default()
1265        };
1266        let peer = FakePeer {
1267            export_stream: format!(
1268                "{}\n{}",
1269                SyncHeader {
1270                    hello: Some(hello(PEER, None)),
1271                    ..SyncHeader::local()
1272                }
1273                .to_line(),
1274                format!("{SEALED}\n").repeat(IMPORT_CHUNK + 1)
1275            ),
1276            ..FakePeer::sending(SyncHeader::local())
1277        };
1278
1279        let imported = pull(&local, &peer, PEER).unwrap();
1280
1281        assert_eq!(imported.max_seq, 20);
1282        assert_eq!(
1283            *local.set_cursors.lock().unwrap(),
1284            vec![(PEER.to_string(), 20)]
1285        );
1286    }
1287
1288    #[test]
1289    fn the_receivers_report_sets_where_a_push_starts() {
1290        assert_eq!(push_start(5, None, None), 5, "no report: the cursor");
1291        assert_eq!(push_start(0, None, Some(9)), 9, "no cursor: the report");
1292        assert_eq!(push_start(5, None, Some(9)), 9);
1293        assert_eq!(push_start(9, Some(4), Some(6)), 9);
1294        assert_eq!(
1295            push_start(9, Some(9), Some(4)),
1296            4,
1297            "a lower report: restored"
1298        );
1299        assert_eq!(
1300            push_start(9, Some(9), None),
1301            0,
1302            "no report after one: restored"
1303        );
1304    }
1305
1306    #[test]
1307    fn a_push_resumes_from_the_receivers_report_and_records_its_acks() {
1308        let local = FakeLocal {
1309            cursors: vec![(format!("push:{PEER}"), 3)],
1310            exports: vec![SEALED.to_string()],
1311            ..FakeLocal::default()
1312        };
1313        let receiver = Hello {
1314            peer_in: Some(6),
1315            ..hello(PEER, Some(2))
1316        };
1317        let peer = FakePeer::taking_pushes(receiving(receiver.clone(), 7));
1318
1319        push(&local, &peer, PEER, &|_| {}).unwrap();
1320
1321        assert_eq!(*local.accepted.lock().unwrap(), vec![receiver]);
1322        assert_eq!(local.export_starts.lock().unwrap()[0], 6);
1323        assert_eq!(*local.last_report.lock().unwrap(), Some(6));
1324        assert_eq!(
1325            *local.acks.lock().unwrap(),
1326            vec![(PEER.to_string(), 6), (PEER.to_string(), 7)],
1327            "the report and then the final watermark are acks"
1328        );
1329    }
1330
1331    #[test]
1332    fn a_receiver_holding_a_failed_record_acks_only_what_it_applied_without_a_gap() {
1333        let local = FakeLocal {
1334            cursors: vec![(format!("push:{PEER}"), 9)],
1335            exports: vec![SEALED.to_string()],
1336            ..FakeLocal::default()
1337        };
1338        let receiver = Hello {
1339            applied: Some(4),
1340            ..hello(PEER, Some(9))
1341        };
1342        let peer = FakePeer::taking_pushes(receiving(receiver, 4));
1343
1344        push(&local, &peer, PEER, &|_| {}).unwrap();
1345
1346        assert_eq!(
1347            local.export_starts.lock().unwrap()[0],
1348            9,
1349            "the push resumes from the receiver's cursor, not from the failed record"
1350        );
1351        assert_eq!(local.acks.lock().unwrap()[0], (PEER.to_string(), 4));
1352    }
1353
1354    #[test]
1355    fn a_push_to_a_receiver_that_closes_without_a_result_fails() {
1356        let local = FakeLocal {
1357            exports: vec![SEALED.to_string()],
1358            ..FakeLocal::default()
1359        };
1360        let header = SyncHeader {
1361            hello: Some(hello(PEER, None)),
1362            ..SyncHeader::local()
1363        };
1364        let peer = FakePeer::taking_pushes(format!("{}\n", header.to_line()));
1365
1366        let error = push(&local, &peer, PEER, &|_| {}).unwrap_err();
1367
1368        assert!(format!("{error:#}").contains("applied"), "{error:#}");
1369        assert!(local.set_cursors.lock().unwrap().is_empty());
1370    }
1371
1372    #[test]
1373    fn serving_a_push_answers_with_this_hello_and_reports_what_it_applied() {
1374        let local = FakeLocal::default();
1375        let pusher = hello(PEER, Some(1));
1376        let input = format!(
1377            "{}\n{SEALED}\n{SEALED}\n",
1378            SyncHeader {
1379                hello: Some(pusher.clone()),
1380                ..SyncHeader::local()
1381            }
1382            .to_line()
1383        );
1384        let mut output = Vec::new();
1385        let mut input = input.as_bytes();
1386        let header = read_checked_header(&mut input, "the sender").unwrap();
1387
1388        let imported = serve_push(&local, &header, input, &mut output).unwrap();
1389
1390        assert_eq!(imported.created, 2);
1391        assert_eq!(*local.accepted.lock().unwrap(), vec![pusher]);
1392        assert_eq!(*local.peer_in.lock().unwrap(), vec![(PEER.to_string(), 8)]);
1393        let mut replies = BufReader::new(output.as_slice());
1394        let reply = read_header(&mut replies).unwrap();
1395        assert_eq!(reply.hello, Some(hello("local-machine", Some(99))));
1396        assert_eq!(
1397            read_push_applied(&mut replies).unwrap(),
1398            PushApplied { applied: 8 }
1399        );
1400    }
1401
1402    #[test]
1403    fn a_restored_receiver_is_pushed_from_its_report_again_after_a_failed_push() {
1404        let local = FakeLocal {
1405            cursors: vec![(format!("push:{PEER}"), 9)],
1406            exports: vec![SEALED.to_string()],
1407            last_report: Mutex::new(Some(9)),
1408            ..FakeLocal::default()
1409        };
1410        let header = SyncHeader {
1411            hello: Some(hello(PEER, Some(4))),
1412            ..SyncHeader::local()
1413        };
1414        let peer = FakePeer::taking_pushes(format!("{}\n", header.to_line()));
1415
1416        push(&local, &peer, PEER, &|_| {}).unwrap_err();
1417
1418        assert_eq!(local.export_starts.lock().unwrap()[0], 4);
1419        assert_eq!(
1420            *local.set_cursors.lock().unwrap(),
1421            vec![(format!("push:{PEER}"), 4)],
1422            "the lowered start outlives the failed push"
1423        );
1424    }
1425
1426    #[test]
1427    fn a_restore_seen_by_a_push_that_fails_before_streaming_is_seen_again() {
1428        let local = FakeLocal {
1429            cursors: vec![(format!("push:{PEER}"), 9)],
1430            exports: vec![SEALED.to_string()],
1431            last_report: Mutex::new(Some(9)),
1432            report_fails: true,
1433            ..FakeLocal::default()
1434        };
1435        let peer = FakePeer::taking_pushes(receiving(hello(PEER, Some(4)), 4));
1436
1437        push(&local, &peer, PEER, &|_| {}).unwrap_err();
1438
1439        assert_eq!(*local.last_report.lock().unwrap(), Some(9));
1440        assert_eq!(
1441            *local.set_cursors.lock().unwrap(),
1442            vec![(format!("push:{PEER}"), 4)]
1443        );
1444        assert!(local.export_starts.lock().unwrap().is_empty());
1445    }
1446
1447    const MISMATCH: &str = "peer-machine holds a different sync identity; pair it with `sb pair`";
1448
1449    fn refused_identity() -> RefusedLine {
1450        RefusedLine {
1451            error: MISMATCH.to_string(),
1452            identity: true,
1453        }
1454    }
1455
1456    fn expect_refused(error: &anyhow::Error) -> &PeerRefused {
1457        error
1458            .downcast_ref::<PeerRefused>()
1459            .unwrap_or_else(|| panic!("not a PeerRefused: {error:#}"))
1460    }
1461
1462    #[test]
1463    fn a_pull_the_peer_refuses_returns_its_named_error_not_its_exit_status() {
1464        let local = FakeLocal::default();
1465        let peer = FakePeer {
1466            export_stream: format!("{}\n", refused_identity().to_line()),
1467            remote_fails: true,
1468            ..FakePeer::sending(SyncHeader::local())
1469        };
1470
1471        let error = pull(&local, &peer, PEER).unwrap_err();
1472
1473        assert_eq!(
1474            expect_refused(&error),
1475            &PeerRefused {
1476                host: "user@peer".to_string(),
1477                error: MISMATCH.to_string(),
1478                identity: true,
1479            }
1480        );
1481        assert!(format!("{error}").contains("sb pair"), "{error}");
1482        assert!(local.events.lock().unwrap().is_empty());
1483        assert!(local.set_cursors.lock().unwrap().is_empty());
1484    }
1485
1486    #[test]
1487    fn a_push_the_receiver_refuses_returns_its_named_error_and_streams_nothing() {
1488        let local = FakeLocal {
1489            exports: vec![SEALED.to_string()],
1490            ..FakeLocal::default()
1491        };
1492        let peer = FakePeer {
1493            remote_fails: true,
1494            ..FakePeer::taking_pushes(format!("\n{}\n", refused_identity().to_line()))
1495        };
1496
1497        let error = push(&local, &peer, PEER, &|_| {}).unwrap_err();
1498
1499        let refusal = expect_refused(&error);
1500        assert!(refusal.identity);
1501        assert_eq!(refusal.error, MISMATCH);
1502        assert!(local.export_starts.lock().unwrap().is_empty());
1503        assert!(local.set_cursors.lock().unwrap().is_empty());
1504        assert!(local.acks.lock().unwrap().is_empty());
1505        assert_eq!(*local.last_report.lock().unwrap(), None);
1506    }
1507
1508    #[test]
1509    fn a_refusal_survives_a_remote_that_cannot_be_waited_on() {
1510        let local = FakeLocal::default();
1511        let puller = FakePeer {
1512            export_stream: format!("{}\n", refused_identity().to_line()),
1513            remote_lost: true,
1514            ..FakePeer::sending(SyncHeader::local())
1515        };
1516        let error = pull(&local, &puller, PEER).unwrap_err();
1517        assert!(expect_refused(&error).identity, "{error:#}");
1518
1519        let pusher = FakePeer {
1520            remote_lost: true,
1521            ..FakePeer::taking_pushes(format!("\n{}\n", refused_identity().to_line()))
1522        };
1523        let error = push(&local, &pusher, PEER, &|_| {}).unwrap_err();
1524        assert!(expect_refused(&error).identity, "{error:#}");
1525    }
1526
1527    #[test]
1528    fn a_push_whose_receiver_fails_without_a_refusal_still_reports_its_exit() {
1529        let local = FakeLocal::default();
1530        let peer = FakePeer {
1531            remote_fails: true,
1532            ..FakePeer::taking_pushes(String::new())
1533        };
1534
1535        let error = push(&local, &peer, PEER, &|_| {}).unwrap_err();
1536
1537        assert!(error.downcast_ref::<PeerRefused>().is_none(), "{error:#}");
1538        assert!(format!("{error:#}").contains("exited with"), "{error:#}");
1539    }
1540
1541    #[test]
1542    fn serving_a_push_whose_hello_is_refused_tells_the_pusher_the_mismatch() {
1543        let receiver = FakeLocal {
1544            refuses_hello: Some(MISMATCH.to_string()),
1545            ..FakeLocal::default()
1546        };
1547        let header = SyncHeader {
1548            hello: Some(hello(PEER, Some(1))),
1549            ..SyncHeader::local()
1550        };
1551        let input = format!("{SEALED}\n");
1552        let mut output = Vec::new();
1553
1554        let error = serve_push(&receiver, &header, input.as_bytes(), &mut output).unwrap_err();
1555
1556        assert!(
1557            error.downcast_ref::<IdentityMismatch>().is_some(),
1558            "{error:#}"
1559        );
1560        assert!(receiver.imported.lock().unwrap().is_empty());
1561        assert!(receiver.peer_in.lock().unwrap().is_empty());
1562        let replies = String::from_utf8(output).unwrap();
1563        assert_eq!(
1564            RefusedLine::parse_line(&replies),
1565            Some(refused_identity()),
1566            "{replies}"
1567        );
1568
1569        let pusher = FakeLocal::default();
1570        let peer = FakePeer {
1571            remote_fails: true,
1572            ..FakePeer::taking_pushes(replies)
1573        };
1574        let error = push(&pusher, &peer, PEER, &|_| {}).unwrap_err();
1575        assert!(expect_refused(&error).identity);
1576    }
1577
1578    #[test]
1579    fn serving_a_push_refused_for_another_reason_sends_the_whole_error() {
1580        let header = SyncHeader {
1581            hello: None,
1582            ..SyncHeader::local()
1583        };
1584        let mut output = Vec::new();
1585
1586        serve_push(&FakeLocal::default(), &header, &b""[..], &mut output).unwrap_err();
1587
1588        let refused = RefusedLine::parse_line(&String::from_utf8(output).unwrap()).unwrap();
1589        assert!(!refused.identity);
1590        assert!(
1591            refused.error.contains("without its hello"),
1592            "{}",
1593            refused.error
1594        );
1595    }
1596}