Skip to main content

second_brain_api/
sync_loop.rs

1use std::collections::{HashMap, HashSet};
2use std::path::PathBuf;
3use std::sync::Arc;
4use std::sync::mpsc::{self, Receiver, SendError, Sender};
5use std::thread::JoinHandle;
6use std::time::{Duration, Instant};
7
8use anyhow::Result;
9use chrono::Utc;
10use second_brain_core::schema::SyncState;
11use second_brain_core::sqlite_store::SqliteStore;
12use second_brain_core::store::StoreAdmin;
13use second_brain_sync::crypto::{XChaChaEncryptor, load_or_create_key};
14use second_brain_sync::export::export_page;
15use second_brain_sync::import::import_values;
16use second_brain_sync::peers::{self, Peer};
17use second_brain_sync::protocol::{SyncHeader, check_store_header, store_header};
18use second_brain_sync::session::{self, Exported, Imported, Local, Transport};
19use second_brain_sync::ssh::SshTransport;
20use serde_json::Value;
21
22use crate::health::Health;
23use crate::value_metrics::PeerOutcome;
24use crate::write_gate::{WriteGate, WriteSlot};
25
26const FULL_INTERVAL: Duration = Duration::from_secs(300);
27const TICK: Duration = Duration::from_secs(1);
28const PUSH_DELAY: Duration = Duration::from_secs(5);
29const FIRST_BACKOFF: Duration = Duration::from_secs(30);
30const MAX_BACKOFF: Duration = Duration::from_secs(30 * 60);
31const ORIGIN_SCAN_PAGE: usize = 500;
32
33pub type Connect = Arc<dyn Fn(&Peer) -> Result<Box<dyn Transport + Send>> + Send + Sync>;
34
35/// Syncs with every peer in `sync-peers`: a full pull and push on start and
36/// every five minutes, and a push shortly after this machine writes.
37pub struct SyncLoop {
38    pub data_dir: PathBuf,
39    pub sqlite: Arc<SqliteStore>,
40    pub gate: Arc<WriteGate>,
41    pub connect: Connect,
42    pub health: Option<Arc<Health>>,
43}
44
45#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
46enum Job {
47    Push,
48    Full,
49}
50
51impl SyncLoop {
52    pub fn over_ssh(data_dir: PathBuf, sqlite: Arc<SqliteStore>, gate: Arc<WriteGate>) -> Self {
53        Self {
54            data_dir,
55            sqlite,
56            gate,
57            connect: Arc::new(|peer: &Peer| {
58                let transport = SshTransport::new(&peer.host, &peer.sb)?.unattended();
59                Ok(Box::new(transport) as Box<dyn Transport + Send>)
60            }),
61            health: None,
62        }
63    }
64
65    pub fn with_health(mut self, health: Arc<Health>) -> Self {
66        self.health = Some(health);
67        self
68    }
69
70    pub fn spawn(self) -> JoinHandle<()> {
71        std::thread::Builder::new()
72            .name("sync-loop".to_string())
73            .spawn(move || Arc::new(self).run())
74            .expect("spawning the sync loop")
75    }
76
77    fn run(self: Arc<Self>) {
78        let mut workers: HashMap<String, Sender<(Job, Peer)>> = HashMap::new();
79        let mut next_full = Instant::now();
80        let mut seen = self.sqlite.max_sync_seq().unwrap_or(0);
81        let mut push_at: Option<Instant> = None;
82        loop {
83            let (peers, repeated) = self.listed_peers();
84            if peers.is_empty() {
85                seen = self.sqlite.max_sync_seq().unwrap_or(seen);
86                push_at = None;
87            } else if self.wrote_locally(&mut seen) {
88                push_at.get_or_insert(Instant::now() + PUSH_DELAY);
89            }
90
91            let now = Instant::now();
92            let job = if now >= next_full {
93                next_full = now + FULL_INTERVAL;
94                Some(Job::Full)
95            } else if push_at.is_some_and(|at| now >= at) {
96                Some(Job::Push)
97            } else {
98                None
99            };
100            if let Some(job) = job {
101                push_at = None;
102                for line in &repeated {
103                    tracing::warn!(
104                        "sync-peers lists {} again; only its first line is used",
105                        line.host
106                    );
107                }
108                for peer in peers {
109                    self.dispatch(&mut workers, job, peer);
110                }
111            }
112            std::thread::sleep(TICK);
113        }
114    }
115
116    /// Workers are keyed by host alone and outlive the host's removal from
117    /// `sync-peers`, so a changed `sb` path or a host re-added mid-sync still
118    /// gets one sync at a time.
119    fn dispatch(
120        self: &Arc<Self>,
121        workers: &mut HashMap<String, Sender<(Job, Peer)>>,
122        job: Job,
123        peer: Peer,
124    ) {
125        let host = peer.host.clone();
126        let worker = workers
127            .entry(host.clone())
128            .or_insert_with(|| self.clone().start_worker(&host));
129        if let Err(SendError(message)) = worker.send((job, peer)) {
130            tracing::warn!("the sync worker for {host} exited; starting another");
131            let worker = self.clone().start_worker(&host);
132            if worker.send(message).is_err() {
133                tracing::warn!("the new sync worker for {host} exited at once");
134            }
135            workers.insert(host, worker);
136        }
137    }
138
139    /// An unreadable `sync-peers` syncs no one but keeps the peers in health,
140    /// so their lag keeps growing instead of vanishing from the report.
141    fn listed_peers(&self) -> (Vec<Peer>, Vec<Peer>) {
142        let lines = match peers::configured_peers(&self.data_dir) {
143            Ok(lines) => lines.unwrap_or_default(),
144            Err(e) => {
145                tracing::warn!("cannot read the sync peers: {e}");
146                return (Vec::new(), Vec::new());
147            }
148        };
149        let (peers, repeated) =
150            one_line_per_host(lines.iter().map(|line| Peer::parse(line)).collect());
151        if let Some(health) = &self.health {
152            let hosts: Vec<String> = peers.iter().map(|peer| peer.host.clone()).collect();
153            health.sync_peers_listed(&hosts, Utc::now());
154        }
155        (peers, repeated)
156    }
157
158    fn report(&self, host: &str, outcome: PeerOutcome) {
159        if let Some(health) = &self.health {
160            health.sync_attempted(host, outcome, Utc::now());
161        }
162    }
163
164    /// Imports also append to sync_log; only a row this machine originated is
165    /// news to its peers.
166    fn wrote_locally(&self, seen: &mut u64) -> bool {
167        let max = match self.sqlite.max_sync_seq() {
168            Ok(max) => max,
169            Err(e) => {
170                tracing::warn!("cannot read the sync log position: {e:#}");
171                return false;
172            }
173        };
174        let own = self.sqlite.machine_id();
175        while *seen < max {
176            let page = match self.sqlite.sync_log_page(*seen, Some(ORIGIN_SCAN_PAGE)) {
177                Ok(page) => page,
178                Err(e) => {
179                    tracing::warn!("cannot read the sync log: {e:#}");
180                    return false;
181                }
182            };
183            let Some(last) = page.last() else {
184                break;
185            };
186            if page.iter().any(|entry| entry.origin_machine_id == own) {
187                *seen = max;
188                return true;
189            }
190            *seen = last.local_seq.max(0) as u64;
191        }
192        *seen = max;
193        false
194    }
195
196    fn start_worker(self: Arc<Self>, host: &str) -> Sender<(Job, Peer)> {
197        let (jobs, inbox) = mpsc::channel();
198        std::thread::Builder::new()
199            .name(format!("sync-{host}"))
200            .spawn(move || self.work(inbox))
201            .expect("spawning a sync worker");
202        jobs
203    }
204
205    /// One sync per peer at a time; a peer that fails waits out its backoff
206    /// on its own thread, so it never holds up the others.
207    fn work(&self, inbox: Receiver<(Job, Peer)>) {
208        let mut backoff = PeerBackoff::default();
209        while let Ok((mut job, mut peer)) = inbox.recv() {
210            while let Ok((queued, latest)) = inbox.try_recv() {
211                job = job.max(queued);
212                peer = latest;
213            }
214            if !backoff.due(&peer, Instant::now()) {
215                continue;
216            }
217            match self.sync(&peer, job) {
218                Ok(()) => {
219                    backoff.succeeded();
220                    let outcome = match job {
221                        Job::Full => PeerOutcome::Synced,
222                        Job::Push => PeerOutcome::Pushed,
223                    };
224                    self.report(&peer.host, outcome);
225                }
226                Err(e) if !counts_against_peer(&e) => {
227                    tracing::info!("sync with {} waits for the next job: {e:#}", peer.host);
228                }
229                Err(e) => {
230                    let wait = backoff.failed(&peer, Instant::now());
231                    self.report(&peer.host, PeerOutcome::Failed);
232                    tracing::warn!(
233                        "sync with {} failed; its jobs are skipped for the next {}s: {e:#}",
234                        peer.host,
235                        wait.as_secs()
236                    );
237                }
238            }
239        }
240    }
241
242    fn sync(&self, peer: &Peer, job: Job) -> Result<()> {
243        let transport = (self.connect)(peer)?;
244        let key = load_or_create_key(&self.data_dir.join("sync.key"))?;
245        let encryptor = XChaChaEncryptor::new(key);
246        let local = StoreLocal {
247            sqlite: &self.sqlite,
248            gate: &self.gate,
249        };
250        let report = |line: &str| tracing::info!("sync with {}: {}", peer.host, line.trim());
251        match job {
252            Job::Full => session::sync(
253                &local,
254                transport.as_ref(),
255                &encryptor,
256                &self.data_dir,
257                &report,
258            ),
259            Job::Push => {
260                let peer_machine_id = transport.whoami()?;
261                session::push(
262                    &local,
263                    transport.as_ref(),
264                    &peer_machine_id,
265                    &encryptor,
266                    &report,
267                )
268                .map(|_| ())
269            }
270        }
271    }
272}
273
274#[derive(Debug)]
275struct MaintenanceRunning;
276
277impl std::fmt::Display for MaintenanceRunning {
278    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
279        f.write_str("store maintenance is running")
280    }
281}
282
283impl std::error::Error for MaintenanceRunning {}
284
285/// Routine store maintenance says nothing about the peer, so it starts no
286/// backoff. Other local faults do back off, which also bounds how often a
287/// persistent one is logged; a daemon restart clears the backoff.
288fn counts_against_peer(error: &anyhow::Error) -> bool {
289    error.downcast_ref::<MaintenanceRunning>().is_none()
290}
291
292/// The same first-line-wins rule `sb sync` applies through `configured_peer`.
293fn one_line_per_host(lines: Vec<Peer>) -> (Vec<Peer>, Vec<Peer>) {
294    let mut hosts = HashSet::new();
295    lines
296        .into_iter()
297        .partition(|peer| hosts.insert(peer.host.clone()))
298}
299
300#[derive(Default)]
301struct PeerBackoff {
302    line: Option<Peer>,
303    failures: u32,
304    retry_at: Option<Instant>,
305}
306
307impl PeerBackoff {
308    /// A changed `sync-peers` line is a new way to reach the host, so it is
309    /// tried at once rather than after the old line's backoff.
310    fn due(&self, peer: &Peer, now: Instant) -> bool {
311        self.line.as_ref() != Some(peer) || self.retry_at.is_none_or(|at| now >= at)
312    }
313
314    fn succeeded(&mut self) {
315        *self = PeerBackoff::default();
316    }
317
318    fn failed(&mut self, peer: &Peer, now: Instant) -> Duration {
319        if self.line.as_ref() != Some(peer) {
320            *self = PeerBackoff {
321                line: Some(peer.clone()),
322                ..PeerBackoff::default()
323            };
324        }
325        self.failures += 1;
326        let wait = backoff(self.failures);
327        self.retry_at = Some(now + wait);
328        wait
329    }
330}
331
332fn backoff(failures: u32) -> Duration {
333    FIRST_BACKOFF
334        .saturating_mul(2u32.saturating_pow(failures.saturating_sub(1)))
335        .min(MAX_BACKOFF)
336}
337
338struct StoreLocal<'a> {
339    sqlite: &'a SqliteStore,
340    gate: &'a WriteGate,
341}
342
343impl StoreLocal<'_> {
344    fn write_slot(&self) -> Result<WriteSlot<'_>> {
345        self.gate
346            .try_enter()
347            .ok_or_else(|| anyhow::Error::new(MaintenanceRunning))
348    }
349}
350
351impl Local for StoreLocal<'_> {
352    fn cursors(&self) -> Result<Vec<(String, u64)>> {
353        Ok(self
354            .sqlite
355            .get_all_sync_states()?
356            .into_iter()
357            .map(|state| (state.peer_id, state.last_seq))
358            .collect())
359    }
360
361    fn set_cursor(&self, peer_id: &str, last_seq: u64) -> Result<()> {
362        let _slot = self.write_slot()?;
363        self.sqlite.set_sync_state(&SyncState {
364            peer_id: peer_id.to_string(),
365            last_seq,
366            last_sync_at: chrono::Utc::now(),
367        })?;
368        Ok(())
369    }
370
371    fn import(&self, header: &SyncHeader, records: &[Value]) -> Result<Imported> {
372        let _slot = self.write_slot()?;
373        check_store_header(self.sqlite, Some(*header), "the sender")?;
374        let (stats, max_seq) = import_values(self.sqlite, records)?;
375        Ok(Imported {
376            created: stats.created,
377            updated: stats.updated,
378            deleted: stats.deleted,
379            skipped: stats.skipped,
380            dangling: Imported::dangling_in(&stats.skipped_by_reason),
381            max_seq,
382        })
383    }
384
385    fn export(&self, after_seq: u64, limit: usize) -> Result<Exported> {
386        store_header(self.sqlite)?;
387        let (records, max_seq) = export_page(self.sqlite, after_seq, limit)?;
388        Ok(Exported { records, max_seq })
389    }
390}
391
392#[cfg(test)]
393mod tests {
394    use std::io::{Read, Write};
395    use std::path::Path;
396    use std::sync::Mutex;
397
398    use anyhow::Context;
399    use second_brain_core::sqlite_store::SCHEMA_VERSION;
400    use second_brain_sync::session::{Exit, Remote};
401
402    use super::*;
403    use crate::value_metrics::PeerSync;
404
405    #[test]
406    fn the_daemon_refuses_to_export_from_a_database_newer_than_its_binary() {
407        let dir = tempfile::tempdir().unwrap();
408        let path = dir.path().join("future.db");
409        drop(SqliteStore::open(&path, "m".to_string()).unwrap());
410        rusqlite::Connection::open(&path)
411            .unwrap()
412            .pragma_update(None, "user_version", SCHEMA_VERSION + 1)
413            .unwrap();
414        let sqlite = SqliteStore::open(&path, "m".to_string()).unwrap();
415        let gate = WriteGate::default();
416        let local = StoreLocal {
417            sqlite: &sqlite,
418            gate: &gate,
419        };
420        let error = local.export(0, 10).err().expect("export refused");
421        assert!(
422            format!("{error:#}").contains("upgrade this daemon"),
423            "{error:#}"
424        );
425    }
426
427    #[test]
428    fn a_host_keeps_one_worker_when_its_sb_path_changes() {
429        let dir = tempfile::tempdir().unwrap();
430        let sync_loop = Arc::new(SyncLoop {
431            data_dir: dir.path().to_path_buf(),
432            sqlite: Arc::new(SqliteStore::in_memory("m".to_string()).unwrap()),
433            gate: Arc::new(WriteGate::default()),
434            connect: Arc::new(|_: &Peer| -> Result<Box<dyn Transport + Send>> {
435                anyhow::bail!("offline")
436            }),
437            health: None,
438        });
439        let mut workers = HashMap::new();
440        sync_loop.dispatch(&mut workers, Job::Push, Peer::parse("user@pi"));
441        sync_loop.dispatch(&mut workers, Job::Push, Peer::parse("user@pi /opt/sb"));
442        assert_eq!(workers.len(), 1);
443    }
444
445    fn scripted_loop(dir: &Path, connect: Connect) -> SyncLoop {
446        SyncLoop {
447            data_dir: dir.to_path_buf(),
448            sqlite: Arc::new(SqliteStore::in_memory("m".to_string()).unwrap()),
449            gate: Arc::new(WriteGate::default()),
450            connect,
451            health: None,
452        }
453    }
454
455    #[test]
456    fn the_worker_takes_the_latest_line_and_backs_off_for_all_but_maintenance() {
457        let dir = tempfile::tempdir().unwrap();
458        let calls: Arc<Mutex<Vec<Peer>>> = Arc::default();
459        let connect: Connect = {
460            let calls = calls.clone();
461            Arc::new(move |peer: &Peer| -> Result<Box<dyn Transport + Send>> {
462                let mut calls = calls.lock().unwrap();
463                calls.push(peer.clone());
464                if calls.len() == 1 {
465                    Err(anyhow::Error::new(MaintenanceRunning))
466                } else {
467                    anyhow::bail!("connection refused")
468                }
469            })
470        };
471        let sync_loop = Arc::new(scripted_loop(dir.path(), connect));
472        let made = |n: usize| {
473            let deadline = Instant::now() + Duration::from_secs(5);
474            while Instant::now() < deadline {
475                if calls.lock().unwrap().len() >= n {
476                    return true;
477                }
478                std::thread::sleep(Duration::from_millis(10));
479            }
480            false
481        };
482        let bare = Peer::parse("user@pi");
483        let fixed = Peer::parse("user@pi /opt/sb");
484        let (jobs, inbox) = mpsc::channel();
485        jobs.send((Job::Push, Peer::parse("user@pi /old/sb")))
486            .unwrap();
487        jobs.send((Job::Full, bare.clone())).unwrap();
488        let worker = {
489            let sync_loop = sync_loop.clone();
490            std::thread::spawn(move || sync_loop.work(inbox))
491        };
492        assert!(made(1), "the queued jobs never ran");
493        jobs.send((Job::Full, bare.clone())).unwrap();
494        assert!(made(2), "local maintenance started a backoff");
495        jobs.send((Job::Full, bare.clone())).unwrap();
496        std::thread::sleep(Duration::from_millis(300));
497        assert_eq!(
498            calls.lock().unwrap().len(),
499            2,
500            "a job inside the backoff ran"
501        );
502        jobs.send((Job::Full, fixed.clone())).unwrap();
503        assert!(made(3), "a changed line waited out the old backoff");
504        drop(jobs);
505        worker.join().unwrap();
506        assert_eq!(*calls.lock().unwrap(), vec![bare.clone(), bare, fixed]);
507    }
508
509    #[test]
510    fn store_maintenance_does_not_count_against_the_peer() {
511        let sqlite = SqliteStore::in_memory("m".to_string()).unwrap();
512        let gate = WriteGate::default();
513        let local = StoreLocal {
514            sqlite: &sqlite,
515            gate: &gate,
516        };
517        let _maintenance = gate.close();
518        let error = local
519            .set_cursor("peer", 1)
520            .context("updating sync state")
521            .unwrap_err();
522        assert!(!counts_against_peer(&error), "{error:#}");
523        assert!(counts_against_peer(&anyhow::anyhow!("connection refused")));
524    }
525
526    #[test]
527    fn a_changed_peer_line_is_tried_without_the_old_lines_backoff() {
528        let now = Instant::now();
529        let later = now + Duration::from_secs(1);
530        let bare = Peer::parse("user@pi");
531        let fixed = Peer::parse("user@pi /opt/sb");
532        let mut state = PeerBackoff::default();
533        assert!(state.due(&bare, now));
534        assert_eq!(state.failed(&bare, now), FIRST_BACKOFF);
535        assert!(!state.due(&bare, later));
536        assert!(state.due(&fixed, later));
537        assert_eq!(state.failed(&fixed, later), FIRST_BACKOFF);
538        assert_eq!(state.failed(&fixed, later), FIRST_BACKOFF * 2);
539        assert!(state.due(&bare, later));
540        state.succeeded();
541        assert_eq!(state.failed(&fixed, later), FIRST_BACKOFF);
542    }
543
544    #[test]
545    fn a_host_listed_twice_keeps_its_first_line() {
546        let (kept, repeated) = one_line_per_host(vec![
547            Peer::parse("user@pi"),
548            Peer::parse("user@mac"),
549            Peer::parse("user@pi /opt/sb"),
550        ]);
551        assert_eq!(kept, vec![Peer::parse("user@pi"), Peer::parse("user@mac")]);
552        assert_eq!(repeated, vec![Peer::parse("user@pi /opt/sb")]);
553    }
554
555    #[test]
556    fn backoff_doubles_from_thirty_seconds_up_to_half_an_hour() {
557        let waits: Vec<u64> = (1..=8).map(|n| backoff(n).as_secs()).collect();
558        assert_eq!(waits, vec![30, 60, 120, 240, 480, 960, 1800, 1800]);
559    }
560
561    struct EmptyPeer;
562
563    impl Transport for EmptyPeer {
564        fn host(&self) -> &str {
565            "user@pi"
566        }
567
568        fn whoami(&self) -> Result<String> {
569            Ok("peer".to_string())
570        }
571
572        fn export(&self, _after_seq: u64) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)> {
573            let stream = format!("{}\n", SyncHeader::local().to_line()).into_bytes();
574            Ok((Box::new(std::io::Cursor::new(stream)), Box::new(Exited)))
575        }
576
577        fn import(&self) -> Result<(Box<dyn Write + Send>, Box<dyn Remote>)> {
578            Ok((Box::new(std::io::sink()), Box::new(Exited)))
579        }
580    }
581
582    struct Exited;
583
584    impl Remote for Exited {
585        fn wait(&mut self) -> Result<Exit> {
586            Ok(Exit {
587                success: true,
588                status: "exit status: 0".to_string(),
589            })
590        }
591
592        fn stop(&mut self) -> Result<()> {
593            Ok(())
594        }
595    }
596
597    fn reachable() -> Connect {
598        Arc::new(|_: &Peer| -> Result<Box<dyn Transport + Send>> { Ok(Box::new(EmptyPeer)) })
599    }
600
601    fn refusing() -> Connect {
602        Arc::new(|_: &Peer| -> Result<Box<dyn Transport + Send>> {
603            anyhow::bail!("connection refused")
604        })
605    }
606
607    fn watched_loop(dir: &Path, connect: Connect) -> (Arc<SyncLoop>, Arc<Health>) {
608        let health = Arc::new(Health::new(Utc::now(), None));
609        let sync_loop = Arc::new(scripted_loop(dir, connect).with_health(health.clone()));
610        std::fs::write(dir.join(peers::PEERS_FILE), "user@pi\n").unwrap();
611        assert_eq!(sync_loop.listed_peers().0, vec![Peer::parse("user@pi")]);
612        (sync_loop, health)
613    }
614
615    fn the_one_peer(health: &Health) -> PeerSync {
616        let mut peers = health.report().value.sync_peers;
617        assert_eq!(peers.len(), 1, "{peers:?}");
618        peers.remove(0)
619    }
620
621    fn run_job(sync_loop: &Arc<SyncLoop>, job: Job, peer: Peer) {
622        let (jobs, inbox) = mpsc::channel();
623        jobs.send((job, peer)).unwrap();
624        drop(jobs);
625        let worker = {
626            let sync_loop = sync_loop.clone();
627            std::thread::spawn(move || sync_loop.work(inbox))
628        };
629        let deadline = Instant::now() + Duration::from_secs(10);
630        while !worker.is_finished() {
631            assert!(
632                Instant::now() < deadline,
633                "the worker never finished its job"
634            );
635            std::thread::sleep(Duration::from_millis(10));
636        }
637        worker.join().unwrap();
638    }
639
640    #[test]
641    fn a_full_sync_reports_the_peer_synced() {
642        let dir = tempfile::tempdir().unwrap();
643        let (sync_loop, health) = watched_loop(dir.path(), reachable());
644        let before = Utc::now();
645        run_job(&sync_loop, Job::Full, Peer::parse("user@pi"));
646        let peer = the_one_peer(&health);
647        assert_eq!(peer.peer, "user@pi");
648        let synced = peer.last_success_at.expect("a full sync is a success");
649        assert!(synced >= before, "{synced} predates the sync");
650        assert_eq!(peer.consecutive_failures, 0);
651    }
652
653    #[test]
654    fn a_failed_sync_counts_a_failure_and_no_success() {
655        let dir = tempfile::tempdir().unwrap();
656        let (sync_loop, health) = watched_loop(dir.path(), refusing());
657        run_job(&sync_loop, Job::Full, Peer::parse("user@pi"));
658        let peer = the_one_peer(&health);
659        assert_eq!(peer.peer, "user@pi");
660        assert_eq!(peer.consecutive_failures, 1);
661        assert_eq!(peer.last_success_at, None);
662        assert!(peer.last_attempt_at.is_some(), "the failure is an attempt");
663    }
664
665    #[test]
666    fn a_push_ends_a_failure_streak_without_counting_as_a_sync() {
667        let dir = tempfile::tempdir().unwrap();
668        let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
669        let connect: Connect = {
670            let calls = calls.clone();
671            Arc::new(move |_: &Peer| -> Result<Box<dyn Transport + Send>> {
672                if calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0 {
673                    anyhow::bail!("connection refused");
674                }
675                Ok(Box::new(EmptyPeer))
676            })
677        };
678        let (sync_loop, health) = watched_loop(dir.path(), connect);
679        run_job(&sync_loop, Job::Full, Peer::parse("user@pi"));
680        assert_eq!(the_one_peer(&health).consecutive_failures, 1);
681        run_job(&sync_loop, Job::Push, Peer::parse("user@pi"));
682        let peer = the_one_peer(&health);
683        assert_eq!(peer.last_success_at, None);
684        assert!(peer.last_attempt_at.is_some());
685        assert_eq!(peer.consecutive_failures, 0);
686    }
687
688    #[test]
689    fn maintenance_reports_no_sync_outcome() {
690        let dir = tempfile::tempdir().unwrap();
691        let connect: Connect = Arc::new(|_: &Peer| -> Result<Box<dyn Transport + Send>> {
692            Err(anyhow::Error::new(MaintenanceRunning))
693        });
694        let (sync_loop, health) = watched_loop(dir.path(), connect);
695        run_job(&sync_loop, Job::Full, Peer::parse("user@pi"));
696        let peer = the_one_peer(&health);
697        assert_eq!(peer.last_success_at, None);
698        assert_eq!(peer.last_attempt_at, None);
699        assert_eq!(peer.consecutive_failures, 0);
700    }
701
702    #[test]
703    fn a_peer_removed_from_sync_peers_drops_out_of_health() {
704        let dir = tempfile::tempdir().unwrap();
705        let (sync_loop, health) = watched_loop(dir.path(), refusing());
706        let listed = |text: &str| {
707            std::fs::write(dir.path().join(peers::PEERS_FILE), text).unwrap();
708            let (kept, _) = sync_loop.listed_peers();
709            let mut reported: Vec<String> = health
710                .report()
711                .value
712                .sync_peers
713                .into_iter()
714                .map(|peer| peer.peer)
715                .collect();
716            reported.sort();
717            (kept.len(), reported)
718        };
719        assert_eq!(
720            listed("user@pi\nuser@mac\nuser@pi /opt/sb\n"),
721            (2, vec!["user@mac".to_string(), "user@pi".to_string()])
722        );
723        assert_eq!(listed("user@mac\n"), (1, vec!["user@mac".to_string()]));
724    }
725
726    #[test]
727    fn an_unreadable_sync_peers_keeps_its_peers_in_health() {
728        let dir = tempfile::tempdir().unwrap();
729        let (sync_loop, health) = watched_loop(dir.path(), refusing());
730        let file = dir.path().join(peers::PEERS_FILE);
731        std::fs::remove_file(&file).unwrap();
732        std::fs::create_dir(&file).unwrap();
733        assert!(peers::configured_peers(dir.path()).is_err());
734
735        let (kept, repeated) = sync_loop.listed_peers();
736
737        assert_eq!(kept, Vec::<Peer>::new());
738        assert_eq!(repeated, Vec::<Peer>::new());
739        assert_eq!(the_one_peer(&health).peer, "user@pi");
740    }
741}