1use std::collections::{HashMap, HashSet};
2use std::path::PathBuf;
3use std::sync::Arc;
4use std::sync::mpsc::{self, Receiver, SendError, Sender};
5use std::thread::JoinHandle;
6use std::time::{Duration, Instant};
7
8use anyhow::Result;
9use chrono::Utc;
10use second_brain_core::schema::SyncState;
11use second_brain_core::sqlite_store::SqliteStore;
12use second_brain_core::store::StoreAdmin;
13use second_brain_sync::crypto::{XChaChaEncryptor, load_or_create_key};
14use second_brain_sync::export::export_page;
15use second_brain_sync::import::import_values;
16use second_brain_sync::peers::{self, Peer};
17use second_brain_sync::protocol::{SyncHeader, check_store_header, store_header};
18use second_brain_sync::session::{self, Exported, Imported, Local, Transport};
19use second_brain_sync::ssh::SshTransport;
20use serde_json::Value;
21
22use crate::health::Health;
23use crate::value_metrics::PeerOutcome;
24use crate::write_gate::{WriteGate, WriteSlot};
25
26const FULL_INTERVAL: Duration = Duration::from_secs(300);
27const TICK: Duration = Duration::from_secs(1);
28const PUSH_DELAY: Duration = Duration::from_secs(5);
29const FIRST_BACKOFF: Duration = Duration::from_secs(30);
30const MAX_BACKOFF: Duration = Duration::from_secs(30 * 60);
31const ORIGIN_SCAN_PAGE: usize = 500;
32
33pub type Connect = Arc<dyn Fn(&Peer) -> Result<Box<dyn Transport + Send>> + Send + Sync>;
34
35pub struct SyncLoop {
38 pub data_dir: PathBuf,
39 pub sqlite: Arc<SqliteStore>,
40 pub gate: Arc<WriteGate>,
41 pub connect: Connect,
42 pub health: Option<Arc<Health>>,
43}
44
45#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
46enum Job {
47 Push,
48 Full,
49}
50
51impl SyncLoop {
52 pub fn over_ssh(data_dir: PathBuf, sqlite: Arc<SqliteStore>, gate: Arc<WriteGate>) -> Self {
53 Self {
54 data_dir,
55 sqlite,
56 gate,
57 connect: Arc::new(|peer: &Peer| {
58 let transport = SshTransport::new(&peer.host, &peer.sb)?.unattended();
59 Ok(Box::new(transport) as Box<dyn Transport + Send>)
60 }),
61 health: None,
62 }
63 }
64
65 pub fn with_health(mut self, health: Arc<Health>) -> Self {
66 self.health = Some(health);
67 self
68 }
69
70 pub fn spawn(self) -> JoinHandle<()> {
71 std::thread::Builder::new()
72 .name("sync-loop".to_string())
73 .spawn(move || Arc::new(self).run())
74 .expect("spawning the sync loop")
75 }
76
77 fn run(self: Arc<Self>) {
78 let mut workers: HashMap<String, Sender<(Job, Peer)>> = HashMap::new();
79 let mut next_full = Instant::now();
80 let mut seen = self.sqlite.max_sync_seq().unwrap_or(0);
81 let mut push_at: Option<Instant> = None;
82 loop {
83 let (peers, repeated) = self.listed_peers();
84 if peers.is_empty() {
85 seen = self.sqlite.max_sync_seq().unwrap_or(seen);
86 push_at = None;
87 } else if self.wrote_locally(&mut seen) {
88 push_at.get_or_insert(Instant::now() + PUSH_DELAY);
89 }
90
91 let now = Instant::now();
92 let job = if now >= next_full {
93 next_full = now + FULL_INTERVAL;
94 Some(Job::Full)
95 } else if push_at.is_some_and(|at| now >= at) {
96 Some(Job::Push)
97 } else {
98 None
99 };
100 if let Some(job) = job {
101 push_at = None;
102 for line in &repeated {
103 tracing::warn!(
104 "sync-peers lists {} again; only its first line is used",
105 line.host
106 );
107 }
108 for peer in peers {
109 self.dispatch(&mut workers, job, peer);
110 }
111 }
112 std::thread::sleep(TICK);
113 }
114 }
115
116 fn dispatch(
120 self: &Arc<Self>,
121 workers: &mut HashMap<String, Sender<(Job, Peer)>>,
122 job: Job,
123 peer: Peer,
124 ) {
125 let host = peer.host.clone();
126 let worker = workers
127 .entry(host.clone())
128 .or_insert_with(|| self.clone().start_worker(&host));
129 if let Err(SendError(message)) = worker.send((job, peer)) {
130 tracing::warn!("the sync worker for {host} exited; starting another");
131 let worker = self.clone().start_worker(&host);
132 if worker.send(message).is_err() {
133 tracing::warn!("the new sync worker for {host} exited at once");
134 }
135 workers.insert(host, worker);
136 }
137 }
138
139 fn listed_peers(&self) -> (Vec<Peer>, Vec<Peer>) {
142 let lines = match peers::configured_peers(&self.data_dir) {
143 Ok(lines) => lines.unwrap_or_default(),
144 Err(e) => {
145 tracing::warn!("cannot read the sync peers: {e}");
146 return (Vec::new(), Vec::new());
147 }
148 };
149 let (peers, repeated) =
150 one_line_per_host(lines.iter().map(|line| Peer::parse(line)).collect());
151 if let Some(health) = &self.health {
152 let hosts: Vec<String> = peers.iter().map(|peer| peer.host.clone()).collect();
153 health.sync_peers_listed(&hosts, Utc::now());
154 }
155 (peers, repeated)
156 }
157
158 fn report(&self, host: &str, outcome: PeerOutcome) {
159 if let Some(health) = &self.health {
160 health.sync_attempted(host, outcome, Utc::now());
161 }
162 }
163
164 fn wrote_locally(&self, seen: &mut u64) -> bool {
167 let max = match self.sqlite.max_sync_seq() {
168 Ok(max) => max,
169 Err(e) => {
170 tracing::warn!("cannot read the sync log position: {e:#}");
171 return false;
172 }
173 };
174 let own = self.sqlite.machine_id();
175 while *seen < max {
176 let page = match self.sqlite.sync_log_page(*seen, Some(ORIGIN_SCAN_PAGE)) {
177 Ok(page) => page,
178 Err(e) => {
179 tracing::warn!("cannot read the sync log: {e:#}");
180 return false;
181 }
182 };
183 let Some(last) = page.last() else {
184 break;
185 };
186 if page.iter().any(|entry| entry.origin_machine_id == own) {
187 *seen = max;
188 return true;
189 }
190 *seen = last.local_seq.max(0) as u64;
191 }
192 *seen = max;
193 false
194 }
195
196 fn start_worker(self: Arc<Self>, host: &str) -> Sender<(Job, Peer)> {
197 let (jobs, inbox) = mpsc::channel();
198 std::thread::Builder::new()
199 .name(format!("sync-{host}"))
200 .spawn(move || self.work(inbox))
201 .expect("spawning a sync worker");
202 jobs
203 }
204
205 fn work(&self, inbox: Receiver<(Job, Peer)>) {
208 let mut backoff = PeerBackoff::default();
209 while let Ok((mut job, mut peer)) = inbox.recv() {
210 while let Ok((queued, latest)) = inbox.try_recv() {
211 job = job.max(queued);
212 peer = latest;
213 }
214 if !backoff.due(&peer, Instant::now()) {
215 continue;
216 }
217 match self.sync(&peer, job) {
218 Ok(()) => {
219 backoff.succeeded();
220 let outcome = match job {
221 Job::Full => PeerOutcome::Synced,
222 Job::Push => PeerOutcome::Pushed,
223 };
224 self.report(&peer.host, outcome);
225 }
226 Err(e) if !counts_against_peer(&e) => {
227 tracing::info!("sync with {} waits for the next job: {e:#}", peer.host);
228 }
229 Err(e) => {
230 let wait = backoff.failed(&peer, Instant::now());
231 self.report(&peer.host, PeerOutcome::Failed);
232 tracing::warn!(
233 "sync with {} failed; its jobs are skipped for the next {}s: {e:#}",
234 peer.host,
235 wait.as_secs()
236 );
237 }
238 }
239 }
240 }
241
242 fn sync(&self, peer: &Peer, job: Job) -> Result<()> {
243 let transport = (self.connect)(peer)?;
244 let key = load_or_create_key(&self.data_dir.join("sync.key"))?;
245 let encryptor = XChaChaEncryptor::new(key);
246 let local = StoreLocal {
247 sqlite: &self.sqlite,
248 gate: &self.gate,
249 };
250 let report = |line: &str| tracing::info!("sync with {}: {}", peer.host, line.trim());
251 match job {
252 Job::Full => session::sync(
253 &local,
254 transport.as_ref(),
255 &encryptor,
256 &self.data_dir,
257 &report,
258 ),
259 Job::Push => {
260 let peer_machine_id = transport.whoami()?;
261 session::push(
262 &local,
263 transport.as_ref(),
264 &peer_machine_id,
265 &encryptor,
266 &report,
267 )
268 .map(|_| ())
269 }
270 }
271 }
272}
273
274#[derive(Debug)]
275struct MaintenanceRunning;
276
277impl std::fmt::Display for MaintenanceRunning {
278 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
279 f.write_str("store maintenance is running")
280 }
281}
282
283impl std::error::Error for MaintenanceRunning {}
284
285fn counts_against_peer(error: &anyhow::Error) -> bool {
289 error.downcast_ref::<MaintenanceRunning>().is_none()
290}
291
292fn one_line_per_host(lines: Vec<Peer>) -> (Vec<Peer>, Vec<Peer>) {
294 let mut hosts = HashSet::new();
295 lines
296 .into_iter()
297 .partition(|peer| hosts.insert(peer.host.clone()))
298}
299
300#[derive(Default)]
301struct PeerBackoff {
302 line: Option<Peer>,
303 failures: u32,
304 retry_at: Option<Instant>,
305}
306
307impl PeerBackoff {
308 fn due(&self, peer: &Peer, now: Instant) -> bool {
311 self.line.as_ref() != Some(peer) || self.retry_at.is_none_or(|at| now >= at)
312 }
313
314 fn succeeded(&mut self) {
315 *self = PeerBackoff::default();
316 }
317
318 fn failed(&mut self, peer: &Peer, now: Instant) -> Duration {
319 if self.line.as_ref() != Some(peer) {
320 *self = PeerBackoff {
321 line: Some(peer.clone()),
322 ..PeerBackoff::default()
323 };
324 }
325 self.failures += 1;
326 let wait = backoff(self.failures);
327 self.retry_at = Some(now + wait);
328 wait
329 }
330}
331
332fn backoff(failures: u32) -> Duration {
333 FIRST_BACKOFF
334 .saturating_mul(2u32.saturating_pow(failures.saturating_sub(1)))
335 .min(MAX_BACKOFF)
336}
337
338struct StoreLocal<'a> {
339 sqlite: &'a SqliteStore,
340 gate: &'a WriteGate,
341}
342
343impl StoreLocal<'_> {
344 fn write_slot(&self) -> Result<WriteSlot<'_>> {
345 self.gate
346 .try_enter()
347 .ok_or_else(|| anyhow::Error::new(MaintenanceRunning))
348 }
349}
350
351impl Local for StoreLocal<'_> {
352 fn cursors(&self) -> Result<Vec<(String, u64)>> {
353 Ok(self
354 .sqlite
355 .get_all_sync_states()?
356 .into_iter()
357 .map(|state| (state.peer_id, state.last_seq))
358 .collect())
359 }
360
361 fn set_cursor(&self, peer_id: &str, last_seq: u64) -> Result<()> {
362 let _slot = self.write_slot()?;
363 self.sqlite.set_sync_state(&SyncState {
364 peer_id: peer_id.to_string(),
365 last_seq,
366 last_sync_at: chrono::Utc::now(),
367 })?;
368 Ok(())
369 }
370
371 fn import(&self, header: &SyncHeader, records: &[Value]) -> Result<Imported> {
372 let _slot = self.write_slot()?;
373 check_store_header(self.sqlite, Some(*header), "the sender")?;
374 let (stats, max_seq) = import_values(self.sqlite, records)?;
375 Ok(Imported {
376 created: stats.created,
377 updated: stats.updated,
378 deleted: stats.deleted,
379 skipped: stats.skipped,
380 dangling: Imported::dangling_in(&stats.skipped_by_reason),
381 max_seq,
382 })
383 }
384
385 fn export(&self, after_seq: u64, limit: usize) -> Result<Exported> {
386 store_header(self.sqlite)?;
387 let (records, max_seq) = export_page(self.sqlite, after_seq, limit)?;
388 Ok(Exported { records, max_seq })
389 }
390}
391
392#[cfg(test)]
393mod tests {
394 use std::io::{Read, Write};
395 use std::path::Path;
396 use std::sync::Mutex;
397
398 use anyhow::Context;
399 use second_brain_core::sqlite_store::SCHEMA_VERSION;
400 use second_brain_sync::session::{Exit, Remote};
401
402 use super::*;
403 use crate::value_metrics::PeerSync;
404
405 #[test]
406 fn the_daemon_refuses_to_export_from_a_database_newer_than_its_binary() {
407 let dir = tempfile::tempdir().unwrap();
408 let path = dir.path().join("future.db");
409 drop(SqliteStore::open(&path, "m".to_string()).unwrap());
410 rusqlite::Connection::open(&path)
411 .unwrap()
412 .pragma_update(None, "user_version", SCHEMA_VERSION + 1)
413 .unwrap();
414 let sqlite = SqliteStore::open(&path, "m".to_string()).unwrap();
415 let gate = WriteGate::default();
416 let local = StoreLocal {
417 sqlite: &sqlite,
418 gate: &gate,
419 };
420 let error = local.export(0, 10).err().expect("export refused");
421 assert!(
422 format!("{error:#}").contains("upgrade this daemon"),
423 "{error:#}"
424 );
425 }
426
427 #[test]
428 fn a_host_keeps_one_worker_when_its_sb_path_changes() {
429 let dir = tempfile::tempdir().unwrap();
430 let sync_loop = Arc::new(SyncLoop {
431 data_dir: dir.path().to_path_buf(),
432 sqlite: Arc::new(SqliteStore::in_memory("m".to_string()).unwrap()),
433 gate: Arc::new(WriteGate::default()),
434 connect: Arc::new(|_: &Peer| -> Result<Box<dyn Transport + Send>> {
435 anyhow::bail!("offline")
436 }),
437 health: None,
438 });
439 let mut workers = HashMap::new();
440 sync_loop.dispatch(&mut workers, Job::Push, Peer::parse("user@pi"));
441 sync_loop.dispatch(&mut workers, Job::Push, Peer::parse("user@pi /opt/sb"));
442 assert_eq!(workers.len(), 1);
443 }
444
445 fn scripted_loop(dir: &Path, connect: Connect) -> SyncLoop {
446 SyncLoop {
447 data_dir: dir.to_path_buf(),
448 sqlite: Arc::new(SqliteStore::in_memory("m".to_string()).unwrap()),
449 gate: Arc::new(WriteGate::default()),
450 connect,
451 health: None,
452 }
453 }
454
455 #[test]
456 fn the_worker_takes_the_latest_line_and_backs_off_for_all_but_maintenance() {
457 let dir = tempfile::tempdir().unwrap();
458 let calls: Arc<Mutex<Vec<Peer>>> = Arc::default();
459 let connect: Connect = {
460 let calls = calls.clone();
461 Arc::new(move |peer: &Peer| -> Result<Box<dyn Transport + Send>> {
462 let mut calls = calls.lock().unwrap();
463 calls.push(peer.clone());
464 if calls.len() == 1 {
465 Err(anyhow::Error::new(MaintenanceRunning))
466 } else {
467 anyhow::bail!("connection refused")
468 }
469 })
470 };
471 let sync_loop = Arc::new(scripted_loop(dir.path(), connect));
472 let made = |n: usize| {
473 let deadline = Instant::now() + Duration::from_secs(5);
474 while Instant::now() < deadline {
475 if calls.lock().unwrap().len() >= n {
476 return true;
477 }
478 std::thread::sleep(Duration::from_millis(10));
479 }
480 false
481 };
482 let bare = Peer::parse("user@pi");
483 let fixed = Peer::parse("user@pi /opt/sb");
484 let (jobs, inbox) = mpsc::channel();
485 jobs.send((Job::Push, Peer::parse("user@pi /old/sb")))
486 .unwrap();
487 jobs.send((Job::Full, bare.clone())).unwrap();
488 let worker = {
489 let sync_loop = sync_loop.clone();
490 std::thread::spawn(move || sync_loop.work(inbox))
491 };
492 assert!(made(1), "the queued jobs never ran");
493 jobs.send((Job::Full, bare.clone())).unwrap();
494 assert!(made(2), "local maintenance started a backoff");
495 jobs.send((Job::Full, bare.clone())).unwrap();
496 std::thread::sleep(Duration::from_millis(300));
497 assert_eq!(
498 calls.lock().unwrap().len(),
499 2,
500 "a job inside the backoff ran"
501 );
502 jobs.send((Job::Full, fixed.clone())).unwrap();
503 assert!(made(3), "a changed line waited out the old backoff");
504 drop(jobs);
505 worker.join().unwrap();
506 assert_eq!(*calls.lock().unwrap(), vec![bare.clone(), bare, fixed]);
507 }
508
509 #[test]
510 fn store_maintenance_does_not_count_against_the_peer() {
511 let sqlite = SqliteStore::in_memory("m".to_string()).unwrap();
512 let gate = WriteGate::default();
513 let local = StoreLocal {
514 sqlite: &sqlite,
515 gate: &gate,
516 };
517 let _maintenance = gate.close();
518 let error = local
519 .set_cursor("peer", 1)
520 .context("updating sync state")
521 .unwrap_err();
522 assert!(!counts_against_peer(&error), "{error:#}");
523 assert!(counts_against_peer(&anyhow::anyhow!("connection refused")));
524 }
525
526 #[test]
527 fn a_changed_peer_line_is_tried_without_the_old_lines_backoff() {
528 let now = Instant::now();
529 let later = now + Duration::from_secs(1);
530 let bare = Peer::parse("user@pi");
531 let fixed = Peer::parse("user@pi /opt/sb");
532 let mut state = PeerBackoff::default();
533 assert!(state.due(&bare, now));
534 assert_eq!(state.failed(&bare, now), FIRST_BACKOFF);
535 assert!(!state.due(&bare, later));
536 assert!(state.due(&fixed, later));
537 assert_eq!(state.failed(&fixed, later), FIRST_BACKOFF);
538 assert_eq!(state.failed(&fixed, later), FIRST_BACKOFF * 2);
539 assert!(state.due(&bare, later));
540 state.succeeded();
541 assert_eq!(state.failed(&fixed, later), FIRST_BACKOFF);
542 }
543
544 #[test]
545 fn a_host_listed_twice_keeps_its_first_line() {
546 let (kept, repeated) = one_line_per_host(vec![
547 Peer::parse("user@pi"),
548 Peer::parse("user@mac"),
549 Peer::parse("user@pi /opt/sb"),
550 ]);
551 assert_eq!(kept, vec![Peer::parse("user@pi"), Peer::parse("user@mac")]);
552 assert_eq!(repeated, vec![Peer::parse("user@pi /opt/sb")]);
553 }
554
555 #[test]
556 fn backoff_doubles_from_thirty_seconds_up_to_half_an_hour() {
557 let waits: Vec<u64> = (1..=8).map(|n| backoff(n).as_secs()).collect();
558 assert_eq!(waits, vec![30, 60, 120, 240, 480, 960, 1800, 1800]);
559 }
560
561 struct EmptyPeer;
562
563 impl Transport for EmptyPeer {
564 fn host(&self) -> &str {
565 "user@pi"
566 }
567
568 fn whoami(&self) -> Result<String> {
569 Ok("peer".to_string())
570 }
571
572 fn export(&self, _after_seq: u64) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)> {
573 let stream = format!("{}\n", SyncHeader::local().to_line()).into_bytes();
574 Ok((Box::new(std::io::Cursor::new(stream)), Box::new(Exited)))
575 }
576
577 fn import(&self) -> Result<(Box<dyn Write + Send>, Box<dyn Remote>)> {
578 Ok((Box::new(std::io::sink()), Box::new(Exited)))
579 }
580 }
581
582 struct Exited;
583
584 impl Remote for Exited {
585 fn wait(&mut self) -> Result<Exit> {
586 Ok(Exit {
587 success: true,
588 status: "exit status: 0".to_string(),
589 })
590 }
591
592 fn stop(&mut self) -> Result<()> {
593 Ok(())
594 }
595 }
596
597 fn reachable() -> Connect {
598 Arc::new(|_: &Peer| -> Result<Box<dyn Transport + Send>> { Ok(Box::new(EmptyPeer)) })
599 }
600
601 fn refusing() -> Connect {
602 Arc::new(|_: &Peer| -> Result<Box<dyn Transport + Send>> {
603 anyhow::bail!("connection refused")
604 })
605 }
606
607 fn watched_loop(dir: &Path, connect: Connect) -> (Arc<SyncLoop>, Arc<Health>) {
608 let health = Arc::new(Health::new(Utc::now(), None));
609 let sync_loop = Arc::new(scripted_loop(dir, connect).with_health(health.clone()));
610 std::fs::write(dir.join(peers::PEERS_FILE), "user@pi\n").unwrap();
611 assert_eq!(sync_loop.listed_peers().0, vec![Peer::parse("user@pi")]);
612 (sync_loop, health)
613 }
614
615 fn the_one_peer(health: &Health) -> PeerSync {
616 let mut peers = health.report().value.sync_peers;
617 assert_eq!(peers.len(), 1, "{peers:?}");
618 peers.remove(0)
619 }
620
621 fn run_job(sync_loop: &Arc<SyncLoop>, job: Job, peer: Peer) {
622 let (jobs, inbox) = mpsc::channel();
623 jobs.send((job, peer)).unwrap();
624 drop(jobs);
625 let worker = {
626 let sync_loop = sync_loop.clone();
627 std::thread::spawn(move || sync_loop.work(inbox))
628 };
629 let deadline = Instant::now() + Duration::from_secs(10);
630 while !worker.is_finished() {
631 assert!(
632 Instant::now() < deadline,
633 "the worker never finished its job"
634 );
635 std::thread::sleep(Duration::from_millis(10));
636 }
637 worker.join().unwrap();
638 }
639
640 #[test]
641 fn a_full_sync_reports_the_peer_synced() {
642 let dir = tempfile::tempdir().unwrap();
643 let (sync_loop, health) = watched_loop(dir.path(), reachable());
644 let before = Utc::now();
645 run_job(&sync_loop, Job::Full, Peer::parse("user@pi"));
646 let peer = the_one_peer(&health);
647 assert_eq!(peer.peer, "user@pi");
648 let synced = peer.last_success_at.expect("a full sync is a success");
649 assert!(synced >= before, "{synced} predates the sync");
650 assert_eq!(peer.consecutive_failures, 0);
651 }
652
653 #[test]
654 fn a_failed_sync_counts_a_failure_and_no_success() {
655 let dir = tempfile::tempdir().unwrap();
656 let (sync_loop, health) = watched_loop(dir.path(), refusing());
657 run_job(&sync_loop, Job::Full, Peer::parse("user@pi"));
658 let peer = the_one_peer(&health);
659 assert_eq!(peer.peer, "user@pi");
660 assert_eq!(peer.consecutive_failures, 1);
661 assert_eq!(peer.last_success_at, None);
662 assert!(peer.last_attempt_at.is_some(), "the failure is an attempt");
663 }
664
665 #[test]
666 fn a_push_ends_a_failure_streak_without_counting_as_a_sync() {
667 let dir = tempfile::tempdir().unwrap();
668 let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
669 let connect: Connect = {
670 let calls = calls.clone();
671 Arc::new(move |_: &Peer| -> Result<Box<dyn Transport + Send>> {
672 if calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0 {
673 anyhow::bail!("connection refused");
674 }
675 Ok(Box::new(EmptyPeer))
676 })
677 };
678 let (sync_loop, health) = watched_loop(dir.path(), connect);
679 run_job(&sync_loop, Job::Full, Peer::parse("user@pi"));
680 assert_eq!(the_one_peer(&health).consecutive_failures, 1);
681 run_job(&sync_loop, Job::Push, Peer::parse("user@pi"));
682 let peer = the_one_peer(&health);
683 assert_eq!(peer.last_success_at, None);
684 assert!(peer.last_attempt_at.is_some());
685 assert_eq!(peer.consecutive_failures, 0);
686 }
687
688 #[test]
689 fn maintenance_reports_no_sync_outcome() {
690 let dir = tempfile::tempdir().unwrap();
691 let connect: Connect = Arc::new(|_: &Peer| -> Result<Box<dyn Transport + Send>> {
692 Err(anyhow::Error::new(MaintenanceRunning))
693 });
694 let (sync_loop, health) = watched_loop(dir.path(), connect);
695 run_job(&sync_loop, Job::Full, Peer::parse("user@pi"));
696 let peer = the_one_peer(&health);
697 assert_eq!(peer.last_success_at, None);
698 assert_eq!(peer.last_attempt_at, None);
699 assert_eq!(peer.consecutive_failures, 0);
700 }
701
702 #[test]
703 fn a_peer_removed_from_sync_peers_drops_out_of_health() {
704 let dir = tempfile::tempdir().unwrap();
705 let (sync_loop, health) = watched_loop(dir.path(), refusing());
706 let listed = |text: &str| {
707 std::fs::write(dir.path().join(peers::PEERS_FILE), text).unwrap();
708 let (kept, _) = sync_loop.listed_peers();
709 let mut reported: Vec<String> = health
710 .report()
711 .value
712 .sync_peers
713 .into_iter()
714 .map(|peer| peer.peer)
715 .collect();
716 reported.sort();
717 (kept.len(), reported)
718 };
719 assert_eq!(
720 listed("user@pi\nuser@mac\nuser@pi /opt/sb\n"),
721 (2, vec!["user@mac".to_string(), "user@pi".to_string()])
722 );
723 assert_eq!(listed("user@mac\n"), (1, vec!["user@mac".to_string()]));
724 }
725
726 #[test]
727 fn an_unreadable_sync_peers_keeps_its_peers_in_health() {
728 let dir = tempfile::tempdir().unwrap();
729 let (sync_loop, health) = watched_loop(dir.path(), refusing());
730 let file = dir.path().join(peers::PEERS_FILE);
731 std::fs::remove_file(&file).unwrap();
732 std::fs::create_dir(&file).unwrap();
733 assert!(peers::configured_peers(dir.path()).is_err());
734
735 let (kept, repeated) = sync_loop.listed_peers();
736
737 assert_eq!(kept, Vec::<Peer>::new());
738 assert_eq!(repeated, Vec::<Peer>::new());
739 assert_eq!(the_one_peer(&health).peer, "user@pi");
740 }
741}