1use std::collections::HashMap;
2use std::io::Read;
3use std::path::{Path, PathBuf};
4use std::sync::mpsc::{self, Receiver, SendError, Sender};
5use std::sync::{Arc, Mutex, PoisonError};
6use std::thread::JoinHandle;
7use std::time::{Duration, Instant};
8
9use anyhow::{Context, Result};
10use chrono::Utc;
11use second_brain_core::keys::{DeviceKeys, load_device_keys};
12use second_brain_core::schema::SyncState;
13use second_brain_core::sqlite_store::SqliteStore;
14use second_brain_core::store::StoreAdmin;
15use second_brain_sync::crypto::seal_line;
16use second_brain_sync::dial::{self, Dial, Route};
17use second_brain_sync::export::export_page;
18use second_brain_sync::keyring::Keyring;
19use second_brain_sync::mdns::{self, Net};
20use second_brain_sync::protocol::{Hello, Merging, SyncHeader, check_store_header, store_header};
21use second_brain_sync::session::{self, Exported, Imported, Local, Merged, Transport};
22use second_brain_sync::ssh::{SshTransport, device_alias, known_hosts_path_in};
23use second_brain_sync::ssh_identity::{EphemeralIdentity, sweep_ephemeral_identities};
24
25use crate::bootstrap::Bootstrap;
26use crate::clock_gate::{ClockGate, is_untrusted};
27use crate::device_access::DeviceAccess;
28use crate::health::Health;
29use crate::snapshot_merge;
30use crate::sync_hello;
31use crate::value_metrics::PeerOutcome;
32use crate::write_gate::{WriteGate, WriteSlot};
33
34const FULL_INTERVAL: Duration = Duration::from_secs(300);
35const TICK: Duration = Duration::from_secs(1);
36const PUSH_DELAY: Duration = Duration::from_secs(5);
37const FIRST_BACKOFF: Duration = Duration::from_secs(30);
38const MAX_BACKOFF: Duration = Duration::from_secs(30 * 60);
39const ORIGIN_SCAN_PAGE: usize = 500;
40
41pub type Connect = Arc<dyn Fn(&Route) -> Result<Box<dyn Transport + Send>> + Send + Sync>;
42
43pub const SSH_KEY_FILE: &str = "sync-ssh-key";
45
46pub struct SyncLoop {
51 pub data_dir: PathBuf,
52 pub sqlite: Arc<SqliteStore>,
53 pub gate: Arc<WriteGate>,
54 pub clock: Arc<ClockGate>,
55 pub health: Arc<Health>,
56 pub access: Arc<DeviceAccess>,
57 pub bootstrap: Arc<Bootstrap>,
58 pub connect: Connect,
59 pub discovery: Option<Net>,
62}
63
64#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
65enum Job {
66 Push,
67 Full,
68}
69
70impl SyncLoop {
71 pub fn over_ssh(
72 data_dir: PathBuf,
73 sqlite: Arc<SqliteStore>,
74 gate: Arc<WriteGate>,
75 clock: Arc<ClockGate>,
76 health: Arc<Health>,
77 access: Arc<DeviceAccess>,
78 bootstrap: Arc<Bootstrap>,
79 ) -> Self {
80 if let Err(e) = sweep_ephemeral_identities(&data_dir) {
81 tracing::warn!("removing ssh key files a killed daemon left: {e:#}");
82 }
83 match retire_plaintext_ssh_key(&data_dir) {
84 Ok(true) => tracing::info!("removed the plaintext {SSH_KEY_FILE}"),
85 Ok(false) => {}
86 Err(e) => tracing::warn!("kept the plaintext {SSH_KEY_FILE}: {e:#}"),
87 }
88 let connect = ssh_connect(data_dir.clone(), sqlite.machine_id().to_string());
89 Self {
90 data_dir,
91 sqlite,
92 gate,
93 clock,
94 health,
95 access,
96 bootstrap,
97 connect,
98 discovery: None,
99 }
100 }
101
102 pub fn with_health(mut self, health: Arc<Health>) -> Self {
103 self.health = health;
104 self
105 }
106
107 pub fn with_discovery(mut self, discovery: Net) -> Self {
108 self.discovery = Some(discovery);
109 self
110 }
111
112 pub fn spawn(self) -> JoinHandle<()> {
113 std::thread::Builder::new()
114 .name("sync-loop".to_string())
115 .spawn(move || Arc::new(self).run())
116 .expect("spawning the sync loop")
117 }
118
119 fn run(self: Arc<Self>) {
120 let mut workers: HashMap<String, Sender<(Job, Dial)>> = HashMap::new();
121 let mut next_full = Instant::now();
122 let mut seen = self.sqlite.max_sync_seq().unwrap_or(0);
123 let mut push_at: Option<Instant> = None;
124 loop {
125 let dials = self.listed_dials();
126 if dials.is_empty() {
127 seen = self.sqlite.max_sync_seq().unwrap_or(seen);
128 push_at = None;
129 } else if self.wrote_locally(&mut seen) {
130 push_at.get_or_insert(Instant::now() + PUSH_DELAY);
131 }
132
133 let now = Instant::now();
134 if self.access.take_kick() {
135 next_full = now;
136 }
137 let job = if now >= next_full {
138 next_full = now + FULL_INTERVAL;
139 Some(Job::Full)
140 } else if push_at.is_some_and(|at| now >= at) {
141 Some(Job::Push)
142 } else {
143 None
144 };
145 if let Some(job) = job {
146 push_at = None;
147 let mut dials = dials;
148 if job == Job::Full {
149 self.add_found(&mut dials);
150 }
151 for dial in dials {
152 self.dispatch(&mut workers, job, dial);
153 }
154 }
155 std::thread::sleep(TICK);
156 }
157 }
158
159 fn dispatch(
163 self: &Arc<Self>,
164 workers: &mut HashMap<String, Sender<(Job, Dial)>>,
165 job: Job,
166 dial: Dial,
167 ) {
168 let key = dial.key.clone();
169 let worker = workers
170 .entry(key.clone())
171 .or_insert_with(|| self.clone().start_worker(&key));
172 if let Err(SendError(message)) = worker.send((job, dial)) {
173 tracing::warn!("the sync worker for {key} exited; starting another");
174 let worker = self.clone().start_worker(&key);
175 if worker.send(message).is_err() {
176 tracing::warn!("the new sync worker for {key} exited at once");
177 }
178 workers.insert(key, worker);
179 }
180 }
181
182 fn listed_dials(&self) -> Vec<Dial> {
186 let mut dials = match dial::device_dials(&self.sqlite) {
187 Ok(dials) => dials,
188 Err(e) => {
189 tracing::warn!("cannot read the devices to sync with: {e:#}");
190 return Vec::new();
191 }
192 };
193 match dial::line_dials(&self.sqlite, &self.data_dir) {
194 Ok(lines) => {
195 dials.extend(lines);
196 let labels: Vec<String> = dials.iter().map(|dial| dial.label.clone()).collect();
197 self.health.sync_peers_listed(&labels, Utc::now());
198 }
199 Err(e) => tracing::warn!(
200 "cannot read {}: {e:#}",
201 second_brain_sync::peers::PEERS_FILE
202 ),
203 }
204 dials
205 }
206
207 fn report(&self, label: &str, outcome: PeerOutcome) {
208 self.health.sync_attempted(label, outcome, Utc::now());
209 }
210
211 fn wrote_locally(&self, seen: &mut u64) -> bool {
214 let max = match self.sqlite.max_sync_seq() {
215 Ok(max) => max,
216 Err(e) => {
217 tracing::warn!("cannot read the sync log position: {e:#}");
218 return false;
219 }
220 };
221 let own = self.sqlite.machine_id();
222 while *seen < max {
223 let page = match self.sqlite.sync_log_page(*seen, Some(ORIGIN_SCAN_PAGE)) {
224 Ok(page) => page,
225 Err(e) => {
226 tracing::warn!("cannot read the sync log: {e:#}");
227 return false;
228 }
229 };
230 let Some(last) = page.last() else {
231 break;
232 };
233 if page.iter().any(|entry| entry.origin_machine_id == own) {
234 *seen = max;
235 return true;
236 }
237 *seen = last.local_seq.max(0) as u64;
238 }
239 *seen = max;
240 false
241 }
242
243 fn start_worker(self: Arc<Self>, key: &str) -> Sender<(Job, Dial)> {
244 let (jobs, inbox) = mpsc::channel();
245 std::thread::Builder::new()
246 .name(format!("sync-{key}"))
247 .spawn(move || self.work(inbox))
248 .expect("spawning a sync worker");
249 jobs
250 }
251
252 fn work(&self, inbox: Receiver<(Job, Dial)>) {
255 let mut backoff = PeerBackoff::default();
256 while let Ok((mut job, mut dial)) = inbox.recv() {
257 while let Ok((queued, latest)) = inbox.try_recv() {
258 job = job.max(queued);
259 dial = latest;
260 }
261 if !backoff.due(&dial, Instant::now()) {
262 continue;
263 }
264 if job == Job::Push && !self.holds_identity() {
265 tracing::info!(
266 "not pushing to {}: this device holds no sync identity yet; pair it with \
267 `sb pair`",
268 dial.label
269 );
270 continue;
271 }
272 match self.sync(&dial, job) {
273 Ok(()) => {
274 backoff.succeeded();
275 let outcome = match job {
276 Job::Full => PeerOutcome::Synced,
277 Job::Push => PeerOutcome::Pushed,
278 };
279 self.report(&dial.label, outcome);
280 }
281 Err(e) if !counts_against_peer(&e) => {
282 tracing::info!("sync with {} waits for the next job: {e:#}", dial.label);
283 }
284 Err(e) => {
285 let wait = backoff.failed(&dial, Instant::now());
286 self.report(&dial.label, PeerOutcome::Failed(format!("{e:#}")));
287 tracing::warn!(
288 "sync with {} failed; its jobs are skipped for the next {}s: {e:#}",
289 dial.label,
290 wait.as_secs()
291 );
292 }
293 }
294 }
295 }
296
297 fn holds_identity(&self) -> bool {
301 match self.sqlite.device_record(self.sqlite.machine_id()) {
302 Ok(row) => row.is_some(),
303 Err(e) => {
304 tracing::warn!("cannot read this device's sync identity row: {e:#}");
305 true
306 }
307 }
308 }
309
310 fn sync(&self, dial: &Dial, job: Job) -> Result<()> {
317 let failed = match self.sync_via(dial, &dial.routes, job)? {
318 Reached::Synced => return Ok(()),
319 Reached::Not(failed) => failed,
320 };
321 let found: Vec<Route> = match &dial.machine_id {
322 Some(machine_id) => self
323 .lookup()
324 .into_iter()
325 .filter(|found| found.key == *machine_id)
326 .flat_map(|found| found.routes)
327 .filter(|route| !dial.routes.contains(route))
328 .collect(),
329 None => Vec::new(),
330 };
331 let found_failed = if found.is_empty() {
332 None
333 } else {
334 match self.sync_via(dial, &found, job)? {
335 Reached::Synced => return Ok(()),
336 Reached::Not(failed) => failed,
337 }
338 };
339 Err(failed
340 .or(found_failed)
341 .unwrap_or_else(|| anyhow::anyhow!("{} has no route to dial", dial.label)))
342 }
343
344 fn add_found(&self, dials: &mut Vec<Dial>) {
347 if self.discovery.is_none() {
348 return;
349 }
350 match dial::undialled(&self.sqlite, dials) {
351 Ok(true) => {}
352 Ok(false) => return,
353 Err(e) => {
354 tracing::warn!("cannot read the devices to sync with: {e:#}");
355 return;
356 }
357 }
358 for dial in self.lookup() {
359 if !dials.iter().any(|listed| listed.key == dial.key) {
360 dials.push(dial);
361 }
362 }
363 }
364
365 fn lookup(&self) -> Vec<Dial> {
368 let Some(net) = &self.discovery else {
369 return Vec::new();
370 };
371 let found = match mdns::browse(net, mdns::SERVICE, mdns::LOOKUP_WINDOW) {
372 Ok(found) => found,
373 Err(e) => {
374 tracing::info!("cannot look devices up on the network: {e:#}");
375 return Vec::new();
376 }
377 };
378 dial::found_dials(&self.sqlite, &found).unwrap_or_else(|e| {
379 tracing::warn!("cannot read the devices to match a lookup against: {e:#}");
380 Vec::new()
381 })
382 }
383
384 fn sync_via(&self, dial: &Dial, routes: &[Route], job: Job) -> Result<Reached> {
385 let mut failed: Option<anyhow::Error> = None;
386 for route in routes {
387 let transport = match (self.connect)(route) {
388 Ok(transport) => transport,
389 Err(e) if !counts_against_peer(&e) => return Err(e),
390 Err(e) => {
391 failed = Some(e.context(format!("reaching {}", route.describe())));
392 continue;
393 }
394 };
395 let answered = match transport.whoami() {
396 Ok(answered) => answered,
397 Err(e) => {
398 failed = Some(e.context(format!(
399 "resolving the machine id at {} (is its daemon running?)",
400 route.describe()
401 )));
402 continue;
403 }
404 };
405 if let Route::Line(peer) = route
406 && let Some(_slot) = self.gate.try_enter()
407 {
408 dial::line_answered(&self.sqlite, peer, &answered)?;
409 }
410 if let Err(e) = dial::refuse_answer(&self.sqlite, dial, &answered) {
411 if dial.machine_id.is_some() && !self.is_revoked(&answered) {
412 failed = Some(e);
413 continue;
414 }
415 return Err(e);
416 }
417 self.record_answered(route);
418 return self
419 .sync_through(transport.as_ref(), &answered, job)
420 .map(|()| Reached::Synced);
421 }
422 Ok(Reached::Not(failed))
423 }
424
425 fn record_answered(&self, route: &Route) {
428 let Some(_slot) = self.gate.try_enter() else {
429 return;
430 };
431 match dial::record_answered(&self.sqlite, route) {
432 Ok(true) => tracing::info!("recorded {} as a route to its device", route.describe()),
433 Ok(false) => {}
434 Err(e) => tracing::warn!("cannot record the route {}: {e:#}", route.describe()),
435 }
436 }
437
438 fn is_revoked(&self, machine_id: &str) -> bool {
439 self.sqlite
440 .device_record(machine_id)
441 .ok()
442 .flatten()
443 .is_some_and(|row| row.revoked)
444 }
445
446 fn sync_through(&self, transport: &dyn Transport, peer: &str, job: Job) -> Result<()> {
447 let local = StoreLocal {
448 sqlite: &self.sqlite,
449 gate: &self.gate,
450 clock: &self.clock,
451 health: &self.health,
452 access: &self.access,
453 bootstrap: &self.bootstrap,
454 data_dir: &self.data_dir,
455 };
456 let host = transport.host().to_string();
457 let report = |line: &str| tracing::info!("sync with {host}: {}", line.trim());
458 match job {
459 Job::Full => session::sync_with(&local, transport, peer, &self.data_dir, &report),
460 Job::Push => session::push(&local, transport, peer, &report).map(|_| ()),
461 }
462 }
463}
464
465fn ssh_connect(data_dir: PathBuf, own: String) -> Connect {
470 let device_keys: Arc<Mutex<Option<DeviceKeys>>> = Arc::default();
471 Arc::new(move |route: &Route| -> Result<Box<dyn Transport + Send>> {
472 let transport = match route {
473 Route::Device {
474 machine_id,
475 login,
476 address,
477 port,
478 } => {
479 let identity = device_identity(&data_dir, &own, &device_keys)?;
480 SshTransport::new(&format!("{login}@{address}"), "sb")?
481 .known_hosts(known_hosts_path_in(&data_dir))
482 .port(*port)
483 .host_key_alias(&device_alias(machine_id))
484 .ephemeral_identity(identity)
485 }
486 Route::Line(peer) => SshTransport::new(&peer.host, &peer.sb)?,
487 };
488 Ok(Box::new(transport.unattended()) as Box<dyn Transport + Send>)
489 })
490}
491
492fn device_identity(
493 data_dir: &Path,
494 own: &str,
495 cached: &Mutex<Option<DeviceKeys>>,
496) -> Result<EphemeralIdentity> {
497 let mut cached = cached.lock().unwrap_or_else(PoisonError::into_inner);
498 if cached.is_none() {
499 let keys = load_device_keys(data_dir)?
500 .context("this device has no device key to sign in to its peers with")?;
501 *cached = Some(keys);
502 }
503 let keys = cached.as_ref().expect("the keys were loaded above");
504 EphemeralIdentity::create(
505 data_dir,
506 keys.ssh_signing_key(),
507 &format!("second-brain:{own}"),
508 )
509}
510
511fn retire_plaintext_ssh_key(data_dir: &Path) -> Result<bool> {
515 let path = data_dir.join(SSH_KEY_FILE);
516 if std::fs::symlink_metadata(&path).is_err() {
517 return Ok(false);
518 }
519 if load_device_keys(data_dir)?.is_none() {
520 return Ok(false);
521 }
522 std::fs::remove_file(&path).with_context(|| format!("removing {}", path.display()))?;
523 Ok(true)
524}
525
526#[derive(Debug)]
527struct MaintenanceRunning;
528
529impl std::fmt::Display for MaintenanceRunning {
530 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
531 f.write_str("store maintenance is running")
532 }
533}
534
535impl std::error::Error for MaintenanceRunning {}
536
537fn counts_against_peer(error: &anyhow::Error) -> bool {
543 error.downcast_ref::<MaintenanceRunning>().is_none() && !is_untrusted(error)
544}
545
546enum Reached {
547 Synced,
548 Not(Option<anyhow::Error>),
551}
552
553#[derive(Default)]
554struct PeerBackoff {
555 dial: Option<Dial>,
556 failures: u32,
557 retry_at: Option<Instant>,
558}
559
560impl PeerBackoff {
561 fn due(&self, dial: &Dial, now: Instant) -> bool {
564 self.dial.as_ref() != Some(dial) || self.retry_at.is_none_or(|at| now >= at)
565 }
566
567 fn succeeded(&mut self) {
568 *self = PeerBackoff::default();
569 }
570
571 fn failed(&mut self, dial: &Dial, now: Instant) -> Duration {
572 if self.dial.as_ref() != Some(dial) {
573 *self = PeerBackoff {
574 dial: Some(dial.clone()),
575 ..PeerBackoff::default()
576 };
577 }
578 self.failures += 1;
579 let wait = backoff(self.failures);
580 self.retry_at = Some(now + wait);
581 wait
582 }
583}
584
585fn backoff(failures: u32) -> Duration {
586 FIRST_BACKOFF
587 .saturating_mul(2u32.saturating_pow(failures.saturating_sub(1)))
588 .min(MAX_BACKOFF)
589}
590
591struct StoreLocal<'a> {
592 sqlite: &'a Arc<SqliteStore>,
593 gate: &'a WriteGate,
594 clock: &'a ClockGate,
595 health: &'a Health,
596 access: &'a DeviceAccess,
597 bootstrap: &'a Bootstrap,
598 data_dir: &'a std::path::Path,
599}
600
601impl StoreLocal<'_> {
602 fn write_slot(&self) -> Result<WriteSlot<'_>> {
603 self.gate
604 .try_enter()
605 .ok_or_else(|| anyhow::Error::new(MaintenanceRunning))
606 }
607}
608
609impl Local for StoreLocal<'_> {
610 fn cursors(&self) -> Result<Vec<(String, u64)>> {
611 Ok(self
612 .sqlite
613 .get_all_sync_states()?
614 .into_iter()
615 .map(|state| (state.peer_id, state.last_seq))
616 .collect())
617 }
618
619 fn set_cursor(&self, peer_id: &str, last_seq: u64) -> Result<()> {
620 let _slot = self.write_slot()?;
621 self.sqlite.set_sync_state(&SyncState {
622 peer_id: peer_id.to_string(),
623 last_seq,
624 last_sync_at: chrono::Utc::now(),
625 })?;
626 Ok(())
627 }
628
629 fn import(&self, header: &SyncHeader, lines: &[String]) -> Result<Imported> {
630 let _slot = self.write_slot()?;
631 check_store_header(self.sqlite, Some(header), "the sender")?;
632 let records = Keyring::load(self.sqlite, self.data_dir)?
633 .open_lines(header.hello.as_ref().map(|h| h.machine_id.as_str()), lines)?
634 .iter()
635 .map(|opened| Ok(serde_json::from_slice(opened)?))
636 .collect::<Result<Vec<serde_json::Value>>>()?;
637 let source = header.hello.as_ref().map(|h| h.machine_id.as_str());
638 let applied = crate::sync_import::import_from(self.sqlite, source, &records)?;
639 let stats = applied.stats;
640 Ok(Imported {
641 created: stats.created,
642 updated: stats.updated,
643 deleted: stats.deleted,
644 skipped: stats.skipped,
645 dangling: Imported::dangling_in(&stats.skipped_by_reason),
646 errors: stats.errors,
647 max_seq: applied.max_seq,
648 reached: applied.reached,
649 })
650 }
651
652 fn export(&self, peer_id: &str, after_seq: u64, limit: usize) -> Result<Exported> {
653 store_header(self.sqlite)?;
654 let (records, max_seq) = export_page(self.sqlite.as_ref(), after_seq, limit)?;
655 let sealer = Keyring::load(self.sqlite, self.data_dir)?.sealer_for(peer_id, None)?;
656 let lines = records
657 .iter()
658 .map(|record| seal_line(&sealer, &serde_json::to_vec(record)?))
659 .collect::<Result<Vec<String>>>()?;
660 Ok(Exported { lines, max_seq })
661 }
662
663 fn hello_for(&self, peer_id: &str) -> Result<Hello> {
664 sync_hello::hello_for(self.sqlite, self.clock, self.data_dir, peer_id, None)
665 }
666
667 fn accept_hello(&self, hello: &Hello) -> Result<()> {
668 sync_hello::check_identity(self.sqlite, self.data_dir, hello)?;
669 let _slot = self.write_slot()?;
670 sync_hello::accept_handshake(
671 self.sqlite,
672 self.clock,
673 self.health,
674 self.access,
675 self.data_dir,
676 hello,
677 )
678 }
679
680 fn record_ack(&self, peer_id: &str, acked_seq: u64) -> Result<()> {
681 let _slot = self.write_slot()?;
682 self.sqlite
683 .record_peer_ack(peer_id, acked_seq, self.clock.wall_micros())
684 }
685
686 fn record_peer_in(&self, peer_id: &str, seq: u64) -> Result<()> {
687 let _slot = self.write_slot()?;
688 self.sqlite.record_peer_in(peer_id, seq)
689 }
690
691 fn push_report(&self, peer_id: &str) -> Result<Option<u64>> {
692 self.sqlite.push_report(peer_id)
693 }
694
695 fn record_push_report(&self, peer_id: &str, reported: Option<u64>) -> Result<()> {
696 let _slot = self.write_slot()?;
697 self.sqlite.record_push_report(peer_id, reported)
698 }
699
700 fn merging(&self) -> Result<Option<Merging>> {
701 snapshot_merge::merging(self.data_dir)
702 }
703
704 fn merge_snapshot(&self, exporter: &str, stream: Box<dyn Read + Send>) -> Result<Merged> {
705 let merging = snapshot_merge::receive(self.sqlite, self.data_dir, stream, Some(exporter))?;
706 match snapshot_merge::merge_spooled(self.sqlite, self.data_dir, self.gate)? {
707 Some(_) => Ok(Merged::Done { seq: merging.seq }),
708 None => Ok(Merged::Running(merging)),
709 }
710 }
711
712 fn bootstrap_pending(&self) -> Result<bool> {
713 Ok(self.bootstrap.is_pending())
714 }
715
716 fn finish_bootstrap(&self) -> Result<()> {
717 let _slot = self.write_slot()?;
718 if self.bootstrap.finish(self.sqlite)? {
719 tracing::info!("bootstrap finished; ingesting this device's own sessions");
720 }
721 Ok(())
722 }
723
724 fn export_snapshot(&self, peer: &Hello) -> Result<Box<dyn Read + Send>> {
725 snapshot_merge::pin_for_snapshot(self.sqlite, self.gate, peer)?;
726 let (reader, writer) = std::io::pipe()?;
727 let (sqlite, data_dir, peer) = (
728 self.sqlite.clone(),
729 self.data_dir.to_path_buf(),
730 peer.clone(),
731 );
732 std::thread::Builder::new()
733 .name("snapshot-export".to_string())
734 .spawn(move || {
735 let out = std::io::BufWriter::with_capacity(1024 * 1024, writer);
736 if let Err(e) = snapshot_merge::stream_snapshot(&sqlite, &data_dir, &peer, out) {
737 tracing::warn!("exporting a snapshot for {} failed: {e:#}", peer.machine_id);
738 }
739 })?;
740 Ok(Box::new(reader))
741 }
742}
743
744#[cfg(test)]
745mod tests {
746 use std::io::{Read, Write};
747 use std::path::Path;
748 use std::sync::Mutex;
749
750 use anyhow::Context;
751 use second_brain_core::keys::{
752 DeviceKeys, DeviceRecord, DeviceRoute, Epoch, IdentityKey, IdentityTag, KeyGuard,
753 admit_device,
754 };
755 use second_brain_core::sqlite_store::SCHEMA_VERSION;
756 use second_brain_sync::crypto::XChaChaEncryptor;
757 use second_brain_sync::keyring::{LEGACY_KEY_FILE, migrate_sync_key};
758 use second_brain_sync::mdns::Responder;
759 use second_brain_sync::peers::{self, Peer};
760 use second_brain_sync::protocol::HandshakeKeys;
761 use second_brain_sync::protocol::PushApplied;
762 use second_brain_sync::session::{Exit, Remote};
763 use second_brain_sync::ssh_identity::write_identity_file;
764
765 use super::*;
766 use crate::value_metrics::PeerSync;
767
768 fn line_dial(line: &str) -> Dial {
769 let peer = Peer::parse(line);
770 Dial {
771 key: peer.host.clone(),
772 label: peer.host.clone(),
773 machine_id: None,
774 routes: vec![Route::Line(peer)],
775 }
776 }
777
778 fn fresh_health() -> Arc<Health> {
779 Arc::new(Health::new(chrono::Utc::now(), None))
780 }
781
782 #[test]
783 fn the_daemon_refuses_to_export_from_a_database_newer_than_its_binary() {
784 let dir = tempfile::tempdir().unwrap();
785 let path = dir.path().join("future.db");
786 drop(SqliteStore::open(&path, "m".to_string()).unwrap());
787 rusqlite::Connection::open(&path)
788 .unwrap()
789 .pragma_update(None, "user_version", SCHEMA_VERSION + 1)
790 .unwrap();
791 let sqlite = Arc::new(SqliteStore::open(&path, "m".to_string()).unwrap());
792 let gate = WriteGate::default();
793 let clock = ClockGate::open_for_tests();
794 let health = fresh_health();
795 let local = StoreLocal {
796 sqlite: &sqlite,
797 gate: &gate,
798 clock: &clock,
799 health: &health,
800 access: &DeviceAccess::default(),
801 bootstrap: &Bootstrap::default(),
802 data_dir: dir.path(),
803 };
804 let error = local.export("peer", 0, 10).err().expect("export refused");
805 assert!(
806 format!("{error:#}").contains("upgrade this daemon"),
807 "{error:#}"
808 );
809 }
810
811 #[test]
812 fn a_host_keeps_one_worker_when_its_sb_path_changes() {
813 let dir = tempfile::tempdir().unwrap();
814 let sync_loop = Arc::new(SyncLoop {
815 data_dir: dir.path().to_path_buf(),
816 sqlite: Arc::new(SqliteStore::in_memory("m".to_string()).unwrap()),
817 gate: Arc::new(WriteGate::default()),
818 clock: ClockGate::open_for_tests(),
819 health: fresh_health(),
820 access: Arc::default(),
821 bootstrap: Arc::default(),
822 connect: Arc::new(|_: &Route| -> Result<Box<dyn Transport + Send>> {
823 anyhow::bail!("offline")
824 }),
825 discovery: None,
826 });
827 let mut workers = HashMap::new();
828 sync_loop.dispatch(&mut workers, Job::Push, line_dial("user@pi"));
829 sync_loop.dispatch(&mut workers, Job::Push, line_dial("user@pi /opt/sb"));
830 assert_eq!(workers.len(), 1);
831 }
832
833 const SYNC_KEY: [u8; 32] = [4; 32];
834
835 fn shared_identity() -> HandshakeKeys {
838 let key = IdentityKey::from_bytes(KeyGuard::new(SYNC_KEY));
839 HandshakeKeys {
840 identity_tag: Some(IdentityTag::migrated(&key.key_id())),
841 epochs: vec![Epoch {
842 number: 0,
843 key_id: key.key_id(),
844 }],
845 ..HandshakeKeys::default()
846 }
847 }
848
849 fn upgraded_store(dir: &Path) -> SqliteStore {
850 let sqlite = SqliteStore::in_memory("m".to_string()).unwrap();
851 second_brain_core::keys::ensure_device_keys(&sqlite, dir).unwrap();
852 std::fs::write(
853 dir.join(LEGACY_KEY_FILE),
854 XChaChaEncryptor::new(SYNC_KEY).key_base64(),
855 )
856 .unwrap();
857 migrate_sync_key(&sqlite, dir, "m", Utc::now().timestamp_micros()).unwrap();
858 sqlite
859 }
860
861 fn scripted_loop(dir: &Path, connect: Connect) -> SyncLoop {
862 SyncLoop {
863 data_dir: dir.to_path_buf(),
864 sqlite: Arc::new(upgraded_store(dir)),
865 gate: Arc::new(WriteGate::default()),
866 clock: ClockGate::open_for_tests(),
867 health: fresh_health(),
868 access: Arc::default(),
869 bootstrap: Arc::default(),
870 connect,
871 discovery: None,
872 }
873 }
874
875 #[test]
876 fn the_worker_takes_the_latest_line_and_backs_off_for_all_but_maintenance() {
877 let dir = tempfile::tempdir().unwrap();
878 let calls: Arc<Mutex<Vec<Route>>> = Arc::default();
879 let connect: Connect = {
880 let calls = calls.clone();
881 Arc::new(move |route: &Route| -> Result<Box<dyn Transport + Send>> {
882 let mut calls = calls.lock().unwrap();
883 calls.push(route.clone());
884 if calls.len() == 1 {
885 Err(anyhow::Error::new(MaintenanceRunning))
886 } else {
887 anyhow::bail!("connection refused")
888 }
889 })
890 };
891 let sync_loop = Arc::new(scripted_loop(dir.path(), connect));
892 let made = |n: usize| {
893 let deadline = Instant::now() + Duration::from_secs(5);
894 while Instant::now() < deadline {
895 if calls.lock().unwrap().len() >= n {
896 return true;
897 }
898 std::thread::sleep(Duration::from_millis(10));
899 }
900 false
901 };
902 let bare = line_dial("user@pi");
903 let fixed = line_dial("user@pi /opt/sb");
904 let (jobs, inbox) = mpsc::channel();
905 jobs.send((Job::Push, line_dial("user@pi /old/sb")))
906 .unwrap();
907 jobs.send((Job::Full, bare.clone())).unwrap();
908 let worker = {
909 let sync_loop = sync_loop.clone();
910 std::thread::spawn(move || sync_loop.work(inbox))
911 };
912 assert!(made(1), "the queued jobs never ran");
913 jobs.send((Job::Full, bare.clone())).unwrap();
914 assert!(made(2), "local maintenance started a backoff");
915 jobs.send((Job::Full, bare.clone())).unwrap();
916 std::thread::sleep(Duration::from_millis(300));
917 assert_eq!(
918 calls.lock().unwrap().len(),
919 2,
920 "a job inside the backoff ran"
921 );
922 jobs.send((Job::Full, fixed.clone())).unwrap();
923 assert!(made(3), "a changed line waited out the old backoff");
924 drop(jobs);
925 worker.join().unwrap();
926 let route = |dial: &Dial| dial.routes[0].clone();
927 assert_eq!(
928 *calls.lock().unwrap(),
929 vec![route(&bare), route(&bare), route(&fixed)]
930 );
931 }
932
933 #[test]
934 fn an_untrusted_local_clock_does_not_count_against_the_peer() {
935 let error = anyhow::Error::new(second_brain_core::sqlite_store::ClockUntrusted)
936 .context("importing from the peer");
937 assert!(!counts_against_peer(&error), "{error:#}");
938 }
939
940 #[test]
941 fn store_maintenance_does_not_count_against_the_peer() {
942 let sqlite = Arc::new(SqliteStore::in_memory("m".to_string()).unwrap());
943 let gate = WriteGate::default();
944 let clock = ClockGate::open_for_tests();
945 let health = fresh_health();
946 let local = StoreLocal {
947 sqlite: &sqlite,
948 gate: &gate,
949 clock: &clock,
950 health: &health,
951 access: &DeviceAccess::default(),
952 bootstrap: &Bootstrap::default(),
953 data_dir: Path::new("/nonexistent"),
954 };
955 let _maintenance = gate.close();
956 let error = local
957 .set_cursor("peer", 1)
958 .context("updating sync state")
959 .unwrap_err();
960 assert!(!counts_against_peer(&error), "{error:#}");
961 assert!(counts_against_peer(&anyhow::anyhow!("connection refused")));
962 }
963
964 #[test]
965 fn a_changed_peer_line_is_tried_without_the_old_lines_backoff() {
966 let now = Instant::now();
967 let later = now + Duration::from_secs(1);
968 let bare = line_dial("user@pi");
969 let fixed = line_dial("user@pi /opt/sb");
970 let mut state = PeerBackoff::default();
971 assert!(state.due(&bare, now));
972 assert_eq!(state.failed(&bare, now), FIRST_BACKOFF);
973 assert!(!state.due(&bare, later));
974 assert!(state.due(&fixed, later));
975 assert_eq!(state.failed(&fixed, later), FIRST_BACKOFF);
976 assert_eq!(state.failed(&fixed, later), FIRST_BACKOFF * 2);
977 assert!(state.due(&bare, later));
978 state.succeeded();
979 assert_eq!(state.failed(&fixed, later), FIRST_BACKOFF);
980 }
981
982 #[test]
983 fn backoff_doubles_from_thirty_seconds_up_to_half_an_hour() {
984 let waits: Vec<u64> = (1..=8).map(|n| backoff(n).as_secs()).collect();
985 assert_eq!(waits, vec![30, 60, 120, 240, 480, 960, 1800, 1800]);
986 }
987
988 struct EmptyPeer;
989
990 impl EmptyPeer {
991 fn header() -> String {
992 SyncHeader {
993 hello: Some(Hello {
994 machine_id: "peer".to_string(),
995 wall_micros: Utc::now().timestamp_micros(),
996 synced: false,
997 cursor: None,
998 held_origin_seq: None,
999 keys: Some(shared_identity()),
1000 peer_in: None,
1001 log_floor: 0,
1002 head: 0,
1003 applied: None,
1004 merging: None,
1005 }),
1006 ..SyncHeader::local()
1007 }
1008 .to_line()
1009 }
1010 }
1011
1012 impl Transport for EmptyPeer {
1013 fn host(&self) -> &str {
1014 "user@pi"
1015 }
1016
1017 fn whoami(&self) -> Result<String> {
1018 Ok("peer".to_string())
1019 }
1020
1021 fn export(
1022 &self,
1023 _after_seq: u64,
1024 _hello: &Hello,
1025 ) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)> {
1026 let stream = format!("{}\n", Self::header()).into_bytes();
1027 Ok((Box::new(std::io::Cursor::new(stream)), Box::new(Exited)))
1028 }
1029
1030 fn import(&self) -> Result<(Box<dyn Write + Send>, Box<dyn Read + Send>, Box<dyn Remote>)> {
1031 let replies = format!(
1032 "{}\n{}\n",
1033 Self::header(),
1034 PushApplied { applied: 0 }.to_line()
1035 );
1036 Ok((
1037 Box::new(std::io::sink()),
1038 Box::new(std::io::Cursor::new(replies.into_bytes())),
1039 Box::new(Exited),
1040 ))
1041 }
1042 }
1043
1044 struct Exited;
1045
1046 impl Remote for Exited {
1047 fn wait(&mut self) -> Result<Exit> {
1048 Ok(Exit {
1049 success: true,
1050 status: "exit status: 0".to_string(),
1051 })
1052 }
1053
1054 fn stop(&mut self) -> Result<()> {
1055 Ok(())
1056 }
1057 }
1058
1059 fn reachable() -> Connect {
1060 Arc::new(|_: &Route| -> Result<Box<dyn Transport + Send>> { Ok(Box::new(EmptyPeer)) })
1061 }
1062
1063 fn refusing() -> Connect {
1064 Arc::new(|_: &Route| -> Result<Box<dyn Transport + Send>> {
1065 anyhow::bail!("connection refused")
1066 })
1067 }
1068
1069 fn watched_loop(dir: &Path, connect: Connect) -> (Arc<SyncLoop>, Arc<Health>) {
1070 let health = Arc::new(Health::new(Utc::now(), None));
1071 let sync_loop = Arc::new(scripted_loop(dir, connect).with_health(health.clone()));
1072 std::fs::write(dir.join(peers::PEERS_FILE), "user@pi\n").unwrap();
1073 assert_eq!(sync_loop.listed_dials(), vec![line_dial("user@pi")]);
1074 (sync_loop, health)
1075 }
1076
1077 fn the_one_peer(health: &Health) -> PeerSync {
1078 let mut peers = health.report().value.sync_peers;
1079 assert_eq!(peers.len(), 1, "{peers:?}");
1080 peers.remove(0)
1081 }
1082
1083 fn run_job(sync_loop: &Arc<SyncLoop>, job: Job, dial: Dial) {
1084 let (jobs, inbox) = mpsc::channel();
1085 jobs.send((job, dial)).unwrap();
1086 drop(jobs);
1087 let worker = {
1088 let sync_loop = sync_loop.clone();
1089 std::thread::spawn(move || sync_loop.work(inbox))
1090 };
1091 let deadline = Instant::now() + Duration::from_secs(10);
1092 while !worker.is_finished() {
1093 assert!(
1094 Instant::now() < deadline,
1095 "the worker never finished its job"
1096 );
1097 std::thread::sleep(Duration::from_millis(10));
1098 }
1099 worker.join().unwrap();
1100 }
1101
1102 struct Answering(&'static str);
1103
1104 impl Transport for Answering {
1105 fn host(&self) -> &str {
1106 "me@192.0.2.1"
1107 }
1108
1109 fn whoami(&self) -> Result<String> {
1110 Ok(self.0.to_string())
1111 }
1112
1113 fn export(
1114 &self,
1115 after_seq: u64,
1116 hello: &Hello,
1117 ) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)> {
1118 EmptyPeer.export(after_seq, hello)
1119 }
1120
1121 fn import(&self) -> Result<(Box<dyn Write + Send>, Box<dyn Read + Send>, Box<dyn Remote>)> {
1122 EmptyPeer.import()
1123 }
1124 }
1125
1126 fn device_dial(machine_id: &str, addresses: &[&str]) -> Dial {
1127 Dial {
1128 key: machine_id.to_string(),
1129 label: machine_id.to_string(),
1130 machine_id: Some(machine_id.to_string()),
1131 routes: addresses
1132 .iter()
1133 .map(|address| Route::Device {
1134 machine_id: machine_id.to_string(),
1135 login: "me".to_string(),
1136 address: address.to_string(),
1137 port: 22,
1138 })
1139 .collect(),
1140 }
1141 }
1142
1143 fn answering(
1146 by_address: &'static [(&'static str, &'static str)],
1147 ) -> (Connect, Arc<Mutex<Vec<String>>>) {
1148 let dialled: Arc<Mutex<Vec<String>>> = Arc::default();
1149 let connect: Connect = {
1150 let dialled = dialled.clone();
1151 Arc::new(move |route: &Route| -> Result<Box<dyn Transport + Send>> {
1152 let Route::Device { address, .. } = route else {
1153 anyhow::bail!("not a device route")
1154 };
1155 dialled.lock().unwrap().push(address.clone());
1156 let Some((_, id)) = by_address.iter().find(|(a, _)| a == address) else {
1157 anyhow::bail!("connection refused")
1158 };
1159 Ok(Box::new(Answering(id)))
1160 })
1161 };
1162 (connect, dialled)
1163 }
1164
1165 fn admit_new_device(sync_loop: &SyncLoop, dir: &Path, machine_id: &str) {
1166 let device = load_device_keys(dir).unwrap().unwrap();
1167 let identity_tag = sync_loop
1168 .sqlite
1169 .device_records()
1170 .unwrap()
1171 .into_iter()
1172 .find(|row| row.current.record.machine_id == "m")
1173 .unwrap()
1174 .current
1175 .record
1176 .identity_tag;
1177 let keys = DeviceKeys::generate().unwrap();
1178 let record = DeviceRecord {
1179 machine_id: machine_id.to_string(),
1180 identity_tag,
1181 version: 1,
1182 name: machine_id.to_string(),
1183 age_recipient: keys.age_recipient().to_string(),
1184 ssh_public_key: keys.ssh_public_key(),
1185 signing_public_key: keys.verifying_key().to_bytes(),
1186 recovery_fingerprints: Vec::new(),
1187 ssh_login: None,
1188 ssh_port: None,
1189 ssh_host_key: None,
1190 };
1191 admit_device(&sync_loop.sqlite, &device, record).unwrap();
1192 }
1193
1194 fn revoke_new_device(sync_loop: &SyncLoop, dir: &Path, machine_id: &str) {
1195 admit_new_device(sync_loop, dir, machine_id);
1196 Keyring::load(&sync_loop.sqlite, dir)
1197 .unwrap()
1198 .remove_device(machine_id, &std::collections::BTreeSet::new(), 10)
1199 .unwrap();
1200 }
1201
1202 #[test]
1203 fn a_route_answering_as_another_device_moves_on_to_the_next() {
1204 let dir = tempfile::tempdir().unwrap();
1205 let (connect, dialled) = answering(&[("192.0.2.1", "someone-else"), ("192.0.2.2", "peer")]);
1206 let sync_loop = scripted_loop(dir.path(), connect);
1207 sync_loop
1208 .sync(&device_dial("peer", &["192.0.2.1", "192.0.2.2"]), Job::Full)
1209 .unwrap();
1210 assert_eq!(*dialled.lock().unwrap(), vec!["192.0.2.1", "192.0.2.2"]);
1211 }
1212
1213 #[test]
1214 fn a_route_answering_as_a_revoked_device_stops_the_dial() {
1215 let dir = tempfile::tempdir().unwrap();
1216 let (connect, dialled) = answering(&[("192.0.2.1", "stolen"), ("192.0.2.2", "peer")]);
1217 let sync_loop = scripted_loop(dir.path(), connect);
1218 revoke_new_device(&sync_loop, dir.path(), "stolen");
1219 assert!(
1220 sync_loop
1221 .sync(&device_dial("peer", &["192.0.2.1", "192.0.2.2"]), Job::Full)
1222 .is_err()
1223 );
1224 assert_eq!(*dialled.lock().unwrap(), vec!["192.0.2.1"]);
1225 }
1226
1227 fn advertising(ids: &[&str]) -> Responder {
1229 let adverts: Vec<_> = ids
1230 .iter()
1231 .map(|id| mdns::device_advert(id, "me", 22))
1232 .collect();
1233 Responder::spawn(
1234 &Net::Unicast(std::net::SocketAddr::from((
1235 std::net::Ipv4Addr::LOCALHOST,
1236 0,
1237 ))),
1238 Arc::new(move || adverts.clone()),
1239 )
1240 .unwrap()
1241 }
1242
1243 fn looking_up(sync_loop: SyncLoop, responder: &Responder) -> SyncLoop {
1244 SyncLoop {
1245 discovery: Some(Net::Unicast(responder.local_addr())),
1246 ..sync_loop
1247 }
1248 }
1249
1250 fn held_addresses(sync_loop: &SyncLoop, machine_id: &str) -> Vec<String> {
1251 sync_loop
1252 .sqlite
1253 .device_route(machine_id)
1254 .unwrap()
1255 .map(|route| route.addresses)
1256 .unwrap_or_default()
1257 }
1258
1259 fn hold_route(sync_loop: &SyncLoop, machine_id: &str, addresses: &[&str]) {
1260 sync_loop
1261 .sqlite
1262 .set_device_route(DeviceRoute {
1263 machine_id: machine_id.to_string(),
1264 addresses: addresses.iter().map(|a| a.to_string()).collect(),
1265 ssh_login: Some("me".to_string()),
1266 ssh_port: None,
1267 })
1268 .unwrap();
1269 }
1270
1271 #[test]
1272 fn a_device_whose_address_changed_is_found_synced_and_recorded() {
1273 let dir = tempfile::tempdir().unwrap();
1274 let (connect, dialled) = answering(&[("127.0.0.1", "peer")]);
1275 let responder = advertising(&["peer"]);
1276 let sync_loop = looking_up(scripted_loop(dir.path(), connect), &responder);
1277 admit_new_device(&sync_loop, dir.path(), "peer");
1278 hold_route(&sync_loop, "peer", &["192.0.2.1"]);
1279 let dials = sync_loop.listed_dials();
1280 assert_eq!(dials.len(), 1, "{dials:?}");
1281
1282 sync_loop.sync(&dials[0], Job::Full).unwrap();
1283 assert_eq!(*dialled.lock().unwrap(), vec!["192.0.2.1", "127.0.0.1"]);
1284 assert_eq!(
1285 held_addresses(&sync_loop, "peer"),
1286 vec!["127.0.0.1", "192.0.2.1"]
1287 );
1288 let Route::Device { address, .. } = &sync_loop.listed_dials()[0].routes[0] else {
1289 panic!("a device is dialled by its device route");
1290 };
1291 assert_eq!(address, "127.0.0.1");
1292 }
1293
1294 #[test]
1295 fn an_address_now_held_by_another_machine_is_looked_past() {
1296 let dir = tempfile::tempdir().unwrap();
1297 let (connect, dialled) = answering(&[("192.0.2.1", "someone-else"), ("127.0.0.1", "peer")]);
1298 let responder = advertising(&["peer"]);
1299 let sync_loop = looking_up(scripted_loop(dir.path(), connect), &responder);
1300 admit_new_device(&sync_loop, dir.path(), "peer");
1301 hold_route(&sync_loop, "peer", &["192.0.2.1"]);
1302 sync_loop
1303 .sync(&sync_loop.listed_dials()[0], Job::Push)
1304 .unwrap();
1305 assert_eq!(*dialled.lock().unwrap(), vec!["192.0.2.1", "127.0.0.1"]);
1306 assert_eq!(held_addresses(&sync_loop, "peer")[0], "127.0.0.1");
1307 }
1308
1309 #[test]
1310 fn a_device_with_no_address_is_looked_up_on_a_full_pass() {
1311 let dir = tempfile::tempdir().unwrap();
1312 let (connect, dialled) = answering(&[("127.0.0.1", "peer")]);
1313 let responder = advertising(&["peer"]);
1314 let sync_loop = looking_up(scripted_loop(dir.path(), connect), &responder);
1315 admit_new_device(&sync_loop, dir.path(), "peer");
1316 let mut dials = sync_loop.listed_dials();
1317 assert_eq!(dials, Vec::<Dial>::new());
1318
1319 sync_loop.add_found(&mut dials);
1320 assert_eq!(dials.len(), 1, "{dials:?}");
1321 assert_eq!(dials[0].routes[0].describe(), "me@127.0.0.1");
1322 sync_loop.sync(&dials[0], Job::Full).unwrap();
1323 assert_eq!(*dialled.lock().unwrap(), vec!["127.0.0.1"]);
1324 assert_eq!(held_addresses(&sync_loop, "peer"), vec!["127.0.0.1"]);
1325 assert_eq!(sync_loop.listed_dials().len(), 1);
1326 }
1327
1328 #[test]
1329 fn a_responder_under_an_unknown_or_revoked_id_is_never_dialled() {
1330 let dir = tempfile::tempdir().unwrap();
1331 let (connect, dialled) = answering(&[]);
1332 let responder = advertising(&["stranger", "stolen"]);
1333 let sync_loop = looking_up(scripted_loop(dir.path(), connect), &responder);
1334 revoke_new_device(&sync_loop, dir.path(), "stolen");
1335 admit_new_device(&sync_loop, dir.path(), "peer");
1336 hold_route(&sync_loop, "peer", &["192.0.2.1"]);
1337
1338 let mut dials = sync_loop.listed_dials();
1339 assert!(sync_loop.sync(&dials[0], Job::Full).is_err());
1340 assert_eq!(*dialled.lock().unwrap(), vec!["192.0.2.1"]);
1341 sync_loop.add_found(&mut dials);
1342 assert_eq!(
1343 dials.iter().map(|d| d.key.as_str()).collect::<Vec<_>>(),
1344 vec!["peer"]
1345 );
1346 assert_eq!(held_addresses(&sync_loop, "peer"), vec!["192.0.2.1"]);
1347 assert_eq!(held_addresses(&sync_loop, "stolen"), Vec::<String>::new());
1348 assert_eq!(held_addresses(&sync_loop, "stranger"), Vec::<String>::new());
1349 }
1350
1351 #[test]
1352 fn a_full_sync_reports_the_peer_synced() {
1353 let dir = tempfile::tempdir().unwrap();
1354 let (sync_loop, health) = watched_loop(dir.path(), reachable());
1355 let before = Utc::now();
1356 run_job(&sync_loop, Job::Full, line_dial("user@pi"));
1357 let peer = the_one_peer(&health);
1358 assert_eq!(peer.peer, "user@pi");
1359 let synced = peer.last_success_at.expect("a full sync is a success");
1360 assert!(synced >= before, "{synced} predates the sync");
1361 assert_eq!(peer.consecutive_failures, 0);
1362 }
1363
1364 #[test]
1365 fn a_failed_sync_counts_a_failure_and_no_success() {
1366 let dir = tempfile::tempdir().unwrap();
1367 let (sync_loop, health) = watched_loop(dir.path(), refusing());
1368 run_job(&sync_loop, Job::Full, line_dial("user@pi"));
1369 let peer = the_one_peer(&health);
1370 assert_eq!(peer.peer, "user@pi");
1371 assert_eq!(peer.consecutive_failures, 1);
1372 assert_eq!(peer.last_success_at, None);
1373 assert!(peer.last_attempt_at.is_some(), "the failure is an attempt");
1374 let error = peer.last_error.expect("the failure's text is kept");
1375 assert!(error.contains("connection refused"), "{error}");
1376 }
1377
1378 #[test]
1379 fn start_deletes_the_plaintext_ssh_key_once_the_device_key_loads() {
1380 let dir = tempfile::tempdir().unwrap();
1381 let sqlite = SqliteStore::in_memory("m".to_string()).unwrap();
1382 second_brain_core::keys::ensure_device_keys(&sqlite, dir.path()).unwrap();
1383 let keys = load_device_keys(dir.path()).unwrap().unwrap();
1384 let legacy = dir.path().join(SSH_KEY_FILE);
1385 write_identity_file(&legacy, keys.ssh_signing_key(), "second-brain:m").unwrap();
1386 let before = std::fs::read(&legacy).unwrap();
1387
1388 assert!(retire_plaintext_ssh_key(dir.path()).unwrap());
1389 assert!(!legacy.exists());
1390 assert!(!retire_plaintext_ssh_key(dir.path()).unwrap());
1391
1392 let reloaded = load_device_keys(dir.path()).unwrap().unwrap();
1393 let elsewhere = tempfile::tempdir().unwrap();
1394 let rederived = elsewhere.path().join("id");
1395 write_identity_file(&rederived, reloaded.ssh_signing_key(), "second-brain:m").unwrap();
1396 assert_eq!(std::fs::read(&rederived).unwrap(), before);
1397 }
1398
1399 #[test]
1400 fn a_plaintext_ssh_key_stays_while_no_device_key_loads() {
1401 let dir = tempfile::tempdir().unwrap();
1402 let legacy = dir.path().join(SSH_KEY_FILE);
1403 let stray = DeviceKeys::generate().unwrap();
1404 write_identity_file(&legacy, stray.ssh_signing_key(), "second-brain:m").unwrap();
1405 assert!(!retire_plaintext_ssh_key(dir.path()).unwrap());
1406 assert!(legacy.exists());
1407
1408 std::fs::write(
1409 dir.path().join(second_brain_core::keys::DEVICE_KEY_FILE),
1410 b"torn",
1411 )
1412 .unwrap();
1413 assert!(retire_plaintext_ssh_key(dir.path()).is_err());
1414 assert!(legacy.exists());
1415 }
1416
1417 #[test]
1418 fn a_push_from_a_device_without_an_identity_is_not_held_against_the_peer() {
1419 let dir = tempfile::tempdir().unwrap();
1420 let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
1421 let connect: Connect = {
1422 let calls = calls.clone();
1423 Arc::new(move |_: &Route| -> Result<Box<dyn Transport + Send>> {
1424 calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
1425 anyhow::bail!("connection refused")
1426 })
1427 };
1428 let sqlite = SqliteStore::in_memory("m".to_string()).unwrap();
1429 second_brain_core::keys::ensure_device_keys(&sqlite, dir.path()).unwrap();
1430 let health = Arc::new(Health::new(Utc::now(), None));
1431 let sync_loop = Arc::new(SyncLoop {
1432 data_dir: dir.path().to_path_buf(),
1433 sqlite: Arc::new(sqlite),
1434 gate: Arc::new(WriteGate::default()),
1435 clock: ClockGate::open_for_tests(),
1436 health: health.clone(),
1437 access: Arc::default(),
1438 bootstrap: Arc::default(),
1439 connect,
1440 discovery: None,
1441 });
1442 std::fs::write(dir.path().join(peers::PEERS_FILE), "user@pi\n").unwrap();
1443 sync_loop.listed_dials();
1444
1445 run_job(&sync_loop, Job::Push, line_dial("user@pi"));
1446 assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 0);
1447 let peer = the_one_peer(&health);
1448 assert_eq!(peer.last_attempt_at, None);
1449 assert_eq!(peer.consecutive_failures, 0);
1450
1451 run_job(&sync_loop, Job::Full, line_dial("user@pi"));
1452 assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 1);
1453 assert_eq!(the_one_peer(&health).consecutive_failures, 1);
1454 }
1455
1456 #[test]
1457 fn a_push_ends_a_failure_streak_without_counting_as_a_sync() {
1458 let dir = tempfile::tempdir().unwrap();
1459 let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
1460 let connect: Connect = {
1461 let calls = calls.clone();
1462 Arc::new(move |_: &Route| -> Result<Box<dyn Transport + Send>> {
1463 if calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0 {
1464 anyhow::bail!("connection refused");
1465 }
1466 Ok(Box::new(EmptyPeer))
1467 })
1468 };
1469 let (sync_loop, health) = watched_loop(dir.path(), connect);
1470 run_job(&sync_loop, Job::Full, line_dial("user@pi"));
1471 assert_eq!(the_one_peer(&health).consecutive_failures, 1);
1472 run_job(&sync_loop, Job::Push, line_dial("user@pi"));
1473 let peer = the_one_peer(&health);
1474 assert_eq!(peer.last_success_at, None);
1475 assert!(peer.last_attempt_at.is_some());
1476 assert_eq!(peer.consecutive_failures, 0);
1477 }
1478
1479 #[test]
1480 fn maintenance_reports_no_sync_outcome() {
1481 let dir = tempfile::tempdir().unwrap();
1482 let connect: Connect = Arc::new(|_: &Route| -> Result<Box<dyn Transport + Send>> {
1483 Err(anyhow::Error::new(MaintenanceRunning))
1484 });
1485 let (sync_loop, health) = watched_loop(dir.path(), connect);
1486 run_job(&sync_loop, Job::Full, line_dial("user@pi"));
1487 let peer = the_one_peer(&health);
1488 assert_eq!(peer.last_success_at, None);
1489 assert_eq!(peer.last_attempt_at, None);
1490 assert_eq!(peer.consecutive_failures, 0);
1491 }
1492
1493 #[test]
1494 fn a_peer_removed_from_sync_peers_drops_out_of_health() {
1495 let dir = tempfile::tempdir().unwrap();
1496 let (sync_loop, health) = watched_loop(dir.path(), refusing());
1497 let listed = |text: &str| {
1498 std::fs::write(dir.path().join(peers::PEERS_FILE), text).unwrap();
1499 let kept = sync_loop.listed_dials();
1500 let mut reported: Vec<String> = health
1501 .report()
1502 .value
1503 .sync_peers
1504 .into_iter()
1505 .map(|peer| peer.peer)
1506 .collect();
1507 reported.sort();
1508 (kept.len(), reported)
1509 };
1510 assert_eq!(
1511 listed("user@pi\nuser@mac\nuser@pi /opt/sb\n"),
1512 (2, vec!["user@mac".to_string(), "user@pi".to_string()])
1513 );
1514 assert_eq!(listed("user@mac\n"), (1, vec!["user@mac".to_string()]));
1515 }
1516
1517 #[test]
1518 fn an_unreadable_sync_peers_keeps_its_peers_in_health() {
1519 let dir = tempfile::tempdir().unwrap();
1520 let (sync_loop, health) = watched_loop(dir.path(), refusing());
1521 let file = dir.path().join(peers::PEERS_FILE);
1522 std::fs::remove_file(&file).unwrap();
1523 std::fs::create_dir(&file).unwrap();
1524 assert!(peers::configured_peers(dir.path()).is_err());
1525
1526 assert_eq!(sync_loop.listed_dials(), Vec::<Dial>::new());
1527 assert_eq!(the_one_peer(&health).peer, "user@pi");
1528 }
1529}