Skip to main content

second_brain_api/
sync_loop.rs

1use std::collections::HashMap;
2use std::io::Read;
3use std::path::{Path, PathBuf};
4use std::sync::mpsc::{self, Receiver, SendError, Sender};
5use std::sync::{Arc, Mutex, PoisonError};
6use std::thread::JoinHandle;
7use std::time::{Duration, Instant};
8
9use anyhow::{Context, Result};
10use chrono::Utc;
11use second_brain_core::keys::{DeviceKeys, load_device_keys};
12use second_brain_core::schema::SyncState;
13use second_brain_core::sqlite_store::SqliteStore;
14use second_brain_core::store::StoreAdmin;
15use second_brain_sync::crypto::seal_line;
16use second_brain_sync::dial::{self, Dial, Route};
17use second_brain_sync::export::export_page;
18use second_brain_sync::keyring::Keyring;
19use second_brain_sync::mdns::{self, Net};
20use second_brain_sync::protocol::{Hello, Merging, SyncHeader, check_store_header, store_header};
21use second_brain_sync::session::{self, Exported, Imported, Local, Merged, Transport};
22use second_brain_sync::ssh::{SshTransport, device_alias, known_hosts_path_in};
23use second_brain_sync::ssh_identity::{EphemeralIdentity, sweep_ephemeral_identities};
24
25use crate::bootstrap::Bootstrap;
26use crate::clock_gate::{ClockGate, is_untrusted};
27use crate::device_access::DeviceAccess;
28use crate::health::Health;
29use crate::snapshot_merge;
30use crate::sync_hello;
31use crate::value_metrics::PeerOutcome;
32use crate::write_gate::{WriteGate, WriteSlot};
33
34const FULL_INTERVAL: Duration = Duration::from_secs(300);
35const TICK: Duration = Duration::from_secs(1);
36const PUSH_DELAY: Duration = Duration::from_secs(5);
37const FIRST_BACKOFF: Duration = Duration::from_secs(30);
38const MAX_BACKOFF: Duration = Duration::from_secs(30 * 60);
39const ORIGIN_SCAN_PAGE: usize = 500;
40
41pub type Connect = Arc<dyn Fn(&Route) -> Result<Box<dyn Transport + Send>> + Send + Sync>;
42
43/// This device's ssh key, as the file ssh reads for device routes.
44pub const SSH_KEY_FILE: &str = "sync-ssh-key";
45
46/// Syncs with every active device in the signed device list that can be
47/// reached, and with `sync-peers` lines not yet matched to one: a full pull
48/// and push on start, every five minutes and right after a pairing, and a
49/// push shortly after this machine writes.
50pub struct SyncLoop {
51    pub data_dir: PathBuf,
52    pub sqlite: Arc<SqliteStore>,
53    pub gate: Arc<WriteGate>,
54    pub clock: Arc<ClockGate>,
55    pub health: Arc<Health>,
56    pub access: Arc<DeviceAccess>,
57    pub bootstrap: Arc<Bootstrap>,
58    pub connect: Connect,
59    /// Where a device with no address, or none that answers, is looked up
60    /// by mDNS; None looks nothing up.
61    pub discovery: Option<Net>,
62}
63
64#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
65enum Job {
66    Push,
67    Full,
68}
69
70impl SyncLoop {
71    pub fn over_ssh(
72        data_dir: PathBuf,
73        sqlite: Arc<SqliteStore>,
74        gate: Arc<WriteGate>,
75        clock: Arc<ClockGate>,
76        health: Arc<Health>,
77        access: Arc<DeviceAccess>,
78        bootstrap: Arc<Bootstrap>,
79    ) -> Self {
80        if let Err(e) = sweep_ephemeral_identities(&data_dir) {
81            tracing::warn!("removing ssh key files a killed daemon left: {e:#}");
82        }
83        match retire_plaintext_ssh_key(&data_dir) {
84            Ok(true) => tracing::info!("removed the plaintext {SSH_KEY_FILE}"),
85            Ok(false) => {}
86            Err(e) => tracing::warn!("kept the plaintext {SSH_KEY_FILE}: {e:#}"),
87        }
88        let connect = ssh_connect(data_dir.clone(), sqlite.machine_id().to_string());
89        Self {
90            data_dir,
91            sqlite,
92            gate,
93            clock,
94            health,
95            access,
96            bootstrap,
97            connect,
98            discovery: None,
99        }
100    }
101
102    pub fn with_health(mut self, health: Arc<Health>) -> Self {
103        self.health = health;
104        self
105    }
106
107    pub fn with_discovery(mut self, discovery: Net) -> Self {
108        self.discovery = Some(discovery);
109        self
110    }
111
112    pub fn spawn(self) -> JoinHandle<()> {
113        std::thread::Builder::new()
114            .name("sync-loop".to_string())
115            .spawn(move || Arc::new(self).run())
116            .expect("spawning the sync loop")
117    }
118
119    fn run(self: Arc<Self>) {
120        let mut workers: HashMap<String, Sender<(Job, Dial)>> = HashMap::new();
121        let mut next_full = Instant::now();
122        let mut seen = self.sqlite.max_sync_seq().unwrap_or(0);
123        let mut push_at: Option<Instant> = None;
124        loop {
125            let dials = self.listed_dials();
126            if dials.is_empty() {
127                seen = self.sqlite.max_sync_seq().unwrap_or(seen);
128                push_at = None;
129            } else if self.wrote_locally(&mut seen) {
130                push_at.get_or_insert(Instant::now() + PUSH_DELAY);
131            }
132
133            let now = Instant::now();
134            if self.access.take_kick() {
135                next_full = now;
136            }
137            let job = if now >= next_full {
138                next_full = now + FULL_INTERVAL;
139                Some(Job::Full)
140            } else if push_at.is_some_and(|at| now >= at) {
141                Some(Job::Push)
142            } else {
143                None
144            };
145            if let Some(job) = job {
146                push_at = None;
147                let mut dials = dials;
148                if job == Job::Full {
149                    self.add_found(&mut dials);
150                }
151                for dial in dials {
152                    self.dispatch(&mut workers, job, dial);
153                }
154            }
155            std::thread::sleep(TICK);
156        }
157    }
158
159    /// Workers are keyed by peer and outlive its removal from the list, so
160    /// a changed route or a peer re-added mid-sync still gets one sync at a
161    /// time.
162    fn dispatch(
163        self: &Arc<Self>,
164        workers: &mut HashMap<String, Sender<(Job, Dial)>>,
165        job: Job,
166        dial: Dial,
167    ) {
168        let key = dial.key.clone();
169        let worker = workers
170            .entry(key.clone())
171            .or_insert_with(|| self.clone().start_worker(&key));
172        if let Err(SendError(message)) = worker.send((job, dial)) {
173            tracing::warn!("the sync worker for {key} exited; starting another");
174            let worker = self.clone().start_worker(&key);
175            if worker.send(message).is_err() {
176                tracing::warn!("the new sync worker for {key} exited at once");
177            }
178            workers.insert(key, worker);
179        }
180    }
181
182    /// A device list or `sync-peers` that can't be read keeps the peers in
183    /// health, so their lag keeps growing instead of vanishing from the
184    /// report; an unreadable `sync-peers` still leaves the devices dialled.
185    fn listed_dials(&self) -> Vec<Dial> {
186        let mut dials = match dial::device_dials(&self.sqlite) {
187            Ok(dials) => dials,
188            Err(e) => {
189                tracing::warn!("cannot read the devices to sync with: {e:#}");
190                return Vec::new();
191            }
192        };
193        match dial::line_dials(&self.sqlite, &self.data_dir) {
194            Ok(lines) => {
195                dials.extend(lines);
196                let labels: Vec<String> = dials.iter().map(|dial| dial.label.clone()).collect();
197                self.health.sync_peers_listed(&labels, Utc::now());
198            }
199            Err(e) => tracing::warn!(
200                "cannot read {}: {e:#}",
201                second_brain_sync::peers::PEERS_FILE
202            ),
203        }
204        dials
205    }
206
207    fn report(&self, label: &str, outcome: PeerOutcome) {
208        self.health.sync_attempted(label, outcome, Utc::now());
209    }
210
211    /// Imports also append to sync_log; only a row this machine originated is
212    /// news to its peers.
213    fn wrote_locally(&self, seen: &mut u64) -> bool {
214        let max = match self.sqlite.max_sync_seq() {
215            Ok(max) => max,
216            Err(e) => {
217                tracing::warn!("cannot read the sync log position: {e:#}");
218                return false;
219            }
220        };
221        let own = self.sqlite.machine_id();
222        while *seen < max {
223            let page = match self.sqlite.sync_log_page(*seen, Some(ORIGIN_SCAN_PAGE)) {
224                Ok(page) => page,
225                Err(e) => {
226                    tracing::warn!("cannot read the sync log: {e:#}");
227                    return false;
228                }
229            };
230            let Some(last) = page.last() else {
231                break;
232            };
233            if page.iter().any(|entry| entry.origin_machine_id == own) {
234                *seen = max;
235                return true;
236            }
237            *seen = last.local_seq.max(0) as u64;
238        }
239        *seen = max;
240        false
241    }
242
243    fn start_worker(self: Arc<Self>, key: &str) -> Sender<(Job, Dial)> {
244        let (jobs, inbox) = mpsc::channel();
245        std::thread::Builder::new()
246            .name(format!("sync-{key}"))
247            .spawn(move || self.work(inbox))
248            .expect("spawning a sync worker");
249        jobs
250    }
251
252    /// One sync per peer at a time; a peer that fails waits out its backoff
253    /// on its own thread, so it never holds up the others.
254    fn work(&self, inbox: Receiver<(Job, Dial)>) {
255        let mut backoff = PeerBackoff::default();
256        while let Ok((mut job, mut dial)) = inbox.recv() {
257            while let Ok((queued, latest)) = inbox.try_recv() {
258                job = job.max(queued);
259                dial = latest;
260            }
261            if !backoff.due(&dial, Instant::now()) {
262                continue;
263            }
264            if job == Job::Push && !self.holds_identity() {
265                tracing::info!(
266                    "not pushing to {}: this device holds no sync identity yet; pair it with \
267                     `sb pair`",
268                    dial.label
269                );
270                continue;
271            }
272            match self.sync(&dial, job) {
273                Ok(()) => {
274                    backoff.succeeded();
275                    let outcome = match job {
276                        Job::Full => PeerOutcome::Synced,
277                        Job::Push => PeerOutcome::Pushed,
278                    };
279                    self.report(&dial.label, outcome);
280                }
281                Err(e) if !counts_against_peer(&e) => {
282                    tracing::info!("sync with {} waits for the next job: {e:#}", dial.label);
283                }
284                Err(e) => {
285                    let wait = backoff.failed(&dial, Instant::now());
286                    self.report(&dial.label, PeerOutcome::Failed(format!("{e:#}")));
287                    tracing::warn!(
288                        "sync with {} failed; its jobs are skipped for the next {}s: {e:#}",
289                        dial.label,
290                        wait.as_secs()
291                    );
292                }
293            }
294        }
295    }
296
297    /// Every way a store gets an identity (founding, migrating a sync.key,
298    /// pairing) writes this device's own row, so its absence is the cheap
299    /// test that needs no device key.
300    fn holds_identity(&self) -> bool {
301        match self.sqlite.device_record(self.sqlite.machine_id()) {
302            Ok(row) => row.is_some(),
303            Err(e) => {
304                tracing::warn!("cannot read this device's sync identity row: {e:#}");
305                true
306            }
307        }
308    }
309
310    /// Tries the peer's routes in order until one answers as the device it
311    /// was dialled for; a route that answers is the one synced through. A
312    /// device none of whose routes answers is looked up by mDNS, and the
313    /// routes found are tried next. A failure there is reported only when
314    /// the device had no route of its own, so an advert cannot replace the
315    /// held route's error in health.
316    fn sync(&self, dial: &Dial, job: Job) -> Result<()> {
317        let failed = match self.sync_via(dial, &dial.routes, job)? {
318            Reached::Synced => return Ok(()),
319            Reached::Not(failed) => failed,
320        };
321        let found: Vec<Route> = match &dial.machine_id {
322            Some(machine_id) => self
323                .lookup()
324                .into_iter()
325                .filter(|found| found.key == *machine_id)
326                .flat_map(|found| found.routes)
327                .filter(|route| !dial.routes.contains(route))
328                .collect(),
329            None => Vec::new(),
330        };
331        let found_failed = if found.is_empty() {
332            None
333        } else {
334            match self.sync_via(dial, &found, job)? {
335                Reached::Synced => return Ok(()),
336                Reached::Not(failed) => failed,
337            }
338        };
339        Err(failed
340            .or(found_failed)
341            .unwrap_or_else(|| anyhow::anyhow!("{} has no route to dial", dial.label)))
342    }
343
344    /// Adds dials for the devices of this identity that `dials` lacks and a
345    /// lookup finds, as a device with no recorded address.
346    fn add_found(&self, dials: &mut Vec<Dial>) {
347        if self.discovery.is_none() {
348            return;
349        }
350        match dial::undialled(&self.sqlite, dials) {
351            Ok(true) => {}
352            Ok(false) => return,
353            Err(e) => {
354                tracing::warn!("cannot read the devices to sync with: {e:#}");
355                return;
356            }
357        }
358        for dial in self.lookup() {
359            if !dials.iter().any(|listed| listed.key == dial.key) {
360                dials.push(dial);
361            }
362        }
363    }
364
365    /// Dials for the devices of this identity an mDNS lookup finds; none
366    /// without `discovery` or when the lookup fails.
367    fn lookup(&self) -> Vec<Dial> {
368        let Some(net) = &self.discovery else {
369            return Vec::new();
370        };
371        let found = match mdns::browse(net, mdns::SERVICE, mdns::LOOKUP_WINDOW) {
372            Ok(found) => found,
373            Err(e) => {
374                tracing::info!("cannot look devices up on the network: {e:#}");
375                return Vec::new();
376            }
377        };
378        dial::found_dials(&self.sqlite, &found).unwrap_or_else(|e| {
379            tracing::warn!("cannot read the devices to match a lookup against: {e:#}");
380            Vec::new()
381        })
382    }
383
384    fn sync_via(&self, dial: &Dial, routes: &[Route], job: Job) -> Result<Reached> {
385        let mut failed: Option<anyhow::Error> = None;
386        for route in routes {
387            let transport = match (self.connect)(route) {
388                Ok(transport) => transport,
389                Err(e) if !counts_against_peer(&e) => return Err(e),
390                Err(e) => {
391                    failed = Some(e.context(format!("reaching {}", route.describe())));
392                    continue;
393                }
394            };
395            let answered = match transport.whoami() {
396                Ok(answered) => answered,
397                Err(e) => {
398                    failed = Some(e.context(format!(
399                        "resolving the machine id at {} (is its daemon running?)",
400                        route.describe()
401                    )));
402                    continue;
403                }
404            };
405            if let Route::Line(peer) = route
406                && let Some(_slot) = self.gate.try_enter()
407            {
408                dial::line_answered(&self.sqlite, peer, &answered)?;
409            }
410            if let Err(e) = dial::refuse_answer(&self.sqlite, dial, &answered) {
411                if dial.machine_id.is_some() && !self.is_revoked(&answered) {
412                    failed = Some(e);
413                    continue;
414                }
415                return Err(e);
416            }
417            self.record_answered(route);
418            return self
419                .sync_through(transport.as_ref(), &answered, job)
420                .map(|()| Reached::Synced);
421        }
422        Ok(Reached::Not(failed))
423    }
424
425    /// A route that answered as its device and is not yet held, as one an
426    /// mDNS lookup found, becomes the first the device is dialled at.
427    fn record_answered(&self, route: &Route) {
428        let Some(_slot) = self.gate.try_enter() else {
429            return;
430        };
431        match dial::record_answered(&self.sqlite, route) {
432            Ok(true) => tracing::info!("recorded {} as a route to its device", route.describe()),
433            Ok(false) => {}
434            Err(e) => tracing::warn!("cannot record the route {}: {e:#}", route.describe()),
435        }
436    }
437
438    fn is_revoked(&self, machine_id: &str) -> bool {
439        self.sqlite
440            .device_record(machine_id)
441            .ok()
442            .flatten()
443            .is_some_and(|row| row.revoked)
444    }
445
446    fn sync_through(&self, transport: &dyn Transport, peer: &str, job: Job) -> Result<()> {
447        let local = StoreLocal {
448            sqlite: &self.sqlite,
449            gate: &self.gate,
450            clock: &self.clock,
451            health: &self.health,
452            access: &self.access,
453            bootstrap: &self.bootstrap,
454            data_dir: &self.data_dir,
455        };
456        let host = transport.host().to_string();
457        let report = |line: &str| tracing::info!("sync with {host}: {}", line.trim());
458        match job {
459            Job::Full => session::sync_with(&local, transport, peer, &self.data_dir, &report),
460            Job::Push => session::push(&local, transport, peer, &report).map(|_| ()),
461        }
462    }
463}
464
465/// Device routes go in with this device's own ssh key, derived from its
466/// device key, loaded once per daemon and kept in memory, and trust only the host
467/// keys pinned under the device's machine id; `sync-peers` lines keep the
468/// user's ssh setup.
469fn ssh_connect(data_dir: PathBuf, own: String) -> Connect {
470    let device_keys: Arc<Mutex<Option<DeviceKeys>>> = Arc::default();
471    Arc::new(move |route: &Route| -> Result<Box<dyn Transport + Send>> {
472        let transport = match route {
473            Route::Device {
474                machine_id,
475                login,
476                address,
477                port,
478            } => {
479                let identity = device_identity(&data_dir, &own, &device_keys)?;
480                SshTransport::new(&format!("{login}@{address}"), "sb")?
481                    .known_hosts(known_hosts_path_in(&data_dir))
482                    .port(*port)
483                    .host_key_alias(&device_alias(machine_id))
484                    .ephemeral_identity(identity)
485            }
486            Route::Line(peer) => SshTransport::new(&peer.host, &peer.sb)?,
487        };
488        Ok(Box::new(transport.unattended()) as Box<dyn Transport + Send>)
489    })
490}
491
492fn device_identity(
493    data_dir: &Path,
494    own: &str,
495    cached: &Mutex<Option<DeviceKeys>>,
496) -> Result<EphemeralIdentity> {
497    let mut cached = cached.lock().unwrap_or_else(PoisonError::into_inner);
498    if cached.is_none() {
499        let keys = load_device_keys(data_dir)?
500            .context("this device has no device key to sign in to its peers with")?;
501        *cached = Some(keys);
502    }
503    let keys = cached.as_ref().expect("the keys were loaded above");
504    EphemeralIdentity::create(
505        data_dir,
506        keys.ssh_signing_key(),
507        &format!("second-brain:{own}"),
508    )
509}
510
511/// `SSH_KEY_FILE` is a plaintext copy of the ssh key that older daemons kept.
512/// It is deleted only once the device key it derives from loads, so a device
513/// key that fails to open never leaves the device without its ssh key.
514fn retire_plaintext_ssh_key(data_dir: &Path) -> Result<bool> {
515    let path = data_dir.join(SSH_KEY_FILE);
516    if std::fs::symlink_metadata(&path).is_err() {
517        return Ok(false);
518    }
519    if load_device_keys(data_dir)?.is_none() {
520        return Ok(false);
521    }
522    std::fs::remove_file(&path).with_context(|| format!("removing {}", path.display()))?;
523    Ok(true)
524}
525
526#[derive(Debug)]
527struct MaintenanceRunning;
528
529impl std::fmt::Display for MaintenanceRunning {
530    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
531        f.write_str("store maintenance is running")
532    }
533}
534
535impl std::error::Error for MaintenanceRunning {}
536
537/// Routine store maintenance and an untrusted local clock say nothing about
538/// the peer, so they start no backoff; the clock may open at any moment and a
539/// peer's handshake is what opens it. Other local faults do back off, which
540/// also bounds how often a persistent one is logged; a daemon restart clears
541/// the backoff.
542fn counts_against_peer(error: &anyhow::Error) -> bool {
543    error.downcast_ref::<MaintenanceRunning>().is_none() && !is_untrusted(error)
544}
545
546enum Reached {
547    Synced,
548    /// No route answered as the device; the last failure, if any route was
549    /// tried.
550    Not(Option<anyhow::Error>),
551}
552
553#[derive(Default)]
554struct PeerBackoff {
555    dial: Option<Dial>,
556    failures: u32,
557    retry_at: Option<Instant>,
558}
559
560impl PeerBackoff {
561    /// A changed route is a new way to reach the peer, so it is tried at
562    /// once rather than after the old routes' backoff.
563    fn due(&self, dial: &Dial, now: Instant) -> bool {
564        self.dial.as_ref() != Some(dial) || self.retry_at.is_none_or(|at| now >= at)
565    }
566
567    fn succeeded(&mut self) {
568        *self = PeerBackoff::default();
569    }
570
571    fn failed(&mut self, dial: &Dial, now: Instant) -> Duration {
572        if self.dial.as_ref() != Some(dial) {
573            *self = PeerBackoff {
574                dial: Some(dial.clone()),
575                ..PeerBackoff::default()
576            };
577        }
578        self.failures += 1;
579        let wait = backoff(self.failures);
580        self.retry_at = Some(now + wait);
581        wait
582    }
583}
584
585fn backoff(failures: u32) -> Duration {
586    FIRST_BACKOFF
587        .saturating_mul(2u32.saturating_pow(failures.saturating_sub(1)))
588        .min(MAX_BACKOFF)
589}
590
591struct StoreLocal<'a> {
592    sqlite: &'a Arc<SqliteStore>,
593    gate: &'a WriteGate,
594    clock: &'a ClockGate,
595    health: &'a Health,
596    access: &'a DeviceAccess,
597    bootstrap: &'a Bootstrap,
598    data_dir: &'a std::path::Path,
599}
600
601impl StoreLocal<'_> {
602    fn write_slot(&self) -> Result<WriteSlot<'_>> {
603        self.gate
604            .try_enter()
605            .ok_or_else(|| anyhow::Error::new(MaintenanceRunning))
606    }
607}
608
609impl Local for StoreLocal<'_> {
610    fn cursors(&self) -> Result<Vec<(String, u64)>> {
611        Ok(self
612            .sqlite
613            .get_all_sync_states()?
614            .into_iter()
615            .map(|state| (state.peer_id, state.last_seq))
616            .collect())
617    }
618
619    fn set_cursor(&self, peer_id: &str, last_seq: u64) -> Result<()> {
620        let _slot = self.write_slot()?;
621        self.sqlite.set_sync_state(&SyncState {
622            peer_id: peer_id.to_string(),
623            last_seq,
624            last_sync_at: chrono::Utc::now(),
625        })?;
626        Ok(())
627    }
628
629    fn import(&self, header: &SyncHeader, lines: &[String]) -> Result<Imported> {
630        let _slot = self.write_slot()?;
631        check_store_header(self.sqlite, Some(header), "the sender")?;
632        let records = Keyring::load(self.sqlite, self.data_dir)?
633            .open_lines(header.hello.as_ref().map(|h| h.machine_id.as_str()), lines)?
634            .iter()
635            .map(|opened| Ok(serde_json::from_slice(opened)?))
636            .collect::<Result<Vec<serde_json::Value>>>()?;
637        let source = header.hello.as_ref().map(|h| h.machine_id.as_str());
638        let applied = crate::sync_import::import_from(self.sqlite, source, &records)?;
639        let stats = applied.stats;
640        Ok(Imported {
641            created: stats.created,
642            updated: stats.updated,
643            deleted: stats.deleted,
644            skipped: stats.skipped,
645            dangling: Imported::dangling_in(&stats.skipped_by_reason),
646            errors: stats.errors,
647            max_seq: applied.max_seq,
648            reached: applied.reached,
649        })
650    }
651
652    fn export(&self, peer_id: &str, after_seq: u64, limit: usize) -> Result<Exported> {
653        store_header(self.sqlite)?;
654        let (records, max_seq) = export_page(self.sqlite.as_ref(), after_seq, limit)?;
655        let sealer = Keyring::load(self.sqlite, self.data_dir)?.sealer_for(peer_id, None)?;
656        let lines = records
657            .iter()
658            .map(|record| seal_line(&sealer, &serde_json::to_vec(record)?))
659            .collect::<Result<Vec<String>>>()?;
660        Ok(Exported { lines, max_seq })
661    }
662
663    fn hello_for(&self, peer_id: &str) -> Result<Hello> {
664        sync_hello::hello_for(self.sqlite, self.clock, self.data_dir, peer_id, None)
665    }
666
667    fn accept_hello(&self, hello: &Hello) -> Result<()> {
668        sync_hello::check_identity(self.sqlite, self.data_dir, hello)?;
669        let _slot = self.write_slot()?;
670        sync_hello::accept_handshake(
671            self.sqlite,
672            self.clock,
673            self.health,
674            self.access,
675            self.data_dir,
676            hello,
677        )
678    }
679
680    fn record_ack(&self, peer_id: &str, acked_seq: u64) -> Result<()> {
681        let _slot = self.write_slot()?;
682        self.sqlite
683            .record_peer_ack(peer_id, acked_seq, self.clock.wall_micros())
684    }
685
686    fn record_peer_in(&self, peer_id: &str, seq: u64) -> Result<()> {
687        let _slot = self.write_slot()?;
688        self.sqlite.record_peer_in(peer_id, seq)
689    }
690
691    fn push_report(&self, peer_id: &str) -> Result<Option<u64>> {
692        self.sqlite.push_report(peer_id)
693    }
694
695    fn record_push_report(&self, peer_id: &str, reported: Option<u64>) -> Result<()> {
696        let _slot = self.write_slot()?;
697        self.sqlite.record_push_report(peer_id, reported)
698    }
699
700    fn merging(&self) -> Result<Option<Merging>> {
701        snapshot_merge::merging(self.data_dir)
702    }
703
704    fn merge_snapshot(&self, exporter: &str, stream: Box<dyn Read + Send>) -> Result<Merged> {
705        let merging = snapshot_merge::receive(self.sqlite, self.data_dir, stream, Some(exporter))?;
706        match snapshot_merge::merge_spooled(self.sqlite, self.data_dir, self.gate)? {
707            Some(_) => Ok(Merged::Done { seq: merging.seq }),
708            None => Ok(Merged::Running(merging)),
709        }
710    }
711
712    fn bootstrap_pending(&self) -> Result<bool> {
713        Ok(self.bootstrap.is_pending())
714    }
715
716    fn finish_bootstrap(&self) -> Result<()> {
717        let _slot = self.write_slot()?;
718        if self.bootstrap.finish(self.sqlite)? {
719            tracing::info!("bootstrap finished; ingesting this device's own sessions");
720        }
721        Ok(())
722    }
723
724    fn export_snapshot(&self, peer: &Hello) -> Result<Box<dyn Read + Send>> {
725        snapshot_merge::pin_for_snapshot(self.sqlite, self.gate, peer)?;
726        let (reader, writer) = std::io::pipe()?;
727        let (sqlite, data_dir, peer) = (
728            self.sqlite.clone(),
729            self.data_dir.to_path_buf(),
730            peer.clone(),
731        );
732        std::thread::Builder::new()
733            .name("snapshot-export".to_string())
734            .spawn(move || {
735                let out = std::io::BufWriter::with_capacity(1024 * 1024, writer);
736                if let Err(e) = snapshot_merge::stream_snapshot(&sqlite, &data_dir, &peer, out) {
737                    tracing::warn!("exporting a snapshot for {} failed: {e:#}", peer.machine_id);
738                }
739            })?;
740        Ok(Box::new(reader))
741    }
742}
743
744#[cfg(test)]
745mod tests {
746    use std::io::{Read, Write};
747    use std::path::Path;
748    use std::sync::Mutex;
749
750    use anyhow::Context;
751    use second_brain_core::keys::{
752        DeviceKeys, DeviceRecord, DeviceRoute, Epoch, IdentityKey, IdentityTag, KeyGuard,
753        admit_device,
754    };
755    use second_brain_core::sqlite_store::SCHEMA_VERSION;
756    use second_brain_sync::crypto::XChaChaEncryptor;
757    use second_brain_sync::keyring::{LEGACY_KEY_FILE, migrate_sync_key};
758    use second_brain_sync::mdns::Responder;
759    use second_brain_sync::peers::{self, Peer};
760    use second_brain_sync::protocol::HandshakeKeys;
761    use second_brain_sync::protocol::PushApplied;
762    use second_brain_sync::session::{Exit, Remote};
763    use second_brain_sync::ssh_identity::write_identity_file;
764
765    use super::*;
766    use crate::value_metrics::PeerSync;
767
768    fn line_dial(line: &str) -> Dial {
769        let peer = Peer::parse(line);
770        Dial {
771            key: peer.host.clone(),
772            label: peer.host.clone(),
773            machine_id: None,
774            routes: vec![Route::Line(peer)],
775        }
776    }
777
778    fn fresh_health() -> Arc<Health> {
779        Arc::new(Health::new(chrono::Utc::now(), None))
780    }
781
782    #[test]
783    fn the_daemon_refuses_to_export_from_a_database_newer_than_its_binary() {
784        let dir = tempfile::tempdir().unwrap();
785        let path = dir.path().join("future.db");
786        drop(SqliteStore::open(&path, "m".to_string()).unwrap());
787        rusqlite::Connection::open(&path)
788            .unwrap()
789            .pragma_update(None, "user_version", SCHEMA_VERSION + 1)
790            .unwrap();
791        let sqlite = Arc::new(SqliteStore::open(&path, "m".to_string()).unwrap());
792        let gate = WriteGate::default();
793        let clock = ClockGate::open_for_tests();
794        let health = fresh_health();
795        let local = StoreLocal {
796            sqlite: &sqlite,
797            gate: &gate,
798            clock: &clock,
799            health: &health,
800            access: &DeviceAccess::default(),
801            bootstrap: &Bootstrap::default(),
802            data_dir: dir.path(),
803        };
804        let error = local.export("peer", 0, 10).err().expect("export refused");
805        assert!(
806            format!("{error:#}").contains("upgrade this daemon"),
807            "{error:#}"
808        );
809    }
810
811    #[test]
812    fn a_host_keeps_one_worker_when_its_sb_path_changes() {
813        let dir = tempfile::tempdir().unwrap();
814        let sync_loop = Arc::new(SyncLoop {
815            data_dir: dir.path().to_path_buf(),
816            sqlite: Arc::new(SqliteStore::in_memory("m".to_string()).unwrap()),
817            gate: Arc::new(WriteGate::default()),
818            clock: ClockGate::open_for_tests(),
819            health: fresh_health(),
820            access: Arc::default(),
821            bootstrap: Arc::default(),
822            connect: Arc::new(|_: &Route| -> Result<Box<dyn Transport + Send>> {
823                anyhow::bail!("offline")
824            }),
825            discovery: None,
826        });
827        let mut workers = HashMap::new();
828        sync_loop.dispatch(&mut workers, Job::Push, line_dial("user@pi"));
829        sync_loop.dispatch(&mut workers, Job::Push, line_dial("user@pi /opt/sb"));
830        assert_eq!(workers.len(), 1);
831    }
832
833    const SYNC_KEY: [u8; 32] = [4; 32];
834
835    /// The identity a store upgraded from `SYNC_KEY` holds, as a peer of it
836    /// declares it.
837    fn shared_identity() -> HandshakeKeys {
838        let key = IdentityKey::from_bytes(KeyGuard::new(SYNC_KEY));
839        HandshakeKeys {
840            identity_tag: Some(IdentityTag::migrated(&key.key_id())),
841            epochs: vec![Epoch {
842                number: 0,
843                key_id: key.key_id(),
844            }],
845            ..HandshakeKeys::default()
846        }
847    }
848
849    fn upgraded_store(dir: &Path) -> SqliteStore {
850        let sqlite = SqliteStore::in_memory("m".to_string()).unwrap();
851        second_brain_core::keys::ensure_device_keys(&sqlite, dir).unwrap();
852        std::fs::write(
853            dir.join(LEGACY_KEY_FILE),
854            XChaChaEncryptor::new(SYNC_KEY).key_base64(),
855        )
856        .unwrap();
857        migrate_sync_key(&sqlite, dir, "m", Utc::now().timestamp_micros()).unwrap();
858        sqlite
859    }
860
861    fn scripted_loop(dir: &Path, connect: Connect) -> SyncLoop {
862        SyncLoop {
863            data_dir: dir.to_path_buf(),
864            sqlite: Arc::new(upgraded_store(dir)),
865            gate: Arc::new(WriteGate::default()),
866            clock: ClockGate::open_for_tests(),
867            health: fresh_health(),
868            access: Arc::default(),
869            bootstrap: Arc::default(),
870            connect,
871            discovery: None,
872        }
873    }
874
875    #[test]
876    fn the_worker_takes_the_latest_line_and_backs_off_for_all_but_maintenance() {
877        let dir = tempfile::tempdir().unwrap();
878        let calls: Arc<Mutex<Vec<Route>>> = Arc::default();
879        let connect: Connect = {
880            let calls = calls.clone();
881            Arc::new(move |route: &Route| -> Result<Box<dyn Transport + Send>> {
882                let mut calls = calls.lock().unwrap();
883                calls.push(route.clone());
884                if calls.len() == 1 {
885                    Err(anyhow::Error::new(MaintenanceRunning))
886                } else {
887                    anyhow::bail!("connection refused")
888                }
889            })
890        };
891        let sync_loop = Arc::new(scripted_loop(dir.path(), connect));
892        let made = |n: usize| {
893            let deadline = Instant::now() + Duration::from_secs(5);
894            while Instant::now() < deadline {
895                if calls.lock().unwrap().len() >= n {
896                    return true;
897                }
898                std::thread::sleep(Duration::from_millis(10));
899            }
900            false
901        };
902        let bare = line_dial("user@pi");
903        let fixed = line_dial("user@pi /opt/sb");
904        let (jobs, inbox) = mpsc::channel();
905        jobs.send((Job::Push, line_dial("user@pi /old/sb")))
906            .unwrap();
907        jobs.send((Job::Full, bare.clone())).unwrap();
908        let worker = {
909            let sync_loop = sync_loop.clone();
910            std::thread::spawn(move || sync_loop.work(inbox))
911        };
912        assert!(made(1), "the queued jobs never ran");
913        jobs.send((Job::Full, bare.clone())).unwrap();
914        assert!(made(2), "local maintenance started a backoff");
915        jobs.send((Job::Full, bare.clone())).unwrap();
916        std::thread::sleep(Duration::from_millis(300));
917        assert_eq!(
918            calls.lock().unwrap().len(),
919            2,
920            "a job inside the backoff ran"
921        );
922        jobs.send((Job::Full, fixed.clone())).unwrap();
923        assert!(made(3), "a changed line waited out the old backoff");
924        drop(jobs);
925        worker.join().unwrap();
926        let route = |dial: &Dial| dial.routes[0].clone();
927        assert_eq!(
928            *calls.lock().unwrap(),
929            vec![route(&bare), route(&bare), route(&fixed)]
930        );
931    }
932
933    #[test]
934    fn an_untrusted_local_clock_does_not_count_against_the_peer() {
935        let error = anyhow::Error::new(second_brain_core::sqlite_store::ClockUntrusted)
936            .context("importing from the peer");
937        assert!(!counts_against_peer(&error), "{error:#}");
938    }
939
940    #[test]
941    fn store_maintenance_does_not_count_against_the_peer() {
942        let sqlite = Arc::new(SqliteStore::in_memory("m".to_string()).unwrap());
943        let gate = WriteGate::default();
944        let clock = ClockGate::open_for_tests();
945        let health = fresh_health();
946        let local = StoreLocal {
947            sqlite: &sqlite,
948            gate: &gate,
949            clock: &clock,
950            health: &health,
951            access: &DeviceAccess::default(),
952            bootstrap: &Bootstrap::default(),
953            data_dir: Path::new("/nonexistent"),
954        };
955        let _maintenance = gate.close();
956        let error = local
957            .set_cursor("peer", 1)
958            .context("updating sync state")
959            .unwrap_err();
960        assert!(!counts_against_peer(&error), "{error:#}");
961        assert!(counts_against_peer(&anyhow::anyhow!("connection refused")));
962    }
963
964    #[test]
965    fn a_changed_peer_line_is_tried_without_the_old_lines_backoff() {
966        let now = Instant::now();
967        let later = now + Duration::from_secs(1);
968        let bare = line_dial("user@pi");
969        let fixed = line_dial("user@pi /opt/sb");
970        let mut state = PeerBackoff::default();
971        assert!(state.due(&bare, now));
972        assert_eq!(state.failed(&bare, now), FIRST_BACKOFF);
973        assert!(!state.due(&bare, later));
974        assert!(state.due(&fixed, later));
975        assert_eq!(state.failed(&fixed, later), FIRST_BACKOFF);
976        assert_eq!(state.failed(&fixed, later), FIRST_BACKOFF * 2);
977        assert!(state.due(&bare, later));
978        state.succeeded();
979        assert_eq!(state.failed(&fixed, later), FIRST_BACKOFF);
980    }
981
982    #[test]
983    fn backoff_doubles_from_thirty_seconds_up_to_half_an_hour() {
984        let waits: Vec<u64> = (1..=8).map(|n| backoff(n).as_secs()).collect();
985        assert_eq!(waits, vec![30, 60, 120, 240, 480, 960, 1800, 1800]);
986    }
987
988    struct EmptyPeer;
989
990    impl EmptyPeer {
991        fn header() -> String {
992            SyncHeader {
993                hello: Some(Hello {
994                    machine_id: "peer".to_string(),
995                    wall_micros: Utc::now().timestamp_micros(),
996                    synced: false,
997                    cursor: None,
998                    held_origin_seq: None,
999                    keys: Some(shared_identity()),
1000                    peer_in: None,
1001                    log_floor: 0,
1002                    head: 0,
1003                    applied: None,
1004                    merging: None,
1005                }),
1006                ..SyncHeader::local()
1007            }
1008            .to_line()
1009        }
1010    }
1011
1012    impl Transport for EmptyPeer {
1013        fn host(&self) -> &str {
1014            "user@pi"
1015        }
1016
1017        fn whoami(&self) -> Result<String> {
1018            Ok("peer".to_string())
1019        }
1020
1021        fn export(
1022            &self,
1023            _after_seq: u64,
1024            _hello: &Hello,
1025        ) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)> {
1026            let stream = format!("{}\n", Self::header()).into_bytes();
1027            Ok((Box::new(std::io::Cursor::new(stream)), Box::new(Exited)))
1028        }
1029
1030        fn import(&self) -> Result<(Box<dyn Write + Send>, Box<dyn Read + Send>, Box<dyn Remote>)> {
1031            let replies = format!(
1032                "{}\n{}\n",
1033                Self::header(),
1034                PushApplied { applied: 0 }.to_line()
1035            );
1036            Ok((
1037                Box::new(std::io::sink()),
1038                Box::new(std::io::Cursor::new(replies.into_bytes())),
1039                Box::new(Exited),
1040            ))
1041        }
1042    }
1043
1044    struct Exited;
1045
1046    impl Remote for Exited {
1047        fn wait(&mut self) -> Result<Exit> {
1048            Ok(Exit {
1049                success: true,
1050                status: "exit status: 0".to_string(),
1051            })
1052        }
1053
1054        fn stop(&mut self) -> Result<()> {
1055            Ok(())
1056        }
1057    }
1058
1059    fn reachable() -> Connect {
1060        Arc::new(|_: &Route| -> Result<Box<dyn Transport + Send>> { Ok(Box::new(EmptyPeer)) })
1061    }
1062
1063    fn refusing() -> Connect {
1064        Arc::new(|_: &Route| -> Result<Box<dyn Transport + Send>> {
1065            anyhow::bail!("connection refused")
1066        })
1067    }
1068
1069    fn watched_loop(dir: &Path, connect: Connect) -> (Arc<SyncLoop>, Arc<Health>) {
1070        let health = Arc::new(Health::new(Utc::now(), None));
1071        let sync_loop = Arc::new(scripted_loop(dir, connect).with_health(health.clone()));
1072        std::fs::write(dir.join(peers::PEERS_FILE), "user@pi\n").unwrap();
1073        assert_eq!(sync_loop.listed_dials(), vec![line_dial("user@pi")]);
1074        (sync_loop, health)
1075    }
1076
1077    fn the_one_peer(health: &Health) -> PeerSync {
1078        let mut peers = health.report().value.sync_peers;
1079        assert_eq!(peers.len(), 1, "{peers:?}");
1080        peers.remove(0)
1081    }
1082
1083    fn run_job(sync_loop: &Arc<SyncLoop>, job: Job, dial: Dial) {
1084        let (jobs, inbox) = mpsc::channel();
1085        jobs.send((job, dial)).unwrap();
1086        drop(jobs);
1087        let worker = {
1088            let sync_loop = sync_loop.clone();
1089            std::thread::spawn(move || sync_loop.work(inbox))
1090        };
1091        let deadline = Instant::now() + Duration::from_secs(10);
1092        while !worker.is_finished() {
1093            assert!(
1094                Instant::now() < deadline,
1095                "the worker never finished its job"
1096            );
1097            std::thread::sleep(Duration::from_millis(10));
1098        }
1099        worker.join().unwrap();
1100    }
1101
1102    struct Answering(&'static str);
1103
1104    impl Transport for Answering {
1105        fn host(&self) -> &str {
1106            "me@192.0.2.1"
1107        }
1108
1109        fn whoami(&self) -> Result<String> {
1110            Ok(self.0.to_string())
1111        }
1112
1113        fn export(
1114            &self,
1115            after_seq: u64,
1116            hello: &Hello,
1117        ) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)> {
1118            EmptyPeer.export(after_seq, hello)
1119        }
1120
1121        fn import(&self) -> Result<(Box<dyn Write + Send>, Box<dyn Read + Send>, Box<dyn Remote>)> {
1122            EmptyPeer.import()
1123        }
1124    }
1125
1126    fn device_dial(machine_id: &str, addresses: &[&str]) -> Dial {
1127        Dial {
1128            key: machine_id.to_string(),
1129            label: machine_id.to_string(),
1130            machine_id: Some(machine_id.to_string()),
1131            routes: addresses
1132                .iter()
1133                .map(|address| Route::Device {
1134                    machine_id: machine_id.to_string(),
1135                    login: "me".to_string(),
1136                    address: address.to_string(),
1137                    port: 22,
1138                })
1139                .collect(),
1140        }
1141    }
1142
1143    /// Answers with the id scripted for each address, recording the
1144    /// addresses dialled.
1145    fn answering(
1146        by_address: &'static [(&'static str, &'static str)],
1147    ) -> (Connect, Arc<Mutex<Vec<String>>>) {
1148        let dialled: Arc<Mutex<Vec<String>>> = Arc::default();
1149        let connect: Connect = {
1150            let dialled = dialled.clone();
1151            Arc::new(move |route: &Route| -> Result<Box<dyn Transport + Send>> {
1152                let Route::Device { address, .. } = route else {
1153                    anyhow::bail!("not a device route")
1154                };
1155                dialled.lock().unwrap().push(address.clone());
1156                let Some((_, id)) = by_address.iter().find(|(a, _)| a == address) else {
1157                    anyhow::bail!("connection refused")
1158                };
1159                Ok(Box::new(Answering(id)))
1160            })
1161        };
1162        (connect, dialled)
1163    }
1164
1165    fn admit_new_device(sync_loop: &SyncLoop, dir: &Path, machine_id: &str) {
1166        let device = load_device_keys(dir).unwrap().unwrap();
1167        let identity_tag = sync_loop
1168            .sqlite
1169            .device_records()
1170            .unwrap()
1171            .into_iter()
1172            .find(|row| row.current.record.machine_id == "m")
1173            .unwrap()
1174            .current
1175            .record
1176            .identity_tag;
1177        let keys = DeviceKeys::generate().unwrap();
1178        let record = DeviceRecord {
1179            machine_id: machine_id.to_string(),
1180            identity_tag,
1181            version: 1,
1182            name: machine_id.to_string(),
1183            age_recipient: keys.age_recipient().to_string(),
1184            ssh_public_key: keys.ssh_public_key(),
1185            signing_public_key: keys.verifying_key().to_bytes(),
1186            recovery_fingerprints: Vec::new(),
1187            ssh_login: None,
1188            ssh_port: None,
1189            ssh_host_key: None,
1190        };
1191        admit_device(&sync_loop.sqlite, &device, record).unwrap();
1192    }
1193
1194    fn revoke_new_device(sync_loop: &SyncLoop, dir: &Path, machine_id: &str) {
1195        admit_new_device(sync_loop, dir, machine_id);
1196        Keyring::load(&sync_loop.sqlite, dir)
1197            .unwrap()
1198            .remove_device(machine_id, &std::collections::BTreeSet::new(), 10)
1199            .unwrap();
1200    }
1201
1202    #[test]
1203    fn a_route_answering_as_another_device_moves_on_to_the_next() {
1204        let dir = tempfile::tempdir().unwrap();
1205        let (connect, dialled) = answering(&[("192.0.2.1", "someone-else"), ("192.0.2.2", "peer")]);
1206        let sync_loop = scripted_loop(dir.path(), connect);
1207        sync_loop
1208            .sync(&device_dial("peer", &["192.0.2.1", "192.0.2.2"]), Job::Full)
1209            .unwrap();
1210        assert_eq!(*dialled.lock().unwrap(), vec!["192.0.2.1", "192.0.2.2"]);
1211    }
1212
1213    #[test]
1214    fn a_route_answering_as_a_revoked_device_stops_the_dial() {
1215        let dir = tempfile::tempdir().unwrap();
1216        let (connect, dialled) = answering(&[("192.0.2.1", "stolen"), ("192.0.2.2", "peer")]);
1217        let sync_loop = scripted_loop(dir.path(), connect);
1218        revoke_new_device(&sync_loop, dir.path(), "stolen");
1219        assert!(
1220            sync_loop
1221                .sync(&device_dial("peer", &["192.0.2.1", "192.0.2.2"]), Job::Full)
1222                .is_err()
1223        );
1224        assert_eq!(*dialled.lock().unwrap(), vec!["192.0.2.1"]);
1225    }
1226
1227    /// A loopback responder advertising each id's sshd as the login `me`.
1228    fn advertising(ids: &[&str]) -> Responder {
1229        let adverts: Vec<_> = ids
1230            .iter()
1231            .map(|id| mdns::device_advert(id, "me", 22))
1232            .collect();
1233        Responder::spawn(
1234            &Net::Unicast(std::net::SocketAddr::from((
1235                std::net::Ipv4Addr::LOCALHOST,
1236                0,
1237            ))),
1238            Arc::new(move || adverts.clone()),
1239        )
1240        .unwrap()
1241    }
1242
1243    fn looking_up(sync_loop: SyncLoop, responder: &Responder) -> SyncLoop {
1244        SyncLoop {
1245            discovery: Some(Net::Unicast(responder.local_addr())),
1246            ..sync_loop
1247        }
1248    }
1249
1250    fn held_addresses(sync_loop: &SyncLoop, machine_id: &str) -> Vec<String> {
1251        sync_loop
1252            .sqlite
1253            .device_route(machine_id)
1254            .unwrap()
1255            .map(|route| route.addresses)
1256            .unwrap_or_default()
1257    }
1258
1259    fn hold_route(sync_loop: &SyncLoop, machine_id: &str, addresses: &[&str]) {
1260        sync_loop
1261            .sqlite
1262            .set_device_route(DeviceRoute {
1263                machine_id: machine_id.to_string(),
1264                addresses: addresses.iter().map(|a| a.to_string()).collect(),
1265                ssh_login: Some("me".to_string()),
1266                ssh_port: None,
1267            })
1268            .unwrap();
1269    }
1270
1271    #[test]
1272    fn a_device_whose_address_changed_is_found_synced_and_recorded() {
1273        let dir = tempfile::tempdir().unwrap();
1274        let (connect, dialled) = answering(&[("127.0.0.1", "peer")]);
1275        let responder = advertising(&["peer"]);
1276        let sync_loop = looking_up(scripted_loop(dir.path(), connect), &responder);
1277        admit_new_device(&sync_loop, dir.path(), "peer");
1278        hold_route(&sync_loop, "peer", &["192.0.2.1"]);
1279        let dials = sync_loop.listed_dials();
1280        assert_eq!(dials.len(), 1, "{dials:?}");
1281
1282        sync_loop.sync(&dials[0], Job::Full).unwrap();
1283        assert_eq!(*dialled.lock().unwrap(), vec!["192.0.2.1", "127.0.0.1"]);
1284        assert_eq!(
1285            held_addresses(&sync_loop, "peer"),
1286            vec!["127.0.0.1", "192.0.2.1"]
1287        );
1288        let Route::Device { address, .. } = &sync_loop.listed_dials()[0].routes[0] else {
1289            panic!("a device is dialled by its device route");
1290        };
1291        assert_eq!(address, "127.0.0.1");
1292    }
1293
1294    #[test]
1295    fn an_address_now_held_by_another_machine_is_looked_past() {
1296        let dir = tempfile::tempdir().unwrap();
1297        let (connect, dialled) = answering(&[("192.0.2.1", "someone-else"), ("127.0.0.1", "peer")]);
1298        let responder = advertising(&["peer"]);
1299        let sync_loop = looking_up(scripted_loop(dir.path(), connect), &responder);
1300        admit_new_device(&sync_loop, dir.path(), "peer");
1301        hold_route(&sync_loop, "peer", &["192.0.2.1"]);
1302        sync_loop
1303            .sync(&sync_loop.listed_dials()[0], Job::Push)
1304            .unwrap();
1305        assert_eq!(*dialled.lock().unwrap(), vec!["192.0.2.1", "127.0.0.1"]);
1306        assert_eq!(held_addresses(&sync_loop, "peer")[0], "127.0.0.1");
1307    }
1308
1309    #[test]
1310    fn a_device_with_no_address_is_looked_up_on_a_full_pass() {
1311        let dir = tempfile::tempdir().unwrap();
1312        let (connect, dialled) = answering(&[("127.0.0.1", "peer")]);
1313        let responder = advertising(&["peer"]);
1314        let sync_loop = looking_up(scripted_loop(dir.path(), connect), &responder);
1315        admit_new_device(&sync_loop, dir.path(), "peer");
1316        let mut dials = sync_loop.listed_dials();
1317        assert_eq!(dials, Vec::<Dial>::new());
1318
1319        sync_loop.add_found(&mut dials);
1320        assert_eq!(dials.len(), 1, "{dials:?}");
1321        assert_eq!(dials[0].routes[0].describe(), "me@127.0.0.1");
1322        sync_loop.sync(&dials[0], Job::Full).unwrap();
1323        assert_eq!(*dialled.lock().unwrap(), vec!["127.0.0.1"]);
1324        assert_eq!(held_addresses(&sync_loop, "peer"), vec!["127.0.0.1"]);
1325        assert_eq!(sync_loop.listed_dials().len(), 1);
1326    }
1327
1328    #[test]
1329    fn a_responder_under_an_unknown_or_revoked_id_is_never_dialled() {
1330        let dir = tempfile::tempdir().unwrap();
1331        let (connect, dialled) = answering(&[]);
1332        let responder = advertising(&["stranger", "stolen"]);
1333        let sync_loop = looking_up(scripted_loop(dir.path(), connect), &responder);
1334        revoke_new_device(&sync_loop, dir.path(), "stolen");
1335        admit_new_device(&sync_loop, dir.path(), "peer");
1336        hold_route(&sync_loop, "peer", &["192.0.2.1"]);
1337
1338        let mut dials = sync_loop.listed_dials();
1339        assert!(sync_loop.sync(&dials[0], Job::Full).is_err());
1340        assert_eq!(*dialled.lock().unwrap(), vec!["192.0.2.1"]);
1341        sync_loop.add_found(&mut dials);
1342        assert_eq!(
1343            dials.iter().map(|d| d.key.as_str()).collect::<Vec<_>>(),
1344            vec!["peer"]
1345        );
1346        assert_eq!(held_addresses(&sync_loop, "peer"), vec!["192.0.2.1"]);
1347        assert_eq!(held_addresses(&sync_loop, "stolen"), Vec::<String>::new());
1348        assert_eq!(held_addresses(&sync_loop, "stranger"), Vec::<String>::new());
1349    }
1350
1351    #[test]
1352    fn a_full_sync_reports_the_peer_synced() {
1353        let dir = tempfile::tempdir().unwrap();
1354        let (sync_loop, health) = watched_loop(dir.path(), reachable());
1355        let before = Utc::now();
1356        run_job(&sync_loop, Job::Full, line_dial("user@pi"));
1357        let peer = the_one_peer(&health);
1358        assert_eq!(peer.peer, "user@pi");
1359        let synced = peer.last_success_at.expect("a full sync is a success");
1360        assert!(synced >= before, "{synced} predates the sync");
1361        assert_eq!(peer.consecutive_failures, 0);
1362    }
1363
1364    #[test]
1365    fn a_failed_sync_counts_a_failure_and_no_success() {
1366        let dir = tempfile::tempdir().unwrap();
1367        let (sync_loop, health) = watched_loop(dir.path(), refusing());
1368        run_job(&sync_loop, Job::Full, line_dial("user@pi"));
1369        let peer = the_one_peer(&health);
1370        assert_eq!(peer.peer, "user@pi");
1371        assert_eq!(peer.consecutive_failures, 1);
1372        assert_eq!(peer.last_success_at, None);
1373        assert!(peer.last_attempt_at.is_some(), "the failure is an attempt");
1374        let error = peer.last_error.expect("the failure's text is kept");
1375        assert!(error.contains("connection refused"), "{error}");
1376    }
1377
1378    #[test]
1379    fn start_deletes_the_plaintext_ssh_key_once_the_device_key_loads() {
1380        let dir = tempfile::tempdir().unwrap();
1381        let sqlite = SqliteStore::in_memory("m".to_string()).unwrap();
1382        second_brain_core::keys::ensure_device_keys(&sqlite, dir.path()).unwrap();
1383        let keys = load_device_keys(dir.path()).unwrap().unwrap();
1384        let legacy = dir.path().join(SSH_KEY_FILE);
1385        write_identity_file(&legacy, keys.ssh_signing_key(), "second-brain:m").unwrap();
1386        let before = std::fs::read(&legacy).unwrap();
1387
1388        assert!(retire_plaintext_ssh_key(dir.path()).unwrap());
1389        assert!(!legacy.exists());
1390        assert!(!retire_plaintext_ssh_key(dir.path()).unwrap());
1391
1392        let reloaded = load_device_keys(dir.path()).unwrap().unwrap();
1393        let elsewhere = tempfile::tempdir().unwrap();
1394        let rederived = elsewhere.path().join("id");
1395        write_identity_file(&rederived, reloaded.ssh_signing_key(), "second-brain:m").unwrap();
1396        assert_eq!(std::fs::read(&rederived).unwrap(), before);
1397    }
1398
1399    #[test]
1400    fn a_plaintext_ssh_key_stays_while_no_device_key_loads() {
1401        let dir = tempfile::tempdir().unwrap();
1402        let legacy = dir.path().join(SSH_KEY_FILE);
1403        let stray = DeviceKeys::generate().unwrap();
1404        write_identity_file(&legacy, stray.ssh_signing_key(), "second-brain:m").unwrap();
1405        assert!(!retire_plaintext_ssh_key(dir.path()).unwrap());
1406        assert!(legacy.exists());
1407
1408        std::fs::write(
1409            dir.path().join(second_brain_core::keys::DEVICE_KEY_FILE),
1410            b"torn",
1411        )
1412        .unwrap();
1413        assert!(retire_plaintext_ssh_key(dir.path()).is_err());
1414        assert!(legacy.exists());
1415    }
1416
1417    #[test]
1418    fn a_push_from_a_device_without_an_identity_is_not_held_against_the_peer() {
1419        let dir = tempfile::tempdir().unwrap();
1420        let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
1421        let connect: Connect = {
1422            let calls = calls.clone();
1423            Arc::new(move |_: &Route| -> Result<Box<dyn Transport + Send>> {
1424                calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
1425                anyhow::bail!("connection refused")
1426            })
1427        };
1428        let sqlite = SqliteStore::in_memory("m".to_string()).unwrap();
1429        second_brain_core::keys::ensure_device_keys(&sqlite, dir.path()).unwrap();
1430        let health = Arc::new(Health::new(Utc::now(), None));
1431        let sync_loop = Arc::new(SyncLoop {
1432            data_dir: dir.path().to_path_buf(),
1433            sqlite: Arc::new(sqlite),
1434            gate: Arc::new(WriteGate::default()),
1435            clock: ClockGate::open_for_tests(),
1436            health: health.clone(),
1437            access: Arc::default(),
1438            bootstrap: Arc::default(),
1439            connect,
1440            discovery: None,
1441        });
1442        std::fs::write(dir.path().join(peers::PEERS_FILE), "user@pi\n").unwrap();
1443        sync_loop.listed_dials();
1444
1445        run_job(&sync_loop, Job::Push, line_dial("user@pi"));
1446        assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 0);
1447        let peer = the_one_peer(&health);
1448        assert_eq!(peer.last_attempt_at, None);
1449        assert_eq!(peer.consecutive_failures, 0);
1450
1451        run_job(&sync_loop, Job::Full, line_dial("user@pi"));
1452        assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 1);
1453        assert_eq!(the_one_peer(&health).consecutive_failures, 1);
1454    }
1455
1456    #[test]
1457    fn a_push_ends_a_failure_streak_without_counting_as_a_sync() {
1458        let dir = tempfile::tempdir().unwrap();
1459        let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
1460        let connect: Connect = {
1461            let calls = calls.clone();
1462            Arc::new(move |_: &Route| -> Result<Box<dyn Transport + Send>> {
1463                if calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0 {
1464                    anyhow::bail!("connection refused");
1465                }
1466                Ok(Box::new(EmptyPeer))
1467            })
1468        };
1469        let (sync_loop, health) = watched_loop(dir.path(), connect);
1470        run_job(&sync_loop, Job::Full, line_dial("user@pi"));
1471        assert_eq!(the_one_peer(&health).consecutive_failures, 1);
1472        run_job(&sync_loop, Job::Push, line_dial("user@pi"));
1473        let peer = the_one_peer(&health);
1474        assert_eq!(peer.last_success_at, None);
1475        assert!(peer.last_attempt_at.is_some());
1476        assert_eq!(peer.consecutive_failures, 0);
1477    }
1478
1479    #[test]
1480    fn maintenance_reports_no_sync_outcome() {
1481        let dir = tempfile::tempdir().unwrap();
1482        let connect: Connect = Arc::new(|_: &Route| -> Result<Box<dyn Transport + Send>> {
1483            Err(anyhow::Error::new(MaintenanceRunning))
1484        });
1485        let (sync_loop, health) = watched_loop(dir.path(), connect);
1486        run_job(&sync_loop, Job::Full, line_dial("user@pi"));
1487        let peer = the_one_peer(&health);
1488        assert_eq!(peer.last_success_at, None);
1489        assert_eq!(peer.last_attempt_at, None);
1490        assert_eq!(peer.consecutive_failures, 0);
1491    }
1492
1493    #[test]
1494    fn a_peer_removed_from_sync_peers_drops_out_of_health() {
1495        let dir = tempfile::tempdir().unwrap();
1496        let (sync_loop, health) = watched_loop(dir.path(), refusing());
1497        let listed = |text: &str| {
1498            std::fs::write(dir.path().join(peers::PEERS_FILE), text).unwrap();
1499            let kept = sync_loop.listed_dials();
1500            let mut reported: Vec<String> = health
1501                .report()
1502                .value
1503                .sync_peers
1504                .into_iter()
1505                .map(|peer| peer.peer)
1506                .collect();
1507            reported.sort();
1508            (kept.len(), reported)
1509        };
1510        assert_eq!(
1511            listed("user@pi\nuser@mac\nuser@pi /opt/sb\n"),
1512            (2, vec!["user@mac".to_string(), "user@pi".to_string()])
1513        );
1514        assert_eq!(listed("user@mac\n"), (1, vec!["user@mac".to_string()]));
1515    }
1516
1517    #[test]
1518    fn an_unreadable_sync_peers_keeps_its_peers_in_health() {
1519        let dir = tempfile::tempdir().unwrap();
1520        let (sync_loop, health) = watched_loop(dir.path(), refusing());
1521        let file = dir.path().join(peers::PEERS_FILE);
1522        std::fs::remove_file(&file).unwrap();
1523        std::fs::create_dir(&file).unwrap();
1524        assert!(peers::configured_peers(dir.path()).is_err());
1525
1526        assert_eq!(sync_loop.listed_dials(), Vec::<Dial>::new());
1527        assert_eq!(the_one_peer(&health).peer, "user@pi");
1528    }
1529}