use std::collections::HashMap;
use std::io::Read;
use std::path::{Path, PathBuf};
use std::sync::mpsc::{self, Receiver, SendError, Sender};
use std::sync::{Arc, Mutex, PoisonError};
use std::thread::JoinHandle;
use std::time::{Duration, Instant};
use anyhow::{Context, Result};
use chrono::Utc;
use second_brain_core::keys::{DeviceKeys, load_device_keys};
use second_brain_core::schema::SyncState;
use second_brain_core::sqlite_store::SqliteStore;
use second_brain_core::store::StoreAdmin;
use second_brain_sync::crypto::seal_line;
use second_brain_sync::dial::{self, Dial, Route};
use second_brain_sync::export::export_page;
use second_brain_sync::keyring::Keyring;
use second_brain_sync::mdns::{self, Net};
use second_brain_sync::protocol::{Hello, Merging, SyncHeader, check_store_header, store_header};
use second_brain_sync::session::{self, Exported, Imported, Local, Merged, Transport};
use second_brain_sync::ssh::{SshTransport, device_alias, known_hosts_path_in};
use second_brain_sync::ssh_identity::{EphemeralIdentity, sweep_ephemeral_identities};
use crate::bootstrap::Bootstrap;
use crate::clock_gate::{ClockGate, is_untrusted};
use crate::device_access::DeviceAccess;
use crate::health::Health;
use crate::snapshot_merge;
use crate::sync_hello;
use crate::value_metrics::PeerOutcome;
use crate::write_gate::{WriteGate, WriteSlot};
const FULL_INTERVAL: Duration = Duration::from_secs(300);
const TICK: Duration = Duration::from_secs(1);
const PUSH_DELAY: Duration = Duration::from_secs(5);
const FIRST_BACKOFF: Duration = Duration::from_secs(30);
const MAX_BACKOFF: Duration = Duration::from_secs(30 * 60);
const ORIGIN_SCAN_PAGE: usize = 500;
pub type Connect = Arc<dyn Fn(&Route) -> Result<Box<dyn Transport + Send>> + Send + Sync>;
pub const SSH_KEY_FILE: &str = "sync-ssh-key";
pub struct SyncLoop {
pub data_dir: PathBuf,
pub sqlite: Arc<SqliteStore>,
pub gate: Arc<WriteGate>,
pub clock: Arc<ClockGate>,
pub health: Arc<Health>,
pub access: Arc<DeviceAccess>,
pub bootstrap: Arc<Bootstrap>,
pub connect: Connect,
pub discovery: Option<Net>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
enum Job {
Push,
Full,
}
impl SyncLoop {
pub fn over_ssh(
data_dir: PathBuf,
sqlite: Arc<SqliteStore>,
gate: Arc<WriteGate>,
clock: Arc<ClockGate>,
health: Arc<Health>,
access: Arc<DeviceAccess>,
bootstrap: Arc<Bootstrap>,
) -> Self {
if let Err(e) = sweep_ephemeral_identities(&data_dir) {
tracing::warn!("removing ssh key files a killed daemon left: {e:#}");
}
match retire_plaintext_ssh_key(&data_dir) {
Ok(true) => tracing::info!("removed the plaintext {SSH_KEY_FILE}"),
Ok(false) => {}
Err(e) => tracing::warn!("kept the plaintext {SSH_KEY_FILE}: {e:#}"),
}
let connect = ssh_connect(data_dir.clone(), sqlite.machine_id().to_string());
Self {
data_dir,
sqlite,
gate,
clock,
health,
access,
bootstrap,
connect,
discovery: None,
}
}
pub fn with_health(mut self, health: Arc<Health>) -> Self {
self.health = health;
self
}
pub fn with_discovery(mut self, discovery: Net) -> Self {
self.discovery = Some(discovery);
self
}
pub fn spawn(self) -> JoinHandle<()> {
std::thread::Builder::new()
.name("sync-loop".to_string())
.spawn(move || Arc::new(self).run())
.expect("spawning the sync loop")
}
fn run(self: Arc<Self>) {
let mut workers: HashMap<String, Sender<(Job, Dial)>> = HashMap::new();
let mut next_full = Instant::now();
let mut seen = self.sqlite.max_sync_seq().unwrap_or(0);
let mut push_at: Option<Instant> = None;
loop {
let dials = self.listed_dials();
if dials.is_empty() {
seen = self.sqlite.max_sync_seq().unwrap_or(seen);
push_at = None;
} else if self.wrote_locally(&mut seen) {
push_at.get_or_insert(Instant::now() + PUSH_DELAY);
}
let now = Instant::now();
if self.access.take_kick() {
next_full = now;
}
let job = if now >= next_full {
next_full = now + FULL_INTERVAL;
Some(Job::Full)
} else if push_at.is_some_and(|at| now >= at) {
Some(Job::Push)
} else {
None
};
if let Some(job) = job {
push_at = None;
let mut dials = dials;
if job == Job::Full {
self.add_found(&mut dials);
}
for dial in dials {
self.dispatch(&mut workers, job, dial);
}
}
std::thread::sleep(TICK);
}
}
fn dispatch(
self: &Arc<Self>,
workers: &mut HashMap<String, Sender<(Job, Dial)>>,
job: Job,
dial: Dial,
) {
let key = dial.key.clone();
let worker = workers
.entry(key.clone())
.or_insert_with(|| self.clone().start_worker(&key));
if let Err(SendError(message)) = worker.send((job, dial)) {
tracing::warn!("the sync worker for {key} exited; starting another");
let worker = self.clone().start_worker(&key);
if worker.send(message).is_err() {
tracing::warn!("the new sync worker for {key} exited at once");
}
workers.insert(key, worker);
}
}
fn listed_dials(&self) -> Vec<Dial> {
let mut dials = match dial::device_dials(&self.sqlite) {
Ok(dials) => dials,
Err(e) => {
tracing::warn!("cannot read the devices to sync with: {e:#}");
return Vec::new();
}
};
match dial::line_dials(&self.sqlite, &self.data_dir) {
Ok(lines) => {
dials.extend(lines);
let labels: Vec<String> = dials.iter().map(|dial| dial.label.clone()).collect();
self.health.sync_peers_listed(&labels, Utc::now());
}
Err(e) => tracing::warn!(
"cannot read {}: {e:#}",
second_brain_sync::peers::PEERS_FILE
),
}
dials
}
fn report(&self, label: &str, outcome: PeerOutcome) {
self.health.sync_attempted(label, outcome, Utc::now());
}
fn wrote_locally(&self, seen: &mut u64) -> bool {
let max = match self.sqlite.max_sync_seq() {
Ok(max) => max,
Err(e) => {
tracing::warn!("cannot read the sync log position: {e:#}");
return false;
}
};
let own = self.sqlite.machine_id();
while *seen < max {
let page = match self.sqlite.sync_log_page(*seen, Some(ORIGIN_SCAN_PAGE)) {
Ok(page) => page,
Err(e) => {
tracing::warn!("cannot read the sync log: {e:#}");
return false;
}
};
let Some(last) = page.last() else {
break;
};
if page.iter().any(|entry| entry.origin_machine_id == own) {
*seen = max;
return true;
}
*seen = last.local_seq.max(0) as u64;
}
*seen = max;
false
}
fn start_worker(self: Arc<Self>, key: &str) -> Sender<(Job, Dial)> {
let (jobs, inbox) = mpsc::channel();
std::thread::Builder::new()
.name(format!("sync-{key}"))
.spawn(move || self.work(inbox))
.expect("spawning a sync worker");
jobs
}
fn work(&self, inbox: Receiver<(Job, Dial)>) {
let mut backoff = PeerBackoff::default();
while let Ok((mut job, mut dial)) = inbox.recv() {
while let Ok((queued, latest)) = inbox.try_recv() {
job = job.max(queued);
dial = latest;
}
if !backoff.due(&dial, Instant::now()) {
continue;
}
if job == Job::Push && !self.holds_identity() {
tracing::info!(
"not pushing to {}: this device holds no sync identity yet; pair it with \
`sb pair`",
dial.label
);
continue;
}
match self.sync(&dial, job) {
Ok(()) => {
backoff.succeeded();
let outcome = match job {
Job::Full => PeerOutcome::Synced,
Job::Push => PeerOutcome::Pushed,
};
self.report(&dial.label, outcome);
}
Err(e) if !counts_against_peer(&e) => {
tracing::info!("sync with {} waits for the next job: {e:#}", dial.label);
}
Err(e) => {
let wait = backoff.failed(&dial, Instant::now());
self.report(&dial.label, PeerOutcome::Failed(format!("{e:#}")));
tracing::warn!(
"sync with {} failed; its jobs are skipped for the next {}s: {e:#}",
dial.label,
wait.as_secs()
);
}
}
}
}
fn holds_identity(&self) -> bool {
match self.sqlite.device_record(self.sqlite.machine_id()) {
Ok(row) => row.is_some(),
Err(e) => {
tracing::warn!("cannot read this device's sync identity row: {e:#}");
true
}
}
}
fn sync(&self, dial: &Dial, job: Job) -> Result<()> {
let failed = match self.sync_via(dial, &dial.routes, job)? {
Reached::Synced => return Ok(()),
Reached::Not(failed) => failed,
};
let found: Vec<Route> = match &dial.machine_id {
Some(machine_id) => self
.lookup()
.into_iter()
.filter(|found| found.key == *machine_id)
.flat_map(|found| found.routes)
.filter(|route| !dial.routes.contains(route))
.collect(),
None => Vec::new(),
};
let found_failed = if found.is_empty() {
None
} else {
match self.sync_via(dial, &found, job)? {
Reached::Synced => return Ok(()),
Reached::Not(failed) => failed,
}
};
Err(failed
.or(found_failed)
.unwrap_or_else(|| anyhow::anyhow!("{} has no route to dial", dial.label)))
}
fn add_found(&self, dials: &mut Vec<Dial>) {
if self.discovery.is_none() {
return;
}
match dial::undialled(&self.sqlite, dials) {
Ok(true) => {}
Ok(false) => return,
Err(e) => {
tracing::warn!("cannot read the devices to sync with: {e:#}");
return;
}
}
for dial in self.lookup() {
if !dials.iter().any(|listed| listed.key == dial.key) {
dials.push(dial);
}
}
}
fn lookup(&self) -> Vec<Dial> {
let Some(net) = &self.discovery else {
return Vec::new();
};
let found = match mdns::browse(net, mdns::SERVICE, mdns::LOOKUP_WINDOW) {
Ok(found) => found,
Err(e) => {
tracing::info!("cannot look devices up on the network: {e:#}");
return Vec::new();
}
};
dial::found_dials(&self.sqlite, &found).unwrap_or_else(|e| {
tracing::warn!("cannot read the devices to match a lookup against: {e:#}");
Vec::new()
})
}
fn sync_via(&self, dial: &Dial, routes: &[Route], job: Job) -> Result<Reached> {
let mut failed: Option<anyhow::Error> = None;
for route in routes {
let transport = match (self.connect)(route) {
Ok(transport) => transport,
Err(e) if !counts_against_peer(&e) => return Err(e),
Err(e) => {
failed = Some(e.context(format!("reaching {}", route.describe())));
continue;
}
};
let answered = match transport.whoami() {
Ok(answered) => answered,
Err(e) => {
failed = Some(e.context(format!(
"resolving the machine id at {} (is its daemon running?)",
route.describe()
)));
continue;
}
};
if let Route::Line(peer) = route
&& let Some(_slot) = self.gate.try_enter()
{
dial::line_answered(&self.sqlite, peer, &answered)?;
}
if let Err(e) = dial::refuse_answer(&self.sqlite, dial, &answered) {
if dial.machine_id.is_some() && !self.is_revoked(&answered) {
failed = Some(e);
continue;
}
return Err(e);
}
self.record_answered(route);
return self
.sync_through(transport.as_ref(), &answered, job)
.map(|()| Reached::Synced);
}
Ok(Reached::Not(failed))
}
fn record_answered(&self, route: &Route) {
let Some(_slot) = self.gate.try_enter() else {
return;
};
match dial::record_answered(&self.sqlite, route) {
Ok(true) => tracing::info!("recorded {} as a route to its device", route.describe()),
Ok(false) => {}
Err(e) => tracing::warn!("cannot record the route {}: {e:#}", route.describe()),
}
}
fn is_revoked(&self, machine_id: &str) -> bool {
self.sqlite
.device_record(machine_id)
.ok()
.flatten()
.is_some_and(|row| row.revoked)
}
fn sync_through(&self, transport: &dyn Transport, peer: &str, job: Job) -> Result<()> {
let local = StoreLocal {
sqlite: &self.sqlite,
gate: &self.gate,
clock: &self.clock,
health: &self.health,
access: &self.access,
bootstrap: &self.bootstrap,
data_dir: &self.data_dir,
};
let host = transport.host().to_string();
let report = |line: &str| tracing::info!("sync with {host}: {}", line.trim());
match job {
Job::Full => session::sync_with(&local, transport, peer, &self.data_dir, &report),
Job::Push => session::push(&local, transport, peer, &report).map(|_| ()),
}
}
}
fn ssh_connect(data_dir: PathBuf, own: String) -> Connect {
let device_keys: Arc<Mutex<Option<DeviceKeys>>> = Arc::default();
Arc::new(move |route: &Route| -> Result<Box<dyn Transport + Send>> {
let transport = match route {
Route::Device {
machine_id,
login,
address,
port,
} => {
let identity = device_identity(&data_dir, &own, &device_keys)?;
SshTransport::new(&format!("{login}@{address}"), "sb")?
.known_hosts(known_hosts_path_in(&data_dir))
.port(*port)
.host_key_alias(&device_alias(machine_id))
.ephemeral_identity(identity)
}
Route::Line(peer) => SshTransport::new(&peer.host, &peer.sb)?,
};
Ok(Box::new(transport.unattended()) as Box<dyn Transport + Send>)
})
}
fn device_identity(
data_dir: &Path,
own: &str,
cached: &Mutex<Option<DeviceKeys>>,
) -> Result<EphemeralIdentity> {
let mut cached = cached.lock().unwrap_or_else(PoisonError::into_inner);
if cached.is_none() {
let keys = load_device_keys(data_dir)?
.context("this device has no device key to sign in to its peers with")?;
*cached = Some(keys);
}
let keys = cached.as_ref().expect("the keys were loaded above");
EphemeralIdentity::create(
data_dir,
keys.ssh_signing_key(),
&format!("second-brain:{own}"),
)
}
fn retire_plaintext_ssh_key(data_dir: &Path) -> Result<bool> {
let path = data_dir.join(SSH_KEY_FILE);
if std::fs::symlink_metadata(&path).is_err() {
return Ok(false);
}
if load_device_keys(data_dir)?.is_none() {
return Ok(false);
}
std::fs::remove_file(&path).with_context(|| format!("removing {}", path.display()))?;
Ok(true)
}
#[derive(Debug)]
struct MaintenanceRunning;
impl std::fmt::Display for MaintenanceRunning {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("store maintenance is running")
}
}
impl std::error::Error for MaintenanceRunning {}
fn counts_against_peer(error: &anyhow::Error) -> bool {
error.downcast_ref::<MaintenanceRunning>().is_none() && !is_untrusted(error)
}
enum Reached {
Synced,
Not(Option<anyhow::Error>),
}
#[derive(Default)]
struct PeerBackoff {
dial: Option<Dial>,
failures: u32,
retry_at: Option<Instant>,
}
impl PeerBackoff {
fn due(&self, dial: &Dial, now: Instant) -> bool {
self.dial.as_ref() != Some(dial) || self.retry_at.is_none_or(|at| now >= at)
}
fn succeeded(&mut self) {
*self = PeerBackoff::default();
}
fn failed(&mut self, dial: &Dial, now: Instant) -> Duration {
if self.dial.as_ref() != Some(dial) {
*self = PeerBackoff {
dial: Some(dial.clone()),
..PeerBackoff::default()
};
}
self.failures += 1;
let wait = backoff(self.failures);
self.retry_at = Some(now + wait);
wait
}
}
fn backoff(failures: u32) -> Duration {
FIRST_BACKOFF
.saturating_mul(2u32.saturating_pow(failures.saturating_sub(1)))
.min(MAX_BACKOFF)
}
struct StoreLocal<'a> {
sqlite: &'a Arc<SqliteStore>,
gate: &'a WriteGate,
clock: &'a ClockGate,
health: &'a Health,
access: &'a DeviceAccess,
bootstrap: &'a Bootstrap,
data_dir: &'a std::path::Path,
}
impl StoreLocal<'_> {
fn write_slot(&self) -> Result<WriteSlot<'_>> {
self.gate
.try_enter()
.ok_or_else(|| anyhow::Error::new(MaintenanceRunning))
}
}
impl Local for StoreLocal<'_> {
fn cursors(&self) -> Result<Vec<(String, u64)>> {
Ok(self
.sqlite
.get_all_sync_states()?
.into_iter()
.map(|state| (state.peer_id, state.last_seq))
.collect())
}
fn set_cursor(&self, peer_id: &str, last_seq: u64) -> Result<()> {
let _slot = self.write_slot()?;
self.sqlite.set_sync_state(&SyncState {
peer_id: peer_id.to_string(),
last_seq,
last_sync_at: chrono::Utc::now(),
})?;
Ok(())
}
fn import(&self, header: &SyncHeader, lines: &[String]) -> Result<Imported> {
let _slot = self.write_slot()?;
check_store_header(self.sqlite, Some(header), "the sender")?;
let records = Keyring::load(self.sqlite, self.data_dir)?
.open_lines(header.hello.as_ref().map(|h| h.machine_id.as_str()), lines)?
.iter()
.map(|opened| Ok(serde_json::from_slice(opened)?))
.collect::<Result<Vec<serde_json::Value>>>()?;
let source = header.hello.as_ref().map(|h| h.machine_id.as_str());
let applied = crate::sync_import::import_from(self.sqlite, source, &records)?;
let stats = applied.stats;
Ok(Imported {
created: stats.created,
updated: stats.updated,
deleted: stats.deleted,
skipped: stats.skipped,
dangling: Imported::dangling_in(&stats.skipped_by_reason),
errors: stats.errors,
max_seq: applied.max_seq,
reached: applied.reached,
})
}
fn export(&self, peer_id: &str, after_seq: u64, limit: usize) -> Result<Exported> {
store_header(self.sqlite)?;
let (records, max_seq) = export_page(self.sqlite.as_ref(), after_seq, limit)?;
let sealer = Keyring::load(self.sqlite, self.data_dir)?.sealer_for(peer_id, None)?;
let lines = records
.iter()
.map(|record| seal_line(&sealer, &serde_json::to_vec(record)?))
.collect::<Result<Vec<String>>>()?;
Ok(Exported { lines, max_seq })
}
fn hello_for(&self, peer_id: &str) -> Result<Hello> {
sync_hello::hello_for(self.sqlite, self.clock, self.data_dir, peer_id, None)
}
fn accept_hello(&self, hello: &Hello) -> Result<()> {
sync_hello::check_identity(self.sqlite, self.data_dir, hello)?;
let _slot = self.write_slot()?;
sync_hello::accept_handshake(
self.sqlite,
self.clock,
self.health,
self.access,
self.data_dir,
hello,
)
}
fn record_ack(&self, peer_id: &str, acked_seq: u64) -> Result<()> {
let _slot = self.write_slot()?;
self.sqlite
.record_peer_ack(peer_id, acked_seq, self.clock.wall_micros())
}
fn record_peer_in(&self, peer_id: &str, seq: u64) -> Result<()> {
let _slot = self.write_slot()?;
self.sqlite.record_peer_in(peer_id, seq)
}
fn push_report(&self, peer_id: &str) -> Result<Option<u64>> {
self.sqlite.push_report(peer_id)
}
fn record_push_report(&self, peer_id: &str, reported: Option<u64>) -> Result<()> {
let _slot = self.write_slot()?;
self.sqlite.record_push_report(peer_id, reported)
}
fn merging(&self) -> Result<Option<Merging>> {
snapshot_merge::merging(self.data_dir)
}
fn merge_snapshot(&self, exporter: &str, stream: Box<dyn Read + Send>) -> Result<Merged> {
let merging = snapshot_merge::receive(self.sqlite, self.data_dir, stream, Some(exporter))?;
match snapshot_merge::merge_spooled(self.sqlite, self.data_dir, self.gate)? {
Some(_) => Ok(Merged::Done { seq: merging.seq }),
None => Ok(Merged::Running(merging)),
}
}
fn bootstrap_pending(&self) -> Result<bool> {
Ok(self.bootstrap.is_pending())
}
fn finish_bootstrap(&self) -> Result<()> {
let _slot = self.write_slot()?;
if self.bootstrap.finish(self.sqlite)? {
tracing::info!("bootstrap finished; ingesting this device's own sessions");
}
Ok(())
}
fn export_snapshot(&self, peer: &Hello) -> Result<Box<dyn Read + Send>> {
snapshot_merge::pin_for_snapshot(self.sqlite, self.gate, peer)?;
let (reader, writer) = std::io::pipe()?;
let (sqlite, data_dir, peer) = (
self.sqlite.clone(),
self.data_dir.to_path_buf(),
peer.clone(),
);
std::thread::Builder::new()
.name("snapshot-export".to_string())
.spawn(move || {
let out = std::io::BufWriter::with_capacity(1024 * 1024, writer);
if let Err(e) = snapshot_merge::stream_snapshot(&sqlite, &data_dir, &peer, out) {
tracing::warn!("exporting a snapshot for {} failed: {e:#}", peer.machine_id);
}
})?;
Ok(Box::new(reader))
}
}
#[cfg(test)]
mod tests {
use std::io::{Read, Write};
use std::path::Path;
use std::sync::Mutex;
use anyhow::Context;
use second_brain_core::keys::{
DeviceKeys, DeviceRecord, DeviceRoute, Epoch, IdentityKey, IdentityTag, KeyGuard,
admit_device,
};
use second_brain_core::sqlite_store::SCHEMA_VERSION;
use second_brain_sync::crypto::XChaChaEncryptor;
use second_brain_sync::keyring::{LEGACY_KEY_FILE, migrate_sync_key};
use second_brain_sync::mdns::Responder;
use second_brain_sync::peers::{self, Peer};
use second_brain_sync::protocol::HandshakeKeys;
use second_brain_sync::protocol::PushApplied;
use second_brain_sync::session::{Exit, Remote};
use second_brain_sync::ssh_identity::write_identity_file;
use super::*;
use crate::value_metrics::PeerSync;
fn line_dial(line: &str) -> Dial {
let peer = Peer::parse(line);
Dial {
key: peer.host.clone(),
label: peer.host.clone(),
machine_id: None,
routes: vec![Route::Line(peer)],
}
}
fn fresh_health() -> Arc<Health> {
Arc::new(Health::new(chrono::Utc::now(), None))
}
#[test]
fn the_daemon_refuses_to_export_from_a_database_newer_than_its_binary() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("future.db");
drop(SqliteStore::open(&path, "m".to_string()).unwrap());
rusqlite::Connection::open(&path)
.unwrap()
.pragma_update(None, "user_version", SCHEMA_VERSION + 1)
.unwrap();
let sqlite = Arc::new(SqliteStore::open(&path, "m".to_string()).unwrap());
let gate = WriteGate::default();
let clock = ClockGate::open_for_tests();
let health = fresh_health();
let local = StoreLocal {
sqlite: &sqlite,
gate: &gate,
clock: &clock,
health: &health,
access: &DeviceAccess::default(),
bootstrap: &Bootstrap::default(),
data_dir: dir.path(),
};
let error = local.export("peer", 0, 10).err().expect("export refused");
assert!(
format!("{error:#}").contains("upgrade this daemon"),
"{error:#}"
);
}
#[test]
fn a_host_keeps_one_worker_when_its_sb_path_changes() {
let dir = tempfile::tempdir().unwrap();
let sync_loop = Arc::new(SyncLoop {
data_dir: dir.path().to_path_buf(),
sqlite: Arc::new(SqliteStore::in_memory("m".to_string()).unwrap()),
gate: Arc::new(WriteGate::default()),
clock: ClockGate::open_for_tests(),
health: fresh_health(),
access: Arc::default(),
bootstrap: Arc::default(),
connect: Arc::new(|_: &Route| -> Result<Box<dyn Transport + Send>> {
anyhow::bail!("offline")
}),
discovery: None,
});
let mut workers = HashMap::new();
sync_loop.dispatch(&mut workers, Job::Push, line_dial("user@pi"));
sync_loop.dispatch(&mut workers, Job::Push, line_dial("user@pi /opt/sb"));
assert_eq!(workers.len(), 1);
}
const SYNC_KEY: [u8; 32] = [4; 32];
fn shared_identity() -> HandshakeKeys {
let key = IdentityKey::from_bytes(KeyGuard::new(SYNC_KEY));
HandshakeKeys {
identity_tag: Some(IdentityTag::migrated(&key.key_id())),
epochs: vec![Epoch {
number: 0,
key_id: key.key_id(),
}],
..HandshakeKeys::default()
}
}
fn upgraded_store(dir: &Path) -> SqliteStore {
let sqlite = SqliteStore::in_memory("m".to_string()).unwrap();
second_brain_core::keys::ensure_device_keys(&sqlite, dir).unwrap();
std::fs::write(
dir.join(LEGACY_KEY_FILE),
XChaChaEncryptor::new(SYNC_KEY).key_base64(),
)
.unwrap();
migrate_sync_key(&sqlite, dir, "m", Utc::now().timestamp_micros()).unwrap();
sqlite
}
fn scripted_loop(dir: &Path, connect: Connect) -> SyncLoop {
SyncLoop {
data_dir: dir.to_path_buf(),
sqlite: Arc::new(upgraded_store(dir)),
gate: Arc::new(WriteGate::default()),
clock: ClockGate::open_for_tests(),
health: fresh_health(),
access: Arc::default(),
bootstrap: Arc::default(),
connect,
discovery: None,
}
}
#[test]
fn the_worker_takes_the_latest_line_and_backs_off_for_all_but_maintenance() {
let dir = tempfile::tempdir().unwrap();
let calls: Arc<Mutex<Vec<Route>>> = Arc::default();
let connect: Connect = {
let calls = calls.clone();
Arc::new(move |route: &Route| -> Result<Box<dyn Transport + Send>> {
let mut calls = calls.lock().unwrap();
calls.push(route.clone());
if calls.len() == 1 {
Err(anyhow::Error::new(MaintenanceRunning))
} else {
anyhow::bail!("connection refused")
}
})
};
let sync_loop = Arc::new(scripted_loop(dir.path(), connect));
let made = |n: usize| {
let deadline = Instant::now() + Duration::from_secs(5);
while Instant::now() < deadline {
if calls.lock().unwrap().len() >= n {
return true;
}
std::thread::sleep(Duration::from_millis(10));
}
false
};
let bare = line_dial("user@pi");
let fixed = line_dial("user@pi /opt/sb");
let (jobs, inbox) = mpsc::channel();
jobs.send((Job::Push, line_dial("user@pi /old/sb")))
.unwrap();
jobs.send((Job::Full, bare.clone())).unwrap();
let worker = {
let sync_loop = sync_loop.clone();
std::thread::spawn(move || sync_loop.work(inbox))
};
assert!(made(1), "the queued jobs never ran");
jobs.send((Job::Full, bare.clone())).unwrap();
assert!(made(2), "local maintenance started a backoff");
jobs.send((Job::Full, bare.clone())).unwrap();
std::thread::sleep(Duration::from_millis(300));
assert_eq!(
calls.lock().unwrap().len(),
2,
"a job inside the backoff ran"
);
jobs.send((Job::Full, fixed.clone())).unwrap();
assert!(made(3), "a changed line waited out the old backoff");
drop(jobs);
worker.join().unwrap();
let route = |dial: &Dial| dial.routes[0].clone();
assert_eq!(
*calls.lock().unwrap(),
vec![route(&bare), route(&bare), route(&fixed)]
);
}
#[test]
fn an_untrusted_local_clock_does_not_count_against_the_peer() {
let error = anyhow::Error::new(second_brain_core::sqlite_store::ClockUntrusted)
.context("importing from the peer");
assert!(!counts_against_peer(&error), "{error:#}");
}
#[test]
fn store_maintenance_does_not_count_against_the_peer() {
let sqlite = Arc::new(SqliteStore::in_memory("m".to_string()).unwrap());
let gate = WriteGate::default();
let clock = ClockGate::open_for_tests();
let health = fresh_health();
let local = StoreLocal {
sqlite: &sqlite,
gate: &gate,
clock: &clock,
health: &health,
access: &DeviceAccess::default(),
bootstrap: &Bootstrap::default(),
data_dir: Path::new("/nonexistent"),
};
let _maintenance = gate.close();
let error = local
.set_cursor("peer", 1)
.context("updating sync state")
.unwrap_err();
assert!(!counts_against_peer(&error), "{error:#}");
assert!(counts_against_peer(&anyhow::anyhow!("connection refused")));
}
#[test]
fn a_changed_peer_line_is_tried_without_the_old_lines_backoff() {
let now = Instant::now();
let later = now + Duration::from_secs(1);
let bare = line_dial("user@pi");
let fixed = line_dial("user@pi /opt/sb");
let mut state = PeerBackoff::default();
assert!(state.due(&bare, now));
assert_eq!(state.failed(&bare, now), FIRST_BACKOFF);
assert!(!state.due(&bare, later));
assert!(state.due(&fixed, later));
assert_eq!(state.failed(&fixed, later), FIRST_BACKOFF);
assert_eq!(state.failed(&fixed, later), FIRST_BACKOFF * 2);
assert!(state.due(&bare, later));
state.succeeded();
assert_eq!(state.failed(&fixed, later), FIRST_BACKOFF);
}
#[test]
fn backoff_doubles_from_thirty_seconds_up_to_half_an_hour() {
let waits: Vec<u64> = (1..=8).map(|n| backoff(n).as_secs()).collect();
assert_eq!(waits, vec![30, 60, 120, 240, 480, 960, 1800, 1800]);
}
struct EmptyPeer;
impl EmptyPeer {
fn header() -> String {
SyncHeader {
hello: Some(Hello {
machine_id: "peer".to_string(),
wall_micros: Utc::now().timestamp_micros(),
synced: false,
cursor: None,
held_origin_seq: None,
keys: Some(shared_identity()),
peer_in: None,
log_floor: 0,
head: 0,
applied: None,
merging: None,
}),
..SyncHeader::local()
}
.to_line()
}
}
impl Transport for EmptyPeer {
fn host(&self) -> &str {
"user@pi"
}
fn whoami(&self) -> Result<String> {
Ok("peer".to_string())
}
fn export(
&self,
_after_seq: u64,
_hello: &Hello,
) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)> {
let stream = format!("{}\n", Self::header()).into_bytes();
Ok((Box::new(std::io::Cursor::new(stream)), Box::new(Exited)))
}
fn import(&self) -> Result<(Box<dyn Write + Send>, Box<dyn Read + Send>, Box<dyn Remote>)> {
let replies = format!(
"{}\n{}\n",
Self::header(),
PushApplied { applied: 0 }.to_line()
);
Ok((
Box::new(std::io::sink()),
Box::new(std::io::Cursor::new(replies.into_bytes())),
Box::new(Exited),
))
}
}
struct Exited;
impl Remote for Exited {
fn wait(&mut self) -> Result<Exit> {
Ok(Exit {
success: true,
status: "exit status: 0".to_string(),
})
}
fn stop(&mut self) -> Result<()> {
Ok(())
}
}
fn reachable() -> Connect {
Arc::new(|_: &Route| -> Result<Box<dyn Transport + Send>> { Ok(Box::new(EmptyPeer)) })
}
fn refusing() -> Connect {
Arc::new(|_: &Route| -> Result<Box<dyn Transport + Send>> {
anyhow::bail!("connection refused")
})
}
fn watched_loop(dir: &Path, connect: Connect) -> (Arc<SyncLoop>, Arc<Health>) {
let health = Arc::new(Health::new(Utc::now(), None));
let sync_loop = Arc::new(scripted_loop(dir, connect).with_health(health.clone()));
std::fs::write(dir.join(peers::PEERS_FILE), "user@pi\n").unwrap();
assert_eq!(sync_loop.listed_dials(), vec![line_dial("user@pi")]);
(sync_loop, health)
}
fn the_one_peer(health: &Health) -> PeerSync {
let mut peers = health.report().value.sync_peers;
assert_eq!(peers.len(), 1, "{peers:?}");
peers.remove(0)
}
fn run_job(sync_loop: &Arc<SyncLoop>, job: Job, dial: Dial) {
let (jobs, inbox) = mpsc::channel();
jobs.send((job, dial)).unwrap();
drop(jobs);
let worker = {
let sync_loop = sync_loop.clone();
std::thread::spawn(move || sync_loop.work(inbox))
};
let deadline = Instant::now() + Duration::from_secs(10);
while !worker.is_finished() {
assert!(
Instant::now() < deadline,
"the worker never finished its job"
);
std::thread::sleep(Duration::from_millis(10));
}
worker.join().unwrap();
}
struct Answering(&'static str);
impl Transport for Answering {
fn host(&self) -> &str {
"me@192.0.2.1"
}
fn whoami(&self) -> Result<String> {
Ok(self.0.to_string())
}
fn export(
&self,
after_seq: u64,
hello: &Hello,
) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)> {
EmptyPeer.export(after_seq, hello)
}
fn import(&self) -> Result<(Box<dyn Write + Send>, Box<dyn Read + Send>, Box<dyn Remote>)> {
EmptyPeer.import()
}
}
fn device_dial(machine_id: &str, addresses: &[&str]) -> Dial {
Dial {
key: machine_id.to_string(),
label: machine_id.to_string(),
machine_id: Some(machine_id.to_string()),
routes: addresses
.iter()
.map(|address| Route::Device {
machine_id: machine_id.to_string(),
login: "me".to_string(),
address: address.to_string(),
port: 22,
})
.collect(),
}
}
fn answering(
by_address: &'static [(&'static str, &'static str)],
) -> (Connect, Arc<Mutex<Vec<String>>>) {
let dialled: Arc<Mutex<Vec<String>>> = Arc::default();
let connect: Connect = {
let dialled = dialled.clone();
Arc::new(move |route: &Route| -> Result<Box<dyn Transport + Send>> {
let Route::Device { address, .. } = route else {
anyhow::bail!("not a device route")
};
dialled.lock().unwrap().push(address.clone());
let Some((_, id)) = by_address.iter().find(|(a, _)| a == address) else {
anyhow::bail!("connection refused")
};
Ok(Box::new(Answering(id)))
})
};
(connect, dialled)
}
fn admit_new_device(sync_loop: &SyncLoop, dir: &Path, machine_id: &str) {
let device = load_device_keys(dir).unwrap().unwrap();
let identity_tag = sync_loop
.sqlite
.device_records()
.unwrap()
.into_iter()
.find(|row| row.current.record.machine_id == "m")
.unwrap()
.current
.record
.identity_tag;
let keys = DeviceKeys::generate().unwrap();
let record = DeviceRecord {
machine_id: machine_id.to_string(),
identity_tag,
version: 1,
name: machine_id.to_string(),
age_recipient: keys.age_recipient().to_string(),
ssh_public_key: keys.ssh_public_key(),
signing_public_key: keys.verifying_key().to_bytes(),
recovery_fingerprints: Vec::new(),
ssh_login: None,
ssh_port: None,
ssh_host_key: None,
};
admit_device(&sync_loop.sqlite, &device, record).unwrap();
}
fn revoke_new_device(sync_loop: &SyncLoop, dir: &Path, machine_id: &str) {
admit_new_device(sync_loop, dir, machine_id);
Keyring::load(&sync_loop.sqlite, dir)
.unwrap()
.remove_device(machine_id, &std::collections::BTreeSet::new(), 10)
.unwrap();
}
#[test]
fn a_route_answering_as_another_device_moves_on_to_the_next() {
let dir = tempfile::tempdir().unwrap();
let (connect, dialled) = answering(&[("192.0.2.1", "someone-else"), ("192.0.2.2", "peer")]);
let sync_loop = scripted_loop(dir.path(), connect);
sync_loop
.sync(&device_dial("peer", &["192.0.2.1", "192.0.2.2"]), Job::Full)
.unwrap();
assert_eq!(*dialled.lock().unwrap(), vec!["192.0.2.1", "192.0.2.2"]);
}
#[test]
fn a_route_answering_as_a_revoked_device_stops_the_dial() {
let dir = tempfile::tempdir().unwrap();
let (connect, dialled) = answering(&[("192.0.2.1", "stolen"), ("192.0.2.2", "peer")]);
let sync_loop = scripted_loop(dir.path(), connect);
revoke_new_device(&sync_loop, dir.path(), "stolen");
assert!(
sync_loop
.sync(&device_dial("peer", &["192.0.2.1", "192.0.2.2"]), Job::Full)
.is_err()
);
assert_eq!(*dialled.lock().unwrap(), vec!["192.0.2.1"]);
}
fn advertising(ids: &[&str]) -> Responder {
let adverts: Vec<_> = ids
.iter()
.map(|id| mdns::device_advert(id, "me", 22))
.collect();
Responder::spawn(
&Net::Unicast(std::net::SocketAddr::from((
std::net::Ipv4Addr::LOCALHOST,
0,
))),
Arc::new(move || adverts.clone()),
)
.unwrap()
}
fn looking_up(sync_loop: SyncLoop, responder: &Responder) -> SyncLoop {
SyncLoop {
discovery: Some(Net::Unicast(responder.local_addr())),
..sync_loop
}
}
fn held_addresses(sync_loop: &SyncLoop, machine_id: &str) -> Vec<String> {
sync_loop
.sqlite
.device_route(machine_id)
.unwrap()
.map(|route| route.addresses)
.unwrap_or_default()
}
fn hold_route(sync_loop: &SyncLoop, machine_id: &str, addresses: &[&str]) {
sync_loop
.sqlite
.set_device_route(DeviceRoute {
machine_id: machine_id.to_string(),
addresses: addresses.iter().map(|a| a.to_string()).collect(),
ssh_login: Some("me".to_string()),
ssh_port: None,
})
.unwrap();
}
#[test]
fn a_device_whose_address_changed_is_found_synced_and_recorded() {
let dir = tempfile::tempdir().unwrap();
let (connect, dialled) = answering(&[("127.0.0.1", "peer")]);
let responder = advertising(&["peer"]);
let sync_loop = looking_up(scripted_loop(dir.path(), connect), &responder);
admit_new_device(&sync_loop, dir.path(), "peer");
hold_route(&sync_loop, "peer", &["192.0.2.1"]);
let dials = sync_loop.listed_dials();
assert_eq!(dials.len(), 1, "{dials:?}");
sync_loop.sync(&dials[0], Job::Full).unwrap();
assert_eq!(*dialled.lock().unwrap(), vec!["192.0.2.1", "127.0.0.1"]);
assert_eq!(
held_addresses(&sync_loop, "peer"),
vec!["127.0.0.1", "192.0.2.1"]
);
let Route::Device { address, .. } = &sync_loop.listed_dials()[0].routes[0] else {
panic!("a device is dialled by its device route");
};
assert_eq!(address, "127.0.0.1");
}
#[test]
fn an_address_now_held_by_another_machine_is_looked_past() {
let dir = tempfile::tempdir().unwrap();
let (connect, dialled) = answering(&[("192.0.2.1", "someone-else"), ("127.0.0.1", "peer")]);
let responder = advertising(&["peer"]);
let sync_loop = looking_up(scripted_loop(dir.path(), connect), &responder);
admit_new_device(&sync_loop, dir.path(), "peer");
hold_route(&sync_loop, "peer", &["192.0.2.1"]);
sync_loop
.sync(&sync_loop.listed_dials()[0], Job::Push)
.unwrap();
assert_eq!(*dialled.lock().unwrap(), vec!["192.0.2.1", "127.0.0.1"]);
assert_eq!(held_addresses(&sync_loop, "peer")[0], "127.0.0.1");
}
#[test]
fn a_device_with_no_address_is_looked_up_on_a_full_pass() {
let dir = tempfile::tempdir().unwrap();
let (connect, dialled) = answering(&[("127.0.0.1", "peer")]);
let responder = advertising(&["peer"]);
let sync_loop = looking_up(scripted_loop(dir.path(), connect), &responder);
admit_new_device(&sync_loop, dir.path(), "peer");
let mut dials = sync_loop.listed_dials();
assert_eq!(dials, Vec::<Dial>::new());
sync_loop.add_found(&mut dials);
assert_eq!(dials.len(), 1, "{dials:?}");
assert_eq!(dials[0].routes[0].describe(), "me@127.0.0.1");
sync_loop.sync(&dials[0], Job::Full).unwrap();
assert_eq!(*dialled.lock().unwrap(), vec!["127.0.0.1"]);
assert_eq!(held_addresses(&sync_loop, "peer"), vec!["127.0.0.1"]);
assert_eq!(sync_loop.listed_dials().len(), 1);
}
#[test]
fn a_responder_under_an_unknown_or_revoked_id_is_never_dialled() {
let dir = tempfile::tempdir().unwrap();
let (connect, dialled) = answering(&[]);
let responder = advertising(&["stranger", "stolen"]);
let sync_loop = looking_up(scripted_loop(dir.path(), connect), &responder);
revoke_new_device(&sync_loop, dir.path(), "stolen");
admit_new_device(&sync_loop, dir.path(), "peer");
hold_route(&sync_loop, "peer", &["192.0.2.1"]);
let mut dials = sync_loop.listed_dials();
assert!(sync_loop.sync(&dials[0], Job::Full).is_err());
assert_eq!(*dialled.lock().unwrap(), vec!["192.0.2.1"]);
sync_loop.add_found(&mut dials);
assert_eq!(
dials.iter().map(|d| d.key.as_str()).collect::<Vec<_>>(),
vec!["peer"]
);
assert_eq!(held_addresses(&sync_loop, "peer"), vec!["192.0.2.1"]);
assert_eq!(held_addresses(&sync_loop, "stolen"), Vec::<String>::new());
assert_eq!(held_addresses(&sync_loop, "stranger"), Vec::<String>::new());
}
#[test]
fn a_full_sync_reports_the_peer_synced() {
let dir = tempfile::tempdir().unwrap();
let (sync_loop, health) = watched_loop(dir.path(), reachable());
let before = Utc::now();
run_job(&sync_loop, Job::Full, line_dial("user@pi"));
let peer = the_one_peer(&health);
assert_eq!(peer.peer, "user@pi");
let synced = peer.last_success_at.expect("a full sync is a success");
assert!(synced >= before, "{synced} predates the sync");
assert_eq!(peer.consecutive_failures, 0);
}
#[test]
fn a_failed_sync_counts_a_failure_and_no_success() {
let dir = tempfile::tempdir().unwrap();
let (sync_loop, health) = watched_loop(dir.path(), refusing());
run_job(&sync_loop, Job::Full, line_dial("user@pi"));
let peer = the_one_peer(&health);
assert_eq!(peer.peer, "user@pi");
assert_eq!(peer.consecutive_failures, 1);
assert_eq!(peer.last_success_at, None);
assert!(peer.last_attempt_at.is_some(), "the failure is an attempt");
let error = peer.last_error.expect("the failure's text is kept");
assert!(error.contains("connection refused"), "{error}");
}
#[test]
fn start_deletes_the_plaintext_ssh_key_once_the_device_key_loads() {
let dir = tempfile::tempdir().unwrap();
let sqlite = SqliteStore::in_memory("m".to_string()).unwrap();
second_brain_core::keys::ensure_device_keys(&sqlite, dir.path()).unwrap();
let keys = load_device_keys(dir.path()).unwrap().unwrap();
let legacy = dir.path().join(SSH_KEY_FILE);
write_identity_file(&legacy, keys.ssh_signing_key(), "second-brain:m").unwrap();
let before = std::fs::read(&legacy).unwrap();
assert!(retire_plaintext_ssh_key(dir.path()).unwrap());
assert!(!legacy.exists());
assert!(!retire_plaintext_ssh_key(dir.path()).unwrap());
let reloaded = load_device_keys(dir.path()).unwrap().unwrap();
let elsewhere = tempfile::tempdir().unwrap();
let rederived = elsewhere.path().join("id");
write_identity_file(&rederived, reloaded.ssh_signing_key(), "second-brain:m").unwrap();
assert_eq!(std::fs::read(&rederived).unwrap(), before);
}
#[test]
fn a_plaintext_ssh_key_stays_while_no_device_key_loads() {
let dir = tempfile::tempdir().unwrap();
let legacy = dir.path().join(SSH_KEY_FILE);
let stray = DeviceKeys::generate().unwrap();
write_identity_file(&legacy, stray.ssh_signing_key(), "second-brain:m").unwrap();
assert!(!retire_plaintext_ssh_key(dir.path()).unwrap());
assert!(legacy.exists());
std::fs::write(
dir.path().join(second_brain_core::keys::DEVICE_KEY_FILE),
b"torn",
)
.unwrap();
assert!(retire_plaintext_ssh_key(dir.path()).is_err());
assert!(legacy.exists());
}
#[test]
fn a_push_from_a_device_without_an_identity_is_not_held_against_the_peer() {
let dir = tempfile::tempdir().unwrap();
let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let connect: Connect = {
let calls = calls.clone();
Arc::new(move |_: &Route| -> Result<Box<dyn Transport + Send>> {
calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
anyhow::bail!("connection refused")
})
};
let sqlite = SqliteStore::in_memory("m".to_string()).unwrap();
second_brain_core::keys::ensure_device_keys(&sqlite, dir.path()).unwrap();
let health = Arc::new(Health::new(Utc::now(), None));
let sync_loop = Arc::new(SyncLoop {
data_dir: dir.path().to_path_buf(),
sqlite: Arc::new(sqlite),
gate: Arc::new(WriteGate::default()),
clock: ClockGate::open_for_tests(),
health: health.clone(),
access: Arc::default(),
bootstrap: Arc::default(),
connect,
discovery: None,
});
std::fs::write(dir.path().join(peers::PEERS_FILE), "user@pi\n").unwrap();
sync_loop.listed_dials();
run_job(&sync_loop, Job::Push, line_dial("user@pi"));
assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 0);
let peer = the_one_peer(&health);
assert_eq!(peer.last_attempt_at, None);
assert_eq!(peer.consecutive_failures, 0);
run_job(&sync_loop, Job::Full, line_dial("user@pi"));
assert_eq!(calls.load(std::sync::atomic::Ordering::SeqCst), 1);
assert_eq!(the_one_peer(&health).consecutive_failures, 1);
}
#[test]
fn a_push_ends_a_failure_streak_without_counting_as_a_sync() {
let dir = tempfile::tempdir().unwrap();
let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let connect: Connect = {
let calls = calls.clone();
Arc::new(move |_: &Route| -> Result<Box<dyn Transport + Send>> {
if calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0 {
anyhow::bail!("connection refused");
}
Ok(Box::new(EmptyPeer))
})
};
let (sync_loop, health) = watched_loop(dir.path(), connect);
run_job(&sync_loop, Job::Full, line_dial("user@pi"));
assert_eq!(the_one_peer(&health).consecutive_failures, 1);
run_job(&sync_loop, Job::Push, line_dial("user@pi"));
let peer = the_one_peer(&health);
assert_eq!(peer.last_success_at, None);
assert!(peer.last_attempt_at.is_some());
assert_eq!(peer.consecutive_failures, 0);
}
#[test]
fn maintenance_reports_no_sync_outcome() {
let dir = tempfile::tempdir().unwrap();
let connect: Connect = Arc::new(|_: &Route| -> Result<Box<dyn Transport + Send>> {
Err(anyhow::Error::new(MaintenanceRunning))
});
let (sync_loop, health) = watched_loop(dir.path(), connect);
run_job(&sync_loop, Job::Full, line_dial("user@pi"));
let peer = the_one_peer(&health);
assert_eq!(peer.last_success_at, None);
assert_eq!(peer.last_attempt_at, None);
assert_eq!(peer.consecutive_failures, 0);
}
#[test]
fn a_peer_removed_from_sync_peers_drops_out_of_health() {
let dir = tempfile::tempdir().unwrap();
let (sync_loop, health) = watched_loop(dir.path(), refusing());
let listed = |text: &str| {
std::fs::write(dir.path().join(peers::PEERS_FILE), text).unwrap();
let kept = sync_loop.listed_dials();
let mut reported: Vec<String> = health
.report()
.value
.sync_peers
.into_iter()
.map(|peer| peer.peer)
.collect();
reported.sort();
(kept.len(), reported)
};
assert_eq!(
listed("user@pi\nuser@mac\nuser@pi /opt/sb\n"),
(2, vec!["user@mac".to_string(), "user@pi".to_string()])
);
assert_eq!(listed("user@mac\n"), (1, vec!["user@mac".to_string()]));
}
#[test]
fn an_unreadable_sync_peers_keeps_its_peers_in_health() {
let dir = tempfile::tempdir().unwrap();
let (sync_loop, health) = watched_loop(dir.path(), refusing());
let file = dir.path().join(peers::PEERS_FILE);
std::fs::remove_file(&file).unwrap();
std::fs::create_dir(&file).unwrap();
assert!(peers::configured_peers(dir.path()).is_err());
assert_eq!(sync_loop.listed_dials(), Vec::<Dial>::new());
assert_eq!(the_one_peer(&health).peer, "user@pi");
}
}