use std::collections::{HashMap, HashSet};
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::mpsc::{self, Receiver, SendError, Sender};
use std::thread::JoinHandle;
use std::time::{Duration, Instant};
use anyhow::Result;
use chrono::Utc;
use second_brain_core::schema::SyncState;
use second_brain_core::sqlite_store::SqliteStore;
use second_brain_core::store::StoreAdmin;
use second_brain_sync::crypto::{XChaChaEncryptor, load_or_create_key};
use second_brain_sync::export::export_page;
use second_brain_sync::import::import_values;
use second_brain_sync::peers::{self, Peer};
use second_brain_sync::protocol::{SyncHeader, check_store_header, store_header};
use second_brain_sync::session::{self, Exported, Imported, Local, Transport};
use second_brain_sync::ssh::SshTransport;
use serde_json::Value;
use crate::health::Health;
use crate::value_metrics::PeerOutcome;
use crate::write_gate::{WriteGate, WriteSlot};
const FULL_INTERVAL: Duration = Duration::from_secs(300);
const TICK: Duration = Duration::from_secs(1);
const PUSH_DELAY: Duration = Duration::from_secs(5);
const FIRST_BACKOFF: Duration = Duration::from_secs(30);
const MAX_BACKOFF: Duration = Duration::from_secs(30 * 60);
const ORIGIN_SCAN_PAGE: usize = 500;
pub type Connect = Arc<dyn Fn(&Peer) -> Result<Box<dyn Transport + Send>> + Send + Sync>;
pub struct SyncLoop {
pub data_dir: PathBuf,
pub sqlite: Arc<SqliteStore>,
pub gate: Arc<WriteGate>,
pub connect: Connect,
pub health: Option<Arc<Health>>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
enum Job {
Push,
Full,
}
impl SyncLoop {
pub fn over_ssh(data_dir: PathBuf, sqlite: Arc<SqliteStore>, gate: Arc<WriteGate>) -> Self {
Self {
data_dir,
sqlite,
gate,
connect: Arc::new(|peer: &Peer| {
let transport = SshTransport::new(&peer.host, &peer.sb)?.unattended();
Ok(Box::new(transport) as Box<dyn Transport + Send>)
}),
health: None,
}
}
pub fn with_health(mut self, health: Arc<Health>) -> Self {
self.health = Some(health);
self
}
pub fn spawn(self) -> JoinHandle<()> {
std::thread::Builder::new()
.name("sync-loop".to_string())
.spawn(move || Arc::new(self).run())
.expect("spawning the sync loop")
}
fn run(self: Arc<Self>) {
let mut workers: HashMap<String, Sender<(Job, Peer)>> = HashMap::new();
let mut next_full = Instant::now();
let mut seen = self.sqlite.max_sync_seq().unwrap_or(0);
let mut push_at: Option<Instant> = None;
loop {
let (peers, repeated) = self.listed_peers();
if peers.is_empty() {
seen = self.sqlite.max_sync_seq().unwrap_or(seen);
push_at = None;
} else if self.wrote_locally(&mut seen) {
push_at.get_or_insert(Instant::now() + PUSH_DELAY);
}
let now = Instant::now();
let job = if now >= next_full {
next_full = now + FULL_INTERVAL;
Some(Job::Full)
} else if push_at.is_some_and(|at| now >= at) {
Some(Job::Push)
} else {
None
};
if let Some(job) = job {
push_at = None;
for line in &repeated {
tracing::warn!(
"sync-peers lists {} again; only its first line is used",
line.host
);
}
for peer in peers {
self.dispatch(&mut workers, job, peer);
}
}
std::thread::sleep(TICK);
}
}
fn dispatch(
self: &Arc<Self>,
workers: &mut HashMap<String, Sender<(Job, Peer)>>,
job: Job,
peer: Peer,
) {
let host = peer.host.clone();
let worker = workers
.entry(host.clone())
.or_insert_with(|| self.clone().start_worker(&host));
if let Err(SendError(message)) = worker.send((job, peer)) {
tracing::warn!("the sync worker for {host} exited; starting another");
let worker = self.clone().start_worker(&host);
if worker.send(message).is_err() {
tracing::warn!("the new sync worker for {host} exited at once");
}
workers.insert(host, worker);
}
}
fn listed_peers(&self) -> (Vec<Peer>, Vec<Peer>) {
let lines = match peers::configured_peers(&self.data_dir) {
Ok(lines) => lines.unwrap_or_default(),
Err(e) => {
tracing::warn!("cannot read the sync peers: {e}");
return (Vec::new(), Vec::new());
}
};
let (peers, repeated) =
one_line_per_host(lines.iter().map(|line| Peer::parse(line)).collect());
if let Some(health) = &self.health {
let hosts: Vec<String> = peers.iter().map(|peer| peer.host.clone()).collect();
health.sync_peers_listed(&hosts, Utc::now());
}
(peers, repeated)
}
fn report(&self, host: &str, outcome: PeerOutcome) {
if let Some(health) = &self.health {
health.sync_attempted(host, outcome, Utc::now());
}
}
fn wrote_locally(&self, seen: &mut u64) -> bool {
let max = match self.sqlite.max_sync_seq() {
Ok(max) => max,
Err(e) => {
tracing::warn!("cannot read the sync log position: {e:#}");
return false;
}
};
let own = self.sqlite.machine_id();
while *seen < max {
let page = match self.sqlite.sync_log_page(*seen, Some(ORIGIN_SCAN_PAGE)) {
Ok(page) => page,
Err(e) => {
tracing::warn!("cannot read the sync log: {e:#}");
return false;
}
};
let Some(last) = page.last() else {
break;
};
if page.iter().any(|entry| entry.origin_machine_id == own) {
*seen = max;
return true;
}
*seen = last.local_seq.max(0) as u64;
}
*seen = max;
false
}
fn start_worker(self: Arc<Self>, host: &str) -> Sender<(Job, Peer)> {
let (jobs, inbox) = mpsc::channel();
std::thread::Builder::new()
.name(format!("sync-{host}"))
.spawn(move || self.work(inbox))
.expect("spawning a sync worker");
jobs
}
fn work(&self, inbox: Receiver<(Job, Peer)>) {
let mut backoff = PeerBackoff::default();
while let Ok((mut job, mut peer)) = inbox.recv() {
while let Ok((queued, latest)) = inbox.try_recv() {
job = job.max(queued);
peer = latest;
}
if !backoff.due(&peer, Instant::now()) {
continue;
}
match self.sync(&peer, job) {
Ok(()) => {
backoff.succeeded();
let outcome = match job {
Job::Full => PeerOutcome::Synced,
Job::Push => PeerOutcome::Pushed,
};
self.report(&peer.host, outcome);
}
Err(e) if !counts_against_peer(&e) => {
tracing::info!("sync with {} waits for the next job: {e:#}", peer.host);
}
Err(e) => {
let wait = backoff.failed(&peer, Instant::now());
self.report(&peer.host, PeerOutcome::Failed);
tracing::warn!(
"sync with {} failed; its jobs are skipped for the next {}s: {e:#}",
peer.host,
wait.as_secs()
);
}
}
}
}
fn sync(&self, peer: &Peer, job: Job) -> Result<()> {
let transport = (self.connect)(peer)?;
let key = load_or_create_key(&self.data_dir.join("sync.key"))?;
let encryptor = XChaChaEncryptor::new(key);
let local = StoreLocal {
sqlite: &self.sqlite,
gate: &self.gate,
};
let report = |line: &str| tracing::info!("sync with {}: {}", peer.host, line.trim());
match job {
Job::Full => session::sync(
&local,
transport.as_ref(),
&encryptor,
&self.data_dir,
&report,
),
Job::Push => {
let peer_machine_id = transport.whoami()?;
session::push(
&local,
transport.as_ref(),
&peer_machine_id,
&encryptor,
&report,
)
.map(|_| ())
}
}
}
}
#[derive(Debug)]
struct MaintenanceRunning;
impl std::fmt::Display for MaintenanceRunning {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("store maintenance is running")
}
}
impl std::error::Error for MaintenanceRunning {}
fn counts_against_peer(error: &anyhow::Error) -> bool {
error.downcast_ref::<MaintenanceRunning>().is_none()
}
fn one_line_per_host(lines: Vec<Peer>) -> (Vec<Peer>, Vec<Peer>) {
let mut hosts = HashSet::new();
lines
.into_iter()
.partition(|peer| hosts.insert(peer.host.clone()))
}
#[derive(Default)]
struct PeerBackoff {
line: Option<Peer>,
failures: u32,
retry_at: Option<Instant>,
}
impl PeerBackoff {
fn due(&self, peer: &Peer, now: Instant) -> bool {
self.line.as_ref() != Some(peer) || self.retry_at.is_none_or(|at| now >= at)
}
fn succeeded(&mut self) {
*self = PeerBackoff::default();
}
fn failed(&mut self, peer: &Peer, now: Instant) -> Duration {
if self.line.as_ref() != Some(peer) {
*self = PeerBackoff {
line: Some(peer.clone()),
..PeerBackoff::default()
};
}
self.failures += 1;
let wait = backoff(self.failures);
self.retry_at = Some(now + wait);
wait
}
}
fn backoff(failures: u32) -> Duration {
FIRST_BACKOFF
.saturating_mul(2u32.saturating_pow(failures.saturating_sub(1)))
.min(MAX_BACKOFF)
}
struct StoreLocal<'a> {
sqlite: &'a SqliteStore,
gate: &'a WriteGate,
}
impl StoreLocal<'_> {
fn write_slot(&self) -> Result<WriteSlot<'_>> {
self.gate
.try_enter()
.ok_or_else(|| anyhow::Error::new(MaintenanceRunning))
}
}
impl Local for StoreLocal<'_> {
fn cursors(&self) -> Result<Vec<(String, u64)>> {
Ok(self
.sqlite
.get_all_sync_states()?
.into_iter()
.map(|state| (state.peer_id, state.last_seq))
.collect())
}
fn set_cursor(&self, peer_id: &str, last_seq: u64) -> Result<()> {
let _slot = self.write_slot()?;
self.sqlite.set_sync_state(&SyncState {
peer_id: peer_id.to_string(),
last_seq,
last_sync_at: chrono::Utc::now(),
})?;
Ok(())
}
fn import(&self, header: &SyncHeader, records: &[Value]) -> Result<Imported> {
let _slot = self.write_slot()?;
check_store_header(self.sqlite, Some(*header), "the sender")?;
let (stats, max_seq) = import_values(self.sqlite, records)?;
Ok(Imported {
created: stats.created,
updated: stats.updated,
deleted: stats.deleted,
skipped: stats.skipped,
dangling: Imported::dangling_in(&stats.skipped_by_reason),
max_seq,
})
}
fn export(&self, after_seq: u64, limit: usize) -> Result<Exported> {
store_header(self.sqlite)?;
let (records, max_seq) = export_page(self.sqlite, after_seq, limit)?;
Ok(Exported { records, max_seq })
}
}
#[cfg(test)]
mod tests {
use std::io::{Read, Write};
use std::path::Path;
use std::sync::Mutex;
use anyhow::Context;
use second_brain_core::sqlite_store::SCHEMA_VERSION;
use second_brain_sync::session::{Exit, Remote};
use super::*;
use crate::value_metrics::PeerSync;
#[test]
fn the_daemon_refuses_to_export_from_a_database_newer_than_its_binary() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("future.db");
drop(SqliteStore::open(&path, "m".to_string()).unwrap());
rusqlite::Connection::open(&path)
.unwrap()
.pragma_update(None, "user_version", SCHEMA_VERSION + 1)
.unwrap();
let sqlite = SqliteStore::open(&path, "m".to_string()).unwrap();
let gate = WriteGate::default();
let local = StoreLocal {
sqlite: &sqlite,
gate: &gate,
};
let error = local.export(0, 10).err().expect("export refused");
assert!(
format!("{error:#}").contains("upgrade this daemon"),
"{error:#}"
);
}
#[test]
fn a_host_keeps_one_worker_when_its_sb_path_changes() {
let dir = tempfile::tempdir().unwrap();
let sync_loop = Arc::new(SyncLoop {
data_dir: dir.path().to_path_buf(),
sqlite: Arc::new(SqliteStore::in_memory("m".to_string()).unwrap()),
gate: Arc::new(WriteGate::default()),
connect: Arc::new(|_: &Peer| -> Result<Box<dyn Transport + Send>> {
anyhow::bail!("offline")
}),
health: None,
});
let mut workers = HashMap::new();
sync_loop.dispatch(&mut workers, Job::Push, Peer::parse("user@pi"));
sync_loop.dispatch(&mut workers, Job::Push, Peer::parse("user@pi /opt/sb"));
assert_eq!(workers.len(), 1);
}
fn scripted_loop(dir: &Path, connect: Connect) -> SyncLoop {
SyncLoop {
data_dir: dir.to_path_buf(),
sqlite: Arc::new(SqliteStore::in_memory("m".to_string()).unwrap()),
gate: Arc::new(WriteGate::default()),
connect,
health: None,
}
}
#[test]
fn the_worker_takes_the_latest_line_and_backs_off_for_all_but_maintenance() {
let dir = tempfile::tempdir().unwrap();
let calls: Arc<Mutex<Vec<Peer>>> = Arc::default();
let connect: Connect = {
let calls = calls.clone();
Arc::new(move |peer: &Peer| -> Result<Box<dyn Transport + Send>> {
let mut calls = calls.lock().unwrap();
calls.push(peer.clone());
if calls.len() == 1 {
Err(anyhow::Error::new(MaintenanceRunning))
} else {
anyhow::bail!("connection refused")
}
})
};
let sync_loop = Arc::new(scripted_loop(dir.path(), connect));
let made = |n: usize| {
let deadline = Instant::now() + Duration::from_secs(5);
while Instant::now() < deadline {
if calls.lock().unwrap().len() >= n {
return true;
}
std::thread::sleep(Duration::from_millis(10));
}
false
};
let bare = Peer::parse("user@pi");
let fixed = Peer::parse("user@pi /opt/sb");
let (jobs, inbox) = mpsc::channel();
jobs.send((Job::Push, Peer::parse("user@pi /old/sb")))
.unwrap();
jobs.send((Job::Full, bare.clone())).unwrap();
let worker = {
let sync_loop = sync_loop.clone();
std::thread::spawn(move || sync_loop.work(inbox))
};
assert!(made(1), "the queued jobs never ran");
jobs.send((Job::Full, bare.clone())).unwrap();
assert!(made(2), "local maintenance started a backoff");
jobs.send((Job::Full, bare.clone())).unwrap();
std::thread::sleep(Duration::from_millis(300));
assert_eq!(
calls.lock().unwrap().len(),
2,
"a job inside the backoff ran"
);
jobs.send((Job::Full, fixed.clone())).unwrap();
assert!(made(3), "a changed line waited out the old backoff");
drop(jobs);
worker.join().unwrap();
assert_eq!(*calls.lock().unwrap(), vec![bare.clone(), bare, fixed]);
}
#[test]
fn store_maintenance_does_not_count_against_the_peer() {
let sqlite = SqliteStore::in_memory("m".to_string()).unwrap();
let gate = WriteGate::default();
let local = StoreLocal {
sqlite: &sqlite,
gate: &gate,
};
let _maintenance = gate.close();
let error = local
.set_cursor("peer", 1)
.context("updating sync state")
.unwrap_err();
assert!(!counts_against_peer(&error), "{error:#}");
assert!(counts_against_peer(&anyhow::anyhow!("connection refused")));
}
#[test]
fn a_changed_peer_line_is_tried_without_the_old_lines_backoff() {
let now = Instant::now();
let later = now + Duration::from_secs(1);
let bare = Peer::parse("user@pi");
let fixed = Peer::parse("user@pi /opt/sb");
let mut state = PeerBackoff::default();
assert!(state.due(&bare, now));
assert_eq!(state.failed(&bare, now), FIRST_BACKOFF);
assert!(!state.due(&bare, later));
assert!(state.due(&fixed, later));
assert_eq!(state.failed(&fixed, later), FIRST_BACKOFF);
assert_eq!(state.failed(&fixed, later), FIRST_BACKOFF * 2);
assert!(state.due(&bare, later));
state.succeeded();
assert_eq!(state.failed(&fixed, later), FIRST_BACKOFF);
}
#[test]
fn a_host_listed_twice_keeps_its_first_line() {
let (kept, repeated) = one_line_per_host(vec![
Peer::parse("user@pi"),
Peer::parse("user@mac"),
Peer::parse("user@pi /opt/sb"),
]);
assert_eq!(kept, vec![Peer::parse("user@pi"), Peer::parse("user@mac")]);
assert_eq!(repeated, vec![Peer::parse("user@pi /opt/sb")]);
}
#[test]
fn backoff_doubles_from_thirty_seconds_up_to_half_an_hour() {
let waits: Vec<u64> = (1..=8).map(|n| backoff(n).as_secs()).collect();
assert_eq!(waits, vec![30, 60, 120, 240, 480, 960, 1800, 1800]);
}
struct EmptyPeer;
impl Transport for EmptyPeer {
fn host(&self) -> &str {
"user@pi"
}
fn whoami(&self) -> Result<String> {
Ok("peer".to_string())
}
fn export(&self, _after_seq: u64) -> Result<(Box<dyn Read + Send>, Box<dyn Remote>)> {
let stream = format!("{}\n", SyncHeader::local().to_line()).into_bytes();
Ok((Box::new(std::io::Cursor::new(stream)), Box::new(Exited)))
}
fn import(&self) -> Result<(Box<dyn Write + Send>, Box<dyn Remote>)> {
Ok((Box::new(std::io::sink()), Box::new(Exited)))
}
}
struct Exited;
impl Remote for Exited {
fn wait(&mut self) -> Result<Exit> {
Ok(Exit {
success: true,
status: "exit status: 0".to_string(),
})
}
fn stop(&mut self) -> Result<()> {
Ok(())
}
}
fn reachable() -> Connect {
Arc::new(|_: &Peer| -> Result<Box<dyn Transport + Send>> { Ok(Box::new(EmptyPeer)) })
}
fn refusing() -> Connect {
Arc::new(|_: &Peer| -> Result<Box<dyn Transport + Send>> {
anyhow::bail!("connection refused")
})
}
fn watched_loop(dir: &Path, connect: Connect) -> (Arc<SyncLoop>, Arc<Health>) {
let health = Arc::new(Health::new(Utc::now(), None));
let sync_loop = Arc::new(scripted_loop(dir, connect).with_health(health.clone()));
std::fs::write(dir.join(peers::PEERS_FILE), "user@pi\n").unwrap();
assert_eq!(sync_loop.listed_peers().0, vec![Peer::parse("user@pi")]);
(sync_loop, health)
}
fn the_one_peer(health: &Health) -> PeerSync {
let mut peers = health.report().value.sync_peers;
assert_eq!(peers.len(), 1, "{peers:?}");
peers.remove(0)
}
fn run_job(sync_loop: &Arc<SyncLoop>, job: Job, peer: Peer) {
let (jobs, inbox) = mpsc::channel();
jobs.send((job, peer)).unwrap();
drop(jobs);
let worker = {
let sync_loop = sync_loop.clone();
std::thread::spawn(move || sync_loop.work(inbox))
};
let deadline = Instant::now() + Duration::from_secs(10);
while !worker.is_finished() {
assert!(
Instant::now() < deadline,
"the worker never finished its job"
);
std::thread::sleep(Duration::from_millis(10));
}
worker.join().unwrap();
}
#[test]
fn a_full_sync_reports_the_peer_synced() {
let dir = tempfile::tempdir().unwrap();
let (sync_loop, health) = watched_loop(dir.path(), reachable());
let before = Utc::now();
run_job(&sync_loop, Job::Full, Peer::parse("user@pi"));
let peer = the_one_peer(&health);
assert_eq!(peer.peer, "user@pi");
let synced = peer.last_success_at.expect("a full sync is a success");
assert!(synced >= before, "{synced} predates the sync");
assert_eq!(peer.consecutive_failures, 0);
}
#[test]
fn a_failed_sync_counts_a_failure_and_no_success() {
let dir = tempfile::tempdir().unwrap();
let (sync_loop, health) = watched_loop(dir.path(), refusing());
run_job(&sync_loop, Job::Full, Peer::parse("user@pi"));
let peer = the_one_peer(&health);
assert_eq!(peer.peer, "user@pi");
assert_eq!(peer.consecutive_failures, 1);
assert_eq!(peer.last_success_at, None);
assert!(peer.last_attempt_at.is_some(), "the failure is an attempt");
}
#[test]
fn a_push_ends_a_failure_streak_without_counting_as_a_sync() {
let dir = tempfile::tempdir().unwrap();
let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let connect: Connect = {
let calls = calls.clone();
Arc::new(move |_: &Peer| -> Result<Box<dyn Transport + Send>> {
if calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst) == 0 {
anyhow::bail!("connection refused");
}
Ok(Box::new(EmptyPeer))
})
};
let (sync_loop, health) = watched_loop(dir.path(), connect);
run_job(&sync_loop, Job::Full, Peer::parse("user@pi"));
assert_eq!(the_one_peer(&health).consecutive_failures, 1);
run_job(&sync_loop, Job::Push, Peer::parse("user@pi"));
let peer = the_one_peer(&health);
assert_eq!(peer.last_success_at, None);
assert!(peer.last_attempt_at.is_some());
assert_eq!(peer.consecutive_failures, 0);
}
#[test]
fn maintenance_reports_no_sync_outcome() {
let dir = tempfile::tempdir().unwrap();
let connect: Connect = Arc::new(|_: &Peer| -> Result<Box<dyn Transport + Send>> {
Err(anyhow::Error::new(MaintenanceRunning))
});
let (sync_loop, health) = watched_loop(dir.path(), connect);
run_job(&sync_loop, Job::Full, Peer::parse("user@pi"));
let peer = the_one_peer(&health);
assert_eq!(peer.last_success_at, None);
assert_eq!(peer.last_attempt_at, None);
assert_eq!(peer.consecutive_failures, 0);
}
#[test]
fn a_peer_removed_from_sync_peers_drops_out_of_health() {
let dir = tempfile::tempdir().unwrap();
let (sync_loop, health) = watched_loop(dir.path(), refusing());
let listed = |text: &str| {
std::fs::write(dir.path().join(peers::PEERS_FILE), text).unwrap();
let (kept, _) = sync_loop.listed_peers();
let mut reported: Vec<String> = health
.report()
.value
.sync_peers
.into_iter()
.map(|peer| peer.peer)
.collect();
reported.sort();
(kept.len(), reported)
};
assert_eq!(
listed("user@pi\nuser@mac\nuser@pi /opt/sb\n"),
(2, vec!["user@mac".to_string(), "user@pi".to_string()])
);
assert_eq!(listed("user@mac\n"), (1, vec!["user@mac".to_string()]));
}
#[test]
fn an_unreadable_sync_peers_keeps_its_peers_in_health() {
let dir = tempfile::tempdir().unwrap();
let (sync_loop, health) = watched_loop(dir.path(), refusing());
let file = dir.path().join(peers::PEERS_FILE);
std::fs::remove_file(&file).unwrap();
std::fs::create_dir(&file).unwrap();
assert!(peers::configured_peers(dir.path()).is_err());
let (kept, repeated) = sync_loop.listed_peers();
assert_eq!(kept, Vec::<Peer>::new());
assert_eq!(repeated, Vec::<Peer>::new());
assert_eq!(the_one_peer(&health).peer, "user@pi");
}
}