use crate::agent::plan::restart_backoff;
use std::time::{Duration, Instant};
pub const EVERY: Duration = Duration::from_secs(30);
pub const STALE_AFTER_SECS: u64 = 180;
const MAX_WAIT: Duration = Duration::from_secs(15 * 60);
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Observation {
pub up: bool,
pub has_peers: bool,
pub handshake_age: Option<u64>,
}
pub trait Probe {
fn wanted(&self) -> bool;
fn observe(&self) -> Observation;
fn repair(&self) -> Result<(), String>;
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Verdict {
Healthy,
Down(String),
}
pub fn judge(o: &Observation) -> Verdict {
if !o.up {
return Verdict::Down("the mesh interface is not up".into());
}
if o.has_peers {
match o.handshake_age {
None => return Verdict::Down("no handshake with the mesh yet".into()),
Some(a) if a > STALE_AFTER_SECS => {
return Verdict::Down(format!("the last handshake was {a}s ago"))
}
_ => {}
}
}
Verdict::Healthy
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Outcome {
Idle,
Healthy,
Waiting(Duration),
Repaired(String),
Failed {
error: String,
retry_in: Duration,
},
}
#[derive(Default)]
pub struct Watch {
failures: u32,
next_try: Option<Instant>,
problem: Option<String>,
}
impl Watch {
pub fn problem(&self) -> Option<&str> {
self.problem.as_deref()
}
pub fn tick(&mut self, now: Instant, probe: &dyn Probe) -> (Outcome, bool) {
let before = self.problem.is_some();
let out = self.step(now, probe);
(out, self.problem.is_some() != before)
}
fn step(&mut self, now: Instant, probe: &dyn Probe) -> Outcome {
if !probe.wanted() {
self.reset();
return Outcome::Idle;
}
let Verdict::Down(why) = judge(&probe.observe()) else {
self.reset();
return Outcome::Healthy;
};
if let Some(t) = self.next_try {
if now < t {
return Outcome::Waiting(t - now);
}
}
match probe.repair() {
Ok(()) => {
self.reset();
Outcome::Repaired(why)
}
Err(e) => {
self.failures = self.failures.saturating_add(1);
let wait = (restart_backoff(self.failures) * EVERY.as_secs() as u32).min(MAX_WAIT);
self.next_try = Some(now + wait);
let error = format!("{why}: {e}");
self.problem = Some(error.clone());
Outcome::Failed {
error,
retry_in: wait,
}
}
}
}
fn reset(&mut self) {
self.failures = 0;
self.next_try = None;
self.problem = None;
}
}
pub fn enabled() -> bool {
enabled_from(std::env::var("ZAKURO_VPN_WATCH").ok().as_deref())
}
fn enabled_from(v: Option<&str>) -> bool {
v.map(str::trim) != Some("0")
}
pub fn handshake_age(now: u64, stamps: impl IntoIterator<Item = Option<u64>>) -> Option<u64> {
stamps
.into_iter()
.flatten()
.filter(|&t| t > 0)
.max()
.map(|newest| now.saturating_sub(newest))
}
pub struct HostProbe {
pub sharing: Box<dyn Fn() -> bool + Send + Sync>,
}
impl Probe for HostProbe {
fn wanted(&self) -> bool {
(self.sharing)() && crate::vpn::state::state_path().exists()
}
fn observe(&self) -> Observation {
let up = !matches!(crate::vpn::route(), crate::vpn::Route::None);
let peers = crate::vpn::live_peers().unwrap_or_default();
let now = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| d.as_secs());
Observation {
up,
has_peers: !peers.is_empty(),
handshake_age: handshake_age(now, peers.iter().map(|p| p.last_handshake_secs)),
}
}
fn repair(&self) -> Result<(), String> {
crate::vpn::ensure(crate::vpn::connector::Preference::Auto)
.map(|_| ())
.map_err(|e| e.to_string())
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::cell::{Cell, RefCell};
struct Fake {
wanted: Cell<bool>,
obs: RefCell<Observation>,
repair_ok: Cell<bool>,
repairs: Cell<u32>,
}
impl Fake {
fn new(obs: Observation) -> Self {
Self {
wanted: Cell::new(true),
obs: RefCell::new(obs),
repair_ok: Cell::new(true),
repairs: Cell::new(0),
}
}
}
impl Probe for Fake {
fn wanted(&self) -> bool {
self.wanted.get()
}
fn observe(&self) -> Observation {
self.obs.borrow().clone()
}
fn repair(&self) -> Result<(), String> {
self.repairs.set(self.repairs.get() + 1);
if self.repair_ok.get() {
Ok(())
} else {
Err("wg-quick failed".into())
}
}
}
fn healthy() -> Observation {
Observation {
up: true,
has_peers: true,
handshake_age: Some(40),
}
}
fn down() -> Observation {
Observation {
up: false,
has_peers: false,
handshake_age: None,
}
}
#[test]
fn the_verdict_covers_missing_interface_stale_and_absent_handshakes() {
assert_eq!(judge(&healthy()), Verdict::Healthy);
assert!(matches!(judge(&down()), Verdict::Down(_)));
let mut o = healthy();
o.handshake_age = Some(STALE_AFTER_SECS);
assert_eq!(judge(&o), Verdict::Healthy, "180 s exactly is still fine");
o.handshake_age = Some(STALE_AFTER_SECS + 1);
assert!(matches!(judge(&o), Verdict::Down(m) if m.contains("181s")));
o.handshake_age = None;
assert!(
matches!(judge(&o), Verdict::Down(_)),
"peers but never a handshake"
);
o.has_peers = false;
assert_eq!(
judge(&o),
Verdict::Healthy,
"no peer listed: nothing to judge by"
);
}
#[test]
fn the_handshake_age_comes_from_the_newest_handshake_time() {
let now = 1_794_390_000;
let age = handshake_age(now, [Some(now - 400), Some(now - 9), None]);
assert_eq!(age, Some(9));
let o = Observation {
up: true,
has_peers: true,
handshake_age: age,
};
assert_eq!(judge(&o), Verdict::Healthy);
assert_eq!(handshake_age(now, [Some(now - 600)]), Some(600));
assert_eq!(handshake_age(now, [None, Some(0)]), None);
assert_eq!(handshake_age(now, [Some(now + 5)]), Some(0));
}
#[test]
fn a_healthy_tunnel_is_left_alone() {
let f = Fake::new(healthy());
let mut w = Watch::default();
assert_eq!(w.tick(Instant::now(), &f).0, Outcome::Healthy);
assert_eq!(f.repairs.get(), 0);
assert_eq!(w.problem(), None);
}
#[test]
fn a_node_not_meant_to_be_on_the_mesh_is_never_touched() {
let f = Fake::new(down());
f.wanted.set(false);
let mut w = Watch::default();
assert_eq!(w.tick(Instant::now(), &f).0, Outcome::Idle);
assert_eq!(f.repairs.get(), 0);
}
#[test]
fn a_down_tunnel_is_repaired_at_once() {
let f = Fake::new(down());
let mut w = Watch::default();
assert!(matches!(w.tick(Instant::now(), &f).0, Outcome::Repaired(_)));
assert_eq!(f.repairs.get(), 1);
assert_eq!(w.problem(), None, "a repair that worked raises nothing");
}
#[test]
fn failed_repairs_back_off_exponentially_and_raise_the_problem_once() {
let f = Fake::new(down());
f.repair_ok.set(false);
let mut w = Watch::default();
let t0 = Instant::now();
let (o, changed) = w.tick(t0, &f);
assert!(changed, "raised on the first failure");
let Outcome::Failed { error, retry_in } = o else {
panic!()
};
assert!(
error.contains("wg-quick failed") && error.contains("interface"),
"{error}"
);
assert_eq!(retry_in, Duration::from_secs(30), "1 x the interval");
assert!(w.problem().is_some_and(|p| p.contains("wg-quick failed")));
let (o, changed) = w.tick(t0 + Duration::from_secs(10), &f);
assert!(matches!(o, Outcome::Waiting(d) if d == Duration::from_secs(20)));
assert!(!changed);
assert_eq!(f.repairs.get(), 1);
let (o, changed) = w.tick(t0 + Duration::from_secs(30), &f);
assert!(!changed, "already raised");
assert!(
matches!(o, Outcome::Failed { retry_in, .. } if retry_in == Duration::from_secs(60))
);
let mut last = Duration::ZERO;
let mut t = t0 + Duration::from_secs(30);
for _ in 0..8 {
t += Duration::from_secs(3600);
if let (Outcome::Failed { retry_in, .. }, _) = w.tick(t, &f) {
last = retry_in;
}
}
assert_eq!(last, MAX_WAIT, "capped at 15 minutes");
}
#[test]
fn the_problem_clears_when_the_tunnel_is_healthy_again() {
let f = Fake::new(down());
f.repair_ok.set(false);
let mut w = Watch::default();
let t0 = Instant::now();
w.tick(t0, &f);
assert!(w.problem().is_some());
*f.obs.borrow_mut() = healthy();
let (o, changed) = w.tick(t0 + Duration::from_secs(5), &f);
assert_eq!(o, Outcome::Healthy);
assert!(changed, "cleared");
assert_eq!(w.problem(), None);
*f.obs.borrow_mut() = down();
f.repair_ok.set(true);
assert!(matches!(
w.tick(t0 + Duration::from_secs(6), &f).0,
Outcome::Repaired(_)
));
}
#[test]
fn a_stale_handshake_triggers_a_repair() {
let mut o = healthy();
o.handshake_age = Some(600);
let f = Fake::new(o);
let mut w = Watch::default();
assert!(matches!(w.tick(Instant::now(), &f).0, Outcome::Repaired(_)));
assert_eq!(f.repairs.get(), 1);
}
#[test]
fn the_watch_is_off_only_for_zero() {
assert!(!enabled_from(Some("0")));
assert!(enabled_from(None));
assert!(enabled_from(Some("1")));
assert!(enabled_from(Some("")));
}
}