zc2 0.0.48

P2P compute broker with yen-denominated billing, WAL, and broker mesh support
//! The supervised VPN (zc#513/#514): every ~30 s, when this node is meant to
//! be on the mesh, check the tunnel and repair it. Before this the agent only
//! read `vpn::route()`; a tunnel that dropped (a sleep, a Wi-Fi change, a
//! stale handshake) stayed down until someone ran `zc vpn`.
//!
//! The tunnel counts as down when its interface is gone, or when it has peers
//! and the newest handshake is older than [`STALE_AFTER_SECS`]. A repair is
//! `vpn::ensure` with an exponential backoff between attempts (the series of
//! [`plan::restart_backoff`], scaled to the check interval). A `vpn_down`
//! problem is raised on the first failed repair and cleared once the tunnel
//! is healthy again. `ZAKURO_VPN_WATCH=0` turns the watcher off.
//!
//! All host access is behind [`Probe`]: the tests use fakes, never WireGuard.

use crate::agent::plan::restart_backoff;
use std::time::{Duration, Instant};

pub const EVERY: Duration = Duration::from_secs(30);
/// WireGuard re-handshakes about every two minutes under traffic; past this
/// the peer is not answering.
pub const STALE_AFTER_SECS: u64 = 180;
const MAX_WAIT: Duration = Duration::from_secs(15 * 60);

/// What the tunnel looks like right now.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Observation {
    /// The mesh interface (or the sidecar's proxy) is there.
    pub up: bool,
    /// Whether the tunnel lists any peer to have a handshake with.
    pub has_peers: bool,
    /// Seconds since the newest handshake; `None` = never.
    pub handshake_age: Option<u64>,
}

pub trait Probe {
    /// Is this node meant to be on the mesh (sharing is on, and a tunnel was
    /// set up here)? When not, the watcher leaves the tunnel alone.
    fn wanted(&self) -> bool;
    fn observe(&self) -> Observation;
    /// Bring the tunnel up (`vpn::ensure`).
    fn repair(&self) -> Result<(), String>;
}

#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Verdict {
    Healthy,
    Down(String),
}

pub fn judge(o: &Observation) -> Verdict {
    if !o.up {
        return Verdict::Down("the mesh interface is not up".into());
    }
    if o.has_peers {
        match o.handshake_age {
            None => return Verdict::Down("no handshake with the mesh yet".into()),
            Some(a) if a > STALE_AFTER_SECS => {
                return Verdict::Down(format!("the last handshake was {a}s ago"))
            }
            _ => {}
        }
    }
    Verdict::Healthy
}

/// What one tick did.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Outcome {
    /// Not wanted: nothing checked, nothing raised.
    Idle,
    Healthy,
    /// Down, and a repair is not due yet.
    Waiting(Duration),
    /// Down, repaired.
    Repaired,
    /// Down, repair failed; the next attempt is not before `retry_in`.
    Failed {
        error: String,
        retry_in: Duration,
    },
}

#[derive(Default)]
pub struct Watch {
    failures: u32,
    next_try: Option<Instant>,
    /// The `vpn_down` problem, with why: raised on the first failed repair,
    /// kept while it lasts.
    problem: Option<String>,
}

impl Watch {
    /// Why the `vpn_down` problem is raised; `None` when it is not.
    pub fn problem(&self) -> Option<&str> {
        self.problem.as_deref()
    }

    /// One check. The flag says whether the problem was raised or cleared by
    /// this tick.
    pub fn tick(&mut self, now: Instant, probe: &dyn Probe) -> (Outcome, bool) {
        let before = self.problem.is_some();
        let out = self.step(now, probe);
        (out, self.problem.is_some() != before)
    }

    fn step(&mut self, now: Instant, probe: &dyn Probe) -> Outcome {
        if !probe.wanted() {
            self.reset();
            return Outcome::Idle;
        }
        let Verdict::Down(why) = judge(&probe.observe()) else {
            self.reset();
            return Outcome::Healthy;
        };
        if let Some(t) = self.next_try {
            if now < t {
                return Outcome::Waiting(t - now);
            }
        }
        match probe.repair() {
            Ok(()) => {
                // Verified by `vpn::ensure`; the next tick re-observes it.
                self.reset();
                Outcome::Repaired
            }
            Err(e) => {
                self.failures = self.failures.saturating_add(1);
                let wait = (restart_backoff(self.failures) * EVERY.as_secs() as u32).min(MAX_WAIT);
                self.next_try = Some(now + wait);
                let error = format!("{why}: {e}");
                self.problem = Some(error.clone());
                Outcome::Failed {
                    error,
                    retry_in: wait,
                }
            }
        }
    }

    fn reset(&mut self) {
        self.failures = 0;
        self.next_try = None;
        self.problem = None;
    }
}

pub fn enabled() -> bool {
    enabled_from(std::env::var("ZAKURO_VPN_WATCH").ok().as_deref())
}

fn enabled_from(v: Option<&str>) -> bool {
    v.map(str::trim) != Some("0")
}

/// The real host: the agent's desired state, `vpn::route`/`status`/`ensure`.
pub struct HostProbe {
    pub sharing: Box<dyn Fn() -> bool + Send + Sync>,
}

impl Probe for HostProbe {
    fn wanted(&self) -> bool {
        (self.sharing)() && crate::vpn::state::state_path().exists()
    }

    fn observe(&self) -> Observation {
        let up = !matches!(crate::vpn::route(), crate::vpn::Route::None);
        let peers = crate::vpn::status()
            .ok()
            .flatten()
            .map(|i| i.peers)
            .unwrap_or_default();
        Observation {
            up,
            has_peers: !peers.is_empty(),
            handshake_age: peers.iter().filter_map(|p| p.last_handshake_secs).min(),
        }
    }

    fn repair(&self) -> Result<(), String> {
        crate::vpn::ensure(crate::vpn::connector::Preference::Auto)
            .map(|_| ())
            .map_err(|e| e.to_string())
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::cell::{Cell, RefCell};

    struct Fake {
        wanted: Cell<bool>,
        obs: RefCell<Observation>,
        repair_ok: Cell<bool>,
        repairs: Cell<u32>,
    }

    impl Fake {
        fn new(obs: Observation) -> Self {
            Self {
                wanted: Cell::new(true),
                obs: RefCell::new(obs),
                repair_ok: Cell::new(true),
                repairs: Cell::new(0),
            }
        }
    }

    impl Probe for Fake {
        fn wanted(&self) -> bool {
            self.wanted.get()
        }
        fn observe(&self) -> Observation {
            self.obs.borrow().clone()
        }
        fn repair(&self) -> Result<(), String> {
            self.repairs.set(self.repairs.get() + 1);
            if self.repair_ok.get() {
                Ok(())
            } else {
                Err("wg-quick failed".into())
            }
        }
    }

    fn healthy() -> Observation {
        Observation {
            up: true,
            has_peers: true,
            handshake_age: Some(40),
        }
    }

    fn down() -> Observation {
        Observation {
            up: false,
            has_peers: false,
            handshake_age: None,
        }
    }

    #[test]
    fn the_verdict_covers_missing_interface_stale_and_absent_handshakes() {
        assert_eq!(judge(&healthy()), Verdict::Healthy);
        assert!(matches!(judge(&down()), Verdict::Down(_)));
        let mut o = healthy();
        o.handshake_age = Some(STALE_AFTER_SECS);
        assert_eq!(judge(&o), Verdict::Healthy, "180 s exactly is still fine");
        o.handshake_age = Some(STALE_AFTER_SECS + 1);
        assert!(matches!(judge(&o), Verdict::Down(m) if m.contains("181s")));
        o.handshake_age = None;
        assert!(
            matches!(judge(&o), Verdict::Down(_)),
            "peers but never a handshake"
        );
        o.has_peers = false;
        assert_eq!(
            judge(&o),
            Verdict::Healthy,
            "no peer listed: nothing to judge by"
        );
    }

    #[test]
    fn a_healthy_tunnel_is_left_alone() {
        let f = Fake::new(healthy());
        let mut w = Watch::default();
        assert_eq!(w.tick(Instant::now(), &f).0, Outcome::Healthy);
        assert_eq!(f.repairs.get(), 0);
        assert_eq!(w.problem(), None);
    }

    #[test]
    fn a_node_not_meant_to_be_on_the_mesh_is_never_touched() {
        let f = Fake::new(down());
        f.wanted.set(false);
        let mut w = Watch::default();
        assert_eq!(w.tick(Instant::now(), &f).0, Outcome::Idle);
        assert_eq!(f.repairs.get(), 0);
    }

    #[test]
    fn a_down_tunnel_is_repaired_at_once() {
        let f = Fake::new(down());
        let mut w = Watch::default();
        assert_eq!(w.tick(Instant::now(), &f).0, Outcome::Repaired);
        assert_eq!(f.repairs.get(), 1);
        assert_eq!(w.problem(), None, "a repair that worked raises nothing");
    }

    #[test]
    fn failed_repairs_back_off_exponentially_and_raise_the_problem_once() {
        let f = Fake::new(down());
        f.repair_ok.set(false);
        let mut w = Watch::default();
        let t0 = Instant::now();
        let (o, changed) = w.tick(t0, &f);
        assert!(changed, "raised on the first failure");
        let Outcome::Failed { error, retry_in } = o else {
            panic!()
        };
        assert!(
            error.contains("wg-quick failed") && error.contains("interface"),
            "{error}"
        );
        assert_eq!(retry_in, Duration::from_secs(30), "1 x the interval");
        assert!(w.problem().is_some_and(|p| p.contains("wg-quick failed")));
        // Not due: no new attempt.
        let (o, changed) = w.tick(t0 + Duration::from_secs(10), &f);
        assert!(matches!(o, Outcome::Waiting(d) if d == Duration::from_secs(20)));
        assert!(!changed);
        assert_eq!(f.repairs.get(), 1);
        // Due: fails again, and waits twice as long.
        let (o, changed) = w.tick(t0 + Duration::from_secs(30), &f);
        assert!(!changed, "already raised");
        assert!(
            matches!(o, Outcome::Failed { retry_in, .. } if retry_in == Duration::from_secs(60))
        );
        let mut last = Duration::ZERO;
        let mut t = t0 + Duration::from_secs(30);
        for _ in 0..8 {
            t += Duration::from_secs(3600);
            if let (Outcome::Failed { retry_in, .. }, _) = w.tick(t, &f) {
                last = retry_in;
            }
        }
        assert_eq!(last, MAX_WAIT, "capped at 15 minutes");
    }

    #[test]
    fn the_problem_clears_when_the_tunnel_is_healthy_again() {
        let f = Fake::new(down());
        f.repair_ok.set(false);
        let mut w = Watch::default();
        let t0 = Instant::now();
        w.tick(t0, &f);
        assert!(w.problem().is_some());
        // The tunnel comes back by itself.
        *f.obs.borrow_mut() = healthy();
        let (o, changed) = w.tick(t0 + Duration::from_secs(5), &f);
        assert_eq!(o, Outcome::Healthy);
        assert!(changed, "cleared");
        assert_eq!(w.problem(), None);
        // And the backoff is forgotten: the next drop is repaired at once.
        *f.obs.borrow_mut() = down();
        f.repair_ok.set(true);
        assert_eq!(w.tick(t0 + Duration::from_secs(6), &f).0, Outcome::Repaired);
    }

    #[test]
    fn a_stale_handshake_triggers_a_repair() {
        let mut o = healthy();
        o.handshake_age = Some(600);
        let f = Fake::new(o);
        let mut w = Watch::default();
        assert_eq!(w.tick(Instant::now(), &f).0, Outcome::Repaired);
        assert_eq!(f.repairs.get(), 1);
    }

    #[test]
    fn the_watch_is_off_only_for_zero() {
        assert!(!enabled_from(Some("0")));
        assert!(enabled_from(None));
        assert!(enabled_from(Some("1")));
        assert!(enabled_from(Some("")));
    }
}