nordisk-pairing 0.5.27

//! **The front renews every paired link before it ends: the schedule and the renewal steps**
//! (feature `front-renew`, off by default: the steps dial the estate's boxes over the paths the
//! product hands in; Holger's lane E2 on Gunnar's lifecycle pattern — `gunnar-front` `recharge::renew_once`
//! every 6 h; shared here since P4c, A28 M11).
//!
//! A link (the boxes' own certificates under the installer CA, the CA itself, a twin's
//! replication leaf) is looked at on a schedule ([`Link`], kept in [`STATE_REL`]): a look
//! that went well is due again after [`RENEW_EVERY`]; one that failed is retried after
//! [`retry_after`] (10 min, doubling, at most [`RENEW_EVERY`]); from the [`ALERT_AFTER`]th
//! failure in a row every failed look says `ALERT [front-renew]` at ERROR ([`look`]).
//!
//! The steps: the installer CA renewed at two thirds of its life with the same key and name
//! ([`renew_ca_due`], so every pin holds), then each box's own certificate re-signed for ITS
//! key ([`renew_boxes`], over the installer's paths the product hands in — the WireGuard paths
//! of `kept::paths_for`, feature `kept`). **No downtime**: a renewal never revokes.
//!
//! What stays the product's: its runner (jera jobs, a thread), where its kept material lives,
//! and any link of its own (Holger's twin leaf through `nordisk-twin`).

use std::collections::BTreeMap;
use std::path::Path;
use std::time::Duration;

use serde::{Deserialize, Serialize};

/// How often a link whose last look went well is looked at again (Gunnar's `RENEW_EVERY`).
pub const RENEW_EVERY: Duration = Duration::from_secs(6 * 3600);
/// The first retry after a failed look; it doubles per failure, to [`RENEW_EVERY`].
pub const RETRY_FIRST: Duration = Duration::from_secs(600);
/// From this many failed looks in a row, each failure is an alert.
pub const ALERT_AFTER: u32 = 3;
/// The schedule of every link, under the data root.
pub const STATE_REL: &str = "pairing/renewal.toml";
/// How long a box's renewed certificate lives when the plan names no lifetime: the
/// installer material's (`kept::MATERIAL_VALID`, 180 days).
pub const BOX_VALID: Duration = Duration::from_secs(180 * 24 * 3600);

/// Pure: **the wait after `failures` failed looks in a row**: [`RETRY_FIRST`], doubling,
/// at most [`RENEW_EVERY`].
pub fn retry_after(failures: u32) -> Duration {
    let n = failures.saturating_sub(1).min(16);
    RETRY_FIRST.saturating_mul(1u32 << n).min(RENEW_EVERY)
}

/// **One link's schedule.** Unix seconds throughout.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct Link {
    /// The next look is due at this moment (0: at once).
    #[serde(default)]
    pub next: i64,
    /// The last look that went well (0: none yet).
    #[serde(default)]
    pub last_ok: i64,
    /// Failed looks in a row.
    #[serde(default)]
    pub failures: u32,
    /// What the last failed look said.
    #[serde(default)]
    pub last_error: String,
}

impl Link {
    pub fn due(&self, now: i64) -> bool {
        now >= self.next
    }

    /// A look went well: the next one after [`RENEW_EVERY`].
    pub fn ok(&mut self, now: i64) {
        self.last_ok = now;
        self.failures = 0;
        self.last_error.clear();
        self.next = now + RENEW_EVERY.as_secs() as i64;
    }

    /// A look failed: retried after [`retry_after`].
    pub fn failed(&mut self, now: i64, why: &str) {
        self.failures = self.failures.saturating_add(1);
        self.last_error = why.to_string();
        self.next = now + retry_after(self.failures).as_secs() as i64;
    }

    /// The alert a link in trouble raises, or `None`.
    pub fn alert(&self, name: &str) -> Option<String> {
        (self.failures >= ALERT_AFTER).then(|| {
            format!(
                "ALERT [front-renew] the {name} link's renewal failed {} times in a row (next try in {} s): {}. Its certificate keeps serving until its end; the boxes' health words say `expiring` once the end is near",
                self.failures,
                retry_after(self.failures).as_secs(),
                self.last_error
            )
        })
    }
}

/// **Every link's schedule**, kept under the data root ([`STATE_REL`]).
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct State {
    #[serde(default)]
    pub links: BTreeMap<String, Link>,
}

impl State {
    pub fn load(root: &Path) -> State {
        std::fs::read_to_string(root.join(STATE_REL)).ok().and_then(|t| toml::from_str(&t).ok()).unwrap_or_default()
    }

    pub fn save(&self, root: &Path) -> Result<(), String> {
        let text = toml::to_string(self).map_err(|e| e.to_string())?;
        crate::boxside::write_atomic(root, STATE_REL, text.as_bytes(), false).map_err(|e| format!("{}: {e}", root.join(STATE_REL).display()))
    }
}

/// What one look did.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum Looked {
    /// The link was not due; nothing was asked.
    NotDue,
    /// The work went well; its words.
    Done(String),
}

/// **One look of `link` at `now`**: nothing when it is not due; else `work`, and the
/// schedule moves on ([`Link::ok`]) or backs off ([`Link::failed`]) and is kept. A
/// failure answers `Err` (the job's ledger row says it) and, from the [`ALERT_AFTER`]th
/// in a row, is an alert at ERROR.
pub fn look(root: &Path, link: &str, now: i64, work: impl FnOnce() -> Result<String, String>) -> Result<Looked, String> {
    let mut state = State::load(root);
    let entry = state.links.entry(link.to_string()).or_default();
    if !entry.due(now) {
        return Ok(Looked::NotDue);
    }
    let r = work();
    match &r {
        Ok(_) => entry.ok(now),
        Err(e) => entry.failed(now, e),
    }
    let (alert, failures) = (entry.alert(link), entry.failures);
    state.save(root)?;
    match r {
        Ok(words) => Ok(Looked::Done(words)),
        Err(e) => {
            match alert {
                Some(a) => tracing::error!("{a}"),
                None => tracing::warn!(link, "renew: the look failed; retried in {} s: {e}", retry_after(failures).as_secs()),
            }
            Err(e)
        }
    }
}

// ─── the boxes' own certificates and the installer CA ───────────────────

/// Pure: **the installer CA renewed when due at `now`** (two thirds of its life): the same
/// key and name, a new lifetime of `valid_for`; `None` while it is not due.
pub fn renew_ca_due(ca: &crate::mtls::InstallerCa, valid_for: Duration, now: i64) -> Result<Option<crate::mtls::InstallerCa>, String> {
    if !crate::mtls::renewal_due(&ca.cert_pem, now).map_err(|e| format!("the installer CA certificate: {e}"))? {
        return Ok(None);
    }
    ca.renew_ca(valid_for).map(Some).map_err(|e| format!("the installer CA could not be renewed: {e}"))
}

/// Pure: a renewal refusal said by name: a box whose certificate has ENDED cannot be
/// reached by its installer any more (the handshake refuses it), so the renewal came
/// too late and the box is re-paired (re-admitted from the kept material) instead.
fn named(name: &str, why: String) -> String {
    let l = why.to_ascii_lowercase();
    if l.contains("expired") || l.contains("notvalidyet") || l.contains("not valid") {
        return format!("REFUSED [front-renew-expired] `{name}`: its certificate has ended and was not renewed in time ({why}); re-pair it from the kept material");
    }
    format!("`{name}`: {why}")
}

/// **Renew the boxes' own certificates that are due** at `now` over `paths` (the
/// installer's WireGuard paths in production, loopback in tests), each for `valid_for`.
/// Every box is tried; the answer names what was renewed, and an `Err` names every box
/// that refused (the others are renewed all the same).
pub fn renew_boxes(m: &crate::installer::Material, paths: Vec<(String, Box<dyn crate::installer_link::Path>)>, valid_for: Duration, now: i64) -> Result<String, String> {
    let each = crate::installer::renew(m, paths, valid_for, now);
    let renewed: Vec<&str> = each.iter().filter(|(_, r)| matches!(r, Ok(true))).map(|(n, _)| n.as_str()).collect();
    let refused: Vec<String> = each.iter().filter_map(|(n, r)| r.as_ref().err().map(|e| named(n, e.clone()))).collect();
    if !refused.is_empty() {
        return Err(format!("{} renewed; refused: {}", if renewed.is_empty() { "none".to_string() } else { renewed.join(", ") }, refused.join("; ")));
    }
    Ok(if renewed.is_empty() { format!("{} box(es) looked at, none due", each.len()) } else { format!("renewed: {}", renewed.join(", ")) })
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn the_retry_waits_double_from_ten_minutes_to_the_renewal_period() {
        assert_eq!(retry_after(1), Duration::from_secs(600));
        assert_eq!(retry_after(2), Duration::from_secs(1200));
        assert_eq!(retry_after(3), Duration::from_secs(2400));
        assert_eq!(retry_after(30), RENEW_EVERY);
    }

    /// A look not due asks nothing; a failure backs off and from the third in a row is an
    /// alert; a success moves the next look [`RENEW_EVERY`] on and clears the failures.
    #[test]
    fn a_link_is_looked_at_on_its_schedule_backed_off_and_alerted() {
        let dir = std::env::temp_dir().join(format!("np-renew-{}-{}", std::process::id(), line!()));
        let _ = std::fs::remove_dir_all(&dir);
        std::fs::create_dir_all(&dir).unwrap();
        let r = dir.as_path();
        assert_eq!(look(r, "boxes", 1_000, || Ok("renewed: main".into())).unwrap(), Looked::Done("renewed: main".into()));
        assert_eq!(look(r, "boxes", 1_001, || panic!("asked before the next look")).unwrap(), Looked::NotDue);
        let next = 1_000 + RENEW_EVERY.as_secs() as i64;
        for (n, at) in [(1u32, next), (2, next + 600), (3, next + 600 + 1200)] {
            assert!(look(r, "boxes", at, || Err("the box did not answer".into())).is_err());
            let s = State::load(r).links["boxes"].clone();
            assert_eq!(s.failures, n);
            assert_eq!(s.alert("boxes").is_some(), n >= ALERT_AFTER, "alert from the {ALERT_AFTER}th: {s:?}");
        }
        let at = next + 600 + 1200 + 2400;
        assert_eq!(look(r, "boxes", at, || Ok("renewed: main".into())).unwrap(), Looked::Done("renewed: main".into()));
        let s = State::load(r).links["boxes"].clone();
        assert_eq!((s.failures, s.alert("boxes"), s.next), (0, None, at + RENEW_EVERY.as_secs() as i64));
        let _ = std::fs::remove_dir_all(&dir);
    }

    /// An ended certificate is refused by name: it is re-paired, not renewed.
    #[test]
    fn an_ended_certificate_is_named_for_a_re_pair() {
        assert!(named("vm2", "certificate expired".into()).starts_with("REFUSED [front-renew-expired] `vm2`"));
        assert_eq!(named("vm2", "the door did not answer".into()), "`vm2`: the door did not answer");
    }
}