mock_upcloud/

guest_clock.rs

//! **The clock, which is the provider's most expensive lie — except that it is
//! not the provider's lie at all.**
//!
//! MEASURED on the live estate, to the second:
//!
//! ```text
//! appliance − front  = +7 198 668 ms   (1 h 59 m 58.7 s: the CEST offset, plus ordinary error)
//! front     − laptop = +468 ms         (same hypervisor, same zone, minutes older)
//! ```
//!
//! Two facts, and it is the PAIR that makes this diagnosable rather than
//! mysterious:
//!
//! 1. **The skew is exactly a timezone offset, never a drift.** +2 h in summer,
//!    and therefore **+1 h in winter** — because the mechanism is an RTC that
//!    holds UTC being READ as local time. So it is computed here from the zone
//!    and the date ([`utc_offset_seconds`]), not stored as a constant. A mock
//!    that hardcoded +2 h would let a fix pass in September and fail in
//!    January, which is precisely the trap the real thing sets.
//! 2. **Not every guest gets it.** The front, on the same hypervisor, in the
//!    same zone, minutes older, was correct to under half a second. The
//!    difference is the guest's own software: the front runs an ordinary distro
//!    whose userland establishes that the RTC is UTC (`/etc/adjtime`,
//!    `systemd-timedated`); the appliance runs gunnar as PID 1 with none of
//!    that, and nothing to say so.
//!
//! **So the hypervisor here presents a CORRECT UTC clock and the GUEST gets it
//! wrong.** That is where the bug lives, and a mock that skewed the clock itself
//! would model the symptom and hide the cause — it would let a "fix" that
//! subtracts two hours somewhere pass, which is not a fix, it is the same bug
//! with a second sign error stacked on it.
//!
//! # And it cannot correct itself
//!
//! [`Fault::UdpInboundDropped`] is on by default because it is the provider's
//! normal: inbound UDP replies are dropped, so DNS-over-UDP and NTP do not work
//! and `systemd-timesyncd` is useless up there. That is the whole reason
//! `gunnar-clock` exists and takes signed time from the FRONT and from nowhere
//! else — which is also what makes it airgap-safe. A mock that let NTP through
//! would let a fix that "just uses NTP" look correct here and fail there.
//!
//! # The cascade, which is the thing worth asserting end to end
//!
//! Two hours out, the box refuses the front's signed answer as too skewed, so
//! it can never correct itself; and then every credential falls outside the
//! ±300 000 ms window and nothing authenticates. Six of seven RED rows from one
//! cause. [`cascade`] produces exactly those rows so a test can assert the
//! CHAIN rather than its last link.
//!
//! # The rule, encoded so it cannot be cheated past
//!
//! **The skew window is never widened.** A row that needs a wider window to pass
//! is a RED. [`SkewWindow`] makes that demonstrable rather than a slogan: widen
//! it as far as you like and [`cascade`] still returns the quorum rows, because
//! a box whose clock is wrong is still a box whose clock is wrong. There is no
//! number that turns this green; only fixing the RTC interpretation does.

use crate::faults::{Fault, Faults};

/// How a guest's userland reads the hardware clock.
///
/// This is the whole bug, as a two-variant enum. The hypervisor's RTC holds UTC
/// in both cases.
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub enum RtcInterpretation {
    /// An ordinary distro: `/etc/adjtime` says `UTC`, `systemd-timedated`
    /// agrees, and the wall clock is right. **The front.**
    Utc,
    /// gunnar as PID 1: no `systemd-timedated`, no `/etc/adjtime`, nothing that
    /// establishes what the RTC holds — so it is read as local time and the
    /// wall clock lands one timezone offset ahead. **The appliance.**
    LocalTime,
}

impl RtcInterpretation {
    /// The skew this interpretation produces, in ms, for a zone and a moment.
    /// `Utc` is zero by construction — not "small", zero: the front's measured
    /// +468 ms is ordinary clock error and is modelled separately by whoever
    /// wants it, because a mock that baked half a second of noise into the
    /// correct case would make an exact assertion impossible.
    pub fn skew_ms(self, zone: &str, unix_secs: i64) -> i64 {
        match self {
            RtcInterpretation::Utc => 0,
            RtcInterpretation::LocalTime => utc_offset_seconds(zone, unix_secs) * 1000,
        }
    }
}

/// The zone's UTC offset in seconds at `unix_secs`.
///
/// `se-sto1` is Europe/Stockholm: CET (+1 h) in winter, CEST (+2 h) in summer,
/// switching at 01:00 UTC on the last Sunday of March and the last Sunday of
/// October — the EU rule, which is the same for every European zone this estate
/// buys in. `fi-hel1`, `de-fra1`, `nl-ams1`, `uk-lon1` are here too, because a
/// zone that is not in this table would otherwise silently answer zero and turn
/// the whole behaviour off.
///
/// No `chrono`, no tz database: the rule is twelve lines and a tz database is a
/// dependency that would have to be shipped with an airgapped appliance.
pub fn utc_offset_seconds(zone: &str, unix_secs: i64) -> i64 {
    let (winter, summer) = match zone {
        // Every zone UpCloud runs that this estate has ever bought in.
        "se-sto1" | "de-fra1" | "nl-ams1" | "es-mad1" | "pl-waw1" | "fr-par1" => (3600, 7200),
        "fi-hel1" => (7200, 10800),
        "uk-lon1" | "ie-dub1" | "pt-lis1" => (0, 3600),
        // A zone outside Europe keeps no DST rule here, and says so by being
        // absent rather than by answering a plausible zero.
        _ => return 0,
    };
    if is_eu_summer_time(unix_secs) {
        summer
    } else {
        winter
    }
}

/// The EU rule: summer time runs from 01:00 UTC on the last Sunday of March to
/// 01:00 UTC on the last Sunday of October.
pub fn is_eu_summer_time(unix_secs: i64) -> bool {
    let (y, _, _) = civil_from_days(unix_secs.div_euclid(86_400));
    let start = days_from_civil(y, 3, last_sunday(y, 3)) * 86_400 + 3_600;
    let end = days_from_civil(y, 10, last_sunday(y, 10)) * 86_400 + 3_600;
    unix_secs >= start && unix_secs < end
}

/// The day-of-month of the last Sunday in `month`.
fn last_sunday(y: i64, m: i64) -> i64 {
    let last = days_in_month(y, m);
    // 1970-01-01 was a Thursday, so `days % 7 == 3` is Sunday counting from 0.
    for d in (1..=last).rev() {
        let days = days_from_civil(y, m, d);
        if days.rem_euclid(7) == 3 {
            return d;
        }
    }
    unreachable!("every month has a Sunday")
}

fn days_in_month(y: i64, m: i64) -> i64 {
    match m {
        1 | 3 | 5 | 7 | 8 | 10 | 12 => 31,
        4 | 6 | 9 | 11 => 30,
        _ if (y % 4 == 0 && y % 100 != 0) || y % 400 == 0 => 29,
        _ => 28,
    }
}

/// Howard Hinnant's `days_from_civil`. Days since 1970-01-01.
pub fn days_from_civil(y: i64, m: i64, d: i64) -> i64 {
    let y = if m <= 2 { y - 1 } else { y };
    let era = y.div_euclid(400);
    let yoe = y - era * 400;
    let mp = if m > 2 { m - 3 } else { m + 9 };
    let doy = (153 * mp + 2) / 5 + d - 1;
    let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
    era * 146_097 + doe - 719_468
}

/// Its inverse.
pub fn civil_from_days(z: i64) -> (i64, i64, i64) {
    let z = z + 719_468;
    let era = z.div_euclid(146_097);
    let doe = z - era * 146_097;
    let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365;
    let y = yoe + era * 400;
    let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
    let mp = (5 * doy + 2) / 153;
    let d = doy - (153 * mp + 2) / 5 + 1;
    let m = if mp < 10 { mp + 3 } else { mp - 9 };
    (if m <= 2 { y + 1 } else { y }, m, d)
}

// ── the window, and the rule that it is never widened ────────────────────────

/// **±300 000 ms, and it does not move.**
///
/// The measured refusal, verbatim: *"the credential is 7198751 ms away from this
/// server's clock; the window is ±300000 ms"*.
pub const DEFAULT_WINDOW_MS: u64 = 300_000;

#[derive(Clone, Copy, Debug)]
pub struct SkewWindow {
    pub pm_ms: u64,
}

impl Default for SkewWindow {
    fn default() -> Self {
        SkewWindow { pm_ms: DEFAULT_WINDOW_MS }
    }
}

#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Verdict {
    Accepted,
    /// How far outside the window it fell, and the sentence a caller sees.
    Refused { by_ms: u64, message: String },
}

impl SkewWindow {
    pub fn check(&self, skew_ms: i64) -> Verdict {
        let away = skew_ms.unsigned_abs();
        if away <= self.pm_ms {
            return Verdict::Accepted;
        }
        Verdict::Refused {
            by_ms: away - self.pm_ms,
            message: format!(
                "the credential is {away} ms away from this server's clock; the window is ±{} ms",
                self.pm_ms
            ),
        }
    }
}

/// One row of the cascade: what went red, and why.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Red {
    pub row: &'static str,
    pub why: String,
}

/// **The whole chain, from one wrong clock.**
///
/// Six of seven RED rows on the live estate came from this single cause, and a
/// test that only asserts the last one (`monetize-poll: tenants=0`) would chase
/// the wrong thing for an afternoon. The order here is the order of causation,
/// not the order they were noticed in.
///
/// The `window` argument exists to make the rule demonstrable: **widening it
/// does not clear the chain.** The first three rows are about the box refusing
/// to CORRECT itself, and no window makes a wrong clock right — so a fix that
/// reaches for a bigger number still has a red, by construction. See
/// [`widening_the_window_still_leaves_a_red`].
pub fn cascade(skew_ms: i64, window: SkewWindow) -> Vec<Red> {
    let mut reds = vec![];
    if skew_ms.unsigned_abs() <= 1_000 {
        return reds;
    }
    // (1)–(3): the box cannot fix itself. The front's signed answer is refused
    // for being further away than the box will accept, one refusal is not a
    // quorum, and so the clock is left exactly as wrong as it was.
    reds.push(Red {
        row: "skew-refusal",
        why: format!("the front's signed time is {} ms away; refused as too skewed", skew_ms.unsigned_abs()),
    });
    reds.push(Red { row: "no-quorum", why: "one refused answer is not a quorum".into() });
    reds.push(Red { row: "clock-unchanged", why: "nothing was accepted, so nothing was set".into() });

    // (4)–(6): and now nothing authenticates, because every credential is
    // stamped by a clock that is right.
    if let Verdict::Refused { message, .. } = window.check(skew_ms) {
        reds.push(Red { row: "console-key", why: message });
        reds.push(Red { row: "banner", why: "git.gunnar.rs:2222 connection timed out".into() });
        reds.push(Red { row: "monetize-poll", why: "tenants=0 failures=1".into() });
    }
    reds
}

// ── the network half ─────────────────────────────────────────────────────────

/// **Can a UDP reply get back in?** No, and it is on by default.
///
/// `systemd-timesyncd` sends its NTP request and waits forever; a DNS query over
/// UDP does the same. That is why `gunnar-clock` takes signed time from the
/// front over TCP and from nowhere else.
pub fn udp_reply_arrives(faults: &Faults) -> bool {
    !faults.fires(Fault::UdpInboundDropped)
}

/// What an NTP query gets. `None` is not an error — it is silence, which is the
/// harder thing to handle and the thing that actually happens.
pub fn ntp_answer(faults: &Faults, true_unix_secs: i64) -> Option<i64> {
    udp_reply_arrives(faults).then_some(true_unix_secs)
}

#[cfg(test)]
mod tests {
    use super::*;

    /// **The measured pair, both halves.** The appliance is one timezone offset
    /// ahead; the front, on the same hypervisor in the same zone, is not. If
    /// this test ever collapses to one row it has stopped modelling the thing
    /// that made the bug findable.
    #[test]
    fn the_appliance_is_skewed_and_the_front_is_not() {
        // 2026-09-20, the day it was measured. CEST.
        let t = days_from_civil(2026, 9, 20) * 86_400;
        let appliance = RtcInterpretation::LocalTime.skew_ms("se-sto1", t);
        let front = RtcInterpretation::Utc.skew_ms("se-sto1", t);
        assert_eq!(appliance, 7_200_000, "1 h 59 m 58.7 s was measured; the mechanism is exactly 2 h");
        assert_eq!(front, 0, "same hypervisor, same zone, correct — that is the half that names the cause");
    }

    /// **+2 h in summer and +1 h in winter, because it is an OFFSET.** A mock
    /// that hardcoded two hours would let a fix pass in September and fail in
    /// January.
    #[test]
    fn the_skew_follows_the_calendar_not_a_constant() {
        let summer = days_from_civil(2026, 9, 20) * 86_400;
        let winter = days_from_civil(2026, 1, 15) * 86_400;
        assert_eq!(RtcInterpretation::LocalTime.skew_ms("se-sto1", summer), 7_200_000);
        assert_eq!(RtcInterpretation::LocalTime.skew_ms("se-sto1", winter), 3_600_000);
        // And Helsinki is an hour further out again, both halves of the year.
        assert_eq!(RtcInterpretation::LocalTime.skew_ms("fi-hel1", summer), 10_800_000);
        assert_eq!(RtcInterpretation::LocalTime.skew_ms("fi-hel1", winter), 7_200_000);
        // London is zero in winter — a zone where this bug is INVISIBLE for
        // half the year, which is its own trap.
        assert_eq!(RtcInterpretation::LocalTime.skew_ms("uk-lon1", winter), 0);
        assert_eq!(RtcInterpretation::LocalTime.skew_ms("uk-lon1", summer), 3_600_000);
    }

    /// The EU switch is 01:00 UTC on the last Sunday of March and of October.
    /// 2026: 29 March and 25 October.
    #[test]
    fn the_switch_is_on_the_last_sunday() {
        assert_eq!(last_sunday(2026, 3), 29);
        assert_eq!(last_sunday(2026, 10), 25);
        assert_eq!(last_sunday(2027, 3), 28);
        let just_before = days_from_civil(2026, 3, 29) * 86_400 + 3_599;
        let just_after = days_from_civil(2026, 3, 29) * 86_400 + 3_601;
        assert!(!is_eu_summer_time(just_before));
        assert!(is_eu_summer_time(just_after));
        let oct_before = days_from_civil(2026, 10, 25) * 86_400 + 3_599;
        let oct_after = days_from_civil(2026, 10, 25) * 86_400 + 3_601;
        assert!(is_eu_summer_time(oct_before));
        assert!(!is_eu_summer_time(oct_after));
    }

    #[test]
    fn the_civil_conversions_round_trip() {
        for (y, m, d) in [(1970, 1, 1), (2000, 2, 29), (2026, 9, 20), (2027, 12, 31)] {
            assert_eq!(civil_from_days(days_from_civil(y, m, d)), (y, m, d));
        }
    }

    /// The measured refusal, word for word.
    #[test]
    fn the_credential_refusal_is_the_measured_sentence() {
        let w = SkewWindow::default();
        match w.check(7_198_751) {
            Verdict::Refused { message, .. } => assert_eq!(
                message,
                "the credential is 7198751 ms away from this server's clock; the window is ±300000 ms"
            ),
            v => panic!("{v:?}"),
        }
        assert_eq!(w.check(299_999), Verdict::Accepted);
        assert_eq!(w.check(-299_999), Verdict::Accepted);
    }

    /// **Six of seven reds from one cause**, in the order of causation.
    #[test]
    fn one_wrong_clock_is_six_red_rows() {
        let reds = cascade(7_198_668, SkewWindow::default());
        let rows: Vec<&str> = reds.iter().map(|r| r.row).collect();
        assert_eq!(
            rows,
            vec!["skew-refusal", "no-quorum", "clock-unchanged", "console-key", "banner", "monetize-poll"]
        );
        assert!(reds[3].why.contains("±300000 ms"), "{}", reds[3].why);
        // And a correct clock is no reds at all, or the cascade would be
        // reporting weather instead of a cause.
        assert!(cascade(468, SkewWindow::default()).is_empty(), "the front's 468 ms is fine");
    }

    /// **THE RULE.** Widen the window as far as you like: the box still cannot
    /// correct itself, so there is still a red. There is no number that buys a
    /// green here, and this test is what stops anyone trying.
    #[test]
    fn widening_the_window_still_leaves_a_red() {
        for window_ms in [300_000u64, 1_000_000, 7_200_000, 86_400_000] {
            let reds = cascade(7_198_668, SkewWindow { pm_ms: window_ms });
            assert!(
                !reds.is_empty(),
                "a ±{window_ms} ms window turned a wrong clock green, which must never be possible"
            );
            assert_eq!(reds[0].row, "skew-refusal", "the first red is always the box refusing to fix itself");
        }
        // The only thing that clears it is the clock being right — which is the
        // RTC interpretation, and nothing else.
        assert!(cascade(RtcInterpretation::Utc.skew_ms("se-sto1", 0), SkewWindow::default()).is_empty());
    }

    /// NTP is silence, not an error, and silence is the harder thing to handle.
    #[test]
    fn no_udp_reply_ever_comes_back() {
        let f = Faults::none();
        assert!(!udp_reply_arrives(&f), "dropped inbound UDP is the provider's NORMAL, so it is the default");
        assert_eq!(ntp_answer(&f, 1_788_436_800), None);
        f.disarm(Fault::UdpInboundDropped);
        assert_eq!(ntp_answer(&f, 1_788_436_800), Some(1_788_436_800));
    }
}