mock_upcloud/

net.rs

//! **The network, and the two ways it is asymmetric.**
//!
//! Both of these were diagnosed as something else first, and both cost real
//! time, because they break traffic in ONE direction while leaving the other
//! looking healthy. A mock that gives full connectivity or no connectivity
//! cannot reproduce either, and a test written against such a mock proves
//! nothing about the estate it is meant to protect.
//!
//! # 1. DHCP option 121, and the guest that does not read it
//!
//! The appliance's utility NIC gets its address by DHCP. The front is on a
//! **different /22**. The route between them arrives only as **classless static
//! routes — DHCP option 121** — and not as a default gateway.
//!
//! A guest whose DHCP client ignores option 121 therefore comes up with a
//! perfectly good address, answers inbound traffic fine, and **cannot reach the
//! front at all**. Its clock sync and its boot narration die OUTBOUND while
//! every inbound probe says the box is healthy. It reads exactly like a
//! two-hour clock bug and is not one — that is how it took an afternoon.
//! Measured, and fixed in gunnar `35ac0c3`.
//!
//! The asymmetry IS the behaviour. [`Reach::NoRouteOutbound`] is returned for
//! the guest's own outbound traffic while [`inbound_reaches`] stays `true`.
//!
//! # 2. Hairpin NAT does not exist
//!
//! The front DNATs `:2222` to the appliance's utility address. From OUTSIDE,
//! `git.gunnar.rs:2222` works. **From the front itself, to its own public
//! address, it is `connection refused`** — locally-generated traffic never
//! traverses `prerouting`, so the DNAT it would need is never applied. A
//! healthy forge was diagnosed as broken on exactly this.
//!
//! So a reachability question here always names WHO is asking. There is no such
//! thing as "is `git.gunnar.rs:2222` up"; there is only "is it up from here".

use std::fmt;

/// One classless static route, as DHCP option 121 carries it:
/// destination prefix → gateway.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Route {
    pub dest: Ipv4Net,
    pub via: String,
}

impl fmt::Display for Route {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{} via {}", self.dest, self.via)
    }
}

/// An IPv4 prefix. Small, because the whole need is "are these two addresses in
/// the same /22", and a CIDR crate is a dependency for one comparison.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Ipv4Net {
    pub addr: u32,
    pub prefix: u8,
}

impl Ipv4Net {
    pub fn new(a: u32, prefix: u8) -> Ipv4Net {
        Ipv4Net { addr: a & mask(prefix), prefix }
    }

    pub fn parse(s: &str, prefix: u8) -> Option<Ipv4Net> {
        Some(Ipv4Net::new(parse_v4(s)?, prefix))
    }

    pub fn contains(&self, ip: &str) -> bool {
        parse_v4(ip).is_some_and(|v| v & mask(self.prefix) == self.addr)
    }

    /// The `.1` of the prefix: what a DHCP offer names as the gateway.
    pub fn gateway(&self) -> String {
        render_v4(self.addr | 1)
    }
}

impl fmt::Display for Ipv4Net {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{}/{}", render_v4(self.addr), self.prefix)
    }
}

fn mask(prefix: u8) -> u32 {
    if prefix == 0 {
        0
    } else {
        u32::MAX << (32 - prefix.min(32))
    }
}

pub fn parse_v4(s: &str) -> Option<u32> {
    let mut out: u32 = 0;
    let mut n = 0;
    for part in s.split('.') {
        let b: u8 = part.parse().ok()?;
        out = (out << 8) | b as u32;
        n += 1;
    }
    (n == 4).then_some(out)
}

pub fn render_v4(a: u32) -> String {
    format!("{}.{}.{}.{}", a >> 24, (a >> 16) & 255, (a >> 8) & 255, a & 255)
}

/// **The two /22s the estate's utility network is laid across.**
///
/// The appliance and the front land in different ones — that is what makes
/// option 121 load-bearing rather than decorative. If every server shared a
/// prefix the guest that ignores the option would work perfectly and the bug
/// would be invisible.
pub const UTILITY_A: (u32, u8) = (0x0A0D_0800, 22); // 10.13.8.0/22
pub const UTILITY_B: (u32, u8) = (0x0A0D_0C00, 22); // 10.13.12.0/22

pub fn utility_nets() -> [Ipv4Net; 2] {
    [Ipv4Net::new(UTILITY_A.0, UTILITY_A.1), Ipv4Net::new(UTILITY_B.0, UTILITY_B.1)]
}

/// The net an address belongs to, if it is one of the estate's utility nets.
pub fn net_of(ip: &str) -> Option<Ipv4Net> {
    utility_nets().into_iter().find(|n| n.contains(ip))
}

/// **What the DHCP server offers on the utility NIC.**
///
/// Note what is NOT here: `router` is `None`. The utility network has **no
/// default gateway** — the only way off your own /22 is the classless static
/// routes, and that is precisely why ignoring them is fatal and looks like
/// something else.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct DhcpOffer {
    pub address: String,
    pub prefix: u8,
    /// Option 3. Absent on the utility network.
    pub router: Option<String>,
    /// **Option 121.** The routes to every other utility prefix.
    pub classless_static_routes: Vec<Route>,
}

impl DhcpOffer {
    /// The offer a server on `ip` receives: its own prefix, no default gateway,
    /// and a route to every OTHER utility prefix via its own gateway.
    pub fn for_address(ip: &str) -> DhcpOffer {
        let own = net_of(ip);
        let routes = own
            .map(|o| {
                utility_nets()
                    .into_iter()
                    .filter(|n| *n != o)
                    .map(|n| Route { dest: n, via: o.gateway() })
                    .collect()
            })
            .unwrap_or_default();
        DhcpOffer {
            address: ip.to_string(),
            prefix: own.map(|o| o.prefix).unwrap_or(24),
            router: None,
            classless_static_routes: routes,
        }
    }
}

/// Whether a guest's DHCP client reads option 121.
///
/// This is the whole bug, as a two-variant enum — the same shape as
/// [`crate::guest_clock::RtcInterpretation`], and for the same reason: the
/// provider offers the right thing and the GUEST does or does not take it.
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
pub enum DhcpClient {
    /// Reads option 121 and installs the routes. The fixed appliance, and every
    /// ordinary distro.
    ReadsOption121,
    /// Takes the address, ignores the routes, and has no way off its own /22.
    /// **The appliance before gunnar `35ac0c3`.**
    IgnoresOption121,
}

/// The answer to "can THIS box reach THAT address, from where it is standing".
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum Reach {
    Ok,
    /// **The packet had nowhere to go.** The destination is off this guest's own
    /// prefix and it never installed the route that would have taken it there.
    /// Inbound to this same guest still works — see [`inbound_reaches`].
    NoRouteOutbound { dest: String, needed: Route },
    /// **Locally-generated traffic does not traverse `prerouting`.** This box
    /// DNATs that port on that address for everyone else, and not for itself.
    NoHairpin { dest: String, port: u16 },
    /// Nothing is listening.
    Refused { dest: String, port: u16 },
    /// **Behaviour 62: the firewall dropped it.** Silence until the caller's
    /// own timeout — NOT a refusal. MEASURED 2026-09-20: the twin's :2222 was
    /// refused from one host (admitted, nothing listening) and timed out from
    /// the other (dropped), and the timeout read as "slow".
    Dropped { dest: String, port: u16 },
}

impl Reach {
    pub fn is_ok(&self) -> bool {
        matches!(self, Reach::Ok)
    }

    /// The sentence a caller sees. `NoHairpin` deliberately reads
    /// `connection refused`, because that is what the kernel actually returns
    /// and what sent somebody looking at a healthy forge.
    pub fn why(&self) -> String {
        match self {
            Reach::Ok => "ok".into(),
            Reach::NoRouteOutbound { dest, needed } => {
                format!("no route to {dest}: option 121 offered `{needed}` and this guest did not install it")
            }
            Reach::NoHairpin { dest, port } => {
                format!("connect {dest}:{port}: connection refused (locally-generated traffic does not traverse prerouting)")
            }
            Reach::Refused { dest, port } => format!("connect {dest}:{port}: connection refused"),
            Reach::Dropped { dest, port } => format!("connect {dest}:{port}: timed out (dropped by the firewall; nothing comes back)"),
        }
    }
}

/// One destination-NAT rule, as the front holds for the forge.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Dnat {
    /// The server whose PUBLIC address carries the rule.
    pub on_server: String,
    pub port: u16,
    /// The address it is forwarded to.
    pub to_address: String,
    pub to_port: u16,
}

/// **Inbound always works**, whatever is wrong with the guest's routing table.
///
/// It is its own function, with no arguments about routes, so the asymmetry is
/// visible in the API and not just in the behaviour: a caller cannot ask one
/// question and get both answers, because there is no one answer.
pub fn inbound_reaches(listening: bool) -> bool {
    listening
}

/// Can `from` reach `dest:port`?
///
/// `from_ip` is the asking box's own utility address, `from_public` its public
/// one, `client` what its DHCP client does with option 121, and `dnats` every
/// rule in the estate.
pub fn outbound_reach(
    from_ip: &str,
    from_public: &str,
    from_uuid: &str,
    client: DhcpClient,
    dest: &str,
    port: u16,
    dnats: &[Dnat],
    listening: impl Fn(&str, u16) -> bool,
) -> Reach {
    // (1) Hairpin, first, because it fires even when the routing is perfect —
    // and because the box asking is the box that holds the rule, so every
    // route-based explanation looks fine.
    if dest == from_public {
        if dnats.iter().any(|d| d.on_server == from_uuid && d.port == port) {
            return Reach::NoHairpin { dest: dest.to_string(), port };
        }
        if !listening(dest, port) {
            return Reach::Refused { dest: dest.to_string(), port };
        }
        return Reach::Ok;
    }

    // (2) Routing. Only the utility network needs option 121; a public address
    // is reached over the public interface, which does have a default route.
    if let (Some(dest_net), Some(own_net)) = (net_of(dest), net_of(from_ip)) {
        if dest_net != own_net && client == DhcpClient::IgnoresOption121 {
            return Reach::NoRouteOutbound {
                dest: dest.to_string(),
                needed: Route { dest: dest_net, via: own_net.gateway() },
            };
        }
    }

    // (3) Is anything there.
    if listening(dest, port) {
        Reach::Ok
    } else {
        Reach::Refused { dest: dest.to_string(), port }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn the_two_utility_prefixes_are_different_slash_22s() {
        let [a, b] = utility_nets();
        assert_ne!(a, b);
        assert!(a.contains("10.13.8.101"));
        assert!(a.contains("10.13.11.255"));
        assert!(!a.contains("10.13.12.1"));
        assert!(b.contains("10.13.12.1"));
        assert_eq!(a.gateway(), "10.13.8.1");
        assert_eq!(b.gateway(), "10.13.12.1");
        assert_eq!(a.to_string(), "10.13.8.0/22");
    }

    /// **The offer has no default gateway.** That is why option 121 is
    /// load-bearing: there is no second way off the prefix.
    #[test]
    fn the_offer_carries_option_121_and_no_router() {
        let o = DhcpOffer::for_address("10.13.8.101");
        assert_eq!(o.router, None, "no default gateway on the utility network, by design");
        assert_eq!(o.classless_static_routes.len(), 1);
        assert_eq!(o.classless_static_routes[0].to_string(), "10.13.12.0/22 via 10.13.8.1");
    }

    /// **The whole afternoon, in one test.** A guest that ignores option 121
    /// cannot reach the other /22 — and inbound to that same guest is fine, so
    /// every health check says the box is up.
    #[test]
    fn ignoring_option_121_breaks_outbound_and_leaves_inbound_healthy() {
        let up = |_: &str, _: u16| true;
        let bad = outbound_reach(
            "10.13.8.101",
            "203.0.113.10",
            "appliance",
            DhcpClient::IgnoresOption121,
            "10.13.12.9",
            443,
            &[],
            up,
        );
        match &bad {
            Reach::NoRouteOutbound { needed, .. } => {
                assert_eq!(needed.to_string(), "10.13.12.0/22 via 10.13.8.1");
                assert!(bad.why().contains("did not install it"), "{}", bad.why());
            }
            other => panic!("{other:?}"),
        }
        // And the same guest answers everything sent TO it. That is the
        // asymmetry, and it is why this looked like a clock bug.
        assert!(inbound_reaches(true));

        // Same box, same address, a client that reads the option: fine.
        let good = outbound_reach(
            "10.13.8.101",
            "203.0.113.10",
            "appliance",
            DhcpClient::ReadsOption121,
            "10.13.12.9",
            443,
            &[],
            up,
        );
        assert_eq!(good, Reach::Ok);
    }

    /// Within its OWN /22 the broken guest is fine, which is the other half of
    /// why it is hard to see.
    #[test]
    fn the_broken_guest_reaches_its_own_prefix() {
        let up = |_: &str, _: u16| true;
        assert_eq!(
            outbound_reach("10.13.8.101", "1.2.3.4", "a", DhcpClient::IgnoresOption121, "10.13.8.99", 22, &[], up),
            Reach::Ok
        );
    }

    /// **From outside the front, the forge answers. From the front itself, it
    /// does not.** Same address, same port, same rule.
    #[test]
    fn there_is_no_hairpin() {
        let dnats = vec![Dnat {
            on_server: "front".into(),
            port: 2222,
            to_address: "10.13.8.101".into(),
            to_port: 2222,
        }];
        let up = |_: &str, _: u16| true;
        // The front, to its own public address.
        let r = outbound_reach(
            "10.13.12.9",
            "203.0.113.10",
            "front",
            DhcpClient::ReadsOption121,
            "203.0.113.10",
            2222,
            &dnats,
            up,
        );
        assert!(matches!(r, Reach::NoHairpin { .. }), "{r:?}");
        assert!(r.why().contains("connection refused"), "the kernel's own words: {}", r.why());
        assert!(r.why().contains("prerouting"), "and the reason, so nobody re-diagnoses it: {}", r.why());

        // Anybody else, to the same address and port: through.
        let outside = outbound_reach(
            "10.13.8.101",
            "198.51.100.30",
            "someone-else",
            DhcpClient::ReadsOption121,
            "203.0.113.10",
            2222,
            &dnats,
            up,
        );
        assert_eq!(outside, Reach::Ok);
    }

    /// A public destination needs no option 121 — the public NIC has a default
    /// route. A mock that broke ALL outbound traffic would be a different bug.
    #[test]
    fn a_public_destination_is_not_affected_by_the_missing_route() {
        let up = |_: &str, _: u16| true;
        assert_eq!(
            outbound_reach("10.13.8.101", "1.2.3.4", "a", DhcpClient::IgnoresOption121, "198.51.100.30", 443, &[], up),
            Reach::Ok
        );
    }
}

// ── behaviour 62: the firewall, evaluated ────────────────────────────────────

/// What the provider's firewall does with one inbound packet.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Admit {
    Accept,
    Drop,
}

/// **Behaviour 62.** UpCloud's firewall on one inbound packet, as this estate
/// has measured and relied on it:
///
/// * `firewall` off, or on with NO rules: wide open (private-holger-ops
///   `terraform_gate.rs` — "a server with no rule set is wide open, so every
///   rule set ends in a catch-all drop").
/// * Rules are evaluated top-down and the FIRST match decides (private-gunnar-ops
///   `reimage.rs`: a rule placed after the drop never matches).
/// * No rule matching is an accept — the same fact as the first bullet.
/// * It filters the UTILITY interface too (MEASURED 2026-09-13: :50051 from
///   the front was dropped until a rule named the front's utility address).
/// * It is stateless for UDP: the REPLY to an outbound DNS/NTP query arrives as
///   an inbound packet from port 53/123 and meets the catch-all drop (MEASURED
///   2026-09-14: `dig +tcp` answers, `+notcp` times out). TCP replies pass,
///   which [`crate::estate::Estate::udp_reply_arrives`] does not need to model
///   because only UDP is asked.
///
/// Only `direction == "in"` rules are read. `src_port`/`dst_port` 0 means "not
/// given", and a rule that constrains a port that was not given does not match.
pub fn firewall_admits(
    firewall_on: bool,
    rules: &[crate::estate::Rule],
    proto: &str,
    src_ip: &str,
    src_port: u16,
    dst_port: u16,
) -> Admit {
    if !firewall_on || rules.is_empty() {
        return Admit::Accept;
    }
    let in_range = |v: u32, lo: &str, hi: &str| -> bool {
        match (lo.trim().parse::<u32>().ok(), hi.trim().parse::<u32>().ok()) {
            (None, None) => true,
            (Some(l), None) => v == l,
            (None, Some(h)) => v == h,
            (Some(l), Some(h)) => (l..=h).contains(&v),
        }
    };
    let src = parse_v4(src_ip);
    for r in rules.iter().filter(|r| r.direction.is_empty() || r.direction == "in") {
        if !r.protocol.is_empty() && !r.protocol.eq_ignore_ascii_case(proto) {
            continue;
        }
        let (lo, hi) = (r.source_address_start.trim(), r.source_address_end.trim());
        if !(lo.is_empty() && hi.is_empty()) {
            let (Some(s), Some(l)) = (src, parse_v4(if lo.is_empty() { hi } else { lo })) else { continue };
            let h = parse_v4(if hi.is_empty() { lo } else { hi }).unwrap_or(l);
            if !(l..=h).contains(&s) {
                continue;
            }
        }
        let sp_given = !(r.source_port_start.trim().is_empty() && r.source_port_end.trim().is_empty());
        if sp_given && (src_port == 0 || !in_range(src_port as u32, &r.source_port_start, &r.source_port_end)) {
            continue;
        }
        let dp_given = !(r.destination_port_start.trim().is_empty() && r.destination_port_end.trim().is_empty());
        if dp_given && (dst_port == 0 || !in_range(dst_port as u32, &r.destination_port_start, &r.destination_port_end)) {
            continue;
        }
        return if r.action.eq_ignore_ascii_case("accept") { Admit::Accept } else { Admit::Drop };
    }
    Admit::Accept
}

#[cfg(test)]
mod firewall_tests {
    use super::*;
    use crate::estate::Rule;

    fn rule(action: &str, proto: &str, src: &str, dport: &str) -> Rule {
        Rule {
            direction: "in".into(),
            action: action.into(),
            family: "IPv4".into(),
            protocol: proto.into(),
            source_address_start: src.into(),
            source_address_end: src.into(),
            destination_port_start: dport.into(),
            destination_port_end: dport.into(),
            ..Rule::default()
        }
    }

    /// Off, or on with nothing written: wide open.
    #[test]
    fn no_rules_is_wide_open() {
        assert_eq!(firewall_admits(false, &[rule("drop", "", "", "")], "tcp", "1.2.3.4", 0, 22), Admit::Accept);
        assert_eq!(firewall_admits(true, &[], "tcp", "1.2.3.4", 0, 22), Admit::Accept);
    }

    /// Top-down, first match; a rule after the catch-all never matches.
    #[test]
    fn first_match_wins_and_a_rule_after_the_drop_is_dead() {
        let rules = vec![rule("accept", "tcp", "", "22"), rule("drop", "", "", ""), rule("accept", "tcp", "", "80")];
        assert_eq!(firewall_admits(true, &rules, "tcp", "1.2.3.4", 0, 22), Admit::Accept);
        assert_eq!(firewall_admits(true, &rules, "tcp", "1.2.3.4", 0, 80), Admit::Drop);
    }

    /// The utility network is filtered like any other source.
    #[test]
    fn the_utility_network_is_filtered_too() {
        let rules = vec![rule("accept", "tcp", "10.13.8.99", "50051"), rule("drop", "", "", "")];
        assert_eq!(firewall_admits(true, &rules, "tcp", "10.13.8.99", 0, 50051), Admit::Accept);
        assert_eq!(firewall_admits(true, &rules, "tcp", "10.13.7.210", 0, 50051), Admit::Drop);
    }
}