Skip to main content

zlayer_overlayd/
server.rs

1//! The overlayd server engine.
2//!
3//! [`OverlaydServer`] is a near 1:1 migration of the *mechanics* half of the
4//! agent's `OverlayManager`: it owns the single cluster `WireGuard`
5//! [`OverlayTransport`], the per-service Linux bridges (Linux) / HCN Internal
6//! network + endpoints (Windows), the per-node IP allocator, DNS config, and
7//! NAT traversal. The cluster-brain half (Raft, scheduler, service registry)
8//! stays in the main daemon, which drives this server over the IPC contract in
9//! [`zlayer_types::overlayd`].
10//!
11//! Every [`OverlaydRequest`] maps to a method here via [`OverlaydServer::handle`].
12
13use std::collections::HashMap;
14use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
15#[cfg(target_os = "linux")]
16use std::os::fd::AsFd;
17use std::path::{Path, PathBuf};
18use std::sync::atomic::{AtomicU64, Ordering};
19
20use ipnetwork::IpNetwork;
21use zlayer_overlay::nat::{RelayServerConfig, StunServerConfig, TurnServerConfig};
22use zlayer_overlay::{
23    Candidate, CandidateType, ConnectionType, NatConfig, NatTraversal, OverlayConfig,
24    OverlayTransport, PeerInfo, RelayServer,
25};
26use zlayer_types::overlayd::{
27    AttachHandle, AttachResult, DedicatedServiceStatus, EdgeConfig, EdgePeerStatus,
28    GuestOverlayConfig, NatCandidateWire, NatConfigSpec, NatPeerWire, NatStatusWire, OverlayMode,
29    OverlaydRequest, OverlaydResponse, PeerScope, PeerSpec, PeerStatus, ServiceOverlayInfo,
30    StatusSnapshot, EDGE_CONFIG_VERSION,
31};
32
33use crate::error::OverlaydError;
34use crate::network_state::{
35    owner_for_service, DedicatedPortAllocator, ManagedNetwork, NetworkState,
36};
37
38/// Maximum length for Linux network interface names (IFNAMSIZ - 1 for null terminator).
39const MAX_IFNAME_LEN: usize = 15;
40
41/// Reserved [`zlayer_overlay::allocator::ServiceSubnetRegistry`] key for the
42/// single node-wide shared bridge (`OverlayMode::Shared`). The leading NUL-like
43/// sentinel can never collide with a real service name (service names come from
44/// deployment specs and are DNS-label-shaped), so the shared bridge always gets
45/// exactly one stable subnet distinct from every per-service subnet.
46#[cfg(target_os = "linux")]
47const SHARED_BRIDGE_REGISTRY_KEY: &str = "__zlayer_shared_bridge__";
48
49/// Generate a Linux-safe interface name guaranteed to be <= 15 chars.
50///
51/// Joins the `parts` with `-` after a `"zl-"` prefix and appends `-{suffix}` if
52/// non-empty. When the result exceeds 15 characters, a deterministic hash of all
53/// parts is used instead to keep the name unique and within the kernel limit.
54#[must_use]
55pub fn make_interface_name(parts: &[&str], suffix: &str) -> String {
56    use std::collections::hash_map::DefaultHasher;
57    use std::hash::{Hash, Hasher};
58
59    let base = format!("zl-{}", parts.join("-"));
60    let candidate = if suffix.is_empty() {
61        base
62    } else {
63        format!("{base}-{suffix}")
64    };
65
66    if candidate.len() <= MAX_IFNAME_LEN {
67        return candidate;
68    }
69
70    // Name is too long -- produce a deterministic hash-based name.
71    let mut hasher = DefaultHasher::new();
72    for part in parts {
73        part.hash(&mut hasher);
74    }
75    suffix.hash(&mut hasher);
76    let hash = format!("{:x}", hasher.finish());
77
78    if suffix.is_empty() {
79        // "zl-" (3) + up to 12 hex chars = 15
80        let budget = MAX_IFNAME_LEN - 3;
81        format!("zl-{}", &hash[..budget.min(hash.len())])
82    } else {
83        // "zl-" (3) + hash + "-" (1) + suffix
84        let suffix_cost = 1 + suffix.len(); // "-" + suffix
85        let hash_budget = MAX_IFNAME_LEN.saturating_sub(3 + suffix_cost);
86        if hash_budget == 0 {
87            let budget = MAX_IFNAME_LEN - 3;
88            format!("zl-{}", &hash[..budget.min(hash.len())])
89        } else {
90            format!("zl-{}-{}", &hash[..hash_budget.min(hash.len())], suffix)
91        }
92    }
93}
94
95/// Pure orphan-selection predicate for [`OverlaydServer::prune_orphan_bridges`].
96///
97/// Returns `true` iff `name` is one of OUR per-service bridge (`zl-…-b`) or
98/// dedicated device (`zl-…-d`) interfaces AND is neither in the `live` set (the
99/// names the daemon says SHOULD exist) nor `protected` (the active global `-g`
100/// device, the node-wide `-sh` shared bridge, and any live in-memory service
101/// bridge/device). The `zl-` prefix gate keeps the sweep off unrelated host
102/// links; the `-b`/`-d` suffix gate keeps it off the global/shared interfaces
103/// and the `veth-…`/`vc-…` container-veth namespace (those are reclaimed by the
104/// PID-keyed `sweep_orphan_veths`, never here).
105#[cfg(target_os = "linux")]
106fn is_orphan_service_bridge(
107    name: &str,
108    live: &std::collections::HashSet<&str>,
109    protected: &std::collections::HashSet<String>,
110) -> bool {
111    if !name.starts_with("zl-") {
112        return false;
113    }
114    if !(name.ends_with("-b") || name.ends_with("-d")) {
115        return false;
116    }
117    !live.contains(name) && !protected.contains(name)
118}
119
120/// First usable host address in `subnet`.
121///
122/// For IPv4 this is `network() + 1` (skipping the network address). For IPv6
123/// the same rule applies — the network address is conventionally reserved.
124fn first_usable_ip(subnet: ipnet::IpNet) -> IpAddr {
125    match subnet {
126        ipnet::IpNet::V4(v4) => {
127            let net = u32::from(v4.network());
128            IpAddr::V4(Ipv4Addr::from(net.wrapping_add(1)))
129        }
130        ipnet::IpNet::V6(v6) => {
131            let net = u128::from(v6.network());
132            IpAddr::V6(Ipv6Addr::from(net.wrapping_add(1)))
133        }
134    }
135}
136
137/// Parameters threaded into [`OverlaydServer::attach_to_interface`] when a
138/// container is being attached to a per-service Linux bridge.
139#[cfg(target_os = "linux")]
140#[derive(Debug)]
141struct BridgeAttachParams<'a> {
142    /// Linux bridge name on the host to enslave the host-side veth into.
143    bridge_name: &'a str,
144    /// Bridge's L3 gateway IP. The container's default route is set here.
145    gateway: IpAddr,
146    /// Prefix length of the bridge's subnet.
147    subnet_prefix_len: u8,
148}
149
150/// Tracking info recorded by [`OverlaydServer::attach_container`] for every
151/// container that successfully attaches on Linux (via the per-PID `attached`
152/// map) and for every macOS host-shared container (via the
153/// `host_shared_attachments` map). Used by `detach_container`. Cross-platform
154/// so the host-shared path — which runs on macOS — can reuse the same record.
155#[derive(Debug, Clone)]
156struct AttachInfo {
157    /// IP allocated on the per-service overlay (eth0 inside the container).
158    service_ip: IpAddr,
159    /// Name of the service whose bridge owns `service_ip`.
160    service_name: Option<String>,
161    /// IP allocated on the global overlay (eth1), if the container joined it.
162    /// `Some` iff the container also attached to the global overlay; the
163    /// detach path now deletes `veth-<pid>-g` unconditionally (idempotent), so
164    /// no separate `joined_global` flag is needed.
165    ///
166    /// Linux-only: this is the per-container global/eth1 IP, allocated and read
167    /// solely by the Linux veth attach/detach paths. Host-shared containers
168    /// (macOS/Windows) share the node's single cluster utun and reach the
169    /// global overlay through their node `/32` alias, so they never allocate a
170    /// separate eth1 IP — it is always `None` off Linux and never read there.
171    #[cfg_attr(not(target_os = "linux"), allow(dead_code))]
172    global_ip: Option<IpAddr>,
173    /// True when this attach asked overlayd to reap the per-service bridge
174    /// once the LAST container detaches (ephemeral/per-job networks). False
175    /// for managed services (bridge persists across scale-to-0).
176    ephemeral: bool,
177    /// `Some(network)` when this container joined the named isolated network;
178    /// drives per-network L3 isolation membership cleanup on detach.
179    isolation_network: Option<String>,
180}
181
182/// Tracking info recorded by [`OverlaydServer::attach_container_guest`] for a
183/// guest-managed attach. Platform-agnostic (no netns/veth/HCN): the guest owns
184/// its own `WireGuard` device; the host only allocated the address + registered
185/// the guest's public key as a global peer.
186#[derive(Debug, Clone)]
187struct GuestAttachInfo {
188    /// Overlay IP allocated for the guest (released on detach).
189    overlay_ip: IpAddr,
190    /// Base64 public key registered on the global transport for the guest
191    /// (removed on detach).
192    public_key: String,
193    /// Service whose bridge pool owns `overlay_ip` (Linux service-bridge path);
194    /// `None` when drawn from the node slice. Mirrors `AttachInfo::service_name`
195    /// so detach returns the IP to the right pool.
196    service_name: Option<String>,
197    /// `Some(network)` when this guest joined the named isolated network;
198    /// drives per-network membership cleanup on detach. The guest's own
199    /// enforcement (`WireGuard` `AllowedIPs`) is wired separately — overlayd only
200    /// maintains the membership map here.
201    isolation_network: Option<String>,
202}
203
204/// Bookkeeping for one minted edge peer
205/// ([`OverlaydRequest::MintEdgePeer`]). Like [`GuestAttachInfo`] this is
206/// platform-agnostic — an edge peer is pure IPAM + keygen + a roaming global
207/// `/32` peer plus an optional node-side L3 fence; there is no netns/veth/HCN.
208/// The record carries everything [`OverlaydServer::revoke_edge_peer`] and the
209/// TTL/silence sweep need so they never have to re-derive it.
210#[derive(Debug, Clone)]
211struct EdgeAttachInfo {
212    /// Overlay `/32` allocated for the edge from the node slice (released on
213    /// revoke).
214    overlay_ip: IpAddr,
215    /// Base64 `WireGuard` public key registered on the global transport for the
216    /// edge (removed on revoke).
217    public_key: String,
218    /// Unix-seconds the peer was minted (the boot-grace clock for the sweep).
219    minted_at_unix: u64,
220    /// Unix-seconds the peer expires and is swept regardless of liveness.
221    expires_at_unix: u64,
222    /// The overlay CIDRs the edge was granted at mint time (the resolved
223    /// `--allow` list). Empty for a guest-equivalent (unfenced) edge. Used to
224    /// reproduce the exact fence `peers` set on revoke and to report `allowed`
225    /// in [`EdgePeerStatus`].
226    allowed_targets: Vec<ipnet::IpNet>,
227    /// `Some(network)` when a node-side L3 fence was installed for this edge
228    /// (network id `edge:<name>`); drives fence teardown on revoke. `None` when
229    /// `--allow` was empty (no fence installed).
230    isolation_network: Option<String>,
231}
232
233/// Per-service Linux bridge state. One bridge per service per node; containers
234/// attach to it via veth pairs and cross-node packets ride the single cluster
235/// `OverlayTransport` with the service subnet plumbed into its `AllowedIPs`.
236#[cfg(target_os = "linux")]
237#[derive(Debug)]
238struct ServiceBridge {
239    /// Linux bridge name, kept under IFNAMSIZ-1 by [`make_interface_name`].
240    name: String,
241    /// CIDR of the service's subnet on this node.
242    subnet: ipnet::IpNet,
243    /// Gateway IP within the subnet (first usable address).
244    gateway: IpAddr,
245    /// Per-service IP allocator covering `subnet`.
246    ip_allocator: zlayer_overlay::allocator::IpAllocator,
247}
248
249/// A dedicated per-service `WireGuard` transport (`OverlayMode::Dedicated`).
250///
251/// Unlike Shared mode — where every service subnet is plumbed onto the single
252/// cluster [`OverlayTransport`] via multi-CIDR `AllowedIPs` — a Dedicated
253/// service owns a *second* real `WireGuard` device with its own crypto context,
254/// listen port, overlay IP, and subnet. The device is portable (boringtun
255/// userspace `WireGuard` works on Linux/macOS/Windows), so this struct is
256/// cross-platform; only the bridge/HCN *attachment* of containers onto it is
257/// platform-gated.
258struct ServiceTransport {
259    /// The live dedicated `WireGuard` device. Dropping it tears down the TUN.
260    transport: OverlayTransport,
261    /// Actual interface name (kernel-assigned `utunN` on macOS).
262    interface: String,
263    /// base64 public key of this dedicated device.
264    public_key: String,
265    /// UDP listen port handed out by [`DedicatedPortAllocator`].
266    listen_port: u16,
267    /// This node's overlay IP on the dedicated device.
268    overlay_ip: std::net::IpAddr,
269    /// The service's subnet carried by the dedicated device.
270    subnet: ipnet::IpNet,
271    /// Guest-attach IPAM bounded to `subnet`. VZ-Linux / WSL2 guests that join
272    /// this Dedicated service draw their overlay IP from here so they land on
273    /// the dedicated device's subnet (own crypto) rather than the node slice.
274    /// The node's own `overlay_ip` is reserved at setup so guests never collide
275    /// with it. Unused on Linux, where dedicated containers attach via a
276    /// per-service bridge that owns its own allocator.
277    #[cfg_attr(target_os = "linux", allow(dead_code))]
278    ip_allocator: zlayer_overlay::allocator::IpAllocator,
279}
280
281/// The overlay daemon engine.
282pub struct OverlaydServer {
283    /// Deployment name (used for network naming). Set by `SetupGlobalOverlay`.
284    deployment: String,
285    /// Per-daemon-process disambiguator included in overlay link names. Set by
286    /// `SetupGlobalOverlay`.
287    instance_id: String,
288    /// Root data directory; HCN markers, IPAM state, etc. live under it.
289    data_dir: PathBuf,
290    /// Global overlay interface name.
291    global_interface: Option<String>,
292    /// Global overlay transport (kept alive for the TUN device lifetime). The
293    /// SINGLE cluster-wide `WireGuard` transport; every service subnet is
294    /// plumbed through its `AllowedIPs`.
295    global_transport: Option<OverlayTransport>,
296    /// Service-name -> per-service Linux bridge / placeholder name.
297    service_interfaces: HashMap<String, String>,
298    /// Service-name -> dedicated per-service `WireGuard` transport (Dedicated
299    /// mode). Coexists with `global_transport`. Empty for Shared-only nodes.
300    service_transports: HashMap<String, ServiceTransport>,
301    /// Port allocator for dedicated devices (band above the global WG port).
302    dedicated_ports: DedicatedPortAllocator,
303    /// Per-service bridge state (Linux only).
304    #[cfg(target_os = "linux")]
305    service_bridges: HashMap<String, ServiceBridge>,
306    /// The SINGLE node-wide shared bridge backing every `OverlayMode::Shared`
307    /// service (Linux only). Created once on the first Shared-service setup and
308    /// reused for all subsequent ones; container ports are exposed via the
309    /// userspace free-port L4 proxy (`proxy_manager.rs`), not per-service
310    /// bridges. `None` until the first Shared service is set up.
311    #[cfg(target_os = "linux")]
312    shared_bridge: Option<ServiceBridge>,
313    /// Resolved per-service overlay mode, recorded at `setup_service_overlay_*`
314    /// time so the container ATTACH path knows which data-plane a service uses
315    /// (per-service bridge for `Auto`/`Dedicated` vs the single shared bridge
316    /// for `Shared`) without re-deriving it. Cross-platform.
317    service_modes: HashMap<String, OverlayMode>,
318    /// Local fallback `ServiceSubnetRegistry`. Used by the Linux Shared bridge
319    /// path and by the cross-platform Dedicated path (subnets stay globally
320    /// unique regardless of mode/OS).
321    service_subnet_registry: Option<zlayer_overlay::allocator::ServiceSubnetRegistry>,
322    /// Local raft node id used as the partition key for service-subnet assign.
323    local_node_id: u64,
324    /// Base64 `WireGuard` public key of THIS node's cluster transport, as told
325    /// by the main daemon via `SetLocalWgPubkey` (used for service-subnet
326    /// `AllowedIPs` plumbing).
327    local_wg_pubkey: Option<String>,
328    /// Public key generated for the live global transport, recorded at
329    /// `setup_global_overlay` time so `Status` can surface it (the transport
330    /// itself exposes no public-key accessor).
331    transport_public_key: Option<String>,
332    /// IP allocator for the node's overlay slice.
333    ip_allocator: IpAllocator,
334    /// This node's IP on the global overlay network.
335    node_ip: Option<IpAddr>,
336    /// `WireGuard` listen port for the overlay network.
337    overlay_port: u16,
338    /// Full cluster CIDR (e.g. `10.200.0.0/16`).
339    cluster_cidr: Option<IpNetwork>,
340    /// Per-node slice CIDR.
341    slice_cidr: Option<IpNetwork>,
342    /// Map of HCN namespace GUID -> (`service_name`, `allocated_ip`,
343    /// `isolation_network`) for autoclean. The trailing `isolation_network` lets
344    /// detach drain the per-network membership map for this container.
345    #[cfg(target_os = "windows")]
346    hcn_cleanup: HashMap<windows::core::GUID, (String, std::net::IpAddr, Option<String>)>,
347    /// Per-service container-IP allocators for Windows dedicated services. Each
348    /// is bounded to that service's subnet (not the node slice) so dedicated
349    /// containers draw addresses from their own isolated network. Keyed by
350    /// service name; created lazily on the first dedicated attach.
351    #[cfg(target_os = "windows")]
352    service_ip_allocators: HashMap<String, IpAllocator>,
353    /// Per-PID tracking of overlay attachments on Linux.
354    #[cfg(target_os = "linux")]
355    attached: HashMap<u32, AttachInfo>,
356    /// Per-isolated-network membership: network name -> the set of member
357    /// overlay (service) IPs currently attached to it. Drives per-network L3
358    /// isolation (a member reaches only its own network's members + node +
359    /// egress). Populated on attach, drained on detach, across all platforms.
360    network_members: std::collections::HashMap<String, std::collections::HashSet<IpAddr>>,
361    /// Peers installed on the GLOBAL transport via `AddPeer { Global }`, keyed by
362    /// base64 public key. Tracked here (in wire-safe [`PeerSpec`] form, with the
363    /// keys kept base64 — the boringtun UAPI dump only exposes hex keys) so a
364    /// guest-managed attach can hand the guest the exact peer set the host's own
365    /// global device carries. Platform-agnostic: the guest path runs on macOS.
366    global_peers: HashMap<String, PeerSpec>,
367    /// Guest-managed overlay attachments, keyed by the opaque container `id` from
368    /// [`AttachHandle::GuestManaged`]. Records the allocated overlay IP and the
369    /// generated public key registered in the mesh so `DetachContainer` can
370    /// release the IP and remove the peer.
371    guest_attachments: HashMap<String, GuestAttachInfo>,
372    /// Minted edge peers, keyed by the caller-chosen `name` from
373    /// [`OverlaydRequest::MintEdgePeer`]. Each records the allocated `/32`, the
374    /// generated public key registered on the global transport, the mint/expiry
375    /// clocks, the granted `--allow` targets, and any node-side fence — enough
376    /// for `revoke_edge_peer` (the single teardown path) and the TTL/silence
377    /// sweep to reverse the mint without re-deriving anything. Minted peers die
378    /// with overlayd: this map is never persisted.
379    edge_attachments: HashMap<String, EdgeAttachInfo>,
380    /// Host-shared overlay attachments, keyed by the opaque container `id` from
381    /// [`AttachHandle::HostShared`] (macOS Seatbelt / native-VZ / libkrun
382    /// containers that share the node's host network namespace and its single
383    /// cluster `utun`). Records the distinct overlay `/32` allocated for the
384    /// container so `DetachContainer` can remove the utun alias, drain the
385    /// per-network L3 isolation membership, and release the IP. Cross-platform
386    /// (the host-shared path compiles everywhere; it is exercised on macOS).
387    host_shared_attachments: HashMap<String, AttachInfo>,
388    /// Overlay DNS server listen address, if one was bootstrapped.
389    dns_server_addr: Option<SocketAddr>,
390    /// DNS domain for overlay service discovery.
391    dns_domain: Option<String>,
392    /// Overlay DNS A/AAAA records this node owns (name -> ip).
393    dns_records: HashMap<String, IpAddr>,
394    /// NAT traversal configuration threaded into every `OverlayConfig`.
395    nat_config: Option<NatConfig>,
396    /// Override for `OverlayConfig::uapi_sock_dir`.
397    uapi_sock_dir: Option<PathBuf>,
398    /// Live NAT traversal orchestrator.
399    nat_traversal: Option<NatTraversal>,
400    /// Unix-epoch seconds of the last successful candidate gather / STUN refresh.
401    nat_last_refresh: AtomicU64,
402    /// NAT-traversal candidates each peer advertised, keyed by base64 public
403    /// key. Populated from `AddPeer { Global }` (the join-time candidate
404    /// exchange); the NAT maintenance tick feeds these into
405    /// `NatTraversal::connect_to_peer` to hole-punch / relay toward a peer whose
406    /// direct endpoint has not produced a recent `WireGuard` handshake.
407    peer_candidates: HashMap<String, Vec<Candidate>>,
408    /// The [`ConnectionType`] last negotiated to each peer (keyed by base64
409    /// public key), recorded by the connect loop so `NatStatus` can report
410    /// direct / hole-punched / relayed per peer.
411    peer_connection_type: HashMap<String, ConnectionType>,
412    /// Built-in relay server, started lazily on the first NAT tick when the
413    /// resolved [`NatConfig::relay_server`] is `Some`. Kept alive for the
414    /// daemon's lifetime so its background accept loop keeps running.
415    relay_server: Option<RelayServer>,
416    /// The address the built-in [`Self::relay_server`] actually bound (the real
417    /// port when `listen_port == 0`).
418    relay_bound_addr: Option<SocketAddr>,
419    /// Cluster-shared credential used to derive the built-in relay server's
420    /// `BLAKE2b` auth key. Carried in `NatConfigSpec.relay_server.auth_credential`
421    /// (the main daemon sets it from the cluster HS256 secret) so every node's
422    /// relay client derives the *same* key. `None` when no credential was
423    /// supplied (the relay then derives a key from the empty string — only nodes
424    /// that likewise have no credential can use it).
425    cluster_relay_credential: Option<String>,
426    /// Set when a `Shutdown` request has been received.
427    shutdown_requested: bool,
428    /// IPv4 `net.ipv4.ip_forward` value observed BEFORE the daemon first
429    /// enabled forwarding for an overlay container attach. `Some(prev)` is
430    /// recorded exactly once (the first time we flip it to `1`); teardown
431    /// restores `prev` so a clean shutdown reverts host routing state the
432    /// daemon turned on without clobbering an operator who set it. `None`
433    /// means the daemon never enabled IPv4 forwarding (nothing to revert).
434    #[cfg(target_os = "linux")]
435    prev_ipv4_forward: Option<String>,
436    /// Per-interface IPv6 `net.ipv6.conf.<dev>.forwarding` was enabled on
437    /// these device names for overlay routing. We enable forwarding
438    /// PER-INTERFACE (never `net.ipv6.conf.all.forwarding`, which has the
439    /// documented side effect of forcing `accept_ra=0` + `autoconf=0` on
440    /// every IPv6 interface — including the public NIC — and silently
441    /// dropping the RA-learned default route / path-MTU, which blackholes
442    /// the host's own larger reply packets). Teardown clears forwarding on
443    /// exactly these devices.
444    #[cfg(target_os = "linux")]
445    ipv6_forward_ifaces: std::collections::HashSet<String>,
446    /// Host-side veth device names THIS daemon created (`veth-<pid>-<tag>`),
447    /// recorded right after a successful `create_veth_pair`. A clean global
448    /// teardown deletes each so no host veth half is left dangling once the
449    /// overlay stops. Per-container detach may delete some of these first;
450    /// deletion is idempotent (a missing device is ignored). Only names this
451    /// daemon created are tracked — never a blanket prefix sweep that could
452    /// catch a concurrent overlay's interfaces.
453    #[cfg(target_os = "linux")]
454    created_veths: std::collections::HashSet<String>,
455    /// `zl-*` bridge device names THIS daemon created (per-service and the
456    /// node-wide shared bridge), recorded right after a successful
457    /// `create_bridge` + address + up. Deleting the bridge link on teardown
458    /// also drops its gateway address and up state, so the name alone is enough
459    /// to fully revert it.
460    #[cfg(target_os = "linux")]
461    created_bridges: std::collections::HashSet<String>,
462    /// Host `/32` (`/128`) routes to a container IP via a host-side veth that
463    /// THIS daemon installed via `replace_route_via_dev` (the bridgeless attach
464    /// path). Each entry is `(dest, prefix_len, dev)` — enough to delete the
465    /// exact route on teardown via `delete_route_via_dev`. Deletion is
466    /// idempotent (a route a prior detach already removed is ignored).
467    #[cfg(target_os = "linux")]
468    created_host_routes: Vec<(IpAddr, u8, String)>,
469}
470
471/// Whether rootless mode forces the `WireGuard` `local_endpoint` to UNSPECIFIED.
472///
473/// In rootless mode `detect_physical_egress()` runs inside the daemon netns and
474/// resolves pasta's in-netns tap IP, which is a meaningless WG source/advertised
475/// endpoint to remote peers. Extracted as a pure fn so the decision is testable
476/// without mutating the process-global `ZLAYER_ROOTLESS` env var (env writes race
477/// across parallel tests).
478fn rootless_forces_unspecified(rootless: bool) -> bool {
479    rootless
480}
481
482/// Whether a failure to create the HOST overlay adapter is fatal for the node.
483///
484/// On Linux the host adapter (a kernel TUN brought up via netlink, with the
485/// rootless userns+netns path as a fallback) IS the container data path, so a
486/// creation failure must abort overlay setup. On macOS/Windows, Linux
487/// containers live in a VZ VM / WSL2 distro that creates its OWN overlay device
488/// and meshes VM-to-VM over UDP — the host adapter (utun/Wintun, which needs
489/// root/Administrator) is only the host's own membership in the overlay and is
490/// NOT on the container data path. So on those platforms a host-adapter failure
491/// must DEGRADE to a VM-only overlay (warn + continue) rather than abort.
492///
493/// Extracted as a `cfg!`-driven pure fn so the degrade decision is unit-testable
494/// on Linux without needing to provoke a real utun/Wintun syscall failure.
495fn host_adapter_failure_is_fatal(host_adapter_mandatory: bool) -> bool {
496    cfg!(target_os = "linux") || host_adapter_mandatory
497}
498
499impl OverlaydServer {
500    /// Create a fresh server bound to `data_dir`. The overlay itself is brought
501    /// up lazily by `SetupGlobalOverlay` (which carries the deployment, slice,
502    /// port, and NAT toggle from the main daemon).
503    ///
504    /// # Panics
505    /// Panics only if the compile-time-constant default CIDR `10.200.0.0/16`
506    /// fails to parse (impossible).
507    #[must_use]
508    pub fn new(data_dir: PathBuf) -> Self {
509        // Until SetupGlobalOverlay arrives, the allocator is bounded to the
510        // default cluster /16. SetupGlobalOverlay re-binds it to the node slice.
511        let default_cidr: IpNetwork = "10.200.0.0/16".parse().expect("compile-time constant CIDR");
512        let overlay_port = zlayer_core::DEFAULT_WG_PORT;
513
514        // Rehydrate the dedicated-port allocator from the on-disk marker so a
515        // service that already owns a dedicated overlay re-binds the exact UDP
516        // port it had before this process started.
517        let marker_path = zlayer_paths::ZLayerDirs::new(data_dir.clone()).agent_network_state();
518        let recorded_dedicated_ports: Vec<u16> = NetworkState::load(&marker_path)
519            .networks
520            .iter()
521            .filter(|n| n.owner.starts_with("service:"))
522            .filter_map(|n| n.wg_port)
523            .collect();
524
525        Self {
526            deployment: String::new(),
527            instance_id: String::new(),
528            data_dir,
529            global_interface: None,
530            global_transport: None,
531            service_interfaces: HashMap::new(),
532            service_transports: HashMap::new(),
533            dedicated_ports: DedicatedPortAllocator::new(overlay_port, recorded_dedicated_ports),
534            #[cfg(target_os = "linux")]
535            service_bridges: HashMap::new(),
536            #[cfg(target_os = "linux")]
537            shared_bridge: None,
538            service_modes: HashMap::new(),
539            service_subnet_registry: None,
540            local_node_id: 0,
541            local_wg_pubkey: None,
542            transport_public_key: None,
543            ip_allocator: IpAllocator::new(default_cidr),
544            node_ip: None,
545            overlay_port,
546            cluster_cidr: Some(default_cidr),
547            slice_cidr: None,
548            #[cfg(target_os = "windows")]
549            hcn_cleanup: HashMap::new(),
550            #[cfg(target_os = "windows")]
551            service_ip_allocators: HashMap::new(),
552            #[cfg(target_os = "linux")]
553            attached: HashMap::new(),
554            network_members: std::collections::HashMap::new(),
555            global_peers: HashMap::new(),
556            guest_attachments: HashMap::new(),
557            edge_attachments: HashMap::new(),
558            host_shared_attachments: HashMap::new(),
559            dns_server_addr: None,
560            dns_domain: None,
561            dns_records: HashMap::new(),
562            nat_config: None,
563            uapi_sock_dir: None,
564            nat_traversal: None,
565            nat_last_refresh: AtomicU64::new(0),
566            peer_candidates: HashMap::new(),
567            peer_connection_type: HashMap::new(),
568            relay_server: None,
569            relay_bound_addr: None,
570            cluster_relay_credential: None,
571            shutdown_requested: false,
572            #[cfg(target_os = "linux")]
573            prev_ipv4_forward: None,
574            #[cfg(target_os = "linux")]
575            ipv6_forward_ifaces: std::collections::HashSet::new(),
576            #[cfg(target_os = "linux")]
577            created_veths: std::collections::HashSet::new(),
578            #[cfg(target_os = "linux")]
579            created_bridges: std::collections::HashSet::new(),
580            #[cfg(target_os = "linux")]
581            created_host_routes: Vec::new(),
582        }
583    }
584
585    /// Override the `WireGuard` UAPI socket directory for every overlay
586    /// transport built by this server.
587    #[must_use]
588    pub fn with_uapi_sock_dir(mut self, dir: impl Into<PathBuf>) -> Self {
589        self.uapi_sock_dir = Some(dir.into());
590        self
591    }
592
593    /// Whether a `Shutdown` request has been received.
594    #[must_use]
595    pub fn shutdown_requested(&self) -> bool {
596        self.shutdown_requested
597    }
598
599    /// The root data directory this server was constructed with. Used by the
600    /// uninstall path (`purge_managed_networks`) and for HCN marker resolution.
601    #[must_use]
602    pub fn data_dir(&self) -> &Path {
603        &self.data_dir
604    }
605
606    // -- request dispatch ----------------------------------------------------
607
608    /// Execute one [`OverlaydRequest`], producing the [`OverlaydResponse`] the
609    /// server sends back over IPC. Any internal error is folded into
610    /// [`OverlaydResponse::Err`].
611    pub async fn handle(&mut self, req: OverlaydRequest) -> OverlaydResponse {
612        match self.dispatch(req).await {
613            Ok(resp) => resp,
614            Err(e) => OverlaydResponse::Err {
615                message: e.to_string(),
616            },
617        }
618    }
619
620    #[allow(clippy::too_many_lines)]
621    async fn dispatch(&mut self, req: OverlaydRequest) -> Result<OverlaydResponse, OverlaydError> {
622        match req {
623            OverlaydRequest::SetLocalNodeId { node_id } => {
624                self.local_node_id = node_id;
625                Ok(OverlaydResponse::Ok)
626            }
627            OverlaydRequest::SetLocalWgPubkey { pubkey } => {
628                self.local_wg_pubkey = Some(pubkey);
629                Ok(OverlaydResponse::Ok)
630            }
631            OverlaydRequest::SetupGlobalOverlay {
632                deployment,
633                instance_id,
634                cluster_cidr,
635                slice_cidr,
636                wg_port,
637                nat,
638                host_adapter_mandatory,
639            } => {
640                let name = self
641                    .setup_global_overlay(
642                        deployment,
643                        instance_id,
644                        &cluster_cidr,
645                        slice_cidr.as_deref(),
646                        wg_port,
647                        nat,
648                        host_adapter_mandatory,
649                    )
650                    .await?;
651                Ok(OverlaydResponse::BridgeName { name })
652            }
653            OverlaydRequest::TeardownGlobalOverlay => {
654                self.teardown_global_overlay();
655                Ok(OverlaydResponse::Ok)
656            }
657            OverlaydRequest::SetupServiceOverlay { service, mode } => {
658                let info = self.setup_service_overlay(&service, mode).await?;
659                Ok(OverlaydResponse::ServiceOverlay(info))
660            }
661            OverlaydRequest::TeardownServiceOverlay { service } => {
662                self.teardown_service_overlay(&service).await;
663                Ok(OverlaydResponse::Ok)
664            }
665            OverlaydRequest::AllocateIp {
666                service,
667                join_global,
668            } => {
669                let ip = self.allocate_ip(&service, join_global)?;
670                Ok(OverlaydResponse::Ip { ip })
671            }
672            OverlaydRequest::ReleaseIp { ip } => {
673                self.release_ip(ip);
674                Ok(OverlaydResponse::Ok)
675            }
676            OverlaydRequest::AttachContainer {
677                handle,
678                service,
679                join_global,
680                dns_server,
681                dns_domain,
682                ephemeral,
683                isolation_network,
684            } => {
685                // A guest-managed attach takes a wholly separate path: it cannot
686                // build a veth/HCN endpoint (the target is a VM, not a host
687                // process), so it allocates the overlay identity + peer set and
688                // returns it as `GuestConfig`. PID/HCN handles keep the existing
689                // veth/HCN attach and return `Attached`.
690                if let AttachHandle::GuestManaged { id } = handle {
691                    // Record the overlay DNS resolver/zone the daemon staged for
692                    // this node so the guest config can fall back to them (same
693                    // bookkeeping `attach_container` does for the other handles).
694                    if let Some(server) = dns_server {
695                        self.dns_server_addr = Some(SocketAddr::new(server, 53));
696                    }
697                    if dns_domain.is_some() {
698                        self.dns_domain.clone_from(&dns_domain);
699                    }
700                    let config = self
701                        .attach_container_guest(
702                            &id,
703                            &service,
704                            join_global,
705                            dns_server,
706                            dns_domain,
707                            isolation_network,
708                        )
709                        .await?;
710                    Ok(OverlaydResponse::GuestConfig(config))
711                } else {
712                    let result = self
713                        .attach_container(
714                            handle,
715                            &service,
716                            join_global,
717                            ephemeral,
718                            dns_server,
719                            dns_domain,
720                            isolation_network,
721                        )
722                        .await?;
723                    Ok(OverlaydResponse::Attached(result))
724                }
725            }
726            OverlaydRequest::DetachContainer { handle } => {
727                if let AttachHandle::GuestManaged { id } = handle {
728                    self.detach_container_guest(&id).await?;
729                } else {
730                    self.detach_container(handle).await?;
731                }
732                Ok(OverlaydResponse::Ok)
733            }
734            // `scope` selects the target device: `Global` (default) = the single
735            // cluster transport; `Service { service }` = that service's
736            // dedicated per-service transport.
737            OverlaydRequest::AddPeer { peer, scope } => {
738                let info = peer_spec_to_info(&peer)?;
739                // VM-only overlay (macOS/Windows host adapter unavailable):
740                // there is no host transport to program for the Global scope, so
741                // WARN-AND-SKIP the on-device install instead of erroring. The
742                // peer is still mirrored into `global_peers` below so guests can
743                // reproduce the global peer set via the separate guest-config
744                // push — the host simply doesn't join. `Some` transports are
745                // unaffected.
746                if matches!(scope, PeerScope::Global) && self.global_transport.is_none() {
747                    tracing::warn!(
748                        peer = %peer.public_key,
749                        "global overlay has no host adapter (VM-only overlay); \
750                         skipping host peer install — guests receive this peer via \
751                         guest-config push"
752                    );
753                } else {
754                    let transport = self.transport_for_scope(&scope)?;
755                    Self::add_peer_on(transport, &info).await?;
756                }
757                // Record the peer's advertised NAT candidates (if any) so the
758                // NAT maintenance tick can hole-punch / relay toward it. Stored
759                // for both scopes keyed by public key (the cluster transport is
760                // the one carrying packets either way). Empty candidate lists
761                // are dropped from the map so the tick's borrow loop stays cheap.
762                if peer.candidates.is_empty() {
763                    self.peer_candidates.remove(&peer.public_key);
764                } else {
765                    let parsed: Vec<Candidate> = peer
766                        .candidates
767                        .iter()
768                        .filter_map(wire_to_candidate)
769                        .collect();
770                    if parsed.is_empty() {
771                        self.peer_candidates.remove(&peer.public_key);
772                    } else {
773                        self.peer_candidates.insert(peer.public_key.clone(), parsed);
774                    }
775                }
776                // Mirror Global peers into `global_peers` so a guest-managed
777                // attach can reproduce the host's global peer set for the guest.
778                if matches!(scope, PeerScope::Global) {
779                    self.global_peers.insert(peer.public_key.clone(), peer);
780                }
781                Ok(OverlaydResponse::Ok)
782            }
783            OverlaydRequest::RemovePeer { pubkey, scope } => {
784                // VM-only overlay: no host transport for the Global scope, so the
785                // on-device removal is a no-op — just drop it from `global_peers`
786                // below. `Some` transports are unaffected.
787                if matches!(scope, PeerScope::Global) && self.global_transport.is_none() {
788                    tracing::warn!(
789                        peer = %pubkey,
790                        "global overlay has no host adapter (VM-only overlay); \
791                         skipping host peer removal"
792                    );
793                } else {
794                    let transport = self.transport_for_scope(&scope)?;
795                    Self::remove_peer_on(transport, &pubkey).await?;
796                }
797                if matches!(scope, PeerScope::Global) {
798                    self.global_peers.remove(&pubkey);
799                }
800                self.peer_candidates.remove(&pubkey);
801                self.peer_connection_type.remove(&pubkey);
802                Ok(OverlaydResponse::Ok)
803            }
804            OverlaydRequest::AddAllowedIp {
805                pubkey,
806                cidr,
807                scope,
808            } => {
809                // VM-only overlay: no host device to plumb AllowedIPs into for the
810                // Global scope — warn-and-skip. `Some` transports are unaffected.
811                if matches!(scope, PeerScope::Global) && self.global_transport.is_none() {
812                    tracing::warn!(
813                        peer = %pubkey,
814                        cidr = %cidr,
815                        "global overlay has no host adapter (VM-only overlay); \
816                         skipping host AllowedIP add"
817                    );
818                } else {
819                    let transport = self.transport_for_scope(&scope)?;
820                    Self::add_allowed_ip_on(transport, &pubkey, &cidr).await?;
821                }
822                Ok(OverlaydResponse::Ok)
823            }
824            OverlaydRequest::RemoveAllowedIp {
825                pubkey,
826                cidr,
827                scope,
828            } => {
829                // VM-only overlay: no host device for the Global scope — the
830                // removal is a no-op. `Some` transports are unaffected.
831                if matches!(scope, PeerScope::Global) && self.global_transport.is_none() {
832                    tracing::warn!(
833                        peer = %pubkey,
834                        cidr = %cidr,
835                        "global overlay has no host adapter (VM-only overlay); \
836                         skipping host AllowedIP removal"
837                    );
838                } else {
839                    let transport = self.transport_for_scope(&scope)?;
840                    Self::remove_allowed_ip_on(transport, &pubkey, &cidr).await?;
841                }
842                Ok(OverlaydResponse::Ok)
843            }
844            OverlaydRequest::MintEdgePeer {
845                name,
846                ttl_secs,
847                allow,
848                node_endpoint,
849            } => {
850                let config = self
851                    .mint_edge_peer(name, ttl_secs, &allow, &node_endpoint)
852                    .await?;
853                Ok(OverlaydResponse::EdgeConfig(config))
854            }
855            OverlaydRequest::RevokeEdgePeer { name } => {
856                // The wire has no dedicated revoke response; success (idempotent,
857                // whether or not the name existed) maps to the generic `Ok`.
858                self.revoke_edge_peer(&name).await?;
859                Ok(OverlaydResponse::Ok)
860            }
861            OverlaydRequest::ListEdgePeers => Ok(OverlaydResponse::EdgePeers {
862                peers: self.edge_peer_statuses().await,
863            }),
864            OverlaydRequest::RegisterDns { name, ip } => {
865                self.register_dns(name, ip);
866                Ok(OverlaydResponse::Ok)
867            }
868            OverlaydRequest::UnregisterDns { name } => {
869                self.unregister_dns(&name);
870                Ok(OverlaydResponse::Ok)
871            }
872            OverlaydRequest::WriteScopedResolver {
873                zone,
874                node_ip,
875                port,
876            } => {
877                #[cfg(target_os = "macos")]
878                {
879                    zlayer_overlay::dns::write_scoped_resolver(&zone, node_ip, port).map_err(
880                        |e| OverlaydError::Overlay(format!("write_scoped_resolver({zone}): {e}")),
881                    )?;
882                    Ok(OverlaydResponse::Ok)
883                }
884                #[cfg(not(target_os = "macos"))]
885                {
886                    let _ = (zone, node_ip, port);
887                    Err(OverlaydError::Overlay(
888                        "scoped resolver is macOS-only".into(),
889                    ))
890                }
891            }
892            OverlaydRequest::RemoveScopedResolver { zone } => {
893                #[cfg(target_os = "macos")]
894                {
895                    zlayer_overlay::dns::remove_scoped_resolver(&zone).map_err(|e| {
896                        OverlaydError::Overlay(format!("remove_scoped_resolver({zone}): {e}"))
897                    })?;
898                    Ok(OverlaydResponse::Ok)
899                }
900                #[cfg(not(target_os = "macos"))]
901                {
902                    let _ = zone;
903                    Err(OverlaydError::Overlay(
904                        "scoped resolver is macOS-only".into(),
905                    ))
906                }
907            }
908            OverlaydRequest::PruneOrphanBridges { live_bridge_names } => {
909                let reclaimed = self.prune_orphan_bridges(&live_bridge_names).await;
910                Ok(OverlaydResponse::PrunedBridges { reclaimed })
911            }
912            OverlaydRequest::Status => Ok(OverlaydResponse::Status(self.status_snapshot().await)),
913            OverlaydRequest::NatTick => {
914                self.nat_maintenance_tick().await?;
915                Ok(OverlaydResponse::Ok)
916            }
917            OverlaydRequest::NatStatus => Ok(OverlaydResponse::NatStatus(
918                self.nat_status_snapshot().await,
919            )),
920            OverlaydRequest::Shutdown => {
921                self.shutdown_requested = true;
922                self.teardown_global_overlay();
923                Ok(OverlaydResponse::Ok)
924            }
925        }
926    }
927
928    // -- global overlay ------------------------------------------------------
929
930    /// Bring up (or reuse) this node's base/global overlay.
931    ///
932    /// Idempotent: if a global transport is already live, reuse it (recreating
933    /// without this guard could yank the kernel TUN out from under the running
934    /// boringtun worker). Re-binds the IP allocator to `slice_cidr` if one is
935    /// supplied so container IPs never collide across nodes.
936    ///
937    /// # Errors
938    /// Returns an error if key generation or interface creation fails.
939    #[allow(clippy::too_many_lines)]
940    #[allow(clippy::too_many_arguments)]
941    async fn setup_global_overlay(
942        &mut self,
943        deployment: String,
944        instance_id: String,
945        cluster_cidr: &str,
946        slice_cidr: Option<&str>,
947        wg_port: u16,
948        nat: Option<NatConfigSpec>,
949        host_adapter_mandatory: bool,
950    ) -> Result<String, OverlaydError> {
951        self.deployment = deployment;
952        self.instance_id = instance_id;
953        self.overlay_port = wg_port;
954
955        let cluster: IpNetwork = cluster_cidr.parse().map_err(|e| {
956            OverlaydError::Other(format!("invalid cluster CIDR {cluster_cidr}: {e}"))
957        })?;
958        self.cluster_cidr = Some(cluster);
959        if let Some(slice) = slice_cidr {
960            let slice_net: IpNetwork = slice
961                .parse()
962                .map_err(|e| OverlaydError::Other(format!("invalid slice CIDR {slice}: {e}")))?;
963            self.slice_cidr = Some(slice_net);
964            self.ip_allocator = IpAllocator::new(slice_net);
965        }
966        // Thread the full operator-supplied NAT config (STUN/TURN servers,
967        // timeouts, relay-server bind + credential) into overlayd. `None` means
968        // the main daemon supplied no explicit config, so overlayd keeps its
969        // built-in `NatConfig::default()` (NAT enabled, Google STUN). A `Some`
970        // spec is converted verbatim — including the relay credential, stashed
971        // separately so the relay server can be stood up with a cluster-shared
972        // auth key on the first NAT tick.
973        if let Some(spec) = nat {
974            self.cluster_relay_credential = spec
975                .relay_server
976                .as_ref()
977                .and_then(|r| r.auth_credential.clone());
978            self.nat_config = Some(nat_config_spec_to_config(spec));
979        }
980
981        if let Some(name) = self.global_interface.clone() {
982            if self.global_transport.is_some() {
983                tracing::debug!(
984                    deployment = %self.deployment,
985                    "Global overlay already active, reusing existing transport"
986                );
987                return Ok(name);
988            }
989        }
990
991        let interface_name = make_interface_name(&[&self.deployment, &self.instance_id], "g");
992
993        let (private_key, public_key) = OverlayTransport::generate_keys()
994            .await
995            .map_err(|e| OverlaydError::Overlay(format!("Failed to generate keys: {e}")))?;
996
997        // The node's own overlay IP is the deterministic first-usable host of
998        // its slice (reserved offset 1), NOT a racy `allocate()` that drifts by
999        // allocation order. Containers draw from offset 2 onward, so the node
1000        // IP is stable across restarts and never collides with a container.
1001        let node_ip = self.ip_allocator.node_ip();
1002        self.transport_public_key = Some(public_key.clone());
1003        let physical_egress_ip = match zlayer_overlay::detect_physical_egress().await {
1004            Ok(egress) => Some(egress.ip),
1005            Err(e) => {
1006                tracing::warn!(
1007                    error = %e,
1008                    "failed to detect physical egress; WireGuard local_endpoint \
1009                     will bind UNSPECIFIED for the global overlay"
1010                );
1011                None
1012            }
1013        };
1014        let config = self.build_config(
1015            private_key,
1016            public_key,
1017            node_ip,
1018            16,
1019            self.overlay_port,
1020            physical_egress_ip,
1021        );
1022        // Remove any stale `-g` interface with this (now deterministic) name
1023        // left by a previous daemon instance, so the create below cleanly
1024        // REPLACES it instead of failing "File exists" or orphaning the old
1025        // one. With a stable per-host instance id the name is constant across
1026        // restarts, so exactly one global interface ever exists.
1027        #[cfg(target_os = "linux")]
1028        let _ = crate::netlink::delete_link_by_name(&interface_name).await;
1029        let mut transport = OverlayTransport::new(config, interface_name);
1030
1031        // Creating the host overlay adapter is fatal on Linux (the kernel TUN IS
1032        // the container data path) but only DEGRADES on macOS/Windows: there,
1033        // Linux containers run in a VZ VM / WSL2 distro that creates its own
1034        // overlay device and meshes VM-to-VM over UDP, so the host adapter
1035        // (utun/Wintun, needs root/Administrator) is just the host's own overlay
1036        // membership and is NOT on the container data path. The allocator and
1037        // `node_ip` are already bound above, so guest-config push + IP allocation
1038        // keep working even when the host adapter is unavailable.
1039        // Map the (non-`Send`) `Box<dyn Error>` to an owned `String` BEFORE the
1040        // match so no non-`Send` value is held across the `configure().await`
1041        // below — the daemon's request handler future must stay `Send`.
1042        let create_result = transport
1043            .create_interface()
1044            .await
1045            .map_err(|e| e.to_string());
1046        let actual_name = match create_result {
1047            Ok(()) => {
1048                transport.configure(&[]).await.map_err(|e| {
1049                    OverlaydError::Overlay(format!("Failed to configure global overlay: {e}"))
1050                })?;
1051                // Read back the actual interface name (on macOS, the kernel
1052                // assigns utunN).
1053                let actual_name = transport.interface_name().to_string();
1054                self.node_ip = Some(node_ip);
1055                self.global_interface = Some(actual_name.clone());
1056                self.global_transport = Some(transport);
1057                actual_name
1058            }
1059            Err(e) if !host_adapter_failure_is_fatal(host_adapter_mandatory) => {
1060                // macOS / Windows: continue with a VM-only overlay. Leave
1061                // `global_transport == None` (the natural "no host adapter"
1062                // signal), keep `node_ip` so allocation/guest config are
1063                // unaffected, and SKIP `configure` (no device to program).
1064                tracing::warn!(
1065                    error = %e,
1066                    "host overlay adapter unavailable (needs root/Administrator); \
1067                     continuing with VM-only overlay — the host will not join the \
1068                     overlay, but containers running in the VM mesh VM-to-VM and IP \
1069                     allocation/guest config are unaffected"
1070                );
1071                self.node_ip = Some(node_ip);
1072                self.global_interface = None;
1073                self.global_transport = None;
1074                // No real device exists; return an honest marker so the IPC
1075                // response is a success without implying a live adapter.
1076                "(host-adapter-disabled)".to_string()
1077            }
1078            Err(e) => {
1079                // Linux (and any future fatal-on-failure target): unchanged —
1080                // a host-adapter creation failure aborts overlay setup.
1081                return Err(OverlaydError::Overlay(format!(
1082                    "Failed to create global overlay: {e}"
1083                )));
1084            }
1085        };
1086
1087        // In rootless mode the daemon runs in its own network namespace and
1088        // `pasta` provides egress NAT + inbound port forwarding; the host-table
1089        // iptables setup below is at best a no-op inside the netns and at worst
1090        // spurious, so skip it entirely. Otherwise install the host firewall
1091        // rules as usual.
1092        if std::env::var_os("ZLAYER_ROOTLESS").is_none() {
1093            // Stop systemd-networkd / NetworkManager from managing the overlay
1094            // links overlayd just created. With a permissive default match they
1095            // try to bring `zl-*` up / run DHCP and (seen on a CI runner)
1096            // SIGABRT on the networkd watchdog while processing a `zl-*` Link
1097            // UP. Best-effort; reverted in `teardown_global_overlay`.
1098            zlayer_overlay::networkd::mark_overlay_interfaces_unmanaged();
1099
1100            // Allow overlay traffic through the host firewall (UFW / firewalld /
1101            // a bare `iptables -P FORWARD DROP`). Without this, a container's DNS
1102            // query to the node overlay IP — and inter-service overlay traffic —
1103            // is dropped by the host's INPUT/FORWARD policy before it reaches
1104            // ZLayer's resolver. Best-effort: a host without `iptables` logs a
1105            // warning rather than aborting overlay setup.
1106            if let Err(e) =
1107                zlayer_overlay::firewall::ensure_overlay_subnet_rules(&cluster.to_string())
1108            {
1109                tracing::warn!(
1110                    error = %e,
1111                    cidr = %cluster,
1112                    "failed to install overlay firewall allow-rules; service DNS / \
1113                     cross-service traffic may be blocked by the host firewall"
1114                );
1115            }
1116
1117            // SNAT overlay-sourced egress so containers can reach the LAN/internet.
1118            // The allow-rules above + `ip_forward` only get the packet *forwarded*
1119            // out the WAN NIC; without masquerade it leaves with a private
1120            // `10.200.0.0/16` source and replies never route back (ENETUNREACH /
1121            // hangs for `wget http://<public-ip>`). Best-effort, same as above.
1122            if let Err(e) =
1123                zlayer_overlay::firewall::ensure_overlay_masquerade(&cluster.to_string())
1124            {
1125                tracing::warn!(
1126                    error = %e,
1127                    cidr = %cluster,
1128                    "failed to install overlay egress masquerade; overlay containers \
1129                     may be unable to reach the LAN / internet"
1130                );
1131            }
1132        } else {
1133            tracing::info!(
1134                "rootless mode: skipping host iptables (pasta provides egress + port forwarding)"
1135            );
1136        }
1137
1138        Ok(actual_name)
1139    }
1140
1141    /// Tear down the node's base overlay (e.g. on full uninstall / shutdown).
1142    fn teardown_global_overlay(&mut self) {
1143        if let Some(mut transport) = self.global_transport.take() {
1144            tracing::info!("Shutting down global overlay");
1145            transport.shutdown();
1146        }
1147        self.global_interface = None;
1148        self.transport_public_key = None;
1149
1150        // Revert host network state this daemon mutated so a clean stop
1151        // recovers connectivity WITHOUT requiring a reboot. Forwarding
1152        // sysctls and the overlay iptables chains are otherwise sticky:
1153        // they survive both the daemon stop and an `iptables -F`, so prior
1154        // to this the only way to undo them was a reboot.
1155        #[cfg(target_os = "linux")]
1156        self.revert_forwarding();
1157        zlayer_overlay::firewall::remove_overlay_masquerade();
1158        zlayer_overlay::firewall::remove_overlay_subnet_rules();
1159        // `remove_member_isolation` deliberately leaves the ZLAYER-OVERLAY-ISO
1160        // chain + its FORWARD jump resident (other members may still use them);
1161        // on a full overlay teardown remove the whole chain so nothing leaks.
1162        zlayer_overlay::firewall::remove_overlay_isolation();
1163        // macOS: strip the pf overlay anchor + the two marked `/etc/pf.conf`
1164        // lines this node installs for the cluster/DNS ports. Without this they
1165        // leak past daemon stop (the anchor file and `/etc/pf.conf` refs are
1166        // sticky on disk). Idempotent: a missing anchor / not-root / disabled-pf
1167        // case is treated as a successful no-op by the backend. cfg-gated so
1168        // Linux/Windows teardown behaviour is unchanged.
1169        #[cfg(target_os = "macos")]
1170        if let Err(e) = zlayer_overlay::firewall::remove_overlay_rules() {
1171            tracing::warn!(error = %e, "failed to remove macOS pf overlay rules during teardown");
1172        }
1173        // Remove the systemd-networkd / NetworkManager "unmanaged" drop-ins we
1174        // installed at setup so a clean stop fully reverts host network state.
1175        zlayer_overlay::networkd::unmark_overlay_interfaces_unmanaged();
1176
1177        // Revert the host-side netlink resources this daemon created (veths,
1178        // host /32 routes, bridges). The netlink helpers are async; this fn must
1179        // keep its sync signature, so bridge to the surrounding multi-thread
1180        // tokio runtime via block_in_place + Handle::block_on. Order matters:
1181        // delete routes first (they reference the veth as their oif), then the
1182        // host-side veths, then the bridges (deleting a bridge link drops its
1183        // address + up state). Every delete is best-effort + idempotent: a
1184        // resource a prior per-container detach already removed surfaces as
1185        // NotFound/ESRCH which the helpers treat as success, and a genuine
1186        // failure is logged and skipped so a partial teardown never aborts the
1187        // rest.
1188        #[cfg(target_os = "linux")]
1189        {
1190            let routes: Vec<(IpAddr, u8, String)> = std::mem::take(&mut self.created_host_routes);
1191            let veths: Vec<String> = self.created_veths.drain().collect();
1192            let bridges: Vec<String> = self.created_bridges.drain().collect();
1193
1194            let delete_all = || async {
1195                for (dest, prefix, dev) in &routes {
1196                    if let Err(e) = crate::netlink::delete_route_via_dev(*dest, *prefix, dev).await
1197                    {
1198                        tracing::warn!(
1199                            dest = %dest, prefix, dev = %dev, error = %e,
1200                            "teardown: failed to delete host route (continuing)"
1201                        );
1202                    }
1203                }
1204                for veth in &veths {
1205                    if let Err(e) = crate::netlink::delete_link_by_name(veth).await {
1206                        tracing::warn!(
1207                            veth = %veth, error = %e,
1208                            "teardown: failed to delete host-side veth (continuing)"
1209                        );
1210                    }
1211                }
1212                for bridge in &bridges {
1213                    if let Err(e) = crate::netlink::delete_link_by_name(bridge).await {
1214                        tracing::warn!(
1215                            bridge = %bridge, error = %e,
1216                            "teardown: failed to delete bridge (continuing)"
1217                        );
1218                    }
1219                }
1220            };
1221
1222            match tokio::runtime::Handle::try_current() {
1223                Ok(handle) => {
1224                    tokio::task::block_in_place(|| handle.block_on(delete_all()));
1225                }
1226                Err(_) => {
1227                    // No ambient runtime (e.g. a non-async shutdown path): spin
1228                    // up a throwaway current-thread runtime to drive the deletes.
1229                    match tokio::runtime::Builder::new_current_thread()
1230                        .enable_all()
1231                        .build()
1232                    {
1233                        Ok(rt) => rt.block_on(delete_all()),
1234                        Err(e) => tracing::warn!(
1235                            error = %e,
1236                            "teardown: could not build a runtime to revert netlink \
1237                             resources; veths/routes/bridges left in place"
1238                        ),
1239                    }
1240                }
1241            }
1242        }
1243    }
1244
1245    /// Enable IP forwarding for an overlay container attach, scoped to the
1246    /// address family in use and (for IPv6) to the specific overlay devices.
1247    ///
1248    /// IPv4 has no per-interface forwarding knob that affects routing the way
1249    /// we need, so `net.ipv4.ip_forward` is global — but that is harmless for
1250    /// the host's own INPUT / reply path (it only permits the box to route
1251    /// transit traffic). We snapshot its prior value once so teardown can
1252    /// restore it.
1253    ///
1254    /// IPv6 is the dangerous case: `net.ipv6.conf.all.forwarding=1` forces
1255    /// `accept_ra=0` + `autoconf=0` on EVERY IPv6 interface, which drops the
1256    /// RA-learned default route and path-MTU on the public NIC and blackholes
1257    /// the host's own larger reply packets. We therefore enable forwarding
1258    /// only on the specific overlay device(s) via
1259    /// `net.ipv6.conf.<dev>.forwarding`, which routes overlay traffic without
1260    /// touching the physical NIC's RA / PMTU state.
1261    #[cfg(target_os = "linux")]
1262    fn enable_forwarding_for_attach(
1263        &mut self,
1264        is_v6: bool,
1265        veth_host: &str,
1266        bridge_name: Option<&str>,
1267    ) {
1268        // IPv4 forwarding (global) — required for v4 overlay egress, benign
1269        // for INPUT. Snapshot the prior value exactly once.
1270        if self.prev_ipv4_forward.is_none() {
1271            let prev = crate::netlink::read_sysctl("net.ipv4.ip_forward")
1272                .unwrap_or_else(|_| "0".to_string());
1273            self.prev_ipv4_forward = Some(prev);
1274        }
1275        let _ = crate::netlink::set_sysctl("net.ipv4.ip_forward", "1");
1276
1277        // IPv6 forwarding — PER-INTERFACE only. Enable on the host-side veth
1278        // and (when bridged) the bridge so the overlay routes, without the
1279        // `all.forwarding` RA/PMTU side effect on the physical NIC. The Linux
1280        // sysctl name uses '/' for the interface segment escaped to '.' by
1281        // set_sysctl's dot-translation — so pass the device name with any
1282        // literal dots intact (overlay device names never contain dots).
1283        if is_v6 {
1284            for dev in std::iter::once(veth_host).chain(bridge_name) {
1285                let key = format!("net.ipv6.conf.{dev}.forwarding");
1286                if crate::netlink::set_sysctl(&key, "1").is_ok() {
1287                    self.ipv6_forward_ifaces.insert(dev.to_string());
1288                }
1289            }
1290        }
1291    }
1292
1293    /// Revert the forwarding sysctls this daemon enabled (counterpart of
1294    /// [`Self::enable_forwarding_for_attach`]). Restores the snapshotted IPv4
1295    /// value and clears per-interface IPv6 forwarding on exactly the devices
1296    /// we touched. Best-effort: a failed write (device already gone, `/proc`
1297    /// not writable) is ignored — the worst case is the pre-existing sticky
1298    /// state, never a crash on shutdown.
1299    #[cfg(target_os = "linux")]
1300    fn revert_forwarding(&mut self) {
1301        if let Some(prev) = self.prev_ipv4_forward.take() {
1302            let _ = crate::netlink::set_sysctl("net.ipv4.ip_forward", &prev);
1303        }
1304        for dev in self.ipv6_forward_ifaces.drain() {
1305            let key = format!("net.ipv6.conf.{dev}.forwarding");
1306            let _ = crate::netlink::set_sysctl(&key, "0");
1307        }
1308    }
1309
1310    // -- service overlay -----------------------------------------------------
1311
1312    /// Set up the per-service Linux bridge that backs `service` on this node.
1313    ///
1314    /// Returns the bridge name on success.
1315    ///
1316    /// # Errors
1317    /// Returns an error if subnet assignment fails (exhaustion), if the bridge
1318    /// cannot be created, or if the cluster transport rejects the `AllowedIPs`
1319    /// update.
1320    #[cfg(target_os = "linux")]
1321    async fn setup_service_overlay(
1322        &mut self,
1323        service: &str,
1324        mode: OverlayMode,
1325    ) -> Result<ServiceOverlayInfo, OverlaydError> {
1326        // Decision surface is the two predicates on `OverlayMode` (see
1327        // `zlayer_types::overlay`), not an ad-hoc variant match:
1328        //   - uses_shared_bridge() -> the single node-wide shared bridge (+ the
1329        //     userspace free-port L4 proxy wired in `proxy_manager.rs`).
1330        //   - uses_per_service_wg() -> a dedicated per-service WireGuard device.
1331        //   - uses_isolation_scope() -> Isolated: Auto topology here; the L3
1332        //     fence is applied at ATTACH time via `isolation_network`.
1333        //   - otherwise (Auto)      -> per-service Linux bridge carried on the
1334        //     single cluster-wide WireGuard interface (today's default).
1335        // Record the resolved mode so the container ATTACH path can branch.
1336        let resolved = mode.resolve();
1337        self.service_modes.insert(service.to_string(), resolved);
1338        if resolved.uses_shared_bridge() {
1339            self.setup_service_overlay_shared_bridge(service).await
1340        } else if resolved.uses_per_service_wg() {
1341            self.setup_service_overlay_dedicated(service).await
1342        } else if resolved.uses_isolation_scope() {
1343            // Isolated == Auto topology (per-service bridge on the cluster-wide
1344            // WireGuard); the L3 fence is applied at ATTACH time via
1345            // `isolation_network`, not in segment setup. Same target as the
1346            // default, made explicit so a new mode can't silently fall through.
1347            self.setup_service_overlay_cluster_wg(service).await
1348        } else {
1349            self.setup_service_overlay_cluster_wg(service).await
1350        }
1351    }
1352
1353    /// `Auto`-mode per-service overlay (Linux): a per-service Linux bridge backed
1354    /// by the SINGLE cluster-wide `WireGuard` transport (the service subnet is
1355    /// plumbed onto the cluster device's `AllowedIPs`). This is the original
1356    /// default `setup_service_overlay` body, returning a [`ServiceOverlayInfo`]
1357    /// with the bridge name and all dedicated-device identity fields `None`
1358    /// (`Auto` shares the cluster device).
1359    ///
1360    /// Returns the bridge name on success.
1361    ///
1362    /// # Errors
1363    /// Returns an error if subnet assignment fails (exhaustion), if the bridge
1364    /// cannot be created, or if the cluster transport rejects the `AllowedIPs`
1365    /// update.
1366    #[cfg(target_os = "linux")]
1367    #[allow(clippy::too_many_lines)]
1368    async fn setup_service_overlay_cluster_wg(
1369        &mut self,
1370        service: &str,
1371    ) -> Result<ServiceOverlayInfo, OverlaydError> {
1372        // 1. Idempotency check.
1373        if let Some(existing) = self.service_bridges.get(service) {
1374            let name = existing.name.clone();
1375            tracing::debug!(service = %service, bridge = %name, "Service bridge already active, reusing");
1376            return Ok(cluster_wg_overlay_info(name));
1377        }
1378
1379        // 2. Assign subnet via the (currently local) ServiceSubnetRegistry.
1380        self.ensure_service_subnet_registry()?;
1381        let subnet: ipnet::IpNet = {
1382            let registry = self
1383                .service_subnet_registry
1384                .as_mut()
1385                .expect("ensure_service_subnet_registry leaves Some");
1386            let node_key = self.local_node_id.to_string();
1387            registry.assign(service, &node_key).map_err(|e| {
1388                OverlaydError::Overlay(format!(
1389                    "ServiceSubnetRegistry::assign({service}, {node_key}) failed: {e}"
1390                ))
1391            })?
1392        };
1393
1394        // 3+4+6. Create the per-service Linux bridge, assign its gateway, bring
1395        // it up, build the per-service IpAllocator, and record it.
1396        let bridge_name = self.create_service_bridge(service, subnet).await?;
1397
1398        // 5. Plumb subnet into the cluster transport's local AllowedIPs so the
1399        // single cluster device carries this service's cross-node traffic
1400        // (Shared mode shares one crypto context for every service).
1401        if let Some(ref cluster) = self.global_transport {
1402            if let Some(ref pubkey) = self.local_wg_pubkey {
1403                if let Err(e) = cluster.add_allowed_ip(pubkey, subnet).await {
1404                    tracing::warn!(
1405                        service = %service,
1406                        subnet = %subnet,
1407                        error = %e,
1408                        "Failed to add service subnet to cluster transport AllowedIPs (non-fatal)"
1409                    );
1410                }
1411            } else {
1412                tracing::debug!(service = %service, "local_wg_pubkey not yet set; skipping cluster AllowedIPs update");
1413            }
1414        }
1415
1416        Ok(cluster_wg_overlay_info(bridge_name))
1417    }
1418
1419    /// `Shared`-mode per-service overlay (Linux): attach `service` onto the
1420    /// SINGLE node-wide shared Linux bridge (created once, reused by every
1421    /// Shared service on this node), carried on the cluster-wide `WireGuard`
1422    /// interface. There is NO per-service bridge and NO per-service `WireGuard`;
1423    /// container ports are exposed via the userspace free-port L4 proxy
1424    /// (`proxy_manager.rs`). Returns the shared bridge name.
1425    ///
1426    /// Idempotent: the shared bridge is allocated a single subnet and brought up
1427    /// exactly once; subsequent Shared services reuse it. The service is recorded
1428    /// in `service_interfaces` (pointing at the shared bridge) so presence checks
1429    /// and the attach path resolve it.
1430    ///
1431    /// # Errors
1432    /// Returns an error if the one-time shared-subnet assignment fails
1433    /// (exhaustion), if the shared bridge cannot be created, or if the cluster
1434    /// transport rejects the `AllowedIPs` update.
1435    #[cfg(target_os = "linux")]
1436    async fn setup_service_overlay_shared_bridge(
1437        &mut self,
1438        service: &str,
1439    ) -> Result<ServiceOverlayInfo, OverlaydError> {
1440        let bridge_name = self.ensure_shared_bridge().await?;
1441        // Point this service at the shared bridge so presence checks succeed and
1442        // the attach path resolves it to the shared bridge.
1443        self.service_interfaces
1444            .insert(service.to_string(), bridge_name.clone());
1445        tracing::info!(service = %service, bridge = %bridge_name, "Service attached to shared node-wide bridge");
1446        Ok(shared_overlay_info(bridge_name))
1447    }
1448
1449    /// Ensure the single node-wide shared Linux bridge exists, returning its
1450    /// name. Created once with its own subnet (drawn from the same
1451    /// `ServiceSubnetRegistry` every service subnet comes from, under a fixed
1452    /// reserved key so it never collides with a real service) and plumbed onto
1453    /// the cluster transport's `AllowedIPs` so shared containers are
1454    /// mesh-reachable across nodes. Subsequent calls return the existing name.
1455    ///
1456    /// # Errors
1457    /// Returns an error if subnet assignment fails or the bridge cannot be
1458    /// created/addressed/brought up.
1459    #[cfg(target_os = "linux")]
1460    async fn ensure_shared_bridge(&mut self) -> Result<String, OverlaydError> {
1461        use zlayer_overlay::allocator::IpAllocator as OverlayIpAllocator;
1462
1463        if let Some(existing) = self.shared_bridge.as_ref() {
1464            return Ok(existing.name.clone());
1465        }
1466
1467        // One subnet for the whole shared bridge. Use a fixed reserved key in the
1468        // registry (never a real service name) so the shared bridge gets exactly
1469        // one stable subnet, distinct from every per-service subnet.
1470        self.ensure_service_subnet_registry()?;
1471        let subnet: ipnet::IpNet = {
1472            let registry = self
1473                .service_subnet_registry
1474                .as_mut()
1475                .expect("ensure_service_subnet_registry leaves Some");
1476            let node_key = self.local_node_id.to_string();
1477            registry.assign(SHARED_BRIDGE_REGISTRY_KEY, &node_key).map_err(|e| {
1478                OverlaydError::Overlay(format!(
1479                    "ServiceSubnetRegistry::assign({SHARED_BRIDGE_REGISTRY_KEY}, {node_key}) failed: {e}"
1480                ))
1481            })?
1482        };
1483
1484        // Deterministic, IFNAMSIZ-safe shared-bridge name (one per node). Use the
1485        // same naming helper as per-service bridges with a fixed key so it stays
1486        // <= 15 chars and is unambiguous (`zl-...-sh`).
1487        let bridge_name =
1488            make_interface_name(&[&self.deployment, &self.instance_id, "shared"], "sh");
1489
1490        if let Err(e) = crate::netlink::create_bridge(&bridge_name).await {
1491            return Err(OverlaydError::Overlay(format!(
1492                "create_bridge({bridge_name}) failed: {e}"
1493            )));
1494        }
1495        if let Err(e) = crate::netlink::set_bridge_stp(&bridge_name, false) {
1496            tracing::warn!(bridge = %bridge_name, error = %e, "set_bridge_stp(off) failed (non-fatal)");
1497        }
1498
1499        // Flush stale addresses first: `create_bridge` is idempotent on EEXIST, so
1500        // a shared bridge that survived a restart would otherwise accumulate a
1501        // second gateway (the same dual-address bug fixed for per-service bridges).
1502        let gateway = first_usable_ip(subnet);
1503        if let Err(e) = crate::netlink::flush_addresses_on_link_by_name(&bridge_name).await {
1504            tracing::warn!(bridge = %bridge_name, error = %e, "flush_addresses_on_link_by_name failed (non-fatal)");
1505        }
1506        if let Err(e) =
1507            crate::netlink::add_address_to_link_by_name(&bridge_name, gateway, subnet.prefix_len())
1508                .await
1509        {
1510            let _ = crate::netlink::delete_bridge(&bridge_name).await;
1511            return Err(OverlaydError::Overlay(format!(
1512                "add_address_to_link_by_name({bridge_name}, {gateway}/{}) failed: {e}",
1513                subnet.prefix_len()
1514            )));
1515        }
1516        if let Err(e) = crate::netlink::set_link_up_by_name(&bridge_name).await {
1517            let _ = crate::netlink::delete_bridge(&bridge_name).await;
1518            return Err(OverlaydError::Overlay(format!(
1519                "set_link_up_by_name({bridge_name}) failed: {e}"
1520            )));
1521        }
1522
1523        // Track the shared bridge for global teardown (deleting the link drops
1524        // its gateway address + up state).
1525        self.created_bridges.insert(bridge_name.clone());
1526
1527        let mut ip_allocator = OverlayIpAllocator::new(&subnet.to_string()).map_err(|e| {
1528            OverlaydError::Overlay(format!("IpAllocator::new({subnet}) failed: {e}"))
1529        })?;
1530        let _ = ip_allocator.allocate_specific(gateway);
1531
1532        // Plumb the shared subnet onto the cluster transport's AllowedIPs so the
1533        // single cluster device carries shared-bridge cross-node traffic (same
1534        // mechanism the cluster-WG per-service path uses).
1535        if let Some(ref cluster) = self.global_transport {
1536            if let Some(ref pubkey) = self.local_wg_pubkey {
1537                if let Err(e) = cluster.add_allowed_ip(pubkey, subnet).await {
1538                    tracing::warn!(
1539                        subnet = %subnet,
1540                        error = %e,
1541                        "Failed to add shared-bridge subnet to cluster transport AllowedIPs (non-fatal)"
1542                    );
1543                }
1544            } else {
1545                tracing::debug!(
1546                    "local_wg_pubkey not yet set; skipping shared-bridge cluster AllowedIPs update"
1547                );
1548            }
1549        }
1550
1551        self.shared_bridge = Some(ServiceBridge {
1552            name: bridge_name.clone(),
1553            subnet,
1554            gateway,
1555            ip_allocator,
1556        });
1557
1558        tracing::info!(bridge = %bridge_name, subnet = %subnet, gateway = %gateway, "Shared node-wide bridge created");
1559        Ok(bridge_name)
1560    }
1561
1562    /// Create the per-service Linux bridge for `service` on `subnet`, assign its
1563    /// gateway, bring it up, build the per-service [`IpAllocator`], and record it
1564    /// in `service_bridges` + `service_interfaces`. Returns the bridge name.
1565    ///
1566    /// Shared and Dedicated mode share this bridge mechanic verbatim — the ONLY
1567    /// difference between the two modes is which `WireGuard` device the service
1568    /// subnet/peers are plumbed onto (the single cluster transport for Shared,
1569    /// the dedicated per-service transport for Dedicated). This helper does NOT
1570    /// touch any transport's `AllowedIPs`; the caller does that against the
1571    /// device it owns.
1572    ///
1573    /// # Errors
1574    /// Returns an error if the bridge cannot be created, addressed, or brought
1575    /// up, or if the per-service `IpAllocator` cannot be built.
1576    #[cfg(target_os = "linux")]
1577    async fn create_service_bridge(
1578        &mut self,
1579        service: &str,
1580        subnet: ipnet::IpNet,
1581    ) -> Result<String, OverlaydError> {
1582        use zlayer_overlay::allocator::IpAllocator as OverlayIpAllocator;
1583
1584        let bridge_name = make_interface_name(&[&self.deployment, &self.instance_id, service], "b");
1585
1586        if let Err(e) = crate::netlink::create_bridge(&bridge_name).await {
1587            return Err(OverlaydError::Overlay(format!(
1588                "create_bridge({bridge_name}) failed: {e}"
1589            )));
1590        }
1591        if let Err(e) = crate::netlink::set_bridge_stp(&bridge_name, false) {
1592            tracing::warn!(bridge = %bridge_name, error = %e, "set_bridge_stp(off) failed (non-fatal)");
1593        }
1594
1595        // Gateway = first usable host in the subnet, assigned to the bridge.
1596        // Flush any pre-existing addresses FIRST: `create_bridge` is idempotent
1597        // on EEXIST, so a bridge that survived a restart would otherwise keep its
1598        // old gateway and we'd stack the new one on top (the observed dual
1599        // /28 + /26 bug). Flushing makes the assignment idempotent and self-heals
1600        // such bridges. Non-fatal: on a brand-new bridge there is nothing to flush.
1601        let gateway = first_usable_ip(subnet);
1602        if let Err(e) = crate::netlink::flush_addresses_on_link_by_name(&bridge_name).await {
1603            tracing::warn!(bridge = %bridge_name, error = %e, "flush_addresses_on_link_by_name failed (non-fatal)");
1604        }
1605        if let Err(e) =
1606            crate::netlink::add_address_to_link_by_name(&bridge_name, gateway, subnet.prefix_len())
1607                .await
1608        {
1609            let _ = crate::netlink::delete_bridge(&bridge_name).await;
1610            return Err(OverlaydError::Overlay(format!(
1611                "add_address_to_link_by_name({bridge_name}, {gateway}/{}) failed: {e}",
1612                subnet.prefix_len()
1613            )));
1614        }
1615        if let Err(e) = crate::netlink::set_link_up_by_name(&bridge_name).await {
1616            let _ = crate::netlink::delete_bridge(&bridge_name).await;
1617            return Err(OverlaydError::Overlay(format!(
1618                "set_link_up_by_name({bridge_name}) failed: {e}"
1619            )));
1620        }
1621
1622        // Track the per-service bridge for global teardown (deleting the link
1623        // drops its gateway address + up state).
1624        self.created_bridges.insert(bridge_name.clone());
1625
1626        // Build per-service IpAllocator, reserve the gateway.
1627        let mut ip_allocator = OverlayIpAllocator::new(&subnet.to_string()).map_err(|e| {
1628            OverlaydError::Overlay(format!("IpAllocator::new({subnet}) failed: {e}"))
1629        })?;
1630        let _ = ip_allocator.allocate_specific(gateway);
1631
1632        self.service_bridges.insert(
1633            service.to_string(),
1634            ServiceBridge {
1635                name: bridge_name.clone(),
1636                subnet,
1637                gateway,
1638                ip_allocator,
1639            },
1640        );
1641        self.service_interfaces
1642            .insert(service.to_string(), bridge_name.clone());
1643
1644        tracing::info!(service = %service, bridge = %bridge_name, subnet = %subnet, gateway = %gateway, "Service bridge created");
1645        Ok(bridge_name)
1646    }
1647
1648    /// Non-Linux variant of `setup_service_overlay`. On Windows the per-service
1649    /// segment is the HCN Internal network created lazily at attach time, and on
1650    /// macOS containers fall through to host networking. Registers the service
1651    /// in `service_interfaces` with a placeholder name so presence checks work.
1652    ///
1653    /// # Errors
1654    /// Infallible on non-Linux; the `Result` is preserved for ABI parity.
1655    #[cfg(not(target_os = "linux"))]
1656    async fn setup_service_overlay(
1657        &mut self,
1658        service: &str,
1659        mode: OverlayMode,
1660    ) -> Result<ServiceOverlayInfo, OverlaydError> {
1661        // Same predicate-driven decision surface as Linux (see
1662        // `zlayer_types::overlay`). The container ATTACH path differentiates the
1663        // modes per-OS; here we only record the resolved mode and register the
1664        // appropriate placeholder/info so presence checks and `Status` work.
1665        //
1666        //   - uses_per_service_wg() -> the cross-platform dedicated path (a real
1667        //     per-service WireGuard device; on Windows it also stands up a
1668        //     per-service HCN Internal network at attach time).
1669        //   - otherwise (`Auto` and `Shared`) -> no per-service WireGuard device.
1670        //     On macOS both rely on VZ NAT + host-port forwarding (the free-port
1671        //     L4 proxy), so they route to the SAME real path — the only honest
1672        //     mapping a VZ guest can express (it has no per-service bridge or WG
1673        //     to differentiate). On Windows the attach path reads the recorded
1674        //     mode to send `Shared` containers onto a shared HCN NAT network and
1675        //     `Auto` containers onto the node's base overlay network.
1676        //   - uses_isolation_scope() -> Isolated: Auto topology here; the L3
1677        //     fence is applied at ATTACH time via `isolation_network`.
1678        let resolved = mode.resolve();
1679        self.service_modes.insert(service.to_string(), resolved);
1680        if resolved.uses_per_service_wg() {
1681            self.setup_service_overlay_dedicated(service).await
1682        } else if resolved.uses_shared_bridge() {
1683            self.setup_service_overlay_shared_bridge(service).await
1684        } else if resolved.uses_isolation_scope() {
1685            // Isolated == Auto topology (per-service bridge on the cluster-wide
1686            // WireGuard); the L3 fence is applied at ATTACH time via
1687            // `isolation_network`, not in segment setup. Same target as the
1688            // default, made explicit so a new mode can't silently fall through.
1689            self.setup_service_overlay_cluster_wg(service).await
1690        } else {
1691            self.setup_service_overlay_cluster_wg(service).await
1692        }
1693    }
1694
1695    /// `Auto`-mode per-service overlay (non-Linux): on Windows the per-service
1696    /// segment is the node's base overlay HCN network used at attach time, and on
1697    /// macOS containers ride VZ NAT. Registers the service in `service_interfaces`
1698    /// with a placeholder name so presence checks work.
1699    ///
1700    /// # Errors
1701    /// Infallible on non-Linux; the `Result` is preserved for ABI parity.
1702    #[cfg(not(target_os = "linux"))]
1703    #[allow(clippy::unused_async)]
1704    async fn setup_service_overlay_cluster_wg(
1705        &mut self,
1706        service: &str,
1707    ) -> Result<ServiceOverlayInfo, OverlaydError> {
1708        let placeholder = make_interface_name(&[&self.deployment, &self.instance_id, service], "b");
1709        self.service_interfaces
1710            .insert(service.to_string(), placeholder.clone());
1711        tracing::debug!(service = %service, "Service overlay bridge setup is Linux-only; using direct networking placeholder");
1712        Ok(cluster_wg_overlay_info(placeholder))
1713    }
1714
1715    /// `Shared`-mode per-service overlay (non-Linux). There is no per-service
1716    /// `WireGuard` device and no per-service bridge:
1717    /// - macOS: the container is a VZ VM behind VZ NAT (a single shared host
1718    ///   adapter with host-port forwarding); its ports are exposed by the
1719    ///   userspace free-port L4 proxy. Nothing to provision here beyond a
1720    ///   placeholder so presence checks succeed.
1721    /// - Windows: containers attach to a SINGLE shared HCN NAT network reused
1722    ///   across all Shared services (created lazily at attach time); a placeholder
1723    ///   interface is registered here.
1724    ///
1725    /// Registers the service in `service_interfaces` with a placeholder name.
1726    ///
1727    /// # Errors
1728    /// Infallible on non-Linux; the `Result` is preserved for ABI parity.
1729    #[cfg(not(target_os = "linux"))]
1730    #[allow(clippy::unused_async)]
1731    async fn setup_service_overlay_shared_bridge(
1732        &mut self,
1733        service: &str,
1734    ) -> Result<ServiceOverlayInfo, OverlaydError> {
1735        // A single placeholder shared by every Shared service on this node (it
1736        // names the shared data-plane, not a per-service interface).
1737        let placeholder =
1738            make_interface_name(&[&self.deployment, &self.instance_id, "shared"], "sh");
1739        self.service_interfaces
1740            .insert(service.to_string(), placeholder.clone());
1741        tracing::debug!(service = %service, "Shared-mode service uses the node-wide shared data-plane (VZ NAT on macOS / shared HCN NAT on Windows)");
1742        Ok(shared_overlay_info(placeholder))
1743    }
1744
1745    /// Dedicated-mode per-service overlay: stand up a *second* real `WireGuard`
1746    /// device for `service` with its own crypto context, listen port, overlay
1747    /// IP, and subnet — distinct from the single cluster transport.
1748    ///
1749    /// The cross-platform core (identity, subnet assign, transport bring-up,
1750    /// marker persist, status) runs on every OS; only the *attachment* of
1751    /// containers onto the device is platform-gated:
1752    /// - Linux: a per-service bridge (same mechanic as Shared) routed over the
1753    ///   dedicated device instead of the cluster device.
1754    /// - Windows: a per-service HCN Internal network (a later task; a clearly
1755    ///   marked seam returns an error here for now).
1756    /// - macOS: nothing further — the utun device is the attachment.
1757    ///
1758    /// # Errors
1759    /// Returns an error if port/key/subnet allocation, transport bring-up,
1760    /// marker persistence, or the platform attachment fails.
1761    #[allow(clippy::too_many_lines)]
1762    async fn setup_service_overlay_dedicated(
1763        &mut self,
1764        service: &str,
1765    ) -> Result<ServiceOverlayInfo, OverlaydError> {
1766        // ----- cross-platform core (runs on every OS) -----
1767
1768        // 1. Idempotency: an existing dedicated transport returns its identity.
1769        if let Some(st) = self.service_transports.get(service) {
1770            return Ok(dedicated_overlay_info(
1771                st.interface.clone(),
1772                &st.public_key,
1773                st.listen_port,
1774                st.overlay_ip,
1775                st.subnet,
1776            ));
1777        }
1778
1779        // 2. Identity: reuse a stable identity from the marker if one exists
1780        //    (so the device re-binds the same key + port across restarts),
1781        //    otherwise mint a fresh port + keypair + interface name.
1782        let marker_path =
1783            zlayer_paths::ZLayerDirs::new(self.data_dir.clone()).agent_network_state();
1784        let recorded = NetworkState::load(&marker_path)
1785            .get(&owner_for_service(service))
1786            .cloned();
1787
1788        let (private_key, public_key, listen_port, iface_hint) = match recorded.as_ref() {
1789            Some(entry)
1790                if entry.wg_private_key.is_some()
1791                    && entry.wg_public_key.is_some()
1792                    && entry.wg_port.is_some()
1793                    && entry.interface.is_some() =>
1794            {
1795                let port = entry.wg_port.expect("checked above");
1796                self.dedicated_ports.reserve(port);
1797                (
1798                    entry.wg_private_key.clone().expect("checked above"),
1799                    entry.wg_public_key.clone().expect("checked above"),
1800                    port,
1801                    entry.interface.clone().expect("checked above"),
1802                )
1803            }
1804            _ => {
1805                let port = self.dedicated_ports.allocate()?;
1806                let (priv_key, pub_key) = OverlayTransport::generate_keys()
1807                    .await
1808                    .map_err(|e| OverlaydError::Overlay(format!("Failed to generate keys: {e}")))?;
1809                let iface =
1810                    make_interface_name(&[&self.deployment, &self.instance_id, service], "d");
1811                (priv_key, pub_key, port, iface)
1812            }
1813        };
1814
1815        // 3. Subnet: assign from the same registry Shared uses, so per-service
1816        //    subnets stay globally unique regardless of mode.
1817        self.ensure_service_subnet_registry()?;
1818        let subnet: ipnet::IpNet = {
1819            let registry = self
1820                .service_subnet_registry
1821                .as_mut()
1822                .expect("ensure_service_subnet_registry leaves Some");
1823            let node_key = self.local_node_id.to_string();
1824            registry.assign(service, &node_key).map_err(|e| {
1825                OverlaydError::Overlay(format!(
1826                    "ServiceSubnetRegistry::assign({service}, {node_key}) failed: {e}"
1827                ))
1828            })?
1829        };
1830        let overlay_ip = first_usable_ip(subnet);
1831
1832        // 4. Build + bring up the dedicated transport. The device's overlay CIDR
1833        //    is the service subnet (so boringtun routes that subnet over THIS
1834        //    device), and its listen port is the dedicated port.
1835        let physical_egress_ip = match zlayer_overlay::detect_physical_egress().await {
1836            Ok(egress) => Some(egress.ip),
1837            Err(e) => {
1838                tracing::warn!(
1839                    error = %e,
1840                    service = %service,
1841                    "failed to detect physical egress; WireGuard local_endpoint \
1842                     will bind UNSPECIFIED for the dedicated overlay"
1843                );
1844                None
1845            }
1846        };
1847        let config = self.build_config(
1848            private_key.clone(),
1849            public_key.clone(),
1850            overlay_ip,
1851            subnet.prefix_len(),
1852            listen_port,
1853            physical_egress_ip,
1854        );
1855        let mut transport = OverlayTransport::new(config, iface_hint);
1856        transport.create_interface().await.map_err(|e| {
1857            OverlaydError::Overlay(format!(
1858                "Failed to create dedicated overlay for {service}: {e}"
1859            ))
1860        })?;
1861        transport.configure(&[]).await.map_err(|e| {
1862            OverlaydError::Overlay(format!(
1863                "Failed to configure dedicated overlay for {service}: {e}"
1864            ))
1865        })?;
1866        let actual_iface = transport.interface_name().to_string();
1867
1868        // 5. Persist the marker so the identity survives restarts. Match the
1869        //    base/Shared entry shape (owner/kind/name/id/subnet) plus the
1870        //    dedicated WG fields.
1871        let mut marker = NetworkState::load(&marker_path);
1872        marker.upsert(ManagedNetwork {
1873            owner: owner_for_service(service),
1874            kind: "wg-dedicated".to_string(),
1875            name: actual_iface.clone(),
1876            id: public_key.clone(),
1877            subnet: subnet.to_string(),
1878            wg_port: Some(listen_port),
1879            wg_private_key: Some(private_key),
1880            wg_public_key: Some(public_key.clone()),
1881            interface: Some(actual_iface.clone()),
1882        });
1883        if let Err(e) = marker.save(&marker_path) {
1884            tracing::warn!(service = %service, error = %e, path = %marker_path.display(), "failed to persist dedicated-overlay marker (device still live)");
1885        }
1886
1887        // 6. Record the live transport. Build the guest-attach IPAM bounded to
1888        //    the service subnet, reserving the node's own dedicated-device IP so
1889        //    a joining guest never draws it.
1890        let mut ip_allocator = zlayer_overlay::allocator::IpAllocator::new(&subnet.to_string())
1891            .map_err(|e| {
1892                OverlaydError::Overlay(format!("IpAllocator::new({subnet}) failed: {e}"))
1893            })?;
1894        let _ = ip_allocator.allocate_specific(overlay_ip);
1895        self.service_transports.insert(
1896            service.to_string(),
1897            ServiceTransport {
1898                transport,
1899                interface: actual_iface.clone(),
1900                public_key: public_key.clone(),
1901                listen_port,
1902                overlay_ip,
1903                subnet,
1904                ip_allocator,
1905            },
1906        );
1907
1908        tracing::info!(
1909            service = %service,
1910            interface = %actual_iface,
1911            listen_port,
1912            subnet = %subnet,
1913            overlay_ip = %overlay_ip,
1914            "Dedicated per-service overlay device created"
1915        );
1916
1917        // ----- platform-gated attachment -----
1918        // `name` in the returned info is the container-attach handle: the bridge
1919        // name on Linux, the dedicated interface elsewhere.
1920        let name = self
1921            .attach_dedicated_service(service, subnet, overlay_ip)
1922            .await?;
1923
1924        Ok(dedicated_overlay_info(
1925            name,
1926            &public_key,
1927            listen_port,
1928            overlay_ip,
1929            subnet,
1930        ))
1931    }
1932
1933    /// Linux attachment for a dedicated per-service overlay: create the same
1934    /// per-service bridge Shared uses, but route the service subnet over the
1935    /// DEDICATED device rather than the cluster device.
1936    ///
1937    /// Concretely, the dedicated transport's overlay CIDR already covers
1938    /// `subnet` (set at `build_config` time in the core), so boringtun routes
1939    /// `subnet` out the dedicated TUN; we additionally plumb `subnet` onto this
1940    /// node's own `AllowedIPs` entry on the dedicated device so locally
1941    /// originated packets to the subnet are accepted. Returns the bridge name.
1942    ///
1943    /// # Errors
1944    /// Returns an error if the bridge cannot be created.
1945    #[cfg(target_os = "linux")]
1946    async fn attach_dedicated_service(
1947        &mut self,
1948        service: &str,
1949        subnet: ipnet::IpNet,
1950        overlay_ip: IpAddr,
1951    ) -> Result<String, OverlaydError> {
1952        let _ = overlay_ip;
1953        let bridge_name = self.create_service_bridge(service, subnet).await?;
1954
1955        // Plumb the service subnet onto the DEDICATED device (not the cluster
1956        // device). The dedicated transport's overlay CIDR already routes the
1957        // subnet out its TUN; adding it to our own pubkey's AllowedIPs keeps the
1958        // local-accept side consistent with the Shared path's cluster plumbing.
1959        if let Some(st) = self.service_transports.get(service) {
1960            if let Some(ref pubkey) = self.local_wg_pubkey {
1961                if let Err(e) = st.transport.add_allowed_ip(pubkey, subnet).await {
1962                    tracing::warn!(
1963                        service = %service,
1964                        subnet = %subnet,
1965                        error = %e,
1966                        "Failed to add service subnet to dedicated transport AllowedIPs (non-fatal)"
1967                    );
1968                }
1969            } else {
1970                tracing::debug!(service = %service, "local_wg_pubkey not yet set; skipping dedicated AllowedIPs update");
1971            }
1972        }
1973
1974        Ok(bridge_name)
1975    }
1976
1977    /// Windows attachment for a dedicated per-service overlay.
1978    ///
1979    /// The cross-platform core has already stood up the dedicated Wintun
1980    /// transport (the encrypted node-to-node path for the service subnet). This
1981    /// adds the *container-facing* side: a per-service HCN **Internal** network
1982    /// onto which the agent's containers attach (instead of the node's shared
1983    /// base overlay network), so dedicated-service traffic is isolated at the
1984    /// vSwitch layer. Returns the per-service network's name, which the caller
1985    /// records as the [`ServiceOverlayInfo::name`] attach handle.
1986    ///
1987    /// # Errors
1988    /// Propagates any error from [`Self::ensure_service_network`].
1989    #[cfg(target_os = "windows")]
1990    async fn attach_dedicated_service(
1991        &mut self,
1992        service: &str,
1993        subnet: ipnet::IpNet,
1994        _overlay_ip: IpAddr,
1995    ) -> Result<String, OverlaydError> {
1996        // Create (or reuse) the per-service Internal HCN network. The returned
1997        // GUID is recorded in the marker under `owner_for_service(service)`;
1998        // the `AttachContainer` handler reuses it via the same marker lookup.
1999        let _net_id = self.ensure_service_network(service, subnet).await?;
2000        // The attach handle reported back is the per-service network's name.
2001        let daemon_name = self.deployment_or_default();
2002        Ok(format!(
2003            "{}-svc-{service}",
2004            overlay_network_name(&daemon_name)
2005        ))
2006    }
2007
2008    /// macOS attachment for a dedicated per-service overlay: the cross-platform
2009    /// core already brought up a utun device; there is no bridge, so the
2010    /// interface name itself is the attach handle.
2011    #[cfg(all(not(target_os = "linux"), not(target_os = "windows")))]
2012    #[allow(clippy::unused_async)]
2013    async fn attach_dedicated_service(
2014        &mut self,
2015        service: &str,
2016        _subnet: ipnet::IpNet,
2017        _overlay_ip: IpAddr,
2018    ) -> Result<String, OverlaydError> {
2019        let iface = self
2020            .service_transports
2021            .get(service)
2022            .map(|st| st.interface.clone())
2023            .unwrap_or_default();
2024        Ok(iface)
2025    }
2026
2027    /// Tear down the per-service segment for `service`. Idempotent.
2028    // Only the Linux body awaits (netlink + cluster AllowedIPs); other targets
2029    // are synchronous (transport shutdown is sync) but must keep the async
2030    // signature for the dispatch call.
2031    #[cfg_attr(not(target_os = "linux"), allow(clippy::unused_async))]
2032    async fn teardown_service_overlay(&mut self, service: &str) {
2033        // Drop the recorded mode; a `Shared` service's containers no longer route
2034        // to the shared bridge once it is gone. The node-wide shared bridge
2035        // itself is deliberately NOT torn down here — other Shared services reuse
2036        // it (it is reclaimed only on full overlay teardown / uninstall).
2037        self.service_modes.remove(service);
2038
2039        // Auto-mode segment teardown (per-service bridge on Linux, placeholder
2040        // elsewhere). A Shared-mode service has no per-service bridge, so
2041        // `service_bridges.remove` is a no-op for it (its `service_interfaces`
2042        // placeholder pointing at the shared bridge is removed below).
2043        #[cfg(target_os = "linux")]
2044        {
2045            let removed = self.service_bridges.remove(service);
2046            self.service_interfaces.remove(service);
2047
2048            // Remove the subnet from the cluster AllowedIPs only when we still
2049            // know it (the in-memory entry survived).
2050            if let Some(ref bridge) = removed {
2051                if let Some(ref cluster) = self.global_transport {
2052                    if let Some(ref pubkey) = self.local_wg_pubkey {
2053                        if let Err(e) = cluster.remove_allowed_ip(pubkey, bridge.subnet).await {
2054                            tracing::warn!(
2055                                service = %service,
2056                                subnet = %bridge.subnet,
2057                                error = %e,
2058                                "Failed to remove service subnet from cluster AllowedIPs (non-fatal)"
2059                            );
2060                        }
2061                    }
2062                }
2063            }
2064
2065            // Delete the physical bridge by its DETERMINISTIC name, regardless of
2066            // whether the in-memory entry survived. After an overlayd restart the
2067            // `service_bridges` map is empty, so a delete gated on `Some(..)` would
2068            // silently leak the `zl-…-b` link forever (the observed orphan/linkdown
2069            // bridges). `delete_bridge` no-ops on ENODEV, so deleting an absent link
2070            // is safe — and the `-b` suffix never collides with a Shared service's
2071            // shared `-sh` bridge, so this can't tear down the wrong thing.
2072            let bridge_name = removed.as_ref().map_or_else(
2073                || make_interface_name(&[&self.deployment, &self.instance_id, service], "b"),
2074                |b| b.name.clone(),
2075            );
2076            if let Err(e) = crate::netlink::delete_bridge(&bridge_name).await {
2077                tracing::warn!(service = %service, bridge = %bridge_name, error = %e, "delete_bridge failed (non-fatal)");
2078            }
2079
2080            // Release the subnet-registry slot by service name (works whether or
2081            // not the in-memory entry survived).
2082            if let Some(registry) = self.service_subnet_registry.as_mut() {
2083                let node_key = self.local_node_id.to_string();
2084                let _ = registry.release(service, &node_key);
2085            }
2086
2087            if removed.is_some() {
2088                tracing::info!(service = %service, bridge = %bridge_name, "Tore down service bridge");
2089            } else {
2090                tracing::debug!(service = %service, bridge = %bridge_name, "best-effort delete of (possibly absent) service bridge by name");
2091            }
2092        }
2093        #[cfg(not(target_os = "linux"))]
2094        {
2095            if let Some(iface) = self.service_interfaces.remove(service) {
2096                tracing::info!(service = %service, interface = %iface, "Removed service overlay interface (placeholder, non-Linux)");
2097            }
2098        }
2099
2100        // Dedicated-mode teardown (cross-platform): tear down the per-service
2101        // transport, free its port, and drop its marker entry. No-op when the
2102        // service ran in Shared mode (nothing in `service_transports`).
2103        if let Some(mut st) = self.service_transports.remove(service) {
2104            st.transport.shutdown();
2105            self.dedicated_ports.release(st.listen_port);
2106
2107            // Release the subnet assignment (Shared releases it inside the
2108            // Linux block above; the dedicated subnet lives in the same
2109            // registry, so release it here for the dedicated case on every OS).
2110            if let Some(registry) = self.service_subnet_registry.as_mut() {
2111                let node_key = self.local_node_id.to_string();
2112                let _ = registry.release(service, &node_key);
2113            }
2114
2115            let marker_path =
2116                zlayer_paths::ZLayerDirs::new(self.data_dir.clone()).agent_network_state();
2117            let mut marker = NetworkState::load(&marker_path);
2118            let removed_entry = marker.remove(&owner_for_service(service));
2119            if removed_entry.is_some() {
2120                if let Err(e) = marker.save(&marker_path) {
2121                    tracing::warn!(service = %service, error = %e, path = %marker_path.display(), "failed to persist dedicated-overlay marker removal");
2122                }
2123            }
2124
2125            // Windows: delete the per-service HCN Internal network this service
2126            // owned. The marker entry's `id` is the bare HCN GUID (set by
2127            // `ensure_service_network`); delete the network so a dedicated
2128            // service tears down cleanly without waiting for a full uninstall.
2129            // Also drop the per-service container-IP allocator.
2130            #[cfg(target_os = "windows")]
2131            {
2132                self.service_ip_allocators.remove(service);
2133                if let Some(entry) = removed_entry.as_ref() {
2134                    if entry.kind == "hcn-internal" {
2135                        if let Ok(guid) = windows::core::GUID::try_from(entry.id.as_str()) {
2136                            match zlayer_hns::network::Network::delete(guid) {
2137                                Ok(()) => {
2138                                    tracing::info!(service = %service, id = %entry.id, "deleted per-service HCN network");
2139                                }
2140                                Err(e) => {
2141                                    tracing::warn!(service = %service, id = %entry.id, error = %e, "failed to delete per-service HCN network (may leak until uninstall)");
2142                                }
2143                            }
2144                        } else {
2145                            tracing::warn!(service = %service, id = %entry.id, "per-service marker has unparseable HCN GUID; skipping network delete");
2146                        }
2147                    }
2148                }
2149            }
2150            #[cfg(not(target_os = "windows"))]
2151            drop(removed_entry);
2152
2153            tracing::info!(
2154                service = %service,
2155                interface = %st.interface,
2156                listen_port = st.listen_port,
2157                "Tore down dedicated per-service overlay device"
2158            );
2159        }
2160    }
2161
2162    /// Reclaim orphaned per-service host bridges (and their stale device veths)
2163    /// that no live deployment still owns. `live_bridge_names` is the full set of
2164    /// `zl-…-b` bridge names every currently-restored service SHOULD own,
2165    /// computed by the main daemon from storage.
2166    ///
2167    /// For every host link whose name looks like one of OUR per-service bridge
2168    /// (`…-b`) or dedicated device (`…-d`) interfaces but is NOT in `live` and is
2169    /// NOT the active global (`-g`) or shared (`-sh`) interface, we:
2170    ///   1. delete the link (idempotent — ENODEV is success),
2171    ///   2. release its service-subnet registry assignment + cluster `AllowedIPs`
2172    ///      when the `(service, node)` key can be recovered from the registry
2173    ///      snapshot by reproducing the deterministic bridge name, and
2174    ///   3. drop any stale in-memory `service_bridges`/`service_interfaces`
2175    ///      entries pointing at it.
2176    ///
2177    /// Best-effort + idempotent: a failure on one link is logged and the sweep
2178    /// continues. Returns the names actually reclaimed.
2179    #[cfg(target_os = "linux")]
2180    async fn prune_orphan_bridges(&mut self, live_bridge_names: &[String]) -> Vec<String> {
2181        use std::collections::HashSet;
2182
2183        let links = match crate::netlink::list_all_links().await {
2184            Ok(links) => links,
2185            Err(e) => {
2186                tracing::warn!(error = %e, "prune_orphan_bridges: failed to list host links");
2187                return Vec::new();
2188            }
2189        };
2190
2191        let live: HashSet<&str> = live_bridge_names.iter().map(String::as_str).collect();
2192
2193        // The interfaces we must NEVER reclaim even though they carry the `zl-`
2194        // prefix: the active global transport device and the node-wide shared
2195        // bridge. (Container veths `veth-…`/`vc-…` are handled by the separate
2196        // PID-keyed `sweep_orphan_veths`; here we only target service bridges +
2197        // dedicated device interfaces, which `sweep_orphan_veths` never touches.)
2198        let mut protected: HashSet<String> = HashSet::new();
2199        if let Some(g) = self.global_interface.clone() {
2200            protected.insert(g);
2201        }
2202        if let Some(ref sh) = self.shared_bridge {
2203            protected.insert(sh.name.clone());
2204        }
2205        // Protect every dedicated-service WireGuard transport (`…-d`) by name. A
2206        // `-d` is a WG device, not a bridge — it has no `brif`, so the zero-member
2207        // guard below treats it as 0 members, and the daemon's `live` set only
2208        // carries `…-b` names; without this it would be reaped as a live device.
2209        //
2210        // We deliberately do NOT blanket-protect `service_bridges` (`…-b`) here.
2211        // That map holds BOTH managed-service bridges AND standalone/per-job
2212        // bridges (e.g. a Runner's per-job network), and overlayd cannot tell
2213        // them apart — a standalone container's `DetachContainer` releases the
2214        // veth/IP but never removes the bridge or its `service_bridges` entry, so
2215        // a blanket protect shielded those orphans forever (only a restart, which
2216        // wipes the map, ever cleared them). Managed bridges stay protected by
2217        // being in the daemon's authoritative `live` set; standalone bridges are
2218        // not in storage, so they fall through to the zero-member guard and are
2219        // reclaimed once idle.
2220        for st in self.service_transports.values() {
2221            protected.insert(st.interface.clone());
2222        }
2223
2224        // Snapshot the subnet registry once so we can recover the `(service,
2225        // node)` key for an orphan by reproducing its deterministic bridge/device
2226        // name. The registry has no release-by-subnet API, so we map name ->
2227        // (service, node) here.
2228        let mut name_to_key: HashMap<String, (String, String, ipnet::IpNet)> = HashMap::new();
2229        if let Some(registry) = self.service_subnet_registry.as_ref() {
2230            for ((service, node), subnet) in registry.snapshot().assignments {
2231                let bridge =
2232                    make_interface_name(&[&self.deployment, &self.instance_id, &service], "b");
2233                let device =
2234                    make_interface_name(&[&self.deployment, &self.instance_id, &service], "d");
2235                name_to_key.insert(bridge, (service.clone(), node.clone(), subnet));
2236                name_to_key.insert(device, (service, node, subnet));
2237            }
2238        }
2239
2240        let mut reclaimed = Vec::new();
2241        for (_index, name) in links {
2242            // Only consider OUR per-service bridge (`-b`) or dedicated device
2243            // (`-d`) interfaces that are neither live nor protected. The pure
2244            // predicate (unit-tested in `orphan_bridge_selection`) keeps us off
2245            // unrelated host links, the global/shared interfaces, and the veth
2246            // namespaces.
2247            if !is_orphan_service_bridge(&name, &live, &protected) {
2248                continue;
2249            }
2250
2251            // Zero-member guard: only reclaim a non-live candidate once it is
2252            // IDLE — no member links. A `-b` bridge with a running container has
2253            // ≥1 veth in its `brif`, so an in-use (or a sub-ms mid-creation,
2254            // pre-attach is the only 0-member window) standalone bridge is left
2255            // alone; an orphan `-d` has no `brif` (0) and is correctly reaped.
2256            // This is what makes dropping the `service_bridges` blanket-protect
2257            // safe — a live managed bridge is already excluded by `live`, and any
2258            // other in-use bridge is excluded here.
2259            if crate::netlink::bridge_member_count(&name).await > 0 {
2260                continue;
2261            }
2262
2263            tracing::info!(link = %name, "prune_orphan_bridges: reclaiming orphan service bridge/device");
2264
2265            // 1. Release the subnet + cluster AllowedIPs when we can recover the
2266            //    owning service key from the registry.
2267            if let Some((service, node, subnet)) = name_to_key.get(&name).cloned() {
2268                if let Some(ref cluster) = self.global_transport {
2269                    if let Some(ref pubkey) = self.local_wg_pubkey {
2270                        if let Err(e) = cluster.remove_allowed_ip(pubkey, subnet).await {
2271                            tracing::warn!(
2272                                link = %name,
2273                                subnet = %subnet,
2274                                error = %e,
2275                                "prune_orphan_bridges: remove_allowed_ip failed (non-fatal)"
2276                            );
2277                        }
2278                    }
2279                }
2280                if let Some(registry) = self.service_subnet_registry.as_mut() {
2281                    let _ = registry.release(&service, &node);
2282                }
2283            }
2284
2285            // 2. Delete the link itself (idempotent).
2286            if let Err(e) = crate::netlink::delete_bridge(&name).await {
2287                tracing::warn!(link = %name, error = %e, "prune_orphan_bridges: delete_bridge failed (non-fatal)");
2288                continue;
2289            }
2290
2291            // 3. Drop any stale in-memory bookkeeping pointing at this link.
2292            self.service_bridges.retain(|_, b| b.name != name);
2293            self.service_interfaces.retain(|_, iface| *iface != name);
2294
2295            reclaimed.push(name);
2296        }
2297
2298        if !reclaimed.is_empty() {
2299            tracing::info!(count = reclaimed.len(), bridges = ?reclaimed, "prune_orphan_bridges: reclaimed orphaned service bridges/devices");
2300        }
2301        reclaimed
2302    }
2303
2304    /// Non-Linux variant: per-service bridges are a Linux-only mechanic (Windows
2305    /// uses HCN networks torn down in `teardown_service_overlay`; macOS rides VZ
2306    /// NAT), so there are no host bridge links to sweep.
2307    #[cfg(not(target_os = "linux"))]
2308    #[allow(clippy::unused_async, clippy::unused_self)]
2309    async fn prune_orphan_bridges(&mut self, _live_bridge_names: &[String]) -> Vec<String> {
2310        Vec::new()
2311    }
2312
2313    /// Initialize the local fallback `ServiceSubnetRegistry` from the configured
2314    /// cluster CIDR. Called on first `setup_service_overlay` use.
2315    ///
2316    /// # Errors
2317    /// Returns an error when no cluster CIDR is configured or the registry
2318    /// cannot be built.
2319    fn ensure_service_subnet_registry(&mut self) -> Result<(), OverlaydError> {
2320        use zlayer_overlay::allocator::ServiceSubnetRegistry;
2321
2322        if self.service_subnet_registry.is_some() {
2323            return Ok(());
2324        }
2325        let cluster_cidr = self.cluster_cidr.ok_or_else(|| {
2326            OverlaydError::Other(
2327                "service subnet registry needs a cluster CIDR (SetupGlobalOverlay first)"
2328                    .to_string(),
2329            )
2330        })?;
2331        let cluster_ipnet: ipnet::IpNet = cluster_cidr.to_string().parse().map_err(|e| {
2332            OverlaydError::Other(format!(
2333                "failed to convert cluster CIDR {cluster_cidr} to ipnet::IpNet: {e}"
2334            ))
2335        })?;
2336        // Per-service bridge slice prefix. `/26` (V4) = ~61 usable container
2337        // IPs per service per node — keep in sync with
2338        // `zlayer_scheduler::raft::DEFAULT_SERVICE_SUBNET_SLICE_PREFIX` (the
2339        // canonical default; not imported here to avoid a dependency cycle).
2340        // The older `/28` (13 usable) exhausted under CI churn.
2341        let slice_prefix: u8 = match cluster_ipnet {
2342            ipnet::IpNet::V4(_) => 26,
2343            ipnet::IpNet::V6(_) => 120,
2344        };
2345        let mut registry =
2346            ServiceSubnetRegistry::new(cluster_ipnet, slice_prefix).map_err(|e| {
2347                OverlaydError::Other(format!("failed to build ServiceSubnetRegistry: {e}"))
2348            })?;
2349        // Reserve the node's own overlay IP so no per-service bridge subnet
2350        // overlaps it — the overlay DNS server listens on `<node_ip>:53`, and a
2351        // bridge subnet containing that IP would black-hole its containers' DNS
2352        // (they'd ARP for the node IP on their bridge, where nothing answers).
2353        if let Some(node_ip) = self.node_ip {
2354            registry.reserve_ip(node_ip);
2355        }
2356        self.service_subnet_registry = Some(registry);
2357        Ok(())
2358    }
2359
2360    // -- IP allocation -------------------------------------------------------
2361
2362    /// Allocate an overlay IP from the per-service bridge (Linux) or the node
2363    /// slice (otherwise). `join_global` reserves a second global-overlay IP too,
2364    /// matching the eth1 attach behavior.
2365    ///
2366    /// # Errors
2367    /// Returns an error if the relevant pool is exhausted.
2368    fn allocate_ip(&mut self, service: &str, join_global: bool) -> Result<IpAddr, OverlaydError> {
2369        // `join_global` does not allocate a second IP here: the companion
2370        // global-overlay IP (eth1) is reserved at attach time. `AllocateIp`
2371        // returns only the primary (service / slice) IP the caller asked for.
2372        let _ = join_global;
2373        #[cfg(target_os = "linux")]
2374        {
2375            // A Shared-mode service draws from the single node-wide shared bridge;
2376            // every other mode draws from its own per-service bridge.
2377            let use_shared = self
2378                .service_modes
2379                .get(service)
2380                .copied()
2381                .unwrap_or_default()
2382                .uses_shared_bridge();
2383            if use_shared {
2384                if let Some(bridge) = self.shared_bridge.as_mut() {
2385                    return bridge.ip_allocator.allocate().ok_or_else(|| {
2386                        OverlaydError::Overlay(format!(
2387                            "shared bridge {} subnet {} exhausted",
2388                            bridge.name, bridge.subnet
2389                        ))
2390                    });
2391                }
2392            } else if let Some(bridge) = self.service_bridges.get_mut(service) {
2393                return bridge.ip_allocator.allocate().ok_or_else(|| {
2394                    OverlaydError::Overlay(format!(
2395                        "service bridge {} subnet {} exhausted",
2396                        bridge.name, bridge.subnet
2397                    ))
2398                });
2399            }
2400        }
2401        let _ = service;
2402        self.ip_allocator.allocate()
2403    }
2404
2405    /// Return an overlay IP to the allocator (service-bridge pool when known,
2406    /// otherwise the node slice).
2407    fn release_ip(&mut self, ip: IpAddr) {
2408        #[cfg(target_os = "linux")]
2409        {
2410            if let Some(bridge) = self.shared_bridge.as_mut() {
2411                if bridge.subnet.contains(&ip) {
2412                    bridge.ip_allocator.release(ip);
2413                    return;
2414                }
2415            }
2416            for bridge in self.service_bridges.values_mut() {
2417                if bridge.subnet.contains(&ip) {
2418                    bridge.ip_allocator.release(ip);
2419                    return;
2420                }
2421            }
2422        }
2423        self.ip_allocator.release(ip);
2424    }
2425
2426    // -- container attach (Linux) -------------------------------------------
2427
2428    /// Wire a container into the overlay and return its [`AttachResult`].
2429    ///
2430    /// # Errors
2431    /// Returns an error if the container cannot be attached.
2432    #[allow(clippy::too_many_arguments)]
2433    async fn attach_container(
2434        &mut self,
2435        handle: AttachHandle,
2436        service: &str,
2437        join_global: bool,
2438        ephemeral: bool,
2439        dns_server: Option<IpAddr>,
2440        dns_domain: Option<String>,
2441        isolation_network: Option<String>,
2442    ) -> Result<AttachResult, OverlaydError> {
2443        // Record the overlay DNS resolver/zone the main daemon staged for this
2444        // node so later attaches (and the Windows HCN endpoint `Dns` schema)
2445        // can fall back to them when a per-attach value isn't supplied.
2446        if let Some(server) = dns_server {
2447            self.dns_server_addr = Some(SocketAddr::new(server, 53));
2448        }
2449        if dns_domain.is_some() {
2450            self.dns_domain.clone_from(&dns_domain);
2451        }
2452        match handle {
2453            AttachHandle::LinuxPid { pid } => {
2454                let ip = self
2455                    .attach_container_linux(pid, service, join_global, ephemeral, isolation_network)
2456                    .await?;
2457                Ok(AttachResult {
2458                    ip,
2459                    namespace_guid: None,
2460                })
2461            }
2462            AttachHandle::WindowsContainer { container_id, ip } => {
2463                self.attach_container_windows(
2464                    &container_id,
2465                    service,
2466                    ip,
2467                    dns_server,
2468                    dns_domain,
2469                    isolation_network,
2470                )
2471                .await
2472            }
2473            AttachHandle::HostShared { id } => {
2474                let ip = self
2475                    .attach_container_host_shared(&id, service, ephemeral, isolation_network)
2476                    .await?;
2477                Ok(AttachResult {
2478                    ip,
2479                    namespace_guid: None,
2480                })
2481            }
2482            AttachHandle::GuestManaged { .. } => Err(OverlaydError::Other(
2483                "guest-managed attach must go through attach_container_guest, not attach_container"
2484                    .to_string(),
2485            )),
2486        }
2487    }
2488
2489    /// Tear down a container's overlay attachment and release its IP.
2490    ///
2491    /// # Errors
2492    /// Returns an error only if a netlink delete fails for a reason other than
2493    /// "link not found".
2494    async fn detach_container(&mut self, handle: AttachHandle) -> Result<(), OverlaydError> {
2495        match handle {
2496            AttachHandle::LinuxPid { pid } => self.detach_container_linux(pid).await,
2497            AttachHandle::WindowsContainer { container_id, .. } => {
2498                self.detach_container_windows(&container_id).await
2499            }
2500            AttachHandle::HostShared { id } => self.detach_container_host_shared(&id).await,
2501            AttachHandle::GuestManaged { .. } => Err(OverlaydError::Other(
2502                "guest-managed detach must go through detach_container_guest, not detach_container"
2503                    .to_string(),
2504            )),
2505        }
2506    }
2507
2508    // -- container attach (guest-managed) -----------------------------------
2509
2510    /// Guest-managed overlay attach: allocate the overlay identity for a VM guest
2511    /// that brings up its own kernel `WireGuard` device.
2512    ///
2513    /// overlayd cannot enter the guest's network namespace (it is a VM, not a
2514    /// host process), so instead of a veth/HCN endpoint it:
2515    /// 1. allocates the overlay IP from the SAME pool the Linux attach uses (the
2516    ///    per-service bridge pool when one exists, otherwise the node slice) so
2517    ///    guest addresses never collide with container addresses;
2518    /// 2. generates a fresh `WireGuard` keypair for the guest;
2519    /// 3. builds the peer set the guest must configure — every GLOBAL peer the
2520    ///    host already knows, plus THIS node itself (so the guest can reach the
2521    ///    host node over the overlay; carries a keepalive so the guest keeps its
2522    ///    NAT mapping open from behind VZ NAT);
2523    /// 4. registers the generated public key as a GLOBAL peer (host route to the
2524    ///    guest, roaming endpoint learned from the guest's keepalive) so remote
2525    ///    nodes and this node route to it;
2526    /// 5. records the attachment keyed by `id` so `DetachContainer` can release
2527    ///    the IP and remove the peer.
2528    ///
2529    /// Platform-agnostic: pure IPAM + keygen + peer bookkeeping (no netns/veth/
2530    /// HCN), so it compiles and runs on macOS (where the overlayd serving a VZ
2531    /// host lives) as well as Linux.
2532    ///
2533    /// # Errors
2534    /// Returns an error if the global overlay is not set up, the IP pool is
2535    /// exhausted, key generation fails, or registering the guest peer fails.
2536    #[allow(clippy::cast_possible_truncation, clippy::too_many_lines)]
2537    async fn attach_container_guest(
2538        &mut self,
2539        id: &str,
2540        service: &str,
2541        join_global: bool,
2542        dns_server: Option<IpAddr>,
2543        dns_domain: Option<String>,
2544        isolation_network: Option<String>,
2545    ) -> Result<GuestOverlayConfig, OverlaydError> {
2546        // The global transport must exist: we both register the guest as a peer
2547        // on it and advertise this node (its public key + listen port) to the
2548        // guest. Resolve both up front so we fail before allocating anything.
2549        let node_public_key = self.transport_public_key.clone().ok_or_else(|| {
2550            OverlaydError::Other(
2551                "guest-managed attach requires the global overlay to be set up first \
2552                 (no node WireGuard public key)"
2553                    .to_string(),
2554            )
2555        })?;
2556        if self.global_transport.is_none() {
2557            return Err(OverlaydError::Other(
2558                "guest-managed attach requires the global overlay to be set up first \
2559                 (no global transport)"
2560                    .to_string(),
2561            ));
2562        }
2563
2564        // 1. Allocate the overlay IP from the same pool the Linux attach uses and
2565        //    derive the prefix length from that pool's network. On Linux a
2566        //    per-service bridge (when present) supplies both the IP and its
2567        //    subnet's prefix; otherwise (and on every non-Linux host) the node
2568        //    slice / cluster CIDR does.
2569        let (overlay_ip, prefix_len, pool_service, dedicated): (IpAddr, u8, Option<String>, bool) = {
2570            #[cfg(target_os = "linux")]
2571            {
2572                let use_shared = self
2573                    .service_modes
2574                    .get(service)
2575                    .copied()
2576                    .unwrap_or_default()
2577                    .uses_shared_bridge();
2578                let bridge = if use_shared {
2579                    self.shared_bridge.as_mut()
2580                } else {
2581                    self.service_bridges.get_mut(service)
2582                };
2583                if let Some(bridge) = bridge {
2584                    let ip = bridge.ip_allocator.allocate().ok_or_else(|| {
2585                        OverlaydError::Overlay(format!(
2586                            "service bridge {} subnet {} exhausted",
2587                            bridge.name, bridge.subnet
2588                        ))
2589                    })?;
2590                    let prefix = bridge.subnet.prefix_len();
2591                    (ip, prefix, Some(service.to_string()), false)
2592                } else {
2593                    let ip = self.ip_allocator.allocate()?;
2594                    (ip, self.slice_prefix_len(), None, false)
2595                }
2596            }
2597            #[cfg(not(target_os = "linux"))]
2598            {
2599                // A Dedicated service owns a second WireGuard device (own crypto +
2600                // subnet); its guest draws from that device's allocator and lands
2601                // on the dedicated subnet, not the global cluster mesh. Every other
2602                // mode hairpins through the node slice on the global transport.
2603                let dedicated = self
2604                    .service_modes
2605                    .get(service)
2606                    .copied()
2607                    .unwrap_or_default()
2608                    .uses_per_service_wg();
2609                if dedicated {
2610                    let st = self.service_transports.get_mut(service).ok_or_else(|| {
2611                        OverlaydError::Other(format!(
2612                            "Dedicated service {service} has no dedicated overlay; \
2613                             call setup_service_overlay first"
2614                        ))
2615                    })?;
2616                    let ip = st.ip_allocator.allocate().ok_or_else(|| {
2617                        OverlaydError::Overlay(format!(
2618                            "dedicated service {service} subnet {} exhausted",
2619                            st.subnet
2620                        ))
2621                    })?;
2622                    (ip, st.subnet.prefix_len(), Some(service.to_string()), true)
2623                } else {
2624                    let ip = self.ip_allocator.allocate()?;
2625                    (ip, self.slice_prefix_len(), None, false)
2626                }
2627            }
2628        };
2629        // `join_global` is informational for a guest-managed attach: the guest's
2630        // single WireGuard device IS its global-overlay endpoint, so there is no
2631        // separate eth1 IP to reserve. Touch it so callers stay consistent with
2632        // the Linux/Windows handles.
2633        let _ = join_global;
2634
2635        // 2. Generate the guest's WireGuard keypair (reuse the transport's
2636        //    native x25519 keygen — never reimplement curve25519 here).
2637        let (private_key, public_key) = OverlayTransport::generate_keys().await.map_err(|e| {
2638            // Roll back the IP allocation so a keygen failure leaks nothing.
2639            self.release_guest_ip(overlay_ip, pool_service.as_deref());
2640            OverlaydError::Overlay(format!("failed to generate guest keys: {e}"))
2641        })?;
2642
2643        // 3. Build the peer set. A VZ guest is behind the host's NAT and can only
2644        //    reach the LOCAL node (via its NAT gateway) — it cannot dial other
2645        //    nodes' or sibling guests' endpoints directly. So it gets exactly ONE
2646        //    peer: this node. ALL overlay traffic (including to sibling containers
2647        //    and remote nodes) routes through this node, which forwards/hairpins it
2648        //    (the node already holds a /32 peer for every container — step 4 — and
2649        //    the real inter-node peers). We deliberately do NOT add the per-guest
2650        //    /32 peers here: a /32 with no reachable endpoint would win
2651        //    longest-prefix routing and black-hole sibling traffic. The endpoint
2652        //    returned here is the node's overlay IP as a placeholder; the VZ
2653        //    runtime rewrites it to the guest's NAT gateway (the only host address
2654        //    the guest can reach) before delivering the config. Keepalive holds the
2655        //    guest's NAT mapping open so the node can reach back.
2656        //
2657        //    Dedicated mode: the single peer is this node's DEDICATED per-service
2658        //    device (its own pubkey + listen port + subnet as AllowedIPs), so the
2659        //    guest joins that service's isolated mesh. Every other mode peers with
2660        //    the global cluster device, AllowedIPs = the whole cluster CIDR.
2661        let (peer_pubkey, peer_listen_port, peer_allowed) = if dedicated {
2662            let st = self
2663                .service_transports
2664                .get(service)
2665                .expect("dedicated transport allocated above");
2666            (st.public_key.clone(), st.listen_port, st.subnet.to_string())
2667        } else {
2668            let node_allowed = self
2669                .cluster_cidr
2670                .or(self.slice_cidr)
2671                .map_or_else(|| String::from("0.0.0.0/0"), |c| c.to_string());
2672            (node_public_key, self.overlay_port, node_allowed)
2673        };
2674        let node_endpoint = self.node_endpoint_for_guest(peer_listen_port);
2675        let peers: Vec<PeerSpec> = vec![PeerSpec {
2676            public_key: peer_pubkey,
2677            endpoint: node_endpoint,
2678            allowed_ips: peer_allowed,
2679            persistent_keepalive_secs: 25,
2680            // The guest reaches the node via its NAT gateway (the only host
2681            // address it can route to); it does not run the host's ICE-lite
2682            // candidate exchange, so no candidates are advertised here.
2683            candidates: Vec::new(),
2684        }];
2685
2686        // 4. Register the guest's public key as a GLOBAL peer (host route to the
2687        //    guest at <overlay_ip>/32, roaming endpoint learned from keepalive).
2688        //    Go through the same internal path `AddPeer { Global }` uses.
2689        let host_route = format!(
2690            "{}/{}",
2691            overlay_ip,
2692            if overlay_ip.is_ipv6() { 128 } else { 32 }
2693        );
2694        let guest_peer = PeerSpec {
2695            public_key: public_key.clone(),
2696            // Empty/roaming: the guest is behind NAT; boringtun learns its source
2697            // endpoint from the guest's first keepalive. `0.0.0.0:0` is the
2698            // wire-safe "unset endpoint" sentinel that still parses as a
2699            // SocketAddr (peer_spec_to_info requires a parseable endpoint).
2700            endpoint: "0.0.0.0:0".to_string(),
2701            allowed_ips: host_route,
2702            persistent_keepalive_secs: 0,
2703            // The guest's roaming endpoint is learned from its first keepalive;
2704            // it advertises no NAT candidates (the host learns the source).
2705            candidates: Vec::new(),
2706        };
2707        let guest_peer_info = peer_spec_to_info(&guest_peer)?;
2708        let scope = if dedicated {
2709            PeerScope::Service {
2710                service: service.to_string(),
2711            }
2712        } else {
2713            PeerScope::Global
2714        };
2715        {
2716            let transport = self.transport_for_scope(&scope)?;
2717            if let Err(e) = Self::add_peer_on(transport, &guest_peer_info).await {
2718                self.release_guest_ip(overlay_ip, pool_service.as_deref());
2719                return Err(e);
2720            }
2721        }
2722        // Track it among the global peers (so a *subsequent* guest attach also
2723        // learns about this guest) and record the attachment for detach.
2724        self.global_peers
2725            .insert(public_key.clone(), guest_peer.clone());
2726        // Per-network membership + node-side L3 isolation: record the guest's
2727        // overlay IP in its isolated network's member set, and enforce the
2728        // cross-platform isolation policy on THIS node. A VZ guest hairpins ALL
2729        // its overlay traffic through this node's WireGuard device, so the node
2730        // is the enforcement point: on macOS this dispatches to pf (a per-network
2731        // table + sub-anchor); on Linux it dispatches to iptables (harmless here
2732        // — guests do not run on Linux). The guest's own WireGuard AllowedIPs are
2733        // the in-guest belt; this is the node-side suspenders.
2734        if let Some(ref net) = isolation_network {
2735            let node_ip = self
2736                .node_ip
2737                .unwrap_or(std::net::IpAddr::V4(std::net::Ipv4Addr::new(10, 200, 0, 1)));
2738            let cidr = self
2739                .cluster_cidr
2740                .map_or_else(|| "10.200.0.0/16".to_string(), |c| c.to_string());
2741            // Peers = current members of the network BEFORE inserting this guest.
2742            let peers: Vec<IpAddr> = self
2743                .network_members
2744                .get(net)
2745                .map(|m| m.iter().copied().collect())
2746                .unwrap_or_default();
2747            if let Err(e) = zlayer_overlay::firewall::ensure_member_isolation(
2748                net, overlay_ip, &peers, node_ip, &cidr,
2749            ) {
2750                tracing::warn!(network = %net, member = %overlay_ip, error = %e, "failed to install per-network L3 isolation for guest (non-fatal)");
2751            }
2752            self.network_members
2753                .entry(net.clone())
2754                .or_default()
2755                .insert(overlay_ip);
2756        }
2757        self.guest_attachments.insert(
2758            id.to_string(),
2759            GuestAttachInfo {
2760                overlay_ip,
2761                public_key: public_key.clone(),
2762                service_name: pool_service,
2763                isolation_network,
2764            },
2765        );
2766
2767        // 5. Return the config the caller ships into the guest.
2768        Ok(GuestOverlayConfig {
2769            overlay_ip,
2770            prefix_len,
2771            private_key,
2772            public_key,
2773            // The guest's device listens on the same port as its single in-guest
2774            // peer (the node device it joins): the node's overlay WG port for the
2775            // global mesh, or the dedicated device's listen port in Dedicated mode.
2776            listen_port: peer_listen_port,
2777            peers,
2778            dns_server: dns_server.or_else(|| self.dns_server_addr.map(|s| s.ip())),
2779            dns_domain: dns_domain.or_else(|| self.dns_domain.clone()),
2780        })
2781    }
2782
2783    /// Release a guest-managed attach by `id`: drop the host route + global peer
2784    /// and return the allocated IP to its pool. Idempotent.
2785    ///
2786    /// # Errors
2787    /// Returns an error only if removing the peer from the global transport fails
2788    /// for a reason other than "peer not found".
2789    async fn detach_container_guest(&mut self, id: &str) -> Result<(), OverlaydError> {
2790        let Some(info) = self.guest_attachments.remove(id) else {
2791            return Ok(());
2792        };
2793        // Remove the guest's peer from the same scope it was registered on: a
2794        // Dedicated guest sits on its service's dedicated device, every other
2795        // guest on the global cluster device. Mirror the attach-time scope choice
2796        // so a dedicated guest peer does not leak on teardown.
2797        let scope = match info.service_name.as_deref() {
2798            Some(svc)
2799                if self
2800                    .service_modes
2801                    .get(svc)
2802                    .copied()
2803                    .unwrap_or_default()
2804                    .uses_per_service_wg() =>
2805            {
2806                PeerScope::Service {
2807                    service: svc.to_string(),
2808                }
2809            }
2810            _ => PeerScope::Global,
2811        };
2812        self.global_peers.remove(&info.public_key);
2813        if let Ok(transport) = self.transport_for_scope(&scope) {
2814            if let Err(e) = Self::remove_peer_on(transport, &info.public_key).await {
2815                tracing::warn!(
2816                    guest = %id,
2817                    pubkey = %info.public_key,
2818                    scope = ?scope,
2819                    error = %e,
2820                    "failed to remove guest peer from its overlay transport"
2821                );
2822            }
2823        }
2824        // Drain the per-network membership set for this guest and tear down the
2825        // node-side L3 isolation rule for it (pf on macOS, iptables on Linux —
2826        // the latter is a no-op for guests, which never run on Linux). Drop the
2827        // network entry once empty.
2828        if let Some(net) = info.isolation_network.as_deref() {
2829            if let Some(set) = self.network_members.get_mut(net) {
2830                set.remove(&info.overlay_ip);
2831            }
2832            let remaining_peers: Vec<IpAddr> = self
2833                .network_members
2834                .get(net)
2835                .map(|m| m.iter().copied().collect())
2836                .unwrap_or_default();
2837            let node_ip = self
2838                .node_ip
2839                .unwrap_or(std::net::IpAddr::V4(std::net::Ipv4Addr::new(10, 200, 0, 1)));
2840            let cidr = self
2841                .cluster_cidr
2842                .map_or_else(|| "10.200.0.0/16".to_string(), |c| c.to_string());
2843            zlayer_overlay::firewall::remove_member_isolation(
2844                net,
2845                info.overlay_ip,
2846                &remaining_peers,
2847                node_ip,
2848                &cidr,
2849            );
2850            if self
2851                .network_members
2852                .get(net)
2853                .is_some_and(std::collections::HashSet::is_empty)
2854            {
2855                self.network_members.remove(net);
2856            }
2857        }
2858        // Return the IP to whichever pool it came from.
2859        self.release_guest_ip(info.overlay_ip, info.service_name.as_deref());
2860        Ok(())
2861    }
2862
2863    // ---- edge peers --------------------------------------------------------
2864    //
2865    // A minted edge peer is a short-lived, unprivileged `WireGuard` endpoint
2866    // (e.g. a remote CI runner) that joins the overlay without any of the
2867    // container plumbing. Mechanically it clones the guest-attach path: allocate
2868    // a node-slice `/32`, generate the peer's OWN keypair, register it as a
2869    // roaming global `/32` peer, and hand back a portable [`EdgeConfig`] whose
2870    // single artifact peer is THIS node. Unlike a guest, an edge peer carries a
2871    // TTL and an optional node-side L3 fence, and is swept when it expires or
2872    // goes silent. Every path is platform-agnostic (no netns/veth/HCN).
2873
2874    /// Grace, in seconds, an edge peer is allowed to complete its first
2875    /// handshake before the silence sweep reaps it. `120` ≈ 3× the artifact's
2876    /// 25s keepalive — enough slack for a remote pod to boot, resolve the node
2877    /// endpoint, and hole-punch — before we assume the mint was abandoned.
2878    const EDGE_CONNECT_GRACE_SECS: u64 = 120;
2879
2880    /// Silence timeout, in seconds: once an edge peer HAS handshaked, it is
2881    /// reaped when no handshake has been seen for this long. Matches
2882    /// `zlayer_overlay::health`'s `HANDSHAKE_TIMEOUT_SECS` (180) so the edge
2883    /// liveness verdict lines up with the rest of the overlay's peer health.
2884    const EDGE_SILENCE_TIMEOUT_SECS: u64 = 180;
2885
2886    /// Mint a short-lived unprivileged edge peer: allocate an overlay `/32`,
2887    /// generate the edge's keypair, register it as a roaming global peer, install
2888    /// the node-side L3 fence when `--allow` is given, and return the portable
2889    /// [`EdgeConfig`] a remote `WireGuard` process needs to join the overlay.
2890    ///
2891    /// Mirrors [`Self::attach_container_guest`]'s keygen/peer bookkeeping but for
2892    /// an off-cluster peer: the EDGE's keypair goes in the config, and the single
2893    /// artifact peer is THIS node (public key from `transport_public_key`).
2894    ///
2895    /// # Errors
2896    /// Returns an error if the name is blank or already live, the global overlay
2897    /// is not up, `node_endpoint` is empty / unspecified / portless, an `--allow`
2898    /// entry resolves to nothing, the slice is exhausted, or keygen / peer
2899    /// registration fails (each of which rolls back any IP already allocated).
2900    async fn mint_edge_peer(
2901        &mut self,
2902        name: String,
2903        ttl_secs: u64,
2904        allow: &[String],
2905        node_endpoint: &str,
2906    ) -> Result<EdgeConfig, OverlaydError> {
2907        let name = name.trim().to_string();
2908        if name.is_empty() {
2909            return Err(OverlaydError::Other(
2910                "edge peer name must not be empty".to_string(),
2911            ));
2912        }
2913        if self.edge_attachments.contains_key(&name) {
2914            return Err(OverlaydError::Other(format!(
2915                "edge peer `{name}` is already minted; revoke it before re-minting"
2916            )));
2917        }
2918        // The global overlay must be up: we register the edge on it AND advertise
2919        // this node's identity to the edge. Resolve both before allocating.
2920        let node_public_key = self.transport_public_key.clone().ok_or_else(|| {
2921            OverlaydError::Other(
2922                "edge mint requires the global overlay to be set up first \
2923                 (no node WireGuard public key)"
2924                    .to_string(),
2925            )
2926        })?;
2927        if self.global_transport.is_none() {
2928            return Err(OverlaydError::Other(
2929                "edge mint requires the global overlay to be set up first \
2930                 (no global transport)"
2931                    .to_string(),
2932            ));
2933        }
2934        validate_edge_node_endpoint(node_endpoint)?;
2935        // Resolve `--allow` up front so an unknown service name fails before any
2936        // allocation happens.
2937        let targets = self.resolve_allow_targets(allow)?;
2938
2939        // Allocate the edge's `/32` from the node slice (mirrors the guest path).
2940        let overlay_ip = self.ip_allocator.allocate()?;
2941
2942        // Generate the EDGE's own keypair — never the node's. Roll the IP back on
2943        // any failure below so a partial mint leaks no address.
2944        let (private_key, public_key) = match OverlayTransport::generate_keys().await {
2945            Ok(keys) => keys,
2946            Err(e) => {
2947                self.ip_allocator.release(overlay_ip);
2948                return Err(OverlaydError::Overlay(format!(
2949                    "failed to generate edge keys: {e}"
2950                )));
2951            }
2952        };
2953
2954        // Register the edge as a roaming global `/32` peer (endpoint learned from
2955        // its first keepalive; keepalive 0 — mirrors guest registration).
2956        if let Err(e) = self
2957            .register_edge_global_peer(&public_key, overlay_ip)
2958            .await
2959        {
2960            self.ip_allocator.release(overlay_ip);
2961            return Err(e);
2962        }
2963
2964        // Node-side L3 fence: installed ONLY when `--allow` was given. Without
2965        // it, the edge gets guest-equivalent trust (cluster-CIDR AllowedIPs).
2966        let isolation_network = if targets.is_empty() {
2967            None
2968        } else {
2969            Some(self.install_edge_fence(&name, overlay_ip, &targets))
2970        };
2971
2972        // The single artifact peer is THIS node. Its AllowedIPs scope what the
2973        // edge routes to: the resolved targets (+ node + DNS `/32`) when fenced,
2974        // else the whole cluster CIDR.
2975        let dns_ip = self.dns_server_addr.map(|s| s.ip());
2976        let node_allowed = self.edge_node_allowed_ips(&targets, dns_ip);
2977        let node_peer = Self::build_edge_peer_spec(node_public_key, node_endpoint, node_allowed);
2978
2979        let minted_at_unix = now_unix();
2980        let expires_at_unix = minted_at_unix.saturating_add(ttl_secs);
2981
2982        self.edge_attachments.insert(
2983            name.clone(),
2984            EdgeAttachInfo {
2985                overlay_ip,
2986                public_key: public_key.clone(),
2987                minted_at_unix,
2988                expires_at_unix,
2989                allowed_targets: targets,
2990                isolation_network,
2991            },
2992        );
2993
2994        Ok(EdgeConfig {
2995            version: EDGE_CONFIG_VERSION,
2996            name,
2997            overlay_ip,
2998            prefix_len: self.slice_prefix_len(),
2999            private_key,
3000            public_key,
3001            peers: vec![node_peer],
3002            dns_server: dns_ip,
3003            dns_domain: self.dns_domain.clone(),
3004            expires_at_unix,
3005        })
3006    }
3007
3008    /// Register the edge's roaming `/32` on the GLOBAL transport (endpoint
3009    /// `0.0.0.0:0`, keepalive 0 — boringtun learns the source from the edge's
3010    /// first keepalive) and mirror it into `global_peers`. Goes through the same
3011    /// internal path `AddPeer { Global }` uses. On error the caller rolls back the
3012    /// IP allocation.
3013    ///
3014    /// # Errors
3015    /// Returns an error if the endpoint sentinel fails to parse, the global
3016    /// transport is missing, or the UAPI peer write fails.
3017    async fn register_edge_global_peer(
3018        &mut self,
3019        public_key: &str,
3020        overlay_ip: IpAddr,
3021    ) -> Result<(), OverlaydError> {
3022        let edge_peer = PeerSpec {
3023            public_key: public_key.to_string(),
3024            // Roaming: the edge is outside the overlay until its first handshake.
3025            // `0.0.0.0:0` is the wire-safe "unset endpoint" sentinel (still parses
3026            // as a `SocketAddr`, which `peer_spec_to_info` requires).
3027            endpoint: "0.0.0.0:0".to_string(),
3028            allowed_ips: edge_host_route(overlay_ip),
3029            persistent_keepalive_secs: 0,
3030            candidates: Vec::new(),
3031        };
3032        let info = peer_spec_to_info(&edge_peer)?;
3033        {
3034            let transport = self.transport_for_scope(&PeerScope::Global)?;
3035            Self::add_peer_on(transport, &info).await?;
3036        }
3037        self.global_peers.insert(public_key.to_string(), edge_peer);
3038        Ok(())
3039    }
3040
3041    /// Resolve an interior overlay subnet for a service set up on THIS node
3042    /// (used by [`Self::resolve_allow_targets`] for a `--allow <service>` entry):
3043    /// the dedicated per-service device's subnet, or on Linux the per-service /
3044    /// node-wide shared bridge subnet. `None` if `service` is unknown here.
3045    fn edge_service_subnet(&self, service: &str) -> Option<ipnet::IpNet> {
3046        // A Dedicated per-service `WireGuard` device owns its subnet (all OSes).
3047        if let Some(st) = self.service_transports.get(service) {
3048            return Some(st.subnet);
3049        }
3050        #[cfg(target_os = "linux")]
3051        {
3052            if let Some(bridge) = self.service_bridges.get(service) {
3053                return Some(bridge.subnet);
3054            }
3055            // A Shared-mode service is backed by the single node-wide shared
3056            // bridge, keyed by subnet rather than by service name.
3057            if self
3058                .service_modes
3059                .get(service)
3060                .copied()
3061                .unwrap_or_default()
3062                .uses_shared_bridge()
3063            {
3064                if let Some(bridge) = self.shared_bridge.as_ref() {
3065                    return Some(bridge.subnet);
3066                }
3067            }
3068        }
3069        None
3070    }
3071
3072    /// Resolve each `--allow` entry to an overlay CIDR: a literal CIDR, a bare
3073    /// overlay IP (→ host `/32` / `/128`), or a service name → its per-node
3074    /// subnet. Blank entries are skipped; an entry that matches none is a hard
3075    /// error naming the CIDR alternative.
3076    ///
3077    /// # Errors
3078    /// Returns [`OverlaydError::Other`] for an entry that is neither a CIDR/IP
3079    /// nor a service set up on this node.
3080    fn resolve_allow_targets(&self, allow: &[String]) -> Result<Vec<ipnet::IpNet>, OverlaydError> {
3081        let mut out: Vec<ipnet::IpNet> = Vec::with_capacity(allow.len());
3082        for raw in allow {
3083            let entry = raw.trim();
3084            if entry.is_empty() {
3085                continue;
3086            }
3087            if let Ok(net) = entry.parse::<ipnet::IpNet>() {
3088                out.push(net);
3089                continue;
3090            }
3091            if let Ok(ip) = entry.parse::<IpAddr>() {
3092                let prefix = if ip.is_ipv6() { 128 } else { 32 };
3093                let net = ipnet::IpNet::new(ip, prefix)
3094                    .expect("a host prefix length (32/128) is always in range");
3095                out.push(net);
3096                continue;
3097            }
3098            if let Some(net) = self.edge_service_subnet(entry) {
3099                out.push(net);
3100                continue;
3101            }
3102            return Err(OverlaydError::Other(format!(
3103                "edge --allow target `{entry}` is neither an overlay CIDR/IP nor a service \
3104                 set up on this node; pass a CIDR (e.g. `10.200.0.0/16`), an overlay IP, or a \
3105                 known service name"
3106            )));
3107        }
3108        Ok(out)
3109    }
3110
3111    /// Build the single artifact peer an edge configures: THIS node, with a 25s
3112    /// keepalive so the edge holds its NAT mapping open from behind its own NAT.
3113    fn build_edge_peer_spec(
3114        node_pubkey: String,
3115        node_endpoint: &str,
3116        allowed_ips_csv: String,
3117    ) -> PeerSpec {
3118        PeerSpec {
3119            public_key: node_pubkey,
3120            endpoint: node_endpoint.to_string(),
3121            allowed_ips: allowed_ips_csv,
3122            persistent_keepalive_secs: 25,
3123            candidates: Vec::new(),
3124        }
3125    }
3126
3127    /// Comma-separated `AllowedIPs` for the artifact's node peer: the resolved
3128    /// `--allow` targets plus the node's own `/32` and the overlay DNS `/32`
3129    /// when fenced, else the whole cluster (guest-equivalent) CIDR.
3130    fn edge_node_allowed_ips(&self, targets: &[ipnet::IpNet], dns_ip: Option<IpAddr>) -> String {
3131        if targets.is_empty() {
3132            return self
3133                .cluster_cidr
3134                .or(self.slice_cidr)
3135                .map_or_else(|| "0.0.0.0/0".to_string(), |c| c.to_string());
3136        }
3137        let mut cidrs: Vec<String> = targets.iter().map(ToString::to_string).collect();
3138        if let Some(node_ip) = self.node_ip {
3139            cidrs.push(edge_host_route(node_ip));
3140        }
3141        if let Some(dns) = dns_ip {
3142            cidrs.push(edge_host_route(dns));
3143        }
3144        cidrs.join(",")
3145    }
3146
3147    /// Install the node-side L3 fence for a fenced edge peer (network id
3148    /// `edge:<name>`): the ACTUAL enforcement of which overlay destinations the
3149    /// edge `/32` may reach. Non-fatal — a firewall hiccup is warned, not
3150    /// propagated, exactly like the guest path — and the network id is always
3151    /// returned so revoke tears the fence down regardless. The caller decides
3152    /// whether a fence exists at all (only when `--allow` is non-empty).
3153    ///
3154    /// `targets` become the fence's permitted `peers` (each target's network
3155    /// address). On Linux those are pairwise ACCEPTs with a catch-all overlay
3156    /// DROP; on macOS `peers` is ignored (the pf table already covers members),
3157    /// so the guarantee is "no stronger than the platform's guest isolation".
3158    fn install_edge_fence(&self, name: &str, edge_ip: IpAddr, targets: &[ipnet::IpNet]) -> String {
3159        let network = format!("edge:{name}");
3160        let node_ip = self
3161            .node_ip
3162            .unwrap_or(IpAddr::V4(Ipv4Addr::new(10, 200, 0, 1)));
3163        let cidr = self
3164            .cluster_cidr
3165            .map_or_else(|| "10.200.0.0/16".to_string(), |c| c.to_string());
3166        let peers: Vec<IpAddr> = targets.iter().map(ipnet::IpNet::network).collect();
3167        if let Err(e) = zlayer_overlay::firewall::ensure_member_isolation(
3168            &network, edge_ip, &peers, node_ip, &cidr,
3169        ) {
3170            tracing::warn!(
3171                network = %network,
3172                member = %edge_ip,
3173                error = %e,
3174                "failed to install per-edge L3 isolation fence (non-fatal)"
3175            );
3176        }
3177        network
3178    }
3179
3180    /// Revoke a minted edge peer by name — the SINGLE teardown path (the sweep
3181    /// reuses it). Idempotent: `Ok(false)` when the name is absent. Removes the
3182    /// global `/32` peer, tears down the fence if one was installed, and returns
3183    /// the `/32` to the node slice.
3184    ///
3185    /// # Errors
3186    /// Never returns `Err` in practice: peer removal is best-effort (warned, not
3187    /// propagated) so a transport hiccup cannot strand the allocation. The
3188    /// `Result` is kept for symmetry with the other teardown methods.
3189    async fn revoke_edge_peer(&mut self, name: &str) -> Result<bool, OverlaydError> {
3190        let Some(info) = self.edge_attachments.remove(name) else {
3191            return Ok(false);
3192        };
3193        // Drop the edge's `/32` peer from the global transport (and the mirror).
3194        self.global_peers.remove(&info.public_key);
3195        if let Ok(transport) = self.transport_for_scope(&PeerScope::Global) {
3196            if let Err(e) = Self::remove_peer_on(transport, &info.public_key).await {
3197                tracing::warn!(
3198                    edge = %name,
3199                    pubkey = %info.public_key,
3200                    error = %e,
3201                    "failed to remove edge peer from the global transport (non-fatal)"
3202                );
3203            }
3204        }
3205        // Tear down the node-side L3 fence, if one was installed. Reproduce the
3206        // exact `peers` set from the stored targets so the removal matches.
3207        if let Some(network) = info.isolation_network.as_deref() {
3208            let node_ip = self
3209                .node_ip
3210                .unwrap_or(IpAddr::V4(Ipv4Addr::new(10, 200, 0, 1)));
3211            let cidr = self
3212                .cluster_cidr
3213                .map_or_else(|| "10.200.0.0/16".to_string(), |c| c.to_string());
3214            let peers: Vec<IpAddr> = info
3215                .allowed_targets
3216                .iter()
3217                .map(ipnet::IpNet::network)
3218                .collect();
3219            zlayer_overlay::firewall::remove_member_isolation(
3220                network,
3221                info.overlay_ip,
3222                &peers,
3223                node_ip,
3224                &cidr,
3225            );
3226        }
3227        // Return the `/32` to the node-slice pool.
3228        self.ip_allocator.release(info.overlay_ip);
3229        Ok(true)
3230    }
3231
3232    /// Pure, unit-testable reap predicate for one edge peer. Reaped when: the TTL
3233    /// has elapsed; OR it never handshaked and the boot-grace window is up; OR it
3234    /// handshaked once and has been silent past the timeout.
3235    fn edge_reap_due(
3236        info: &EdgeAttachInfo,
3237        last_handshake_unix: Option<i64>,
3238        now_unix: u64,
3239    ) -> bool {
3240        // Hard TTL expiry wins regardless of liveness.
3241        if now_unix >= info.expires_at_unix {
3242            return true;
3243        }
3244        match last_handshake_unix.filter(|&h| h > 0) {
3245            // A real handshake was seen: reap after the silence window.
3246            Some(h) => {
3247                let last = u64::try_from(h).unwrap_or(0);
3248                now_unix.saturating_sub(last) >= Self::EDGE_SILENCE_TIMEOUT_SECS
3249            }
3250            // Never handshaked (None / 0 / negative): reap once boot-grace is up.
3251            None => now_unix.saturating_sub(info.minted_at_unix) >= Self::EDGE_CONNECT_GRACE_SECS,
3252        }
3253    }
3254
3255    /// Snapshot the global transport's per-peer last-handshake once, indexed by
3256    /// HEX public key (`parse_peer_status` reports hex; edge keys are base64, so
3257    /// callers convert via [`zlayer_overlay::nat::pubkey_b64_to_hex`]). A missing
3258    /// transport / failed dump yields an empty map (every edge then reads as
3259    /// never-handshaked).
3260    async fn edge_handshake_map(&self) -> HashMap<String, i64> {
3261        let mut map: HashMap<String, i64> = HashMap::new();
3262        if let Some(transport) = self.global_transport.as_ref() {
3263            if let Ok(dump) = transport.status().await {
3264                for p in parse_peer_status(&dump) {
3265                    map.insert(p.public_key, p.last_handshake_unix_secs);
3266                }
3267            }
3268        }
3269        map
3270    }
3271
3272    /// Sweep expired / silent edge peers: snapshot handshakes ONCE, collect the
3273    /// due names, then revoke each (the borrow-safe collect-then-revoke order
3274    /// avoids mutating `edge_attachments` while iterating it).
3275    ///
3276    /// Public because the daemon's serve loop drives it on a timer (the edge
3277    /// sweep task) rather than through a wire request — there is no
3278    /// `SweepEdgePeers` [`OverlaydRequest`]; TTL/silence reaping is overlayd's
3279    /// own background responsibility, like the NAT maintenance tick.
3280    pub async fn sweep_edge_peers(&mut self, now_unix: u64) {
3281        if self.edge_attachments.is_empty() {
3282            return;
3283        }
3284        let handshakes = self.edge_handshake_map().await;
3285        let due: Vec<String> = self
3286            .edge_attachments
3287            .iter()
3288            .filter(|(_, info)| {
3289                let hs = zlayer_overlay::nat::pubkey_b64_to_hex(&info.public_key)
3290                    .and_then(|hex| handshakes.get(&hex).copied());
3291                Self::edge_reap_due(info, hs, now_unix)
3292            })
3293            .map(|(name, _)| name.clone())
3294            .collect();
3295        for name in due {
3296            match self.revoke_edge_peer(&name).await {
3297                Ok(true) => tracing::info!(edge = %name, "swept expired/silent edge peer"),
3298                Ok(false) => {}
3299                Err(e) => {
3300                    tracing::warn!(edge = %name, error = %e, "failed to sweep edge peer");
3301                }
3302            }
3303        }
3304    }
3305
3306    /// Join every live edge peer with its last-handshake from the global
3307    /// transport for [`OverlaydRequest::ListEdgePeers`]. A non-positive handshake
3308    /// (never seen) is reported as `None`.
3309    async fn edge_peer_statuses(&self) -> Vec<EdgePeerStatus> {
3310        let handshakes = self.edge_handshake_map().await;
3311        self.edge_attachments
3312            .iter()
3313            .map(|(name, info)| {
3314                let last_handshake_unix_secs =
3315                    zlayer_overlay::nat::pubkey_b64_to_hex(&info.public_key)
3316                        .and_then(|hex| handshakes.get(&hex).copied())
3317                        .filter(|&h| h > 0);
3318                EdgePeerStatus {
3319                    name: name.clone(),
3320                    overlay_ip: info.overlay_ip,
3321                    public_key: info.public_key.clone(),
3322                    minted_at_unix: info.minted_at_unix,
3323                    expires_at_unix: info.expires_at_unix,
3324                    last_handshake_unix_secs,
3325                    allowed: info
3326                        .allowed_targets
3327                        .iter()
3328                        .map(ToString::to_string)
3329                        .collect(),
3330                }
3331            })
3332            .collect()
3333    }
3334
3335    // -- container attach (macOS host-shared) -------------------------------
3336
3337    /// Host-shared overlay attach: give a macOS host-shared container
3338    /// ([`AttachHandle::HostShared`] — Seatbelt / native-VZ / libkrun) its own
3339    /// first-class L3 overlay membership.
3340    ///
3341    /// A host-shared container shares the node's host network namespace and its
3342    /// single cluster `utun`; it cannot get its own netns/veth (Seatbelt) or its
3343    /// own kernel `WireGuard` device (no guest VM to run one). So instead of a
3344    /// veth/HCN endpoint or a per-guest WG keypair, this:
3345    /// 1. allocates a DISTINCT overlay `/32` from the node slice (never the node
3346    ///    IP — `IpAllocator` reserves offset 1 — and never `None`). The node
3347    ///    slice is already advertised cluster-wide as this node's `AllowedIPs`,
3348    ///    so the `/32` auto-routes to this node with no peer reconfiguration;
3349    /// 2. adds that `/32` as an alias on the node's overlay `utun` so the kernel
3350    ///    delivers inbound overlay packets for it locally (boringtun decrypts
3351    ///    and writes the plaintext packet to the utun, which only delivers to a
3352    ///    configured local address);
3353    /// 3. records per-network membership + installs node-side L3 isolation when
3354    ///    `isolation_network` is set (pf on macOS), exactly like the guest path;
3355    /// 4. records the attachment keyed by `id` so `DetachContainer` can remove
3356    ///    the alias, drain the membership, and release the IP.
3357    ///
3358    /// HONEST CONSTRAINT: host-shared containers share the node's single cluster
3359    /// `utun`, so `OverlayMode::Dedicated`'s per-service `WireGuard` CRYPTO
3360    /// isolation cannot apply to them — there is no per-container WG device
3361    /// without a netns or a guest VM to host one. They still get a distinct
3362    /// overlay IP + L3 isolation (per-network membership / pf) + overlay DNS,
3363    /// which is full first-class L3 overlay membership. This is a real OS
3364    /// constraint of host-shared execution, not a stub.
3365    ///
3366    /// # Errors
3367    /// Returns an error if the node slice is exhausted, or if the global overlay
3368    /// interface is not set up (so there is no `utun` to alias the `/32` on).
3369    async fn attach_container_host_shared(
3370        &mut self,
3371        id: &str,
3372        service: &str,
3373        ephemeral: bool,
3374        isolation_network: Option<String>,
3375    ) -> Result<IpAddr, OverlaydError> {
3376        // 1. Allocate a distinct /32 from the node slice. Never the node IP
3377        //    (reserved at offset 1), never None — exhaustion maps to the same
3378        //    `OverlaydError::Overlay` the other attach paths surface.
3379        let ip = self.ip_allocator.allocate()?;
3380        let prefix_len: u8 = if ip.is_ipv6() { 128 } else { 32 };
3381
3382        // 2. Make the /32 locally deliverable on the node's overlay utun via an
3383        //    alias on the single cluster transport's interface. Roll the IP
3384        //    allocation back on any failure so nothing leaks.
3385        let alias_res = if let Some(transport) = self.global_transport.as_ref() {
3386            transport
3387                .add_alias(ip, prefix_len)
3388                .await
3389                .map_err(|e| OverlaydError::Overlay(e.to_string()))
3390        } else {
3391            Err(OverlaydError::Other(
3392                "host-shared attach requires the global overlay to be set up first \
3393                 (no utun to alias the container /32 on)"
3394                    .to_string(),
3395            ))
3396        };
3397        if let Err(e) = alias_res {
3398            self.ip_allocator.release(ip);
3399            return Err(e);
3400        }
3401
3402        // 3. Per-network membership + node-side L3 isolation (mirror the guest
3403        //    path). The host-shared container hairpins all overlay traffic
3404        //    through this node's WireGuard device, so the node is the
3405        //    enforcement point (pf on macOS).
3406        if let Some(ref net) = isolation_network {
3407            let node_ip = self
3408                .node_ip
3409                .unwrap_or(std::net::IpAddr::V4(std::net::Ipv4Addr::new(10, 200, 0, 1)));
3410            let cidr = self
3411                .cluster_cidr
3412                .map_or_else(|| "10.200.0.0/16".to_string(), |c| c.to_string());
3413            // Peers = current members of the network BEFORE inserting this one.
3414            let peers: Vec<IpAddr> = self
3415                .network_members
3416                .get(net)
3417                .map(|m| m.iter().copied().collect())
3418                .unwrap_or_default();
3419            if let Err(e) =
3420                zlayer_overlay::firewall::ensure_member_isolation(net, ip, &peers, node_ip, &cidr)
3421            {
3422                tracing::warn!(network = %net, member = %ip, error = %e, "failed to install per-network L3 isolation for host-shared container (non-fatal)");
3423            }
3424            self.network_members
3425                .entry(net.clone())
3426                .or_default()
3427                .insert(ip);
3428        }
3429
3430        // 4. Record the attachment so detach can reverse all of the above.
3431        self.host_shared_attachments.insert(
3432            id.to_string(),
3433            AttachInfo {
3434                service_ip: ip,
3435                service_name: Some(service.to_string()),
3436                // No separate global/eth1 IP: a host-shared container reaches the
3437                // global overlay through the SAME /32 aliased on the node utun.
3438                global_ip: None,
3439                ephemeral,
3440                isolation_network,
3441            },
3442        );
3443
3444        Ok(ip)
3445    }
3446
3447    /// Release a host-shared attach by `id`: remove the utun `/32` alias, drain
3448    /// its per-network L3 isolation membership, and return the IP to the node
3449    /// slice. Idempotent. Mirrors [`Self::detach_container_guest`].
3450    ///
3451    /// # Errors
3452    /// Returns `Ok` even when removing the alias fails (best-effort, logged) —
3453    /// the IP is always returned to the pool so it can never leak.
3454    async fn detach_container_host_shared(&mut self, id: &str) -> Result<(), OverlaydError> {
3455        let Some(info) = self.host_shared_attachments.remove(id) else {
3456            return Ok(());
3457        };
3458        // Drain the per-network membership set and tear down the node-side L3
3459        // isolation rule for this container; drop the network entry once empty.
3460        if let Some(net) = info.isolation_network.as_deref() {
3461            if let Some(set) = self.network_members.get_mut(net) {
3462                set.remove(&info.service_ip);
3463            }
3464            let remaining_peers: Vec<IpAddr> = self
3465                .network_members
3466                .get(net)
3467                .map(|m| m.iter().copied().collect())
3468                .unwrap_or_default();
3469            let node_ip = self
3470                .node_ip
3471                .unwrap_or(std::net::IpAddr::V4(std::net::Ipv4Addr::new(10, 200, 0, 1)));
3472            let cidr = self
3473                .cluster_cidr
3474                .map_or_else(|| "10.200.0.0/16".to_string(), |c| c.to_string());
3475            zlayer_overlay::firewall::remove_member_isolation(
3476                net,
3477                info.service_ip,
3478                &remaining_peers,
3479                node_ip,
3480                &cidr,
3481            );
3482            if self
3483                .network_members
3484                .get(net)
3485                .is_some_and(std::collections::HashSet::is_empty)
3486            {
3487                self.network_members.remove(net);
3488            }
3489        }
3490        // Remove the utun /32 alias (best-effort: a failed removal must not
3491        // strand the IP, so we log and still release below).
3492        let prefix_len: u8 = if info.service_ip.is_ipv6() { 128 } else { 32 };
3493        if let Some(transport) = self.global_transport.as_ref() {
3494            if let Err(e) = transport.remove_alias(info.service_ip, prefix_len).await {
3495                tracing::warn!(
3496                    container = %id,
3497                    ip = %info.service_ip,
3498                    error = %e,
3499                    "failed to remove host-shared overlay /32 alias from utun (non-fatal)"
3500                );
3501            }
3502        }
3503        // Return the IP to the node slice.
3504        self.ip_allocator.release(info.service_ip);
3505
3506        // Per-job segment lifecycle observability. Unlike the Linux veth path —
3507        // which reaps a per-service BRIDGE on the last ephemeral detach — a
3508        // host-shared container shares the node's single cluster utun and owns
3509        // no per-service bridge or dedicated WG device to tear down (see the
3510        // HONEST CONSTRAINT note on `attach_container_host_shared`). The only
3511        // per-segment state is its overlay `/32` + per-network membership, both
3512        // already reversed above. So `ephemeral` and `service_name` drive the
3513        // last-leaver TRACE here (mirroring the Linux ephemeral-teardown log)
3514        // rather than a bridge teardown: an ephemeral (per-job) segment's IP
3515        // return is logged at info level for reclamation traceability, a
3516        // managed service's at debug.
3517        let service = info.service_name.as_deref().unwrap_or("<none>");
3518        if info.ephemeral {
3519            tracing::info!(
3520                container = %id,
3521                service = %service,
3522                ip = %info.service_ip,
3523                "ephemeral host-shared overlay member detached — per-job segment /32 returned to node slice"
3524            );
3525        } else {
3526            tracing::debug!(
3527                container = %id,
3528                service = %service,
3529                ip = %info.service_ip,
3530                "host-shared overlay member detached — /32 returned to node slice"
3531            );
3532        }
3533        Ok(())
3534    }
3535
3536    /// Release a guest overlay IP back to the pool it was drawn from: the named
3537    /// service bridge's allocator (Linux) when `service` is set and the bridge
3538    /// still exists, otherwise the node slice allocator.
3539    fn release_guest_ip(&mut self, ip: IpAddr, service: Option<&str>) {
3540        #[cfg(target_os = "linux")]
3541        {
3542            // A Shared-mode service drew from the single node-wide shared bridge,
3543            // which is keyed by subnet, not by service name. Try it first.
3544            if let Some(bridge) = self.shared_bridge.as_mut() {
3545                if bridge.subnet.contains(&ip) {
3546                    bridge.ip_allocator.release(ip);
3547                    return;
3548                }
3549            }
3550            if let Some(svc) = service {
3551                if let Some(bridge) = self.service_bridges.get_mut(svc) {
3552                    bridge.ip_allocator.release(ip);
3553                    return;
3554                }
3555            }
3556        }
3557        #[cfg(not(target_os = "linux"))]
3558        {
3559            // A Dedicated-mode guest drew its IP from the per-service transport's
3560            // allocator (keyed by service name); return it there so the dedicated
3561            // subnet does not leak addresses across guest churn.
3562            if let Some(svc) = service {
3563                if let Some(st) = self.service_transports.get_mut(svc) {
3564                    st.ip_allocator.release(ip);
3565                    return;
3566                }
3567            }
3568        }
3569        let _ = service;
3570        self.ip_allocator.release(ip);
3571    }
3572
3573    /// Prefix length of the address pool guest IPs are drawn from when not using
3574    /// a per-service bridge: the node slice if assigned, else the cluster CIDR.
3575    fn slice_prefix_len(&self) -> u8 {
3576        self.slice_cidr.or(self.cluster_cidr).map_or(
3577            if self.node_ip.is_some_and(|ip| ip.is_ipv6()) {
3578                64
3579            } else {
3580                24
3581            },
3582            |c| c.prefix(),
3583        )
3584    }
3585
3586    /// Reachable `WireGuard` endpoint for THIS node, advertised to a guest as a
3587    /// peer on `listen_port` (the node's global overlay port, or a Dedicated
3588    /// service's per-service device port). overlayd has no public reflexive
3589    /// address at this layer, so it uses the node's overlay-listen identity
3590    /// (`node_ip:listen_port`); the caller (the VZ runtime that ships the config
3591    /// into the guest) rewrites it to the concrete VZ-NAT gateway endpoint the
3592    /// guest can dial. Falls back to the unspecified address when no node IP is
3593    /// assigned yet.
3594    fn node_endpoint_for_guest(&self, listen_port: u16) -> String {
3595        let ip = self.node_ip.unwrap_or(IpAddr::V4(Ipv4Addr::UNSPECIFIED));
3596        SocketAddr::new(ip, listen_port).to_string()
3597    }
3598
3599    /// Linux veth/netns attach. On non-Linux this returns the node's overlay IP
3600    /// (host networking) and is never wired for a `LinuxPid` handle in practice.
3601    #[cfg(target_os = "linux")]
3602    #[allow(clippy::too_many_lines)]
3603    async fn attach_container_linux(
3604        &mut self,
3605        container_pid: u32,
3606        service: &str,
3607        join_global: bool,
3608        ephemeral: bool,
3609        isolation_network: Option<String>,
3610    ) -> Result<IpAddr, OverlaydError> {
3611        // Resolve which bridge backs this service. A `Shared`-mode service
3612        // attaches onto the SINGLE node-wide shared bridge; every other mode
3613        // (`Auto`, `Dedicated`) attaches onto its own per-service bridge. The
3614        // mode was recorded at `setup_service_overlay` time.
3615        let use_shared = self
3616            .service_modes
3617            .get(service)
3618            .copied()
3619            .unwrap_or_default()
3620            .uses_shared_bridge();
3621
3622        let (bridge_name, bridge_subnet, bridge_gateway, container_ip) = if use_shared {
3623            let bridge = self.shared_bridge.as_mut().ok_or_else(|| {
3624                OverlaydError::Other(format!(
3625                    "no shared bridge for Shared-mode service {service}; call setup_service_overlay() first"
3626                ))
3627            })?;
3628            let ip = bridge.ip_allocator.allocate().ok_or_else(|| {
3629                OverlaydError::Overlay(format!(
3630                    "shared bridge {} subnet {} exhausted",
3631                    bridge.name, bridge.subnet
3632                ))
3633            })?;
3634            (bridge.name.clone(), bridge.subnet, bridge.gateway, ip)
3635        } else {
3636            let bridge = self.service_bridges.get_mut(service).ok_or_else(|| {
3637                OverlaydError::Other(format!(
3638                    "no service bridge for service {service}; call setup_service_overlay() first"
3639                ))
3640            })?;
3641            let ip = bridge.ip_allocator.allocate().ok_or_else(|| {
3642                OverlaydError::Overlay(format!(
3643                    "service bridge {} subnet {} exhausted",
3644                    bridge.name, bridge.subnet
3645                ))
3646            })?;
3647            (bridge.name.clone(), bridge.subnet, bridge.gateway, ip)
3648        };
3649
3650        let bridge_params = BridgeAttachParams {
3651            bridge_name: &bridge_name,
3652            gateway: bridge_gateway,
3653            subnet_prefix_len: bridge_subnet.prefix_len(),
3654        };
3655        if let Err(e) = self
3656            .attach_to_interface(
3657                container_pid,
3658                container_ip,
3659                "s",
3660                "eth0",
3661                Some(&bridge_params),
3662            )
3663            .await
3664        {
3665            if use_shared {
3666                if let Some(bridge) = self.shared_bridge.as_mut() {
3667                    bridge.ip_allocator.release(container_ip);
3668                }
3669            } else if let Some(bridge) = self.service_bridges.get_mut(service) {
3670                bridge.ip_allocator.release(container_ip);
3671            }
3672            return Err(e);
3673        }
3674
3675        let mut global_ip: Option<IpAddr> = None;
3676        if join_global && self.global_interface.is_some() {
3677            let g_ip = self.ip_allocator.allocate()?;
3678            self.attach_to_interface(container_pid, g_ip, "g", "eth1", None)
3679                .await?;
3680            global_ip = Some(g_ip);
3681        }
3682
3683        // Per-network L3 isolation: when this attach joins a named isolated
3684        // network, install the Docker-style iptables rules pinning this member
3685        // to its own network's members + node + egress, then record it in the
3686        // membership map. Non-fatal: a host without iptables logs and proceeds.
3687        if let Some(ref net) = isolation_network {
3688            let node_ip = self
3689                .node_ip
3690                .unwrap_or(std::net::IpAddr::V4(std::net::Ipv4Addr::new(10, 200, 0, 1)));
3691            let cidr = self
3692                .cluster_cidr
3693                .map_or_else(|| "10.200.0.0/16".to_string(), |c| c.to_string());
3694            let peers: Vec<IpAddr> = self
3695                .network_members
3696                .get(net)
3697                .map(|m| m.iter().copied().collect())
3698                .unwrap_or_default();
3699            if let Err(e) = zlayer_overlay::firewall::ensure_member_isolation(
3700                net,
3701                container_ip,
3702                &peers,
3703                node_ip,
3704                &cidr,
3705            ) {
3706                tracing::warn!(network = %net, member = %container_ip, error = %e, "failed to install per-network L3 isolation (non-fatal)");
3707            }
3708            self.network_members
3709                .entry(net.clone())
3710                .or_default()
3711                .insert(container_ip);
3712        }
3713
3714        self.attached.insert(
3715            container_pid,
3716            AttachInfo {
3717                service_ip: container_ip,
3718                service_name: Some(service.to_string()),
3719                global_ip,
3720                ephemeral,
3721                isolation_network,
3722            },
3723        );
3724
3725        Ok(container_ip)
3726    }
3727
3728    /// Non-Linux fallback: containers share the host network, so return the
3729    /// node's overlay IP (or loopback).
3730    #[cfg(not(target_os = "linux"))]
3731    #[allow(clippy::unused_async)]
3732    async fn attach_container_linux(
3733        &mut self,
3734        _container_pid: u32,
3735        service: &str,
3736        _join_global: bool,
3737        _ephemeral: bool,
3738        _isolation_network: Option<String>,
3739    ) -> Result<IpAddr, OverlaydError> {
3740        tracing::debug!(service = %service, "LinuxPid attach is a no-op off Linux; using node overlay IP");
3741        Ok(self.node_ip.unwrap_or(IpAddr::V4(Ipv4Addr::LOCALHOST)))
3742    }
3743
3744    /// Release the overlay resources held by a Linux container PID. Idempotent.
3745    #[cfg(target_os = "linux")]
3746    async fn detach_container_linux(&mut self, pid: u32) -> Result<(), OverlaydError> {
3747        // "Process id or not, kill the adapter": the host-side veth name is
3748        // deterministic (`veth-<pid>-{s,g}`), so delete it UNCONDITIONALLY by
3749        // name — even when no attach record survives (a previous daemon crashed
3750        // before recording it, or it was already reaped). Without this, a missing
3751        // record left the host veth orphaned until the PID-keyed periodic sweep
3752        // (which only fires once the PID is dead). The deletes are idempotent
3753        // (ENODEV = success), so the always-on `-g` delete is harmless when the
3754        // container never joined the global overlay.
3755        let info = self.attached.remove(&pid);
3756
3757        let veth_s = format!("veth-{pid}-s");
3758        if let Err(e) = crate::netlink::delete_link_by_name(&veth_s).await {
3759            tracing::warn!(link = %veth_s, pid, error = %e, "Failed to delete service veth");
3760        }
3761        let veth_g = format!("veth-{pid}-g");
3762        if let Err(e) = crate::netlink::delete_link_by_name(&veth_g).await {
3763            tracing::warn!(link = %veth_g, pid, error = %e, "Failed to delete global veth");
3764        }
3765
3766        // No attach record -> nothing more to release (IP/registry bookkeeping
3767        // is keyed off the record). The veths above are already gone.
3768        let Some(info) = info else {
3769            return Ok(());
3770        };
3771
3772        // Return the service IP to whichever pool owns it. A Shared-mode service
3773        // drew its IP from the single node-wide shared bridge (no per-service
3774        // bridge exists for it), so try the shared bridge by subnet containment
3775        // before the named per-service bridge.
3776        if self.shared_bridge.as_mut().is_some_and(|b| {
3777            b.subnet.contains(&info.service_ip) && b.ip_allocator.release(info.service_ip)
3778        }) {
3779            // released into the shared bridge
3780        } else if let Some(svc) = info.service_name.as_deref() {
3781            if let Some(bridge) = self.service_bridges.get_mut(svc) {
3782                bridge.ip_allocator.release(info.service_ip);
3783            } else {
3784                tracing::debug!(service = %svc, ip = %info.service_ip, "detach: service bridge already torn down; dropping service IP release");
3785            }
3786        } else {
3787            self.ip_allocator.release(info.service_ip);
3788        }
3789        if let Some(g) = info.global_ip {
3790            self.ip_allocator.release(g);
3791        }
3792
3793        // Per-network L3 isolation drain: remove this member from its isolated
3794        // network's membership set and tear down its iptables rules against the
3795        // remaining members. Drop the network entry once empty.
3796        if let Some(net) = info.isolation_network.as_deref() {
3797            if let Some(set) = self.network_members.get_mut(net) {
3798                set.remove(&info.service_ip);
3799            }
3800            let still: Vec<IpAddr> = self
3801                .network_members
3802                .get(net)
3803                .map(|m| m.iter().copied().collect())
3804                .unwrap_or_default();
3805            let node_ip = self
3806                .node_ip
3807                .unwrap_or(std::net::IpAddr::V4(std::net::Ipv4Addr::new(10, 200, 0, 1)));
3808            let cidr = self
3809                .cluster_cidr
3810                .map_or_else(|| "10.200.0.0/16".to_string(), |c| c.to_string());
3811            zlayer_overlay::firewall::remove_member_isolation(
3812                net,
3813                info.service_ip,
3814                &still,
3815                node_ip,
3816                &cidr,
3817            );
3818            if self
3819                .network_members
3820                .get(net)
3821                .is_some_and(std::collections::HashSet::is_empty)
3822            {
3823                self.network_members.remove(net);
3824            }
3825        }
3826
3827        // Ephemeral last-leaver teardown: a standalone/per-job bridge is reclaimed
3828        // the moment its LAST container leaves (the periodic prune is only the
3829        // ~300s backstop). Managed attaches use ephemeral=false so their bridge
3830        // persists across scale-to-0. Route through teardown_service_overlay so
3831        // overlayd's in-memory state stays synced — never a hand `ip link del`.
3832        // This container's veth is already removed above, so a 0 member count
3833        // means no containers remain on the bridge.
3834        if info.ephemeral {
3835            if let Some(svc) = info.service_name.clone() {
3836                if let Some(bridge_name) = self.service_bridges.get(&svc).map(|b| b.name.clone()) {
3837                    if crate::netlink::bridge_member_count(&bridge_name).await == 0 {
3838                        tracing::info!(service = %svc, bridge = %bridge_name, "ephemeral overlay bridge idle after last detach — tearing down");
3839                        self.teardown_service_overlay(&svc).await;
3840                    }
3841                }
3842            }
3843        }
3844        Ok(())
3845    }
3846
3847    /// Non-Linux fallback: nothing to detach (host networking).
3848    #[cfg(not(target_os = "linux"))]
3849    #[allow(clippy::unused_async)]
3850    async fn detach_container_linux(&mut self, _pid: u32) -> Result<(), OverlaydError> {
3851        Ok(())
3852    }
3853
3854    /// Best-effort sweep of orphan veth endpoints whose owning container PID is
3855    /// no longer alive. Names matching `veth-<pid>-*` / `vc-<pid>-*` where
3856    /// `/proc/<pid>` does not exist are deleted.
3857    #[cfg(target_os = "linux")]
3858    async fn sweep_orphan_veths() {
3859        let links = match crate::netlink::list_all_links().await {
3860            Ok(links) => links,
3861            Err(e) => {
3862                tracing::warn!(error = %e, "Failed to list links for orphan sweep");
3863                return;
3864            }
3865        };
3866        for (_index, name) in links {
3867            let remainder = if let Some(r) = name.strip_prefix("veth-") {
3868                r
3869            } else if let Some(r) = name.strip_prefix("vc-") {
3870                r
3871            } else {
3872                continue;
3873            };
3874            let Some(pid_str) = remainder.split('-').next() else {
3875                continue;
3876            };
3877            let pid: u32 = match pid_str.parse() {
3878                Ok(p) => p,
3879                Err(_) => continue,
3880            };
3881            if Path::new(&format!("/proc/{pid}")).exists() {
3882                continue;
3883            }
3884            tracing::info!(link = %name, pid = pid, "Deleting orphan veth");
3885            if let Err(e) = crate::netlink::delete_link_by_name(&name).await {
3886                tracing::warn!(link = %name, error = %e, "Failed to delete orphan veth");
3887            }
3888        }
3889    }
3890
3891    #[cfg(target_os = "linux")]
3892    #[allow(clippy::too_many_lines)]
3893    async fn attach_to_interface(
3894        &mut self,
3895        container_pid: u32,
3896        ip: IpAddr,
3897        tag: &str,
3898        container_iface: &str,
3899        bridge: Option<&BridgeAttachParams<'_>>,
3900    ) -> Result<(), OverlaydError> {
3901        // Best-effort cleanup of orphan veths left by a previous daemon crash.
3902        Self::sweep_orphan_veths().await;
3903
3904        let is_v6 = ip.is_ipv6();
3905        let prefix_len: u8 = if let Some(b) = bridge {
3906            b.subnet_prefix_len
3907        } else if is_v6 {
3908            64
3909        } else {
3910            24
3911        };
3912        let host_prefix: u8 = if is_v6 { 128 } else { 32 };
3913
3914        let veth_host = format!("veth-{container_pid}-{tag}");
3915        let veth_pending = format!("vc-{container_pid}-{tag}");
3916        let veth_container = container_iface.to_string();
3917
3918        let container_ns_fd = std::os::fd::OwnedFd::from(
3919            std::fs::File::open(format!("/proc/{container_pid}/ns/net")).map_err(|e| {
3920                OverlaydError::Overlay(format!("Failed to open /proc/{container_pid}/ns/net: {e}"))
3921            })?,
3922        );
3923
3924        crate::netlink::delete_link_by_name(&veth_host)
3925            .await
3926            .map_err(|e| OverlaydError::Overlay(format!("pre-cleanup delete {veth_host}: {e}")))?;
3927        crate::netlink::delete_link_by_name(&veth_pending)
3928            .await
3929            .map_err(|e| {
3930                OverlaydError::Overlay(format!("pre-cleanup delete {veth_pending}: {e}"))
3931            })?;
3932
3933        let bridge_gateway: Option<IpAddr> = bridge.map(|b| b.gateway);
3934        let bridge_name: Option<String> = bridge.map(|b| b.bridge_name.to_string());
3935        let node_ip = self.node_ip;
3936
3937        let result: Result<(), OverlaydError> = async {
3938            crate::netlink::create_veth_pair(&veth_host, &veth_pending)
3939                .await
3940                .map_err(|e| OverlaydError::Overlay(format!("create veth pair: {e}")))?;
3941
3942            crate::netlink::move_link_into_netns_fd_and_rename(
3943                &veth_pending,
3944                AsFd::as_fd(&container_ns_fd),
3945                &veth_container,
3946            )
3947            .map_err(|e| OverlaydError::Overlay(format!("move veth into netns: {e}")))?;
3948
3949            let vc = veth_container.clone();
3950            let bridge_gateway_for_netns = bridge_gateway;
3951            tokio::task::spawn_blocking(move || {
3952                crate::netlink::with_netns_fd_async(container_ns_fd, move || async move {
3953                    crate::netlink::add_address_to_link_by_name(&vc, ip, prefix_len).await?;
3954                    crate::netlink::set_link_up_by_name(&vc).await?;
3955                    crate::netlink::set_link_up_by_name("lo").await?;
3956                    if let Some(gw) = bridge_gateway_for_netns {
3957                        crate::netlink::add_default_route_via_gateway(gw).await?;
3958                    }
3959                    Ok(())
3960                })
3961            })
3962            .await
3963            .map_err(|e| OverlaydError::Overlay(format!("container netns task panicked: {e}")))?
3964            .map_err(|e| OverlaydError::Overlay(format!("container netns ops: {e}")))?;
3965
3966            crate::netlink::set_link_up_by_name(&veth_host)
3967                .await
3968                .map_err(|e| OverlaydError::Overlay(format!("set {veth_host} up: {e}")))?;
3969
3970            if let Some(bname) = bridge_name.as_deref() {
3971                crate::netlink::add_link_to_bridge(&veth_host, bname)
3972                    .await
3973                    .map_err(|e| {
3974                        OverlaydError::Overlay(format!(
3975                            "enslave {veth_host} to bridge {bname}: {e}"
3976                        ))
3977                    })?;
3978            } else {
3979                crate::netlink::replace_route_via_dev(ip, host_prefix, &veth_host, node_ip)
3980                    .await
3981                    .map_err(|e| {
3982                        OverlaydError::Overlay(format!("host route for {ip}/{host_prefix}: {e}"))
3983                    })?;
3984            }
3985
3986            Ok(())
3987        }
3988        .await;
3989
3990        // Enable IP forwarding so the host routes between the overlay device(s)
3991        // and the egress NIC. CRITICAL: this is scoped to the address family
3992        // actually in use and (for IPv6) to the specific overlay devices —
3993        // NEVER `net.ipv6.conf.all.forwarding`, whose documented kernel side
3994        // effect is to force `accept_ra=0` + `autoconf=0` on every IPv6
3995        // interface (including the public NIC), dropping the RA-learned default
3996        // route / path-MTU and blackholing the host's own larger reply packets
3997        // (e.g. inbound SSH stalls after key exchange). Done outside the
3998        // attach `result` block so a forwarding-sysctl failure can never roll
3999        // back a successful veth attach. Tracked so teardown reverts it.
4000        if result.is_ok() {
4001            self.enable_forwarding_for_attach(is_v6, &veth_host, bridge_name.as_deref());
4002
4003            // Track the host-side resources this attach created so a clean
4004            // global teardown reverts every host mutation. The host-side veth
4005            // half exists in both the bridged and bridgeless paths; the host
4006            // `/32`(`/128`) route is installed ONLY on the bridgeless path
4007            // (`replace_route_via_dev` above), so record it only when there was
4008            // no bridge to enslave into. All deletions are idempotent, so a
4009            // resource a later per-container detach removes first is harmless.
4010            self.created_veths.insert(veth_host.clone());
4011            if bridge_name.is_none() {
4012                self.created_host_routes
4013                    .push((ip, host_prefix, veth_host.clone()));
4014            }
4015        }
4016
4017        if result.is_err() {
4018            let _ = crate::netlink::delete_link_by_name(&veth_host).await;
4019            let _ = crate::netlink::delete_link_by_name(&veth_pending).await;
4020        }
4021        result
4022    }
4023
4024    // -- container attach (Windows HCN) -------------------------------------
4025
4026    /// Windows attach: ensure the overlay HCN Internal network exists, allocate
4027    /// or validate the IP, create the per-container HCN endpoint + namespace,
4028    /// and return the bare-lowercase namespace GUID for the agent to embed in
4029    /// the compute-system document.
4030    ///
4031    /// # Errors
4032    /// Returns an error if the network/endpoint cannot be created or the slice
4033    /// is exhausted.
4034    #[cfg(target_os = "windows")]
4035    #[allow(clippy::too_many_lines)]
4036    async fn attach_container_windows(
4037        &mut self,
4038        container_id: &str,
4039        service: &str,
4040        ip_override: Option<IpAddr>,
4041        dns_server: Option<IpAddr>,
4042        dns_domain: Option<String>,
4043        isolation_network: Option<String>,
4044    ) -> Result<AttachResult, OverlaydError> {
4045        // Resolve whether THIS service has a dedicated per-service overlay. It
4046        // does iff a live dedicated transport exists OR a `hcn-internal` marker
4047        // entry is recorded under `owner_for_service(service)` (the network
4048        // survives daemon restarts even if the transport map is empty mid-init).
4049        // Dedicated services attach onto their OWN per-service Internal network
4050        // and draw IPs from the service subnet; everything else uses the node's
4051        // base/shared overlay network and the node slice.
4052        let dedicated_subnet = self.dedicated_service_subnet(service);
4053        // A `Shared`-mode service attaches onto the SINGLE shared HCN NAT network
4054        // reused across all Shared services (container ports are exposed via the
4055        // userspace free-port L4 proxy). The mode was recorded at setup time.
4056        let use_shared_nat = self
4057            .service_modes
4058            .get(service)
4059            .copied()
4060            .unwrap_or_default()
4061            .uses_shared_bridge();
4062
4063        let (net_id, ip, prefix_length) = if let Some(net) = isolation_network.as_deref() {
4064            // ----- per-isolation-network Internal HCN network path -----
4065            //
4066            // An "isolated" ZLayer network routes its members onto a dedicated
4067            // HCN Internal vSwitch keyed by the isolation-network NAME (not the
4068            // service). HCN Internal vSwitches are mutually isolated by default,
4069            // so same-network members share one vSwitch (reach each other +
4070            // egress via the network gateway + the node), while different
4071            // isolation networks land on separate vSwitches and cannot reach
4072            // each other — L3 isolation with NO ACLs and NO per-member churn.
4073            // This mirrors the Dedicated per-service branch below, but keyed by
4074            // the isolation-network name and drawing IPs from a per-network
4075            // subnet carved deterministically from the node slice.
4076            let iso_subnet = self.isolation_network_subnet(net)?;
4077            let net_id = self.ensure_isolation_network(net, iso_subnet).await?;
4078
4079            // Per-network container IPs come from the isolation network's own
4080            // subnet (never the node slice), via a lazily-created allocator
4081            // bounded to that subnet. The allocator is keyed by the isolation
4082            // network's owner key so it never collides with a same-named
4083            // dedicated service's allocator. An `ip_override` is honored only
4084            // when it falls inside the isolation subnet.
4085            let iso_ipnetwork: IpNetwork = iso_subnet.to_string().parse().map_err(|e| {
4086                OverlaydError::Other(format!(
4087                    "failed to parse isolation subnet {iso_subnet}: {e}"
4088                ))
4089            })?;
4090            let alloc_key = crate::network_state::owner_for_isolation_network(net);
4091            let allocator = self
4092                .service_ip_allocators
4093                .entry(alloc_key)
4094                .or_insert_with(|| IpAllocator::new(iso_ipnetwork));
4095            let ip = match ip_override {
4096                Some(ip) if iso_subnet.contains(&ip) => ip,
4097                Some(ip) => {
4098                    return Err(OverlaydError::Other(format!(
4099                        "overridden IP {ip} is not inside isolation network subnet {iso_subnet} for network {net}"
4100                    )));
4101                }
4102                None => allocator.allocate()?,
4103            };
4104            (net_id, ip, iso_subnet.prefix_len())
4105        } else if use_shared_nat {
4106            // ----- shared HCN NAT network path -----
4107            let slice = self.slice_cidr.ok_or_else(|| {
4108                OverlaydError::Other(
4109                    "no node slice assigned yet (SetupGlobalOverlay with slice_cidr first)"
4110                        .to_string(),
4111                )
4112            })?;
4113            let slice_ipnet: ipnet::IpNet = slice.to_string().parse().map_err(|e| {
4114                OverlaydError::Other(format!("failed to parse slice CIDR {slice}: {e}"))
4115            })?;
4116            let net_id = self.ensure_shared_nat_network(slice_ipnet).await?;
4117            let ip = match ip_override {
4118                Some(ip) => ip,
4119                None => self.ip_allocator.allocate()?,
4120            };
4121            (net_id, ip, slice_ipnet.prefix_len())
4122        } else if let Some(svc_subnet) = dedicated_subnet {
4123            // ----- dedicated per-service network path -----
4124            let net_id = self.ensure_service_network(service, svc_subnet).await?;
4125
4126            // Allocate (or validate) the IP from the SERVICE subnet, not the
4127            // node slice. A per-service allocator is created lazily and bounded
4128            // to the service subnet so addresses stay inside the dedicated
4129            // network. An `ip_override` inside the service subnet is honored;
4130            // one outside it is rejected so a slice-allocated IP can't leak onto
4131            // the dedicated network.
4132            let svc_ipnetwork: IpNetwork = svc_subnet.to_string().parse().map_err(|e| {
4133                OverlaydError::Other(format!("failed to parse service subnet {svc_subnet}: {e}"))
4134            })?;
4135            let allocator = self
4136                .service_ip_allocators
4137                .entry(service.to_string())
4138                .or_insert_with(|| IpAllocator::new(svc_ipnetwork));
4139            let ip = match ip_override {
4140                Some(ip) if svc_subnet.contains(&ip) => ip,
4141                Some(ip) => {
4142                    return Err(OverlaydError::Other(format!(
4143                        "overridden IP {ip} is not inside dedicated service subnet {svc_subnet} for service {service}"
4144                    )));
4145                }
4146                None => allocator.allocate()?,
4147            };
4148            (net_id, ip, svc_subnet.prefix_len())
4149        } else {
4150            // ----- shared base overlay network path (unchanged) -----
4151            let slice = self.slice_cidr.ok_or_else(|| {
4152                OverlaydError::Other(
4153                    "no node slice assigned yet (SetupGlobalOverlay with slice_cidr first)"
4154                        .to_string(),
4155                )
4156            })?;
4157            let slice_ipnet: ipnet::IpNet = slice.to_string().parse().map_err(|e| {
4158                OverlaydError::Other(format!("failed to parse slice CIDR {slice}: {e}"))
4159            })?;
4160            let net_id = self.ensure_overlay_network(slice_ipnet).await?;
4161            let ip = match ip_override {
4162                Some(ip) => ip,
4163                None => self.ip_allocator.allocate()?,
4164            };
4165            (net_id, ip, slice_ipnet.prefix_len())
4166        };
4167
4168        // 3. Create the endpoint + per-container namespace on the network.
4169        let dns_server_eff = dns_server.or_else(|| self.dns_server_addr.map(|a| a.ip()));
4170        let dns_domain_for_attach = dns_domain.or_else(|| self.dns_domain.clone());
4171        let cluster_cidr = self.cluster_cidr.map(|c| c.to_string()).unwrap_or_default();
4172        let owner_tag = owner_tag(&self.deployment_or_default());
4173        let cid = container_id.to_string();
4174
4175        let attachment = tokio::task::spawn_blocking(move || {
4176            zlayer_hns::attach::EndpointAttachment::create_overlay(
4177                net_id,
4178                &owner_tag,
4179                cid.as_str(),
4180                ip,
4181                prefix_length,
4182                &cluster_cidr,
4183                dns_server_eff,
4184                dns_domain_for_attach.as_deref(),
4185            )
4186        })
4187        .await
4188        .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?
4189        .map_err(|e| OverlaydError::Overlay(format!("HCN overlay endpoint attach failed: {e}")))?;
4190
4191        let namespace_id = attachment.namespace_id();
4192        let bare_guid = format_guid_bare(namespace_id);
4193
4194        // Per-network membership: record the container's IP in its isolated
4195        // network's member set. Windows enforcement is an HCN ACL — a
4196        // Linux-incompatible mechanism wired separately; overlayd only maintains
4197        // the membership map here and does NOT call the iptables firewall helper.
4198        if let Some(ref net) = isolation_network {
4199            self.network_members
4200                .entry(net.clone())
4201                .or_default()
4202                .insert(ip);
4203        }
4204
4205        // Record for autoclean keyed by namespace GUID.
4206        self.hcn_cleanup
4207            .insert(namespace_id, (service.to_string(), ip, isolation_network));
4208
4209        tracing::info!(
4210            ns = %bare_guid,
4211            service = %service,
4212            ip = %ip,
4213            "Attached container to HCN overlay"
4214        );
4215
4216        Ok(AttachResult {
4217            ip,
4218            namespace_guid: Some(bare_guid),
4219        })
4220    }
4221
4222    /// Non-Windows path: a `WindowsContainer` handle has no meaning off Windows.
4223    #[cfg(not(target_os = "windows"))]
4224    #[allow(clippy::unused_async)]
4225    async fn attach_container_windows(
4226        &mut self,
4227        _container_id: &str,
4228        _service: &str,
4229        _ip_override: Option<IpAddr>,
4230        _dns_server: Option<IpAddr>,
4231        _dns_domain: Option<String>,
4232        _isolation_network: Option<String>,
4233    ) -> Result<AttachResult, OverlaydError> {
4234        Err(OverlaydError::Other(
4235            "WindowsContainer attach is only supported on Windows".to_string(),
4236        ))
4237    }
4238
4239    /// Detach a Windows container by its bare namespace GUID and release its IP.
4240    /// Idempotent: unknown ids are a no-op.
4241    #[cfg(target_os = "windows")]
4242    async fn detach_container_windows(
4243        &mut self,
4244        namespace_guid: &str,
4245    ) -> Result<(), OverlaydError> {
4246        use windows::core::GUID;
4247
4248        let Ok(guid) = GUID::try_from(namespace_guid) else {
4249            tracing::warn!(ns = %namespace_guid, "detach: unparseable namespace GUID");
4250            return Ok(());
4251        };
4252        if let Some((service, ip, isolation_network)) = self.hcn_cleanup.remove(&guid) {
4253            // Release the IP into the pool it was drawn from. An isolation-network
4254            // member drew from the per-network allocator (keyed by the isolation
4255            // owner key), NOT the node slice; release it there so the isolation
4256            // subnet doesn't leak addresses. Everything else came from the node
4257            // slice.
4258            if let Some(net) = isolation_network.as_deref() {
4259                let alloc_key = crate::network_state::owner_for_isolation_network(net);
4260                if let Some(allocator) = self.service_ip_allocators.get_mut(&alloc_key) {
4261                    allocator.release(ip);
4262                } else {
4263                    self.ip_allocator.release(ip);
4264                }
4265            } else {
4266                self.ip_allocator.release(ip);
4267            }
4268            // Drain the per-network membership set.
4269            let mut net_now_empty: Option<String> = None;
4270            if let Some(net) = isolation_network.as_deref() {
4271                if let Some(set) = self.network_members.get_mut(net) {
4272                    set.remove(&ip);
4273                }
4274                if self
4275                    .network_members
4276                    .get(net)
4277                    .is_some_and(std::collections::HashSet::is_empty)
4278                {
4279                    self.network_members.remove(net);
4280                    net_now_empty = Some(net.to_string());
4281                }
4282            }
4283            tracing::info!(ns = %namespace_guid, service = %service, ip = %ip, "Released HCN overlay attachment");
4284
4285            // Last-member teardown: when the final member of an isolation network
4286            // leaves, reclaim its per-network HCN Internal network (mirroring the
4287            // per-service network teardown in `teardown_service_overlay`) so we
4288            // don't leak an HCN vSwitch until the next full uninstall. Drop the
4289            // per-network IP allocator and the marker entry too.
4290            if let Some(net) = net_now_empty {
4291                self.teardown_isolation_network(&net).await;
4292            }
4293        }
4294        Ok(())
4295    }
4296
4297    /// Reclaim the per-isolation-network HCN Internal network for `net`: delete
4298    /// the HCN network by the GUID recorded in the marker, drop its marker entry,
4299    /// and discard the per-network IP allocator. Best-effort and idempotent —
4300    /// called once the last member of the isolation network detaches. Mirrors the
4301    /// per-service network teardown in [`Self::teardown_service_overlay`].
4302    #[cfg(target_os = "windows")]
4303    async fn teardown_isolation_network(&mut self, net: &str) {
4304        let owner = crate::network_state::owner_for_isolation_network(net);
4305
4306        // Drop the per-network container-IP allocator.
4307        self.service_ip_allocators.remove(&owner);
4308
4309        let marker_path =
4310            zlayer_paths::ZLayerDirs::new(self.data_dir.clone()).agent_network_state();
4311        let mut marker = crate::network_state::NetworkState::load(&marker_path);
4312        let removed_entry = marker.remove(&owner);
4313        if removed_entry.is_some() {
4314            if let Err(e) = marker.save(&marker_path) {
4315                tracing::warn!(network = %net, error = %e, path = %marker_path.display(), "failed to persist isolation-network marker removal");
4316            }
4317        }
4318
4319        if let Some(entry) = removed_entry {
4320            if entry.kind == "hcn-internal" {
4321                match windows::core::GUID::try_from(entry.id.as_str()) {
4322                    Ok(guid) => {
4323                        let id_str = entry.id.clone();
4324                        let net_owned = net.to_string();
4325                        let delete = tokio::task::spawn_blocking(move || {
4326                            zlayer_hns::network::Network::delete(guid)
4327                        })
4328                        .await;
4329                        match delete {
4330                            Ok(Ok(())) => {
4331                                tracing::info!(network = %net_owned, id = %id_str, "deleted per-isolation-network HCN network on last detach");
4332                            }
4333                            Ok(Err(e)) => {
4334                                tracing::warn!(network = %net_owned, id = %id_str, error = %e, "failed to delete isolation-network HCN network (may leak until uninstall)");
4335                            }
4336                            Err(e) => {
4337                                tracing::warn!(network = %net_owned, id = %id_str, error = %e, "spawn_blocking join failed deleting isolation-network HCN network");
4338                            }
4339                        }
4340                    }
4341                    Err(_) => {
4342                        tracing::warn!(network = %net, id = %entry.id, "isolation-network marker has unparseable HCN GUID; skipping network delete");
4343                    }
4344                }
4345            }
4346        }
4347    }
4348
4349    /// Non-Windows path.
4350    #[cfg(not(target_os = "windows"))]
4351    #[allow(clippy::unused_async)]
4352    async fn detach_container_windows(
4353        &mut self,
4354        _namespace_guid: &str,
4355    ) -> Result<(), OverlaydError> {
4356        Ok(())
4357    }
4358
4359    /// Ensure the per-daemon HCN overlay (Internal vSwitch, no physical-NIC
4360    /// binding) exists on the host, reusing one recorded in the
4361    /// `{data_dir}/agent_network.json` marker or discoverable by name, and
4362    /// recording it in the marker on create.
4363    ///
4364    /// # Errors
4365    /// Propagates the underlying `zlayer_hns` error on create failure.
4366    #[cfg(target_os = "windows")]
4367    #[allow(clippy::too_many_lines)]
4368    async fn ensure_overlay_network(
4369        &mut self,
4370        slice_cidr: ipnet::IpNet,
4371    ) -> Result<windows::core::GUID, OverlaydError> {
4372        use windows::core::GUID;
4373
4374        let daemon_name = self.deployment_or_default();
4375        let net_name = overlay_network_name(&daemon_name);
4376        let marker_path =
4377            zlayer_paths::ZLayerDirs::new(self.data_dir.clone()).agent_network_state();
4378
4379        // Fast path: marker names a network GUID that still exists; reopen it.
4380        if let Some(recorded_id) = crate::network_state::NetworkState::load(&marker_path)
4381            .get(crate::network_state::OWNER_BASE)
4382            .and_then(|entry| GUID::try_from(entry.id.as_str()).ok())
4383        {
4384            let reopened = tokio::task::spawn_blocking(move || {
4385                zlayer_hns::network::Network::open(recorded_id).ok()
4386            })
4387            .await
4388            .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?;
4389            if reopened.is_some() {
4390                tracing::info!(name = %net_name, "reusing HCN overlay network from marker");
4391                return Ok(recorded_id);
4392            }
4393        }
4394
4395        // Idempotency: reuse a host network whose queried name matches ours.
4396        let target_name = net_name.clone();
4397        let existing = tokio::task::spawn_blocking(move || -> Option<GUID> {
4398            let guids = zlayer_hns::network::list("{}").ok()?;
4399            for guid in guids {
4400                let Ok(network) = zlayer_hns::network::Network::open(guid) else {
4401                    continue;
4402                };
4403                if matches!(network.query("{}"), Ok(props) if props.name == target_name) {
4404                    return Some(guid);
4405                }
4406            }
4407            None
4408        })
4409        .await
4410        .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?;
4411
4412        if let Some(existing_id) = existing {
4413            tracing::info!(name = %net_name, "reusing existing HCN overlay network");
4414            return Ok(existing_id);
4415        }
4416
4417        let net_id = GUID::new()
4418            .map_err(|e| OverlaydError::Other(format!("GUID::new for overlay network: {e}")))?;
4419        let subnet_str = slice_cidr.to_string();
4420
4421        // Default: an HCN Internal network — an internal vSwitch with NO
4422        // physical-NIC binding — so container traffic never touches the
4423        // operator's gateway adapter. Setting ZLAYER_HCN_UPLINK_ADAPTER opts
4424        // into the legacy Transparent model bound to that named uplink.
4425        let use_transparent = std::env::var(zlayer_hns::adapter::ZLAYER_UPLINK_ENV)
4426            .ok()
4427            .is_some_and(|v| !v.trim().is_empty());
4428
4429        let net_name_for_create = net_name.clone();
4430        let subnet_for_create = subnet_str.clone();
4431        if use_transparent {
4432            let uplink = zlayer_hns::adapter::find_primary_adapter()
4433                .map_err(|e| OverlaydError::Other(format!("find_primary_adapter: {e}")))?;
4434            tracing::warn!(uplink = %uplink, "ZLAYER_HCN_UPLINK_ADAPTER set: creating HCN *Transparent* overlay bound to a physical NIC");
4435            tokio::task::spawn_blocking(move || {
4436                zlayer_hns::network::Network::create_transparent(
4437                    net_id,
4438                    &net_name_for_create,
4439                    &subnet_for_create,
4440                    &uplink,
4441                )
4442            })
4443            .await
4444            .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?
4445            .map_err(|e| {
4446                OverlaydError::Overlay(format!("HcnCreateNetwork transparent ({net_name}): {e}"))
4447            })?;
4448        } else {
4449            tokio::task::spawn_blocking(move || {
4450                zlayer_hns::network::Network::create_internal(
4451                    net_id,
4452                    &net_name_for_create,
4453                    &subnet_for_create,
4454                )
4455            })
4456            .await
4457            .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?
4458            .map_err(|e| {
4459                OverlaydError::Overlay(format!("HcnCreateNetwork internal ({net_name}): {e}"))
4460            })?;
4461        }
4462
4463        // HCN's Static IPAM needs ~1-2s after network create to settle its
4464        // address pool; without this the first endpoint frequently fails with
4465        // HCN_E_ADDR_INVALID_OR_RESERVED.
4466        tokio::time::sleep(std::time::Duration::from_secs(2)).await;
4467
4468        tracing::info!(
4469            subnet = %subnet_str,
4470            mode = if use_transparent { "Transparent" } else { "Internal" },
4471            "created HCN overlay network"
4472        );
4473
4474        // Persist the marker so subsequent runs reuse this network by GUID and a
4475        // full uninstall knows to delete it. Best-effort.
4476        let mut marker = crate::network_state::NetworkState::load(&marker_path);
4477        marker.upsert(crate::network_state::ManagedNetwork {
4478            owner: crate::network_state::OWNER_BASE.to_string(),
4479            kind: if use_transparent {
4480                "hcn-transparent"
4481            } else {
4482                "hcn-internal"
4483            }
4484            .to_string(),
4485            name: net_name.clone(),
4486            id: format_guid_bare(net_id),
4487            subnet: subnet_str.clone(),
4488            // Base/Shared HCN network: no dedicated WireGuard identity.
4489            wg_port: None,
4490            wg_private_key: None,
4491            wg_public_key: None,
4492            interface: None,
4493        });
4494        if let Err(e) = marker.save(&marker_path) {
4495            tracing::warn!(error = %e, path = %marker_path.display(), "failed to persist agent network marker (network still reusable by name)");
4496        }
4497
4498        Ok(net_id)
4499    }
4500
4501    /// Ensure the SINGLE shared HCN **NAT** network exists on the host, reusing
4502    /// one recorded under the [`OWNER_SHARED_NAT`] marker owner (or discoverable
4503    /// by its derived name) and recording it on create. Reused across every
4504    /// `OverlayMode::Shared` service on this node.
4505    ///
4506    /// NAT gives Shared containers outbound connectivity and lets the userspace
4507    /// free-port L4 proxy (`proxy_manager.rs`) forward `host:FREEPORT` ->
4508    /// `container_ip:port` without a per-service vSwitch — the Windows analogue
4509    /// of the Linux node-wide shared bridge. Modeled on
4510    /// [`Self::ensure_overlay_network`] but keyed on [`OWNER_SHARED_NAT`] and
4511    /// forced to the NAT network type.
4512    ///
4513    /// Returns the network GUID.
4514    ///
4515    /// # Errors
4516    /// Propagates the underlying `zlayer_hns` error on create failure, or an
4517    /// error if the slice CIDR has no usable gateway host.
4518    #[cfg(target_os = "windows")]
4519    #[allow(clippy::too_many_lines)]
4520    async fn ensure_shared_nat_network(
4521        &mut self,
4522        slice_cidr: ipnet::IpNet,
4523    ) -> Result<windows::core::GUID, OverlaydError> {
4524        use windows::core::GUID;
4525
4526        let daemon_name = self.deployment_or_default();
4527        // Shared NAT network name: `<base overlay name>-shared` so it is
4528        // unambiguously distinct from the base network and per-service networks.
4529        let net_name = format!("{}-shared", overlay_network_name(&daemon_name));
4530        let owner = crate::network_state::OWNER_SHARED_NAT.to_string();
4531        let marker_path =
4532            zlayer_paths::ZLayerDirs::new(self.data_dir.clone()).agent_network_state();
4533
4534        // Fast path: marker names a network GUID that still exists; reopen it.
4535        let recorded_id = crate::network_state::NetworkState::load(&marker_path)
4536            .get(&owner)
4537            .filter(|entry| entry.kind == "hcn-nat")
4538            .and_then(|entry| GUID::try_from(entry.id.as_str()).ok());
4539        if let Some(recorded_id) = recorded_id {
4540            let reopened = tokio::task::spawn_blocking(move || {
4541                zlayer_hns::network::Network::open(recorded_id).ok()
4542            })
4543            .await
4544            .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?;
4545            if reopened.is_some() {
4546                tracing::info!(name = %net_name, "reusing shared HCN NAT network from marker");
4547                return Ok(recorded_id);
4548            }
4549        }
4550
4551        // Idempotency: reuse a host network whose queried name matches ours.
4552        let target_name = net_name.clone();
4553        let existing = tokio::task::spawn_blocking(move || -> Option<GUID> {
4554            let guids = zlayer_hns::network::list("{}").ok()?;
4555            for guid in guids {
4556                let Ok(network) = zlayer_hns::network::Network::open(guid) else {
4557                    continue;
4558                };
4559                if matches!(network.query("{}"), Ok(props) if props.name == target_name) {
4560                    return Some(guid);
4561                }
4562            }
4563            None
4564        })
4565        .await
4566        .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?;
4567
4568        if let Some(existing_id) = existing {
4569            tracing::info!(name = %net_name, "reusing existing shared HCN NAT network");
4570            return Ok(existing_id);
4571        }
4572
4573        let net_id = GUID::new()
4574            .map_err(|e| OverlaydError::Other(format!("GUID::new for shared NAT network: {e}")))?;
4575        let subnet_str = slice_cidr.to_string();
4576        let settings = shared_nat_settings(&net_name, &subnet_str).ok_or_else(|| {
4577            OverlaydError::Other(format!(
4578                "shared NAT network: slice CIDR '{subnet_str}' has no usable gateway host"
4579            ))
4580        })?;
4581
4582        let net_name_for_create = net_name.clone();
4583        tokio::task::spawn_blocking(move || {
4584            zlayer_hns::network::Network::create(net_id, &settings)
4585        })
4586        .await
4587        .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?
4588        .map_err(|e| OverlaydError::Overlay(format!("HcnCreateNetwork NAT ({net_name}): {e}")))?;
4589        let _ = net_name_for_create;
4590
4591        // HCN's IPAM needs ~1-2s after network create to settle its address pool
4592        // (same wait as the base/Internal networks).
4593        tokio::time::sleep(std::time::Duration::from_secs(2)).await;
4594
4595        tracing::info!(subnet = %subnet_str, "created shared HCN NAT network");
4596
4597        let mut marker = crate::network_state::NetworkState::load(&marker_path);
4598        marker.upsert(crate::network_state::ManagedNetwork {
4599            owner,
4600            kind: "hcn-nat".to_string(),
4601            name: net_name.clone(),
4602            id: format_guid_bare(net_id),
4603            subnet: subnet_str.clone(),
4604            wg_port: None,
4605            wg_private_key: None,
4606            wg_public_key: None,
4607            interface: None,
4608        });
4609        if let Err(e) = marker.save(&marker_path) {
4610            tracing::warn!(error = %e, path = %marker_path.display(), "failed to persist shared NAT network marker (network still reusable by name)");
4611        }
4612
4613        Ok(net_id)
4614    }
4615
4616    /// Ensure the per-service HCN **Internal** network for `service` exists on
4617    /// the host, reusing one recorded under the `service:<name>` marker owner
4618    /// (or discoverable by its derived name) and recording it on create.
4619    ///
4620    /// This is the Windows analogue of the Linux per-service bridge: a
4621    /// dedicated (`OverlayMode::Dedicated`) service gets its OWN isolated HCN
4622    /// Internal network — an internal vSwitch with NO physical-NIC binding —
4623    /// distinct from the node's shared base overlay network. Containers attach
4624    /// to it (rather than the base network) so dedicated-service traffic is
4625    /// segregated at the vSwitch layer. Modeled on [`Self::ensure_overlay_network`]
4626    /// but keyed on [`owner_for_service`] and forced to the Internal type (never
4627    /// Transparent — the on-box test asserts zero external vSwitches for
4628    /// dedicated services).
4629    ///
4630    /// Returns the network GUID.
4631    ///
4632    /// # Errors
4633    /// Propagates the underlying `zlayer_hns` error on create failure.
4634    #[cfg(target_os = "windows")]
4635    #[allow(clippy::too_many_lines)]
4636    async fn ensure_service_network(
4637        &mut self,
4638        service: &str,
4639        subnet: ipnet::IpNet,
4640    ) -> Result<windows::core::GUID, OverlaydError> {
4641        use windows::core::GUID;
4642
4643        let daemon_name = self.deployment_or_default();
4644        // Per-service network name: `<base overlay name>-svc-<service>` so it is
4645        // unambiguously distinct from the base network and from other services.
4646        let net_name = format!("{}-svc-{service}", overlay_network_name(&daemon_name));
4647        let owner = owner_for_service(service);
4648        let marker_path =
4649            zlayer_paths::ZLayerDirs::new(self.data_dir.clone()).agent_network_state();
4650
4651        // Fast path: marker names a network GUID that still exists; reopen it.
4652        // Only honor the recorded id when it belongs to an HCN-internal entry —
4653        // a Dedicated WireGuard marker (`kind == "wg-dedicated"`) stores the
4654        // transport public key in `id`, NOT an HCN GUID, so it must be ignored
4655        // for HCN reuse.
4656        let recorded_hcn_id = crate::network_state::NetworkState::load(&marker_path)
4657            .get(&owner)
4658            .filter(|entry| entry.kind == "hcn-internal")
4659            .and_then(|entry| GUID::try_from(entry.id.as_str()).ok());
4660        if let Some(recorded_id) = recorded_hcn_id {
4661            let reopened = tokio::task::spawn_blocking(move || {
4662                zlayer_hns::network::Network::open(recorded_id).ok()
4663            })
4664            .await
4665            .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?;
4666            if reopened.is_some() {
4667                tracing::info!(name = %net_name, service = %service, "reusing per-service HCN network from marker");
4668                return Ok(recorded_id);
4669            }
4670        }
4671
4672        // Idempotency: reuse a host network whose queried name matches ours.
4673        let target_name = net_name.clone();
4674        let existing = tokio::task::spawn_blocking(move || -> Option<GUID> {
4675            let guids = zlayer_hns::network::list("{}").ok()?;
4676            for guid in guids {
4677                let Ok(network) = zlayer_hns::network::Network::open(guid) else {
4678                    continue;
4679                };
4680                if matches!(network.query("{}"), Ok(props) if props.name == target_name) {
4681                    return Some(guid);
4682                }
4683            }
4684            None
4685        })
4686        .await
4687        .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?;
4688
4689        if let Some(existing_id) = existing {
4690            tracing::info!(name = %net_name, service = %service, "reusing existing per-service HCN network");
4691            return Ok(existing_id);
4692        }
4693
4694        let net_id = GUID::new()
4695            .map_err(|e| OverlaydError::Other(format!("GUID::new for per-service network: {e}")))?;
4696        let subnet_str = subnet.to_string();
4697
4698        // ALWAYS Internal for a dedicated service — never Transparent. The
4699        // dedicated requirement is isolation; an Internal network binds NO
4700        // physical NIC (no external vSwitch), which is what the on-box test
4701        // asserts.
4702        let net_name_for_create = net_name.clone();
4703        let subnet_for_create = subnet_str.clone();
4704        tokio::task::spawn_blocking(move || {
4705            zlayer_hns::network::Network::create_internal(
4706                net_id,
4707                &net_name_for_create,
4708                &subnet_for_create,
4709            )
4710        })
4711        .await
4712        .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?
4713        .map_err(|e| {
4714            OverlaydError::Overlay(format!("HcnCreateNetwork internal ({net_name}): {e}"))
4715        })?;
4716
4717        // HCN's Static IPAM needs ~1-2s after network create to settle its
4718        // address pool; without this the first endpoint frequently fails with
4719        // HCN_E_ADDR_INVALID_OR_RESERVED (same wait as the base network).
4720        tokio::time::sleep(std::time::Duration::from_secs(2)).await;
4721
4722        tracing::info!(
4723            service = %service,
4724            subnet = %subnet_str,
4725            "created per-service HCN Internal network"
4726        );
4727
4728        // Persist the marker (owner = `service:<name>`, kind = `hcn-internal`)
4729        // so subsequent runs reuse this network by GUID and a full uninstall
4730        // (`purge_managed_networks`, which sweeps every `kind` starting with
4731        // `hcn`) deletes it. Best-effort.
4732        //
4733        // A dedicated Windows service shares the SAME owner key for two facts:
4734        // the dedicated WireGuard identity (written by the cross-platform core
4735        // in `setup_service_overlay_dedicated`, kind `wg-dedicated`) and this
4736        // HCN network's GUID. The marker is keyed by owner, so carry the WG
4737        // identity fields over when we rewrite the entry to `hcn-internal` — the
4738        // single entry then holds both the HCN GUID (in `id`) and the WG
4739        // identity (in the `wg_*`/`interface` fields), and the WG private key
4740        // survives restarts. (The core re-asserts the `wg-dedicated` shape on
4741        // the next setup; this path re-asserts `hcn-internal` again right after
4742        // — both are self-healing because the network is also reusable by name.)
4743        let mut marker = crate::network_state::NetworkState::load(&marker_path);
4744        let carried = marker.get(&owner).cloned();
4745        marker.upsert(crate::network_state::ManagedNetwork {
4746            owner,
4747            kind: "hcn-internal".to_string(),
4748            name: net_name.clone(),
4749            id: format_guid_bare(net_id),
4750            subnet: subnet_str.clone(),
4751            wg_port: carried.as_ref().and_then(|c| c.wg_port),
4752            wg_private_key: carried.as_ref().and_then(|c| c.wg_private_key.clone()),
4753            wg_public_key: carried.as_ref().and_then(|c| c.wg_public_key.clone()),
4754            interface: carried.as_ref().and_then(|c| c.interface.clone()),
4755        });
4756        if let Err(e) = marker.save(&marker_path) {
4757            tracing::warn!(service = %service, error = %e, path = %marker_path.display(), "failed to persist per-service network marker (network still reusable by name)");
4758        }
4759
4760        Ok(net_id)
4761    }
4762
4763    /// Resolve the per-isolation-network subnet for `net`, carving a fixed-size
4764    /// sub-block out of the node slice deterministically by name hash.
4765    ///
4766    /// Isolation networks attach onto a dedicated HCN Internal vSwitch and need
4767    /// their OWN address pool (never the node slice's shared pool) so a member's
4768    /// IP is on-link with its network's gateway. Unlike dedicated services,
4769    /// isolation networks aren't registered in the cluster's
4770    /// [`ServiceSubnetRegistry`] (a standalone isolated container may use the
4771    /// base overlay, where no `SetupServiceOverlay` ran), so the subnet is
4772    /// derived locally and deterministically: the node slice is split into
4773    /// `/<sub_prefix>` blocks and the network name selects one by hash. The
4774    /// derivation is stable across restarts (same name -> same block) so a
4775    /// reused HCN network keeps the same subnet.
4776    ///
4777    /// # Errors
4778    /// Returns an error if no node slice is assigned yet, the slice CIDR is
4779    /// unparseable, or the slice cannot be subnetted (e.g. already at the host
4780    /// prefix).
4781    #[cfg(target_os = "windows")]
4782    fn isolation_network_subnet(&self, net: &str) -> Result<ipnet::IpNet, OverlaydError> {
4783        use std::hash::{Hash, Hasher};
4784
4785        let slice = self.slice_cidr.ok_or_else(|| {
4786            OverlaydError::Other(
4787                "no node slice assigned yet (SetupGlobalOverlay with slice_cidr first)".to_string(),
4788            )
4789        })?;
4790        let slice_ipnet: ipnet::IpNet = slice.to_string().parse().map_err(|e| {
4791            OverlaydError::Other(format!("failed to parse slice CIDR {slice}: {e}"))
4792        })?;
4793
4794        // Carve the slice into /<sub_prefix> blocks. A `/28` (V4) gives ~13
4795        // usable container IPs per isolation network per node — enough for the
4796        // isolated-container use case — while leaving room for several distinct
4797        // isolation networks inside one node slice. Clamp to the slice prefix so
4798        // a slice already more specific than the target just yields itself.
4799        let sub_prefix: u8 = match slice_ipnet {
4800            ipnet::IpNet::V4(_) => 28u8.max(slice_ipnet.prefix_len()),
4801            ipnet::IpNet::V6(_) => 124u8.max(slice_ipnet.prefix_len()),
4802        };
4803
4804        let blocks: Vec<ipnet::IpNet> = slice_ipnet
4805            .subnets(sub_prefix)
4806            .map_err(|e| {
4807                OverlaydError::Other(format!(
4808                    "failed to subnet slice {slice_ipnet} into /{sub_prefix} blocks: {e}"
4809                ))
4810            })?
4811            .collect();
4812        if blocks.is_empty() {
4813            return Err(OverlaydError::Other(format!(
4814                "slice {slice_ipnet} yielded no /{sub_prefix} blocks for isolation network {net}"
4815            )));
4816        }
4817
4818        let mut hasher = std::collections::hash_map::DefaultHasher::new();
4819        net.hash(&mut hasher);
4820        // `% blocks.len()` is always < blocks.len() <= usize::MAX, so this never
4821        // truncates; `try_from` keeps clippy happy without an unchecked cast.
4822        let idx = usize::try_from(hasher.finish() % blocks.len() as u64).unwrap_or(0);
4823        Ok(blocks[idx])
4824    }
4825
4826    /// Ensure the per-isolation-network HCN **Internal** network for `net` exists
4827    /// on the host, reusing one recorded under the
4828    /// [`owner_for_isolation_network`] marker owner (or discoverable by its
4829    /// derived name) and recording it on create.
4830    ///
4831    /// This is the Windows mechanism for per-network L3 isolation: every
4832    /// `ZLayer` "isolated" network gets its OWN HCN Internal vSwitch — an
4833    /// internal vSwitch with NO physical-NIC binding. HCN Internal vSwitches are
4834    /// mutually isolated by default, so same-network members (sharing this
4835    /// vSwitch) reach each other + egress + the node, while members of a
4836    /// different isolation network land on a different vSwitch and cannot reach
4837    /// them. No ACLs, no per-member churn. Modeled on
4838    /// [`Self::ensure_service_network`] but keyed on
4839    /// [`owner_for_isolation_network`] and named `<overlay>-iso-<net>`.
4840    ///
4841    /// Returns the network GUID.
4842    ///
4843    /// # Errors
4844    /// Propagates the underlying `zlayer_hns` error on create failure.
4845    #[cfg(target_os = "windows")]
4846    async fn ensure_isolation_network(
4847        &mut self,
4848        net: &str,
4849        subnet: ipnet::IpNet,
4850    ) -> Result<windows::core::GUID, OverlaydError> {
4851        use windows::core::GUID;
4852
4853        let daemon_name = self.deployment_or_default();
4854        // Per-isolation-network name: `<base overlay name>-iso-<net>` so it is
4855        // unambiguously distinct from the base network and per-service networks.
4856        let net_name = format!("{}-iso-{net}", overlay_network_name(&daemon_name));
4857        let owner = crate::network_state::owner_for_isolation_network(net);
4858        let marker_path =
4859            zlayer_paths::ZLayerDirs::new(self.data_dir.clone()).agent_network_state();
4860
4861        // Fast path: marker names a network GUID that still exists; reopen it.
4862        let recorded_hcn_id = crate::network_state::NetworkState::load(&marker_path)
4863            .get(&owner)
4864            .filter(|entry| entry.kind == "hcn-internal")
4865            .and_then(|entry| GUID::try_from(entry.id.as_str()).ok());
4866        if let Some(recorded_id) = recorded_hcn_id {
4867            let reopened = tokio::task::spawn_blocking(move || {
4868                zlayer_hns::network::Network::open(recorded_id).ok()
4869            })
4870            .await
4871            .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?;
4872            if reopened.is_some() {
4873                tracing::info!(name = %net_name, network = %net, "reusing per-isolation-network HCN network from marker");
4874                return Ok(recorded_id);
4875            }
4876        }
4877
4878        // Idempotency: reuse a host network whose queried name matches ours.
4879        let target_name = net_name.clone();
4880        let existing = tokio::task::spawn_blocking(move || -> Option<GUID> {
4881            let guids = zlayer_hns::network::list("{}").ok()?;
4882            for guid in guids {
4883                let Ok(network) = zlayer_hns::network::Network::open(guid) else {
4884                    continue;
4885                };
4886                if matches!(network.query("{}"), Ok(props) if props.name == target_name) {
4887                    return Some(guid);
4888                }
4889            }
4890            None
4891        })
4892        .await
4893        .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?;
4894
4895        if let Some(existing_id) = existing {
4896            tracing::info!(name = %net_name, network = %net, "reusing existing per-isolation-network HCN network");
4897            return Ok(existing_id);
4898        }
4899
4900        let net_id = GUID::new().map_err(|e| {
4901            OverlaydError::Other(format!("GUID::new for per-isolation-network network: {e}"))
4902        })?;
4903        let subnet_str = subnet.to_string();
4904
4905        // ALWAYS Internal for an isolation network — never Transparent. The
4906        // isolation requirement is exactly the Internal-vSwitch property: no
4907        // physical-NIC binding, mutually isolated from other Internal vSwitches.
4908        let net_name_for_create = net_name.clone();
4909        let subnet_for_create = subnet_str.clone();
4910        tokio::task::spawn_blocking(move || {
4911            zlayer_hns::network::Network::create_internal(
4912                net_id,
4913                &net_name_for_create,
4914                &subnet_for_create,
4915            )
4916        })
4917        .await
4918        .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?
4919        .map_err(|e| {
4920            OverlaydError::Overlay(format!("HcnCreateNetwork internal ({net_name}): {e}"))
4921        })?;
4922
4923        // HCN's Static IPAM needs ~1-2s after network create to settle its
4924        // address pool; without this the first endpoint frequently fails with
4925        // HCN_E_ADDR_INVALID_OR_RESERVED (same wait as the per-service network).
4926        tokio::time::sleep(std::time::Duration::from_secs(2)).await;
4927
4928        tracing::info!(
4929            network = %net,
4930            subnet = %subnet_str,
4931            "created per-isolation-network HCN Internal network"
4932        );
4933
4934        // Persist the marker (owner = `iso:<net>`, kind = `hcn-internal`) so
4935        // subsequent runs reuse this network by GUID and a full uninstall
4936        // (`purge_managed_networks`, which sweeps every `kind` starting with
4937        // `hcn`) deletes it. Best-effort.
4938        let mut marker = crate::network_state::NetworkState::load(&marker_path);
4939        marker.upsert(crate::network_state::ManagedNetwork {
4940            owner,
4941            kind: "hcn-internal".to_string(),
4942            name: net_name.clone(),
4943            id: format_guid_bare(net_id),
4944            subnet: subnet_str.clone(),
4945            // Isolation HCN network: no dedicated WireGuard identity.
4946            wg_port: None,
4947            wg_private_key: None,
4948            wg_public_key: None,
4949            interface: None,
4950        });
4951        if let Err(e) = marker.save(&marker_path) {
4952            tracing::warn!(network = %net, error = %e, path = %marker_path.display(), "failed to persist per-isolation-network marker (network still reusable by name)");
4953        }
4954
4955        Ok(net_id)
4956    }
4957
4958    /// Resolve the dedicated per-service subnet for `service`, if the service
4959    /// runs in `OverlayMode::Dedicated` on this node.
4960    ///
4961    /// Source of truth, in order:
4962    /// 1. The live [`ServiceTransport`] in `service_transports` (the normal
4963    ///    case once `SetupServiceOverlay` has run this process).
4964    /// 2. A persisted `hcn-internal` marker entry under
4965    ///    [`owner_for_service`]`(service)` — covers the window where the HCN
4966    ///    network exists from a prior run but the transport map is still empty.
4967    ///
4968    /// Returns `None` for Shared-mode services (attach onto the base network).
4969    #[cfg(target_os = "windows")]
4970    fn dedicated_service_subnet(&self, service: &str) -> Option<ipnet::IpNet> {
4971        if let Some(st) = self.service_transports.get(service) {
4972            return Some(st.subnet);
4973        }
4974        let marker_path =
4975            zlayer_paths::ZLayerDirs::new(self.data_dir.clone()).agent_network_state();
4976        crate::network_state::NetworkState::load(&marker_path)
4977            .get(&owner_for_service(service))
4978            .filter(|entry| entry.kind == "hcn-internal")
4979            .and_then(|entry| entry.subnet.parse::<ipnet::IpNet>().ok())
4980    }
4981
4982    /// The daemon name used for HCN network/owner naming, defaulting to
4983    /// `"zlayer"` when no deployment has been set yet.
4984    #[cfg(target_os = "windows")]
4985    fn deployment_or_default(&self) -> String {
4986        if self.deployment.is_empty() {
4987            "zlayer".to_string()
4988        } else {
4989            self.deployment.clone()
4990        }
4991    }
4992
4993    // -- peers ---------------------------------------------------------------
4994
4995    /// Resolve a [`PeerScope`] to the live [`OverlayTransport`] its ops target.
4996    ///
4997    /// `Global` -> the single cluster transport; `Service { service }` -> that
4998    /// service's dedicated per-service transport (Dedicated mode only).
4999    ///
5000    /// # Errors
5001    /// Returns an error if the global overlay is not up (for `Global`) or no
5002    /// dedicated overlay exists for the named service (for `Service`).
5003    fn transport_for_scope(&self, scope: &PeerScope) -> Result<&OverlayTransport, OverlaydError> {
5004        match scope {
5005            PeerScope::Global => self
5006                .global_transport
5007                .as_ref()
5008                .ok_or_else(|| OverlaydError::Other("global overlay not set up".into())),
5009            PeerScope::Service { service } => self
5010                .service_transports
5011                .get(service)
5012                .map(|s| &s.transport)
5013                .ok_or_else(|| {
5014                    OverlaydError::Other(format!("no dedicated overlay for service {service}"))
5015                }),
5016        }
5017    }
5018
5019    /// Add a peer to a resolved transport.
5020    ///
5021    /// # Errors
5022    /// Wraps the underlying transport error.
5023    async fn add_peer_on(
5024        transport: &OverlayTransport,
5025        peer: &PeerInfo,
5026    ) -> Result<(), OverlaydError> {
5027        transport
5028            .add_peer(peer)
5029            .await
5030            .map_err(|e| OverlaydError::Overlay(format!("add_peer failed: {e}")))
5031    }
5032
5033    /// Remove a peer (by base64 public key) from a resolved transport.
5034    ///
5035    /// # Errors
5036    /// Wraps the underlying transport error.
5037    async fn remove_peer_on(
5038        transport: &OverlayTransport,
5039        pubkey: &str,
5040    ) -> Result<(), OverlaydError> {
5041        transport
5042            .remove_peer(pubkey)
5043            .await
5044            .map_err(|e| OverlaydError::Overlay(format!("remove_peer failed: {e}")))
5045    }
5046
5047    /// Plumb a CIDR into a peer's `AllowedIPs` on a resolved transport.
5048    ///
5049    /// # Errors
5050    /// Returns an error when the CIDR is invalid or the UAPI write fails.
5051    async fn add_allowed_ip_on(
5052        transport: &OverlayTransport,
5053        pubkey: &str,
5054        cidr: &str,
5055    ) -> Result<(), OverlaydError> {
5056        let net: ipnet::IpNet = cidr
5057            .parse()
5058            .map_err(|e| OverlaydError::Other(format!("invalid CIDR {cidr}: {e}")))?;
5059        transport
5060            .add_allowed_ip(pubkey, net)
5061            .await
5062            .map_err(|e| OverlaydError::Overlay(format!("add_allowed_ip failed: {e}")))
5063    }
5064
5065    /// Remove a CIDR from a peer's `AllowedIPs` on a resolved transport.
5066    ///
5067    /// # Errors
5068    /// Returns an error when the CIDR is invalid or the UAPI write fails.
5069    async fn remove_allowed_ip_on(
5070        transport: &OverlayTransport,
5071        pubkey: &str,
5072        cidr: &str,
5073    ) -> Result<(), OverlaydError> {
5074        let net: ipnet::IpNet = cidr
5075            .parse()
5076            .map_err(|e| OverlaydError::Other(format!("invalid CIDR {cidr}: {e}")))?;
5077        transport
5078            .remove_allowed_ip(pubkey, net)
5079            .await
5080            .map_err(|e| OverlaydError::Overlay(format!("remove_allowed_ip failed: {e}")))
5081    }
5082
5083    // -- DNS -----------------------------------------------------------------
5084
5085    /// Register an overlay DNS A/AAAA record.
5086    fn register_dns(&mut self, name: String, ip: IpAddr) {
5087        self.dns_records.insert(name, ip);
5088    }
5089
5090    /// Remove an overlay DNS record.
5091    fn unregister_dns(&mut self, name: &str) {
5092        self.dns_records.remove(name);
5093    }
5094
5095    // -- NAT -----------------------------------------------------------------
5096
5097    /// Periodic NAT traversal maintenance: lazily start NAT (and the built-in
5098    /// relay server), re-probe STUN, refresh relays, and run the connect-half —
5099    /// hole-punching / relaying toward every peer whose direct endpoint has not
5100    /// produced a recent `WireGuard` handshake.
5101    ///
5102    /// No-op when NAT traversal is disabled in the resolved [`NatConfig`].
5103    ///
5104    /// # Errors
5105    /// Returns an error when the underlying STUN refresh fails.
5106    async fn nat_maintenance_tick(&mut self) -> Result<(), OverlaydError> {
5107        // Lazily start NAT traversal on the first tick if a config asks for it.
5108        if self.nat_traversal.is_none() {
5109            let config = self.nat_config.clone().unwrap_or_default();
5110            if config.enabled {
5111                // Stand up the built-in relay server here (once) when the
5112                // resolved config carries a `relay_server`. The auth credential
5113                // MUST be cluster-wide-shared (every node's relay *client*
5114                // derives the same BLAKE2b key via `derive_auth_key`), so it
5115                // comes from `cluster_relay_credential` — the cluster HS256
5116                // secret the main daemon stamped into
5117                // `NatConfigSpec.relay_server.auth_credential`, NOT the node's
5118                // per-node WireGuard key. When no credential was supplied the
5119                // relay derives a key from the empty string (only same-config
5120                // nodes can use it).
5121                if let Some(relay_cfg) = config.relay_server.clone() {
5122                    if self.relay_server.is_none() {
5123                        let credential = self.cluster_relay_credential.clone().unwrap_or_default();
5124                        let relay = RelayServer::new(&relay_cfg, &credential);
5125                        match relay.start().await {
5126                            Ok(bound) => {
5127                                tracing::info!(
5128                                    bound = %bound,
5129                                    external = %relay_cfg.external_addr,
5130                                    "Built-in relay server started"
5131                                );
5132                                self.relay_bound_addr = Some(bound);
5133                                self.relay_server = Some(relay);
5134                            }
5135                            Err(e) => {
5136                                tracing::warn!(error = %e, "Built-in relay server failed to start");
5137                            }
5138                        }
5139                    }
5140                }
5141
5142                let mut nat = NatTraversal::new(config, self.overlay_port);
5143                match nat.gather_candidates().await {
5144                    Ok(candidates) => {
5145                        tracing::info!(count = candidates.len(), "Gathered NAT candidates");
5146                        self.nat_last_refresh.store(now_unix(), Ordering::SeqCst);
5147                        self.nat_traversal = Some(nat);
5148                    }
5149                    Err(e) => {
5150                        tracing::warn!(error = %e, "NAT candidate gathering failed");
5151                        return Ok(());
5152                    }
5153                }
5154                // First-tick connect: try to establish toward every already-known
5155                // peer (peers added before NAT came up).
5156                self.nat_connect_known_peers().await;
5157            } else {
5158                return Ok(());
5159            }
5160        }
5161
5162        // Refresh STUN/relay state, then run the connect-half for peers that
5163        // still lack a recent handshake.
5164        if let Some(nat) = self.nat_traversal.as_mut() {
5165            match nat.refresh().await {
5166                Ok(changed) => {
5167                    if changed {
5168                        tracing::info!("NAT reflexive address changed during refresh");
5169                    }
5170                    self.nat_last_refresh.store(now_unix(), Ordering::SeqCst);
5171                }
5172                Err(e) => {
5173                    return Err(OverlaydError::Overlay(format!(
5174                        "NAT maintenance tick failed: {e}"
5175                    )));
5176                }
5177            }
5178        }
5179        self.nat_connect_known_peers().await;
5180        Ok(())
5181    }
5182
5183    /// The NAT connect-half: for every peer with advertised candidates that has
5184    /// no recent `WireGuard` handshake, call [`NatTraversal::connect_to_peer`]
5185    /// (which itself updates the live device's peer endpoint) and record the
5186    /// resulting [`ConnectionType`].
5187    ///
5188    /// Best-effort: a peer with no live global transport, no candidates, or a
5189    /// failed traversal is left untouched (its persistent direct endpoint keeps
5190    /// retrying). Candidate sets are collected into a local `Vec` first so the
5191    /// borrow of `self.nat_traversal` / `self.global_transport` does not overlap
5192    /// the mutable borrow of `self.peer_connection_type`.
5193    async fn nat_connect_known_peers(&mut self) {
5194        // No host transport (VM-only overlay) or no NAT orchestrator → nothing
5195        // to connect on this node.
5196        let (Some(_), Some(_)) = (self.global_transport.as_ref(), self.nat_traversal.as_ref())
5197        else {
5198            return;
5199        };
5200        if self.peer_candidates.is_empty() {
5201            return;
5202        }
5203
5204        // Peers whose handshake is older than this cutoff (or never seen) are
5205        // candidates for a (re)connect attempt. WireGuard's default keepalive is
5206        // 25s; 3× that is a generous "the direct endpoint is clearly not
5207        // establishing" threshold that avoids churning healthy peers.
5208        let cutoff = now_unix().saturating_sub(75);
5209
5210        // Snapshot the (pubkey, candidates) work set up front to satisfy the
5211        // borrow checker (we borrow self.transport + self.nat below).
5212        let work: Vec<(String, Vec<Candidate>)> = self
5213            .peer_candidates
5214            .iter()
5215            .map(|(k, v)| (k.clone(), v.clone()))
5216            .collect();
5217
5218        let transport = self.global_transport.as_ref().expect("checked above");
5219        let nat = self.nat_traversal.as_ref().expect("checked above");
5220        let mut results: Vec<(String, ConnectionType)> = Vec::new();
5221
5222        for (pubkey, candidates) in &work {
5223            // Skip peers that already have a fresh handshake on the live device.
5224            match transport.check_peer_handshake(pubkey, cutoff).await {
5225                Ok(true) => continue,
5226                Ok(false) => {}
5227                Err(e) => {
5228                    tracing::debug!(peer = %pubkey, error = %e, "handshake check failed; attempting connect anyway");
5229                }
5230            }
5231            match nat.connect_to_peer(transport, pubkey, candidates).await {
5232                Ok(connection_type) => {
5233                    tracing::info!(
5234                        peer = %pubkey,
5235                        connection = %connection_type,
5236                        "NAT traversal established connection to peer"
5237                    );
5238                    results.push((pubkey.clone(), connection_type));
5239                }
5240                Err(e) => {
5241                    tracing::debug!(peer = %pubkey, error = %e, "NAT traversal could not connect to peer this tick");
5242                }
5243            }
5244        }
5245
5246        for (pubkey, ct) in results {
5247            self.peer_connection_type.insert(pubkey, ct);
5248        }
5249    }
5250
5251    /// Build a [`NatStatusWire`] from the live NAT orchestrator: this node's
5252    /// local candidates, the per-peer connection types recorded by the connect
5253    /// loop (with each peer's current remote endpoint parsed from the UAPI
5254    /// status dump), and the last STUN-refresh timestamp.
5255    async fn nat_status_snapshot(&self) -> NatStatusWire {
5256        let candidates = self
5257            .nat_traversal
5258            .as_ref()
5259            .map(|n| n.local_candidates().iter().map(candidate_to_wire).collect())
5260            .unwrap_or_default();
5261
5262        // Map hex-pubkey -> current remote endpoint from the live device's UAPI
5263        // dump. The dump keys peers by hex; `peer_connection_type` keys by
5264        // base64, so the join below converts each base64 key to hex.
5265        let mut endpoints: HashMap<String, String> = HashMap::new();
5266        if let Some(transport) = self.global_transport.as_ref() {
5267            if let Ok(dump) = transport.status().await {
5268                for p in parse_peer_status(&dump) {
5269                    if !p.endpoint.is_empty() {
5270                        endpoints.insert(p.public_key, p.endpoint);
5271                    }
5272                }
5273            }
5274        }
5275
5276        let peers = self
5277            .peer_connection_type
5278            .iter()
5279            .map(|(pubkey, ct)| {
5280                let remote_endpoint = zlayer_overlay::nat::pubkey_b64_to_hex(pubkey)
5281                    .and_then(|hex| endpoints.get(&hex).cloned());
5282                NatPeerWire {
5283                    node_id: pubkey.clone(),
5284                    connection_type: ct.to_string(),
5285                    remote_endpoint,
5286                }
5287            })
5288            .collect();
5289
5290        NatStatusWire {
5291            candidates,
5292            peers,
5293            last_refresh: self.nat_last_refresh.load(Ordering::SeqCst),
5294        }
5295    }
5296
5297    // -- status --------------------------------------------------------------
5298
5299    /// Build a [`StatusSnapshot`] from current overlay state.
5300    async fn status_snapshot(&self) -> StatusSnapshot {
5301        let mut peers: Vec<PeerStatus> = Vec::new();
5302        let public_key = self.transport_public_key.clone();
5303
5304        if let Some(transport) = self.global_transport.as_ref() {
5305            // Parse the UAPI dump for per-peer state. Best-effort: a parse
5306            // failure leaves the peer list empty rather than failing Status.
5307            if let Ok(dump) = transport.status().await {
5308                peers = parse_peer_status(&dump);
5309            }
5310        }
5311
5312        let service_count = u32::try_from(self.service_count()).unwrap_or(u32::MAX);
5313        let peer_count = u32::try_from(peers.len()).unwrap_or(u32::MAX);
5314
5315        // Per dedicated per-service overlay device: count its peers the same
5316        // way the global status does (parse the UAPI/status dump).
5317        let mut dedicated_services: Vec<DedicatedServiceStatus> = Vec::new();
5318        for (svc, st) in &self.service_transports {
5319            let peer_count = match st.transport.status().await {
5320                Ok(dump) => u32::try_from(parse_peer_status(&dump).len()).unwrap_or(u32::MAX),
5321                Err(_) => 0,
5322            };
5323            dedicated_services.push(DedicatedServiceStatus {
5324                service: svc.clone(),
5325                interface: st.interface.clone(),
5326                public_key: st.public_key.clone(),
5327                listen_port: st.listen_port,
5328                overlay_ip: st.overlay_ip,
5329                subnet: st.subnet.to_string(),
5330                peer_count,
5331            });
5332        }
5333
5334        StatusSnapshot {
5335            interface: self.global_interface.clone(),
5336            node_ip: self.node_ip,
5337            public_key,
5338            overlay_cidr: self.cluster_cidr.map(|c| c.to_string()),
5339            slice_cidr: self.slice_cidr.map(|c| c.to_string()),
5340            peer_count,
5341            service_count,
5342            peers,
5343            dedicated_services,
5344        }
5345    }
5346
5347    /// Number of per-service overlays set up on this node (Shared bridges /
5348    /// placeholders plus any Dedicated transports not already counted there).
5349    fn service_count(&self) -> usize {
5350        let extra_dedicated = self
5351            .service_transports
5352            .keys()
5353            .filter(|svc| !self.service_interfaces.contains_key(*svc))
5354            .count();
5355        self.service_interfaces.len() + extra_dedicated
5356    }
5357
5358    // -- config helper -------------------------------------------------------
5359
5360    fn build_config(
5361        &self,
5362        private_key: String,
5363        public_key: String,
5364        ip: IpAddr,
5365        mask: u8,
5366        listen_port: u16,
5367        physical_egress_ip: Option<IpAddr>,
5368    ) -> OverlayConfig {
5369        // Pick the source/advertised address for the WireGuard endpoint.
5370        //
5371        // Default is the family-matched UNSPECIFIED (`0.0.0.0` / `::`), which lets
5372        // the kernel pick a source per outgoing packet. When the caller resolved a
5373        // physical-egress IP (see `detect_physical_egress`) *and* its family
5374        // matches the overlay IP's family, we pin `local_endpoint` to that IP so
5375        // boringtun's data socket sources from — and advertises — the real NIC
5376        // rather than whatever the default route (possibly a VPN mesh) would pick.
5377        //
5378        // Family mismatch (e.g. physical egress is v4 but this overlay is v6) is
5379        // unusable for source selection, so we warn and fall back to UNSPECIFIED.
5380        //
5381        // boringtun limitation: boringtun 0.7's `DeviceConfig` exposes no way to
5382        // inject or pin the WireGuard DATA socket (its `uapi_fd` is the UAPI
5383        // CONTROL socket only), so `SO_BINDTODEVICE` on the data socket is
5384        // impossible today. Setting `local_endpoint` to the physical IP governs
5385        // source-address selection and the advertised endpoint, which is the
5386        // realistic scope of control we have.
5387        let unspecified = match ip {
5388            IpAddr::V4(_) => IpAddr::V4(Ipv4Addr::UNSPECIFIED),
5389            IpAddr::V6(_) => IpAddr::V6(Ipv6Addr::UNSPECIFIED),
5390        };
5391        let local_addr =
5392            if rootless_forces_unspecified(std::env::var_os("ZLAYER_ROOTLESS").is_some()) {
5393                // Rootless: detect_physical_egress() resolves pasta's in-netns tap IP
5394                // (e.g. 192.168.68.x), which is useless as a WG source/advertised
5395                // endpoint to remote peers. Force UNSPECIFIED; the kernel picks the
5396                // source per packet and the real reachable endpoint comes from the
5397                // advertise_addr path + pasta forwarding.
5398                unspecified
5399            } else {
5400                match physical_egress_ip {
5401                    Some(egress) if egress.is_ipv4() == ip.is_ipv4() => egress,
5402                    Some(egress) => {
5403                        tracing::warn!(
5404                            physical_egress_ip = %egress,
5405                            overlay_ip = %ip,
5406                            "physical egress IP family does not match overlay IP family; \
5407                             falling back to UNSPECIFIED for WireGuard local_endpoint"
5408                        );
5409                        unspecified
5410                    }
5411                    None => unspecified,
5412                }
5413            };
5414        let mut config = OverlayConfig {
5415            local_endpoint: SocketAddr::new(local_addr, listen_port),
5416            private_key,
5417            public_key,
5418            overlay_cidr: format!("{ip}/{mask}"),
5419            ..OverlayConfig::default()
5420        };
5421        if let Some(nat) = self.nat_config.clone() {
5422            config.nat = nat;
5423        }
5424        if let Some(dir) = self.uapi_sock_dir.clone() {
5425            config.uapi_sock_dir = dir;
5426        }
5427        config
5428    }
5429}
5430
5431/// Build an `Auto`-mode [`ServiceOverlayInfo`]: the per-service bridge/placeholder
5432/// name with every dedicated-device identity field left `None` (`Auto` carries
5433/// the service subnet on the single cluster-wide `WireGuard` device).
5434fn cluster_wg_overlay_info(name: String) -> ServiceOverlayInfo {
5435    ServiceOverlayInfo {
5436        name,
5437        mode: OverlayMode::Auto,
5438        wg_public_key: None,
5439        wg_port: None,
5440        overlay_ip: None,
5441        subnet: None,
5442    }
5443}
5444
5445/// Build a `Shared`-mode [`ServiceOverlayInfo`]: the shared node-wide
5446/// bridge/placeholder name with every dedicated-device identity field left
5447/// `None` (Shared mode shares the single cluster device and the node-wide
5448/// bridge; ports are exposed by the userspace free-port L4 proxy).
5449fn shared_overlay_info(name: String) -> ServiceOverlayInfo {
5450    ServiceOverlayInfo {
5451        name,
5452        mode: OverlayMode::Shared,
5453        wg_public_key: None,
5454        wg_port: None,
5455        overlay_ip: None,
5456        subnet: None,
5457    }
5458}
5459
5460/// Build a Dedicated-mode [`ServiceOverlayInfo`] from a dedicated device's
5461/// identity. `name` is the container-attach handle (bridge name on Linux, the
5462/// dedicated interface elsewhere).
5463fn dedicated_overlay_info(
5464    name: String,
5465    public_key: &str,
5466    listen_port: u16,
5467    overlay_ip: IpAddr,
5468    subnet: ipnet::IpNet,
5469) -> ServiceOverlayInfo {
5470    ServiceOverlayInfo {
5471        name,
5472        mode: OverlayMode::Dedicated,
5473        wg_public_key: Some(public_key.to_string()),
5474        wg_port: Some(listen_port),
5475        overlay_ip: Some(overlay_ip),
5476        subnet: Some(subnet.to_string()),
5477    }
5478}
5479
5480/// Convert a wire [`PeerSpec`] into a `zlayer_overlay::PeerInfo`.
5481///
5482/// # Errors
5483/// Returns an error if `endpoint` cannot be parsed as a `host:port`
5484/// [`SocketAddr`].
5485pub fn peer_spec_to_info(spec: &PeerSpec) -> Result<PeerInfo, OverlaydError> {
5486    let endpoint: SocketAddr = spec.endpoint.parse().map_err(|e| {
5487        OverlaydError::Other(format!("invalid peer endpoint {}: {e}", spec.endpoint))
5488    })?;
5489    Ok(PeerInfo::new(
5490        spec.public_key.clone(),
5491        endpoint,
5492        &spec.allowed_ips,
5493        std::time::Duration::from_secs(spec.persistent_keepalive_secs),
5494    ))
5495}
5496
5497/// Parse a `wg`-style UAPI/`status` dump into [`PeerStatus`] entries.
5498///
5499/// The dump is a series of `key=value` lines; each `public_key=` line starts a
5500/// new peer block, and subsequent `endpoint=` / `allowed_ip=` /
5501/// `latest_handshake=` lines belong to it.
5502fn parse_peer_status(dump: &str) -> Vec<PeerStatus> {
5503    let mut peers: Vec<PeerStatus> = Vec::new();
5504    let mut current: Option<PeerStatus> = None;
5505    let mut allowed: Vec<String> = Vec::new();
5506
5507    let flush = |peers: &mut Vec<PeerStatus>,
5508                 current: &mut Option<PeerStatus>,
5509                 allowed: &mut Vec<String>| {
5510        if let Some(mut p) = current.take() {
5511            p.allowed_ips = allowed.join(",");
5512            peers.push(p);
5513        }
5514        allowed.clear();
5515    };
5516
5517    for line in dump.lines() {
5518        let line = line.trim();
5519        let Some((key, value)) = line.split_once('=') else {
5520            continue;
5521        };
5522        match key.trim() {
5523            "public_key" | "peer" => {
5524                flush(&mut peers, &mut current, &mut allowed);
5525                current = Some(PeerStatus {
5526                    public_key: value.trim().to_string(),
5527                    endpoint: String::new(),
5528                    allowed_ips: String::new(),
5529                    last_handshake_unix_secs: 0,
5530                });
5531            }
5532            "endpoint" => {
5533                if let Some(p) = current.as_mut() {
5534                    p.endpoint = value.trim().to_string();
5535                }
5536            }
5537            "allowed_ip" | "allowed_ips" if current.is_some() => {
5538                allowed.push(value.trim().to_string());
5539            }
5540            "latest_handshake" | "last_handshake_time_sec" => {
5541                if let Some(p) = current.as_mut() {
5542                    p.last_handshake_unix_secs = value.trim().parse().unwrap_or(0);
5543                }
5544            }
5545            _ => {}
5546        }
5547    }
5548    flush(&mut peers, &mut current, &mut allowed);
5549    peers
5550}
5551
5552/// Convert a wire [`NatConfigSpec`] into the live [`NatConfig`] overlayd drives.
5553///
5554/// Sub-fields left at their zero value in the spec fall back to
5555/// [`NatConfig::default`]'s value (so a sparsely-populated spec still gets sane
5556/// STUN servers / timeouts). The `relay_server`'s `auth_credential` is stripped
5557/// here — it is carried separately on the server (`cluster_relay_credential`)
5558/// because `RelayServerConfig` has no credential field; this conversion only
5559/// produces the bind/external/max-sessions triple it does carry.
5560fn nat_config_spec_to_config(spec: NatConfigSpec) -> NatConfig {
5561    let defaults = NatConfig::default();
5562    NatConfig {
5563        enabled: spec.enabled,
5564        stun_servers: if spec.stun_servers.is_empty() {
5565            defaults.stun_servers
5566        } else {
5567            spec.stun_servers
5568                .into_iter()
5569                .map(|address| StunServerConfig {
5570                    address,
5571                    label: None,
5572                })
5573                .collect()
5574        },
5575        turn_servers: spec
5576            .turn_servers
5577            .into_iter()
5578            .map(|t| TurnServerConfig {
5579                address: t.addr,
5580                username: t.username,
5581                credential: t.credential,
5582                region: None,
5583            })
5584            .collect(),
5585        hole_punch_timeout_secs: if spec.hole_punch_timeout_secs == 0 {
5586            defaults.hole_punch_timeout_secs
5587        } else {
5588            spec.hole_punch_timeout_secs
5589        },
5590        stun_refresh_interval_secs: if spec.stun_refresh_interval_secs == 0 {
5591            defaults.stun_refresh_interval_secs
5592        } else {
5593            spec.stun_refresh_interval_secs
5594        },
5595        max_candidate_pairs: if spec.max_candidate_pairs == 0 {
5596            defaults.max_candidate_pairs
5597        } else {
5598            spec.max_candidate_pairs
5599        },
5600        relay_server: spec.relay_server.map(|r| RelayServerConfig {
5601            listen_port: r.listen_port,
5602            external_addr: r.external_addr,
5603            max_sessions: if r.max_sessions == 0 {
5604                default_max_relay_sessions()
5605            } else {
5606                r.max_sessions
5607            },
5608        }),
5609    }
5610}
5611
5612/// Default relay `max_sessions` used when a spec leaves it at `0`. Mirrors
5613/// `zlayer_overlay::nat::config`'s private `default_max_relay_sessions` (100).
5614const fn default_max_relay_sessions() -> usize {
5615    100
5616}
5617
5618/// Parse a wire [`NatCandidateWire`] into a live [`Candidate`].
5619///
5620/// Returns `None` when the address does not parse as a `host:port` socket
5621/// address or the type string is unrecognized. Priority is taken verbatim from
5622/// the wire (the advertiser already computed it) so the receiver honors the
5623/// peer's own preference ordering.
5624fn wire_to_candidate(w: &NatCandidateWire) -> Option<Candidate> {
5625    let address: SocketAddr = w.address.parse().ok()?;
5626    let candidate_type = match w.candidate_type.as_str() {
5627        "host" => CandidateType::Host,
5628        "server-reflexive" => CandidateType::ServerReflexive,
5629        "relay" => CandidateType::Relay,
5630        _ => return None,
5631    };
5632    let mut c = Candidate::new(candidate_type, address);
5633    c.priority = w.priority;
5634    Some(c)
5635}
5636
5637/// Convert a live [`Candidate`] into its wire [`NatCandidateWire`] form for a
5638/// `NatStatus` response.
5639fn candidate_to_wire(c: &Candidate) -> NatCandidateWire {
5640    let candidate_type = match c.candidate_type {
5641        CandidateType::Host => "host",
5642        CandidateType::ServerReflexive => "server-reflexive",
5643        CandidateType::Relay => "relay",
5644    };
5645    NatCandidateWire {
5646        candidate_type: candidate_type.to_string(),
5647        address: c.address.to_string(),
5648        priority: c.priority,
5649    }
5650}
5651
5652/// Current Unix time in whole seconds.
5653fn now_unix() -> u64 {
5654    std::time::SystemTime::now()
5655        .duration_since(std::time::UNIX_EPOCH)
5656        .unwrap_or_default()
5657        .as_secs()
5658}
5659
5660/// Format an overlay address as a host route (`<ip>/32` or `<ip>/128`) — the
5661/// `AllowedIPs` form for a single overlay endpoint (an edge `/32`, the node, or
5662/// the DNS server).
5663fn edge_host_route(ip: IpAddr) -> String {
5664    format!("{ip}/{}", if ip.is_ipv6() { 128 } else { 32 })
5665}
5666
5667/// Validate the `node_endpoint` an edge peer will dial: it must be non-empty,
5668/// carry an explicit non-zero port, and not be the unspecified address (the
5669/// edge sits outside the overlay until its first handshake, so `0.0.0.0`/`::`
5670/// is meaningless as its bootstrap endpoint). A literal `host:port` /
5671/// `[v6]:port` is fully validated; a `hostname:port` (kept textual so a DNS
5672/// name survives the wire) is accepted once its trailing port parses.
5673///
5674/// # Errors
5675/// Returns [`OverlaydError::Other`] for an empty, unspecified, or portless
5676/// endpoint.
5677fn validate_edge_node_endpoint(endpoint: &str) -> Result<(), OverlaydError> {
5678    let ep = endpoint.trim();
5679    if ep.is_empty() {
5680        return Err(OverlaydError::Other(
5681            "edge node_endpoint must not be empty".to_string(),
5682        ));
5683    }
5684    // Preferred path: a literal socket address (IPv4/IPv6 + port).
5685    if let Ok(sa) = ep.parse::<SocketAddr>() {
5686        if sa.ip().is_unspecified() {
5687            return Err(OverlaydError::Other(format!(
5688                "edge node_endpoint `{ep}` must be a reachable address, not the \
5689                 unspecified `0.0.0.0`/`::`"
5690            )));
5691        }
5692        if sa.port() == 0 {
5693            return Err(OverlaydError::Other(format!(
5694                "edge node_endpoint `{ep}` must include a non-zero port"
5695            )));
5696        }
5697        return Ok(());
5698    }
5699    // Fallback: a `hostname:port` endpoint (textual, so a DNS name survives).
5700    // Require a trailing `:port` that parses to a non-zero port.
5701    let Some((host, port)) = ep.rsplit_once(':') else {
5702        return Err(OverlaydError::Other(format!(
5703            "edge node_endpoint `{ep}` must be `host:port`"
5704        )));
5705    };
5706    if host.is_empty() {
5707        return Err(OverlaydError::Other(format!(
5708            "edge node_endpoint `{ep}` is missing a host"
5709        )));
5710    }
5711    match port.parse::<u16>() {
5712        Ok(0) | Err(_) => Err(OverlaydError::Other(format!(
5713            "edge node_endpoint `{ep}` must include a non-zero port"
5714        ))),
5715        Ok(_) => Ok(()),
5716    }
5717}
5718
5719/// Offset (relative to the slice's network address) reserved for the node's
5720/// own overlay IP. Offset 1 is always the first usable host of the slice, so
5721/// the node IP is deterministic (`base + 1`) regardless of allocation order.
5722const NODE_RESERVED_OFFSET: u64 = 1;
5723
5724/// Simple IP address allocator supporting both IPv4 and IPv6, bounded to a
5725/// specific CIDR (typically a per-node `/28` slice). Allocations past the last
5726/// usable host return an exhaustion error.
5727///
5728/// Offset [`NODE_RESERVED_OFFSET`] (the first usable host) is reserved for the
5729/// node's own overlay IP and is never handed out by [`IpAllocator::allocate`],
5730/// so the node IP stays deterministic across restarts and immune to container
5731/// allocation order. Use [`IpAllocator::node_ip`] to read it.
5732struct IpAllocator {
5733    /// CIDR the allocator is bounded to.
5734    cidr: IpNetwork,
5735    /// Base (network) address of the CIDR.
5736    base: IpAddr,
5737    /// Monotonic counter for the next allocation offset relative to `base`.
5738    /// Starts at [`NODE_RESERVED_OFFSET`] + 1 so the node's reserved IP is
5739    /// never returned to a container.
5740    next_offset: AtomicU64,
5741    /// IPs returned by `release(...)`. `allocate()` drains this first before
5742    /// incrementing `next_offset`.
5743    released: parking_lot::Mutex<Vec<IpAddr>>,
5744}
5745
5746impl IpAllocator {
5747    fn new(cidr: IpNetwork) -> Self {
5748        Self {
5749            base: cidr.network(),
5750            cidr,
5751            // Reserve offset 1 for the node's own overlay IP; container
5752            // allocation starts at offset 2.
5753            next_offset: AtomicU64::new(NODE_RESERVED_OFFSET + 1),
5754            released: parking_lot::Mutex::new(Vec::new()),
5755        }
5756    }
5757
5758    /// The node's own overlay IP for this slice: the first usable host
5759    /// (`base + 1`), reserved so no container ever receives it. Deterministic
5760    /// for a given slice CIDR, independent of allocation order or restarts.
5761    fn node_ip(&self) -> IpAddr {
5762        self.compute_addr(NODE_RESERVED_OFFSET)
5763    }
5764
5765    #[allow(clippy::cast_possible_truncation)]
5766    fn compute_addr(&self, offset: u64) -> IpAddr {
5767        match self.base {
5768            IpAddr::V4(base_v4) => {
5769                let base_u32 = u32::from_be_bytes(base_v4.octets());
5770                let addr = base_u32.wrapping_add(offset as u32);
5771                IpAddr::V4(Ipv4Addr::from(addr.to_be_bytes()))
5772            }
5773            IpAddr::V6(base_v6) => {
5774                let base_u128 = u128::from(base_v6);
5775                let addr = base_u128.wrapping_add(u128::from(offset));
5776                IpAddr::V6(Ipv6Addr::from(addr))
5777            }
5778        }
5779    }
5780
5781    /// Allocate the next IP in the slice, reusing released IPs first.
5782    ///
5783    /// # Errors
5784    /// Returns [`OverlaydError::Overlay`] when the CIDR is exhausted.
5785    fn allocate(&self) -> Result<IpAddr, OverlaydError> {
5786        if let Some(ip) = self.released.lock().pop() {
5787            return Ok(ip);
5788        }
5789        let offset = self.next_offset.fetch_add(1, Ordering::SeqCst);
5790        let addr = self.compute_addr(offset);
5791
5792        let in_cidr = self.cidr.contains(addr);
5793        let is_v4_broadcast = matches!(
5794            (&self.cidr, &addr),
5795            (IpNetwork::V4(v4), IpAddr::V4(a)) if *a == v4.broadcast()
5796        );
5797        if !in_cidr || is_v4_broadcast {
5798            return Err(OverlaydError::Overlay(format!(
5799                "IP allocator exhausted: next address {addr} is outside slice {}",
5800                self.cidr
5801            )));
5802        }
5803        Ok(addr)
5804    }
5805
5806    /// Return an IP to the free pool. Idempotent. The node's reserved IP is
5807    /// never accepted back into the pool so it can never be handed to a
5808    /// container by a later `allocate()`.
5809    fn release(&self, ip: IpAddr) {
5810        if ip == self.node_ip() {
5811            return;
5812        }
5813        let mut released = self.released.lock();
5814        if !released.contains(&ip) {
5815            released.push(ip);
5816        }
5817    }
5818}
5819
5820// -- Windows HCN helpers (ported from the agent's hcs runtime) --------------
5821
5822/// Owner tag stamped onto every HCN endpoint this server creates. The legacy
5823/// single-instance value is `"zlayer"`; any other name is used verbatim so two
5824/// daemons running side-by-side never sweep each other's endpoints.
5825#[cfg(target_os = "windows")]
5826fn owner_tag(daemon_name: &str) -> String {
5827    if daemon_name == "zlayer" {
5828        "zlayer".to_string()
5829    } else {
5830        daemon_name.to_string()
5831    }
5832}
5833
5834/// Name of the per-daemon HCN overlay network on the host. Legacy
5835/// single-instance value is `"zlayer-overlay"`; any other name becomes
5836/// `"<daemon_name>-overlay"`.
5837#[cfg(target_os = "windows")]
5838fn overlay_network_name(daemon_name: &str) -> String {
5839    if daemon_name == "zlayer" {
5840        "zlayer-overlay".to_string()
5841    } else {
5842        format!("{daemon_name}-overlay")
5843    }
5844}
5845
5846/// Build the [`zlayer_hns::schema::HostComputeNetwork`] document for the single
5847/// shared HCN **NAT** network. A NAT network gives every attached container
5848/// outbound connectivity and host-port forwarding (driven by the userspace
5849/// free-port L4 proxy), without a per-service vSwitch — the Windows analogue of
5850/// the Linux node-wide shared bridge. The Static IPAM declares a default route
5851/// to the subnet gateway so HCN reserves only the gateway (same
5852/// `HCN_E_ADDR_INVALID_OR_RESERVED` avoidance the Internal/Transparent paths
5853/// use). Returns `None` when `subnet` has no usable gateway host.
5854#[cfg(target_os = "windows")]
5855fn shared_nat_settings(name: &str, subnet: &str) -> Option<zlayer_hns::schema::HostComputeNetwork> {
5856    use zlayer_hns::schema::{HostComputeNetwork, Ipam, NetworkType, Route, SchemaVersion, Subnet};
5857
5858    let net: ipnet::IpNet = subnet.parse().ok()?;
5859    let ipnet::IpNet::V4(v4) = net else {
5860        // HCN's NAT IPAM is IPv4 in the current schema.
5861        return None;
5862    };
5863    if v4.prefix_len() >= 31 {
5864        return None;
5865    }
5866    let gateway = std::net::Ipv4Addr::from(u32::from(v4.network()).checked_add(1)?).to_string();
5867
5868    Some(HostComputeNetwork {
5869        id: None,
5870        name: name.to_string(),
5871        ty: NetworkType::Nat,
5872        policies: Vec::new(),
5873        mac_pool: None,
5874        dns: None,
5875        ipams: vec![Ipam {
5876            ty: "Static".to_string(),
5877            subnets: vec![Subnet {
5878                ip_address_prefix: subnet.to_string(),
5879                routes: vec![Route {
5880                    next_hop: gateway,
5881                    destination_prefix: "0.0.0.0/0".to_string(),
5882                    metric: None,
5883                }],
5884                policies: Vec::new(),
5885            }],
5886        }],
5887        flags: 0,
5888        schema_version: SchemaVersion::default(),
5889    })
5890}
5891
5892/// Format a GUID as the bare, lowercase, un-braced string HCN/HCS use to
5893/// identify a namespace inside a compute-system document's
5894/// `Container.Networking.Namespace` field (e.g. `aabbccdd-eeff-...`).
5895#[cfg(target_os = "windows")]
5896fn format_guid_bare(id: windows::core::GUID) -> String {
5897    format!("{id:?}")
5898        .trim_matches(|c: char| c == '{' || c == '}')
5899        .to_ascii_lowercase()
5900}
5901
5902/// Delete every host-level HCN network this server created for `daemon_name` and
5903/// clear the persistent marker. Called on a full uninstall — never on a routine
5904/// stop/restart. Best-effort throughout. Synchronous (HCN calls are blocking).
5905#[cfg(target_os = "windows")]
5906pub fn purge_managed_networks(data_dir: &Path, daemon_name: &str) {
5907    use windows::core::GUID;
5908
5909    let marker_path = zlayer_paths::ZLayerDirs::new(data_dir.to_path_buf()).agent_network_state();
5910    let state = crate::network_state::NetworkState::load(&marker_path);
5911
5912    // Pass 1: delete recorded HCN networks by GUID.
5913    for entry in &state.networks {
5914        if !entry.kind.starts_with("hcn") {
5915            continue;
5916        }
5917        match GUID::try_from(entry.id.as_str()) {
5918            Ok(guid) => match zlayer_hns::network::Network::delete(guid) {
5919                Ok(()) => {
5920                    tracing::info!(name = %entry.name, id = %entry.id, "deleted managed HCN network");
5921                }
5922                Err(e) => {
5923                    tracing::warn!(name = %entry.name, id = %entry.id, error = %e, "failed to delete managed HCN network");
5924                }
5925            },
5926            Err(e) => {
5927                tracing::warn!(id = %entry.id, error = %e, "managed network marker has unparseable GUID");
5928            }
5929        }
5930    }
5931
5932    // Pass 2: name-sweep fallback for an overlay network whose marker entry was
5933    // lost (crash between create and marker write).
5934    let overlay_name = overlay_network_name(daemon_name);
5935    if let Ok(guids) = zlayer_hns::network::list("{}") {
5936        for guid in guids {
5937            let Ok(network) = zlayer_hns::network::Network::open(guid) else {
5938                continue;
5939            };
5940            let is_ours = matches!(network.query("{}"), Ok(props) if props.name == overlay_name);
5941            drop(network);
5942            if is_ours {
5943                match zlayer_hns::network::Network::delete(guid) {
5944                    Ok(()) => {
5945                        tracing::info!(name = %overlay_name, "deleted overlay HCN network (name sweep)");
5946                    }
5947                    Err(e) => {
5948                        tracing::warn!(name = %overlay_name, error = %e, "failed to delete overlay network (name sweep)");
5949                    }
5950                }
5951            }
5952        }
5953    }
5954
5955    if marker_path.exists() {
5956        if let Err(e) = std::fs::remove_file(&marker_path) {
5957            tracing::warn!(error = %e, path = %marker_path.display(), "failed to remove agent network marker");
5958        }
5959    }
5960}
5961
5962#[cfg(test)]
5963mod tests {
5964    use super::*;
5965
5966    // -- edge peers ----------------------------------------------------------
5967
5968    /// An [`EdgeAttachInfo`] with the given mint/expiry clocks; the other fields
5969    /// are irrelevant to [`OverlaydServer::edge_reap_due`] (a pure predicate over
5970    /// `minted_at_unix` / `expires_at_unix` + the handshake arg).
5971    fn edge_info(minted_at_unix: u64, expires_at_unix: u64) -> EdgeAttachInfo {
5972        EdgeAttachInfo {
5973            overlay_ip: "10.200.0.9".parse().unwrap(),
5974            public_key: "edge-pub-b64".to_string(),
5975            minted_at_unix,
5976            expires_at_unix,
5977            allowed_targets: Vec::new(),
5978            isolation_network: None,
5979        }
5980    }
5981
5982    #[test]
5983    fn edge_reap_due_truth_table() {
5984        // Minted at t=1000 with a 1h TTL (expires at 4600).
5985        let info = edge_info(1000, 4600);
5986
5987        // Fresh, never handshaked, within the 120s boot grace -> retained.
5988        assert!(!OverlaydServer::edge_reap_due(&info, None, 1060));
5989        // Never handshaked, boot grace elapsed -> reaped.
5990        assert!(OverlaydServer::edge_reap_due(&info, None, 1200));
5991        // A `0` (and a nonsensical negative) handshake reads as "never".
5992        assert!(!OverlaydServer::edge_reap_due(&info, Some(0), 1060));
5993        assert!(OverlaydServer::edge_reap_due(&info, Some(0), 1200));
5994        assert!(OverlaydServer::edge_reap_due(&info, Some(-5), 1200));
5995
5996        // Handshaked at t=2000, still fresh (silent 100s < 180) -> retained.
5997        assert!(!OverlaydServer::edge_reap_due(&info, Some(2000), 2100));
5998        // Handshaked then silent past the 180s timeout -> reaped.
5999        assert!(OverlaydServer::edge_reap_due(&info, Some(2000), 2300));
6000
6001        // TTL expiry wins regardless of a healthy recent handshake.
6002        assert!(OverlaydServer::edge_reap_due(&info, Some(4590), 4600));
6003    }
6004
6005    #[test]
6006    fn edge_reap_due_boundaries() {
6007        let info = edge_info(1000, 4600);
6008
6009        // Boot-grace boundary: reaped at exactly +120s, retained at +119s.
6010        assert!(!OverlaydServer::edge_reap_due(&info, None, 1000 + 119));
6011        assert!(OverlaydServer::edge_reap_due(&info, None, 1000 + 120));
6012
6013        // Silence boundary: reaped at exactly +180s after handshake, retained at
6014        // +179s.
6015        assert!(!OverlaydServer::edge_reap_due(
6016            &info,
6017            Some(2000),
6018            2000 + 179
6019        ));
6020        assert!(OverlaydServer::edge_reap_due(&info, Some(2000), 2000 + 180));
6021
6022        // TTL boundary: reaped at exactly expires_at, retained one second before
6023        // (with a fresh handshake so only the TTL can trigger it).
6024        assert!(!OverlaydServer::edge_reap_due(&info, Some(4599), 4599));
6025        assert!(OverlaydServer::edge_reap_due(&info, Some(4599), 4600));
6026    }
6027
6028    #[test]
6029    fn edge_node_endpoint_validation() {
6030        // Valid literal socket addresses (v4 + v6) and a hostname:port.
6031        assert!(validate_edge_node_endpoint("203.0.113.7:51820").is_ok());
6032        assert!(validate_edge_node_endpoint("[2001:db8::1]:51820").is_ok());
6033        assert!(validate_edge_node_endpoint("node.example.com:51820").is_ok());
6034
6035        // Empty / whitespace-only.
6036        assert!(validate_edge_node_endpoint("").is_err());
6037        assert!(validate_edge_node_endpoint("   ").is_err());
6038        // Unspecified address (a literal 0.0.0.0 / ::).
6039        assert!(validate_edge_node_endpoint("0.0.0.0:51820").is_err());
6040        assert!(validate_edge_node_endpoint("[::]:51820").is_err());
6041        // Portless / zero-port.
6042        assert!(validate_edge_node_endpoint("203.0.113.7").is_err());
6043        assert!(validate_edge_node_endpoint("203.0.113.7:0").is_err());
6044        assert!(validate_edge_node_endpoint("node.example.com").is_err());
6045    }
6046
6047    #[test]
6048    fn edge_host_route_formats_v4_and_v6() {
6049        assert_eq!(
6050            edge_host_route("10.200.0.9".parse().unwrap()),
6051            "10.200.0.9/32"
6052        );
6053        assert_eq!(edge_host_route("fd00::9".parse().unwrap()), "fd00::9/128");
6054    }
6055
6056    // -- edge peer lifecycle on a REAL WireGuard transport (A3 proof) ---------
6057    //
6058    // The tests below are the transport-touching half of the edge proof: they
6059    // stand up a live global overlay (a real boringtun `WireGuard` device) and
6060    // assert that `mint_edge_peer` REGISTERS the peer's `/32` on the device
6061    // (visible in the UAPI `status()` dump), that `revoke_edge_peer` and the
6062    // TTL/silence sweep REMOVE it, and that the freed overlay IP is reusable.
6063    //
6064    // A real device needs `CAP_NET_ADMIN` (Linux, via `sudo`) or root (macOS),
6065    // so every test carries `#[ignore]`. They are deliberately NOT
6066    // `#[cfg(target_os = "linux")]`-gated (unlike the crate's older privileged
6067    // e2e tests): the entire edge code path — mint/revoke/sweep, IPAM, keygen,
6068    // the roaming `/32` peer — is platform-agnostic (boringtun userspace
6069    // `WireGuard` runs on Linux/macOS/Windows), so leaving them un-gated lets
6070    // the suite COMPILE-check them on any dev host while they only EXECUTE on a
6071    // privileged host under `--ignored`. Run them with:
6072    //
6073    // ```sh
6074    // # Linux runner (overlayd has no cargo features — do NOT pass --features):
6075    // sudo -E cargo test -p zlayer-overlayd -- --ignored --nocapture edge_lifecycle
6076    // # macOS (as root):
6077    // sudo -E cargo test -p zlayer-overlayd -- --ignored --nocapture edge_lifecycle
6078    // ```
6079
6080    /// Stand up a fresh [`OverlaydServer`] with a LIVE global overlay for an
6081    /// edge-lifecycle test. Each call takes a UNIQUE deployment name, `WireGuard`
6082    /// listen port, and UAPI socket dir so the `#[ignore]`d tests can run in
6083    /// parallel (cargo's default) without clobbering each other's kernel
6084    /// interface, UDP port, or control socket. Requires a real device:
6085    /// `CAP_NET_ADMIN` on Linux (`sudo`) or root on macOS.
6086    async fn edge_lifecycle_server(deployment: &str, wg_port: u16) -> OverlaydServer {
6087        let base = std::env::temp_dir().join(format!(
6088            "zlayer-overlayd-edge-{deployment}-{}-{}",
6089            std::process::id(),
6090            now_unix()
6091        ));
6092        let uapi = base.join("uapi");
6093        std::fs::create_dir_all(&uapi).expect("create per-test UAPI socket dir");
6094
6095        let mut server = OverlaydServer::new(base).with_uapi_sock_dir(uapi);
6096        server
6097            .setup_global_overlay(
6098                deployment.to_string(),
6099                "e".to_string(),
6100                "10.200.0.0/16",
6101                Some("10.200.0.0/28"),
6102                wg_port,
6103                None,
6104                false,
6105            )
6106            .await
6107            .expect("global overlay up (requires a real WireGuard device)");
6108        assert!(
6109            server.global_transport.is_some(),
6110            "edge lifecycle tests require a REAL WireGuard device (a live \
6111             `global_transport`). Run under sudo on Linux (CAP_NET_ADMIN) or as \
6112             root on macOS; a degraded VM-only overlay (host adapter unavailable) \
6113             leaves global_transport == None and cannot exercise the transport."
6114        );
6115        server
6116    }
6117
6118    /// Does the live UAPI `status()` dump carry the edge peer keyed by
6119    /// `edge_pub_b64` (base64, as [`OverlayTransport::generate_keys`] returns it)
6120    /// with `overlay_ip`'s host route in its `AllowedIPs`?
6121    ///
6122    /// The live device reports peer keys in HEX; [`parse_peer_status`] surfaces
6123    /// them verbatim. The edge's key is base64, so we convert it via
6124    /// [`zlayer_overlay::nat::pubkey_b64_to_hex`] — exactly as
6125    /// [`OverlaydServer::edge_handshake_map`] does — before the (case-insensitive)
6126    /// lookup.
6127    fn dump_has_edge_slash32(dump: &str, edge_pub_b64: &str, overlay_ip: IpAddr) -> bool {
6128        let Some(hex) = zlayer_overlay::nat::pubkey_b64_to_hex(edge_pub_b64) else {
6129            return false;
6130        };
6131        let want = edge_host_route(overlay_ip);
6132        parse_peer_status(dump)
6133            .into_iter()
6134            .filter(|p| p.public_key.eq_ignore_ascii_case(&hex))
6135            .any(|p| p.allowed_ips.split(',').any(|cidr| cidr == want))
6136    }
6137
6138    /// Read the live global transport's UAPI dump (the test asserts against a
6139    /// real device, so a failed dump is a hard error, not an empty fallback).
6140    async fn transport_dump(server: &OverlaydServer) -> String {
6141        server
6142            .global_transport
6143            .as_ref()
6144            .expect("global transport present")
6145            .status()
6146            .await
6147            .expect("WireGuard UAPI status dump")
6148    }
6149
6150    /// A3 proof, mint→revoke: a minted edge registers a `/32` on the real WG
6151    /// transport (visible in the UAPI dump under its hex-converted pubkey), and
6152    /// revoke removes it from both the device and the bookkeeping (idempotently).
6153    #[tokio::test]
6154    #[ignore = "requires CAP_NET_ADMIN to create a real WireGuard device"]
6155    async fn edge_lifecycle_mint_registers_slash32_peer_then_revoke_removes_it() {
6156        let mut server = edge_lifecycle_server("edgl1", 53710).await;
6157
6158        let config = server
6159            .mint_edge_peer("t1".to_string(), 3600, &[], "203.0.113.1:51820")
6160            .await
6161            .expect("mint edge peer t1");
6162
6163        // Returned config shape.
6164        assert_eq!(config.version, EDGE_CONFIG_VERSION);
6165        assert_eq!(config.name, "t1");
6166        assert_eq!(
6167            config.peers.len(),
6168            1,
6169            "the edge's single artifact peer is THIS node"
6170        );
6171        let slice: ipnet::IpNet = "10.200.0.0/28".parse().unwrap();
6172        assert!(
6173            slice.contains(&config.overlay_ip),
6174            "edge overlay_ip {} must fall inside the node slice {slice}",
6175            config.overlay_ip
6176        );
6177
6178        // Bookkeeping recorded.
6179        assert!(
6180            server.edge_attachments.contains_key("t1"),
6181            "mint must record the edge attachment"
6182        );
6183
6184        // The minted /32 must be LIVE on the real transport.
6185        let dump = transport_dump(&server).await;
6186        assert!(
6187            dump_has_edge_slash32(&dump, &config.public_key, config.overlay_ip),
6188            "minted edge pubkey (hex) with {} must appear in the WireGuard UAPI \
6189             dump after mint:\n{dump}",
6190            edge_host_route(config.overlay_ip)
6191        );
6192
6193        // Revoke drops it from the device AND the bookkeeping.
6194        let removed = server.revoke_edge_peer("t1").await.expect("revoke t1");
6195        assert!(removed, "revoke of a live edge returns Ok(true)");
6196        assert!(
6197            server.edge_attachments.is_empty(),
6198            "revoke must drop the edge attachment"
6199        );
6200        let dump = transport_dump(&server).await;
6201        assert!(
6202            !dump_has_edge_slash32(&dump, &config.public_key, config.overlay_ip),
6203            "revoked edge pubkey must be GONE from the WireGuard UAPI dump:\n{dump}"
6204        );
6205
6206        // Idempotent: a second revoke reports nothing removed.
6207        let removed_again = server
6208            .revoke_edge_peer("t1")
6209            .await
6210            .expect("second revoke does not error");
6211        assert!(
6212            !removed_again,
6213            "second revoke of an absent edge is Ok(false)"
6214        );
6215    }
6216
6217    /// A3 proof, TTL sweep: a peer minted with a 1s TTL is reaped by
6218    /// `sweep_edge_peers(now)` for any `now` past its expiry — removed from the
6219    /// device and the bookkeeping. `now_unix` is injected, so no real waiting.
6220    #[tokio::test]
6221    #[ignore = "requires CAP_NET_ADMIN to create a real WireGuard device"]
6222    async fn edge_lifecycle_ttl_sweep_reaps_expired_peer() {
6223        let mut server = edge_lifecycle_server("edgl2", 53711).await;
6224
6225        let config = server
6226            .mint_edge_peer("ttl".to_string(), 1, &[], "203.0.113.1:51820")
6227            .await
6228            .expect("mint edge peer ttl");
6229
6230        // Present on the device right after mint.
6231        let dump = transport_dump(&server).await;
6232        assert!(dump_has_edge_slash32(
6233            &dump,
6234            &config.public_key,
6235            config.overlay_ip
6236        ));
6237
6238        // Sweep at a clock strictly past the 1s TTL (minted_at + 2).
6239        let minted_at = server
6240            .edge_attachments
6241            .get("ttl")
6242            .expect("attachment recorded")
6243            .minted_at_unix;
6244        server.sweep_edge_peers(minted_at + 2).await;
6245
6246        assert!(
6247            server.edge_attachments.is_empty(),
6248            "TTL-expired edge must be swept from the bookkeeping"
6249        );
6250        let dump = transport_dump(&server).await;
6251        assert!(
6252            !dump_has_edge_slash32(&dump, &config.public_key, config.overlay_ip),
6253            "TTL-expired edge pubkey must be GONE from the WireGuard UAPI dump:\n{dump}"
6254        );
6255    }
6256
6257    /// A3 proof, silence sweep: an edge that never handshakes (no real client
6258    /// connects) is retained through its boot-grace window and reaped once the
6259    /// grace elapses — even though its TTL is far in the future, so ONLY the
6260    /// silence path can trigger it. `now_unix` is injected.
6261    #[tokio::test]
6262    #[ignore = "requires CAP_NET_ADMIN to create a real WireGuard device"]
6263    async fn edge_lifecycle_silence_sweep_reaps_never_handshaked_peer() {
6264        let mut server = edge_lifecycle_server("edgl3", 53712).await;
6265
6266        // Long TTL so the TTL path can never be what reaps it.
6267        let config = server
6268            .mint_edge_peer("silent".to_string(), 3600, &[], "203.0.113.1:51820")
6269            .await
6270            .expect("mint edge peer silent");
6271        let minted_at = server
6272            .edge_attachments
6273            .get("silent")
6274            .expect("attachment recorded")
6275            .minted_at_unix;
6276        let grace = OverlaydServer::EDGE_CONNECT_GRACE_SECS;
6277
6278        // Control: one second BEFORE the boot grace elapses, the never-handshaked
6279        // edge is still retained (nothing to reap yet).
6280        server.sweep_edge_peers(minted_at + grace - 1).await;
6281        assert!(
6282            server.edge_attachments.contains_key("silent"),
6283            "edge within its boot-grace window must survive the sweep"
6284        );
6285        let dump = transport_dump(&server).await;
6286        assert!(
6287            dump_has_edge_slash32(&dump, &config.public_key, config.overlay_ip),
6288            "edge within its boot-grace window must still be on the device"
6289        );
6290
6291        // One second AFTER the grace: the never-handshaked edge is reaped.
6292        server.sweep_edge_peers(minted_at + grace + 1).await;
6293        assert!(
6294            server.edge_attachments.is_empty(),
6295            "silent edge past its boot-grace window must be swept"
6296        );
6297        let dump = transport_dump(&server).await;
6298        assert!(
6299            !dump_has_edge_slash32(&dump, &config.public_key, config.overlay_ip),
6300            "swept silent edge pubkey must be GONE from the WireGuard UAPI dump:\n{dump}"
6301        );
6302    }
6303
6304    /// A3 proof, IP reuse: the `/32` freed by revoking one edge is reused by the
6305    /// next mint (the allocator drains its released free-list before advancing
6306    /// the monotonic counter), and the new peer is live under a fresh key.
6307    #[tokio::test]
6308    #[ignore = "requires CAP_NET_ADMIN to create a real WireGuard device"]
6309    async fn edge_lifecycle_freed_ip_is_reusable() {
6310        let mut server = edge_lifecycle_server("edgl4", 53713).await;
6311
6312        let a = server
6313            .mint_edge_peer("a".to_string(), 3600, &[], "203.0.113.1:51820")
6314            .await
6315            .expect("mint a");
6316        let ip_a = a.overlay_ip;
6317
6318        let removed = server.revoke_edge_peer("a").await.expect("revoke a");
6319        assert!(removed, "revoke of the live edge `a` returns Ok(true)");
6320
6321        let b = server
6322            .mint_edge_peer("b".to_string(), 3600, &[], "203.0.113.2:51820")
6323            .await
6324            .expect("mint b");
6325
6326        assert_eq!(
6327            b.overlay_ip, ip_a,
6328            "the /32 freed by revoking `a` must be reused by the next mint `b`"
6329        );
6330        assert_ne!(
6331            a.public_key, b.public_key,
6332            "each mint generates its own fresh keypair"
6333        );
6334
6335        // The reused /32 is live on the device under b's (new) key.
6336        let dump = transport_dump(&server).await;
6337        assert!(
6338            dump_has_edge_slash32(&dump, &b.public_key, b.overlay_ip),
6339            "the reused /32 must be live under b's fresh pubkey:\n{dump}"
6340        );
6341    }
6342
6343    #[cfg(target_os = "linux")]
6344    #[test]
6345    fn orphan_bridge_selection() {
6346        use std::collections::HashSet;
6347
6348        // Two live per-service bridges the daemon says SHOULD exist.
6349        let live: HashSet<&str> = ["zl-prod-0-web-b", "zl-prod-0-api-b"].into_iter().collect();
6350        // The active global device and node-wide shared bridge are protected,
6351        // plus a live in-memory dedicated device.
6352        let protected: HashSet<String> = ["zl-prod-0-g", "zl-prod-0-shared-sh", "zl-prod-0-db-d"]
6353            .into_iter()
6354            .map(String::from)
6355            .collect();
6356
6357        // The full set of host links the kernel would report.
6358        let host_links = [
6359            // Live -> keep.
6360            "zl-prod-0-web-b",
6361            "zl-prod-0-api-b",
6362            // Protected global / shared / live dedicated device -> keep.
6363            "zl-prod-0-g",
6364            "zl-prod-0-shared-sh",
6365            "zl-prod-0-db-d",
6366            // Orphan bridges (the user's observed leaks) -> reclaim.
6367            "zl-1ca4568944-b",
6368            "zl-81c6bc17c7-b",
6369            // Orphan dedicated device -> reclaim.
6370            "zl-prod-0-gone-d",
6371            // Container veths owned by the PID-keyed sweep, never here -> skip.
6372            "veth-4242-s",
6373            "vc-4242-g",
6374            // Unrelated host links -> skip.
6375            "eth0",
6376            "lo",
6377            "docker0",
6378            "zl-not-a-bridge",
6379        ];
6380
6381        let orphans: Vec<&str> = host_links
6382            .into_iter()
6383            .filter(|n| is_orphan_service_bridge(n, &live, &protected))
6384            .collect();
6385
6386        assert_eq!(
6387            orphans,
6388            vec!["zl-1ca4568944-b", "zl-81c6bc17c7-b", "zl-prod-0-gone-d"],
6389            "only orphaned -b/-d service bridges/devices are selected; \
6390             live, protected (-g/-sh/live -d), veth, and unrelated links are excluded"
6391        );
6392    }
6393
6394    #[test]
6395    fn peer_spec_to_info_parses_endpoint_and_keepalive() {
6396        let spec = PeerSpec {
6397            public_key: "base64key".to_string(),
6398            endpoint: "1.2.3.4:51820".to_string(),
6399            allowed_ips: "10.200.0.5/32,10.200.1.0/24".to_string(),
6400            persistent_keepalive_secs: 25,
6401            candidates: Vec::new(),
6402        };
6403        let info = peer_spec_to_info(&spec).expect("valid spec");
6404        assert_eq!(info.public_key, "base64key");
6405        assert_eq!(info.endpoint, "1.2.3.4:51820".parse().unwrap());
6406        assert_eq!(info.allowed_ips, "10.200.0.5/32,10.200.1.0/24");
6407        assert_eq!(
6408            info.persistent_keepalive_interval,
6409            std::time::Duration::from_secs(25)
6410        );
6411    }
6412
6413    #[test]
6414    fn peer_spec_to_info_rejects_bad_endpoint() {
6415        let spec = PeerSpec {
6416            public_key: "k".to_string(),
6417            endpoint: "not-a-socket-addr".to_string(),
6418            allowed_ips: String::new(),
6419            persistent_keepalive_secs: 0,
6420            candidates: Vec::new(),
6421        };
6422        assert!(peer_spec_to_info(&spec).is_err());
6423    }
6424
6425    #[test]
6426    fn interface_name_never_exceeds_limit() {
6427        let cases: Vec<(&[&str], &str)> = vec![
6428            (&["a"], "g"),
6429            (&["zlayer-manager"], "g"),
6430            (&["my-very-long-deployment-name-that-goes-on-and-on"], "g"),
6431            (&["zlayer", "manager"], "s"),
6432            (
6433                &["abcdefghijklmnopqrstuvwxyz", "abcdefghijklmnopqrstuvwxyz"],
6434                "s",
6435            ),
6436            (&["x"], ""),
6437        ];
6438        for (parts, suffix) in &cases {
6439            let name = make_interface_name(parts, suffix);
6440            assert!(name.len() <= MAX_IFNAME_LEN, "Name '{name}' too long");
6441            assert!(name.starts_with("zl-"));
6442        }
6443    }
6444
6445    #[test]
6446    fn node_ip_is_first_usable_and_reserved() {
6447        let cidr: IpNetwork = "10.200.0.0/26".parse().unwrap();
6448        let alloc = IpAllocator::new(cidr);
6449
6450        // The node IP is the deterministic first-usable host of the slice.
6451        let expected_node_ip: IpAddr = "10.200.0.1".parse().unwrap();
6452        assert_eq!(alloc.node_ip(), expected_node_ip);
6453
6454        // Several container allocations must NEVER hand out the node IP, and
6455        // the node IP stays put regardless of allocation order.
6456        let mut handed_out = Vec::new();
6457        for _ in 0..10 {
6458            let ip = alloc.allocate().expect("slice not exhausted");
6459            assert_ne!(
6460                ip, expected_node_ip,
6461                "allocate() returned the reserved node IP"
6462            );
6463            handed_out.push(ip);
6464        }
6465        // Reservation holds after the allocations.
6466        assert_eq!(alloc.node_ip(), expected_node_ip);
6467
6468        // First container allocation is offset 2 (base + 2), proving offset 1
6469        // (the node) was reserved and skipped.
6470        assert_eq!(handed_out[0], "10.200.0.2".parse::<IpAddr>().unwrap());
6471
6472        // Releasing the node IP must not pollute the free pool with it.
6473        alloc.release(expected_node_ip);
6474        let next = alloc.allocate().expect("slice not exhausted");
6475        assert_ne!(
6476            next, expected_node_ip,
6477            "node IP leaked back into the pool via release()"
6478        );
6479    }
6480
6481    #[test]
6482    fn node_ip_ipv6_is_first_usable() {
6483        let cidr: IpNetwork = "fd00:200::/64".parse().unwrap();
6484        let alloc = IpAllocator::new(cidr);
6485        let expected: IpAddr = "fd00:200::1".parse().unwrap();
6486        assert_eq!(alloc.node_ip(), expected);
6487        for _ in 0..5 {
6488            assert_ne!(alloc.allocate().unwrap(), expected);
6489        }
6490        assert_eq!(alloc.node_ip(), expected);
6491    }
6492
6493    #[test]
6494    fn interface_name_is_deterministic() {
6495        assert_eq!(
6496            make_interface_name(&["zlayer-manager"], "g"),
6497            make_interface_name(&["zlayer-manager"], "g")
6498        );
6499    }
6500
6501    #[test]
6502    fn parse_peer_status_splits_blocks() {
6503        let dump = "\
6504public_key=AAA
6505endpoint=1.2.3.4:51820
6506allowed_ip=10.200.0.2/32
6507allowed_ip=10.200.1.0/24
6508latest_handshake=1700000000
6509public_key=BBB
6510endpoint=5.6.7.8:51820
6511allowed_ip=10.200.0.3/32
6512latest_handshake=0
6513";
6514        let peers = parse_peer_status(dump);
6515        assert_eq!(peers.len(), 2);
6516        assert_eq!(peers[0].public_key, "AAA");
6517        assert_eq!(peers[0].endpoint, "1.2.3.4:51820");
6518        assert_eq!(peers[0].allowed_ips, "10.200.0.2/32,10.200.1.0/24");
6519        assert_eq!(peers[0].last_handshake_unix_secs, 1_700_000_000);
6520        assert_eq!(peers[1].public_key, "BBB");
6521        assert_eq!(peers[1].last_handshake_unix_secs, 0);
6522    }
6523
6524    #[tokio::test]
6525    async fn status_snapshot_before_setup_is_empty() {
6526        let server = OverlaydServer::new(std::path::PathBuf::from("/tmp/zlayer-overlayd-test"));
6527        let snap = server.status_snapshot().await;
6528        assert!(snap.interface.is_none());
6529        assert!(snap.node_ip.is_none());
6530        assert!(snap.public_key.is_none());
6531        assert_eq!(snap.peer_count, 0);
6532        assert_eq!(snap.service_count, 0);
6533        assert!(snap.peers.is_empty());
6534    }
6535
6536    #[tokio::test]
6537    async fn allocate_and_release_ip_round_trip() {
6538        let mut server = OverlaydServer::new(std::path::PathBuf::from("/tmp/zlayer-overlayd-test"));
6539        let a = server.allocate_ip("svc", false).expect("alloc a");
6540        let b = server.allocate_ip("svc", false).expect("alloc b");
6541        assert_ne!(a, b);
6542        server.release_ip(a);
6543        // Released IP is handed back before the monotonic counter advances.
6544        let c = server.allocate_ip("svc", false).expect("alloc c");
6545        assert_eq!(c, a);
6546    }
6547
6548    /// Build a throwaway server bound to a unique temp data dir so the marker
6549    /// file (rehydrated in `new`) never collides between tests.
6550    fn test_server() -> OverlaydServer {
6551        let dir = std::env::temp_dir().join(format!(
6552            "zlayer-overlayd-scope-{}-{}",
6553            std::process::id(),
6554            now_unix()
6555        ));
6556        OverlaydServer::new(dir)
6557    }
6558
6559    /// `nat_config_spec_to_config` fills sparse fields from `NatConfig::default`
6560    /// and copies populated ones verbatim (the Step-0 wire-config threading).
6561    #[test]
6562    fn nat_config_spec_to_config_fills_defaults_and_copies() {
6563        // Empty spec → defaults (default STUN servers, default timeouts).
6564        let cfg = nat_config_spec_to_config(NatConfigSpec::default());
6565        let d = NatConfig::default();
6566        assert_eq!(cfg.stun_servers.len(), d.stun_servers.len());
6567        assert_eq!(cfg.hole_punch_timeout_secs, d.hole_punch_timeout_secs);
6568        assert_eq!(cfg.max_candidate_pairs, d.max_candidate_pairs);
6569        assert!(cfg.relay_server.is_none());
6570
6571        // Populated spec → copied verbatim; relay credential is NOT on the
6572        // produced RelayServerConfig (it is carried separately on the server).
6573        let spec = NatConfigSpec {
6574            enabled: true,
6575            stun_servers: vec!["stun.example:3478".to_string()],
6576            turn_servers: vec![zlayer_types::nat_wire::TurnServerSpec {
6577                addr: "turn.example:3478".to_string(),
6578                username: "u".to_string(),
6579                credential: "p".to_string(),
6580            }],
6581            hole_punch_timeout_secs: 9,
6582            stun_refresh_interval_secs: 40,
6583            max_candidate_pairs: 3,
6584            relay_server: Some(zlayer_types::nat_wire::RelayServerSpec {
6585                listen_port: 3478,
6586                external_addr: "1.2.3.4:3478".to_string(),
6587                max_sessions: 7,
6588                auth_credential: Some("cluster-secret".to_string()),
6589            }),
6590        };
6591        let cfg = nat_config_spec_to_config(spec);
6592        assert_eq!(cfg.stun_servers.len(), 1);
6593        assert_eq!(cfg.stun_servers[0].address, "stun.example:3478");
6594        assert_eq!(cfg.turn_servers.len(), 1);
6595        assert_eq!(cfg.hole_punch_timeout_secs, 9);
6596        assert_eq!(cfg.max_candidate_pairs, 3);
6597        let relay = cfg.relay_server.expect("relay present");
6598        assert_eq!(relay.listen_port, 3478);
6599        assert_eq!(relay.max_sessions, 7);
6600    }
6601
6602    /// `wire_to_candidate` parses valid candidates and rejects bad ones;
6603    /// `candidate_to_wire` is its inverse for the type/address/priority triple.
6604    #[test]
6605    fn candidate_wire_conversions_round_trip() {
6606        let w = NatCandidateWire {
6607            candidate_type: "server-reflexive".to_string(),
6608            address: "203.0.113.5:51820".to_string(),
6609            priority: 50,
6610        };
6611        let c = wire_to_candidate(&w).expect("valid candidate");
6612        assert_eq!(c.candidate_type, CandidateType::ServerReflexive);
6613        assert_eq!(c.priority, 50);
6614        let back = candidate_to_wire(&c);
6615        assert_eq!(back, w);
6616
6617        // Bad address / type → None.
6618        assert!(wire_to_candidate(&NatCandidateWire {
6619            candidate_type: "host".to_string(),
6620            address: "not-an-addr".to_string(),
6621            priority: 1,
6622        })
6623        .is_none());
6624        assert!(wire_to_candidate(&NatCandidateWire {
6625            candidate_type: "bogus".to_string(),
6626            address: "1.2.3.4:5".to_string(),
6627            priority: 1,
6628        })
6629        .is_none());
6630    }
6631
6632    /// `AddPeer` carrying candidates records them in `peer_candidates`; a
6633    /// candidate-free add (or one with only-invalid candidates) leaves no entry,
6634    /// and `RemovePeer` clears them.
6635    #[tokio::test]
6636    async fn add_peer_records_candidates_and_remove_clears_them() {
6637        let mut server = test_server();
6638        let pubkey = "base64key".to_string();
6639        let resp = server
6640            .handle(OverlaydRequest::AddPeer {
6641                peer: PeerSpec {
6642                    public_key: pubkey.clone(),
6643                    endpoint: "1.2.3.4:51820".to_string(),
6644                    allowed_ips: "10.200.0.2/32".to_string(),
6645                    persistent_keepalive_secs: 25,
6646                    candidates: vec![NatCandidateWire {
6647                        candidate_type: "host".to_string(),
6648                        address: "192.168.1.5:51820".to_string(),
6649                        priority: 100,
6650                    }],
6651                },
6652                scope: PeerScope::Global,
6653            })
6654            .await;
6655        assert!(matches!(resp, OverlaydResponse::Ok));
6656        assert_eq!(
6657            server.peer_candidates.get(&pubkey).map(Vec::len),
6658            Some(1),
6659            "candidates must be recorded"
6660        );
6661
6662        // Remove clears the candidate + connection-type bookkeeping.
6663        let resp = server
6664            .handle(OverlaydRequest::RemovePeer {
6665                pubkey: pubkey.clone(),
6666                scope: PeerScope::Global,
6667            })
6668            .await;
6669        assert!(matches!(resp, OverlaydResponse::Ok));
6670        assert!(!server.peer_candidates.contains_key(&pubkey));
6671    }
6672
6673    /// `NatStatus` returns a `NatStatusWire` (empty before any tick) — proving
6674    /// the new IPC pair is wired through `dispatch`.
6675    #[tokio::test]
6676    async fn nat_status_request_returns_wire_snapshot() {
6677        let mut server = test_server();
6678        let resp = server.handle(OverlaydRequest::NatStatus).await;
6679        match resp {
6680            OverlaydResponse::NatStatus(wire) => {
6681                assert!(wire.candidates.is_empty());
6682                assert!(wire.peers.is_empty());
6683            }
6684            other => panic!("expected NatStatus response, got {other:?}"),
6685        }
6686    }
6687
6688    /// True when the process can mutate netlink + `/proc/sys` (root). The
6689    /// teardown-completeness test below is `#[ignore]`d and additionally skips
6690    /// (not fails) when run via `--ignored` without privileges, matching the
6691    /// crate's "skip gracefully when not root" convention.
6692    #[cfg(target_os = "linux")]
6693    fn is_root() -> bool {
6694        // SAFETY: `geteuid` is a pure read of the caller's effective uid.
6695        #[allow(unsafe_code)]
6696        let euid = unsafe { libc::geteuid() };
6697        euid == 0
6698    }
6699
6700    /// End-to-end teardown completeness: populate the server's
6701    /// `created_veths` / `created_bridges` / `created_host_routes` tracking sets
6702    /// with REAL host resources created via netlink, snapshot
6703    /// `net.ipv4.ip_forward`, force it to `1` (recording the prior value in
6704    /// `prev_ipv4_forward` exactly as `enable_forwarding_for_attach` does), then
6705    /// drive the same teardown the `Shutdown` request triggers
6706    /// (`handle(OverlaydRequest::Shutdown)`), and assert: every tracked veth /
6707    /// bridge / route is gone at the kernel level AND `ip_forward` is restored to
6708    /// the snapshot.
6709    ///
6710    /// This is the regression for the full teardown fix (revert routes + veths +
6711    /// bridges + forwarding sysctl on shutdown). Names are unique and <=15 chars;
6712    /// a belt-and-braces cleanup runs before the asserts so a failed assertion
6713    /// still leaves the host clean. Skips (returns) when not root.
6714    #[cfg(target_os = "linux")]
6715    #[tokio::test(flavor = "multi_thread")]
6716    #[ignore = "needs CAP_NET_ADMIN + /proc/sys write; run on a privileged Linux host"]
6717    async fn shutdown_teardown_reverts_resources_and_ip_forward() {
6718        if !is_root() {
6719            eprintln!("skipping shutdown_teardown_reverts_resources_and_ip_forward: requires root");
6720            return;
6721        }
6722
6723        let suffix = format!("{:x}", now_unix() & 0xff_ffff);
6724        let veth_host = format!("vh-{suffix}");
6725        let veth_peer = format!("vp-{suffix}");
6726        let bridge = format!("zlb-{suffix}");
6727        assert!(veth_host.len() <= 15, "veth host name exceeds IFNAMSIZ");
6728        assert!(veth_peer.len() <= 15, "veth peer name exceeds IFNAMSIZ");
6729        assert!(bridge.len() <= 15, "bridge name exceeds IFNAMSIZ");
6730
6731        let dest = IpAddr::V4(Ipv4Addr::new(10, 233, 0, 9));
6732        let prefix: u8 = 32;
6733
6734        // --- create real host resources and register them with the server's
6735        // teardown-tracking sets, exactly as the attach paths do. ---
6736        crate::netlink::create_veth_pair(&veth_host, &veth_peer)
6737            .await
6738            .expect("create_veth_pair");
6739        crate::netlink::create_bridge(&bridge)
6740            .await
6741            .expect("create_bridge");
6742        crate::netlink::replace_route_via_dev(dest, prefix, &veth_host, None)
6743            .await
6744            .expect("replace_route_via_dev");
6745
6746        let mut server = test_server();
6747        server.created_veths.insert(veth_host.clone());
6748        server.created_bridges.insert(bridge.clone());
6749        server
6750            .created_host_routes
6751            .push((dest, prefix, veth_host.clone()));
6752
6753        // Snapshot ip_forward, then flip it to 1 and record the prior value the
6754        // way enable_forwarding_for_attach does so revert_forwarding restores it.
6755        let snapshot =
6756            crate::netlink::read_sysctl("net.ipv4.ip_forward").unwrap_or_else(|_| "0".to_string());
6757        server.prev_ipv4_forward = Some(snapshot.clone());
6758        crate::netlink::set_sysctl("net.ipv4.ip_forward", "1").expect("set ip_forward=1");
6759
6760        // --- drive teardown via the real Shutdown dispatch path ---
6761        let resp = server.handle(OverlaydRequest::Shutdown).await;
6762        assert!(
6763            matches!(resp, OverlaydResponse::Ok),
6764            "Shutdown should return Ok, got {resp:?}"
6765        );
6766
6767        // Snapshot kernel state AFTER teardown.
6768        let veth_gone = !std::path::Path::new(&format!("/sys/class/net/{veth_host}")).exists();
6769        let bridge_gone = !std::path::Path::new(&format!("/sys/class/net/{bridge}")).exists();
6770        let route_gone = {
6771            let target = format!("10.233.0.9/{prefix}");
6772            std::process::Command::new("ip")
6773                .args(["route", "show", &target, "dev", &veth_host])
6774                .output()
6775                .map_or(true, |o| !o.status.success() || o.stdout.is_empty())
6776        };
6777        let ip_forward_after = crate::netlink::read_sysctl("net.ipv4.ip_forward")
6778            .unwrap_or_else(|_| "unknown".to_string());
6779
6780        // Belt-and-braces cleanup before asserting so the host stays clean even
6781        // if an assertion fails (teardown should have done all of this already).
6782        let _ = crate::netlink::delete_route_via_dev(dest, prefix, &veth_host).await;
6783        let _ = crate::netlink::delete_link_by_name(&veth_host).await;
6784        let _ = crate::netlink::delete_link_by_name(&veth_peer).await;
6785        let _ = crate::netlink::delete_link_by_name(&bridge).await;
6786        // Restore ip_forward to the snapshot regardless of teardown outcome.
6787        let _ = crate::netlink::set_sysctl("net.ipv4.ip_forward", &snapshot);
6788
6789        // --- assertions ---
6790        assert!(veth_gone, "teardown should delete the tracked host veth");
6791        assert!(bridge_gone, "teardown should delete the tracked bridge");
6792        assert!(
6793            route_gone,
6794            "teardown should delete the tracked /32 host route"
6795        );
6796        assert_eq!(
6797            ip_forward_after.trim(),
6798            snapshot.trim(),
6799            "teardown should restore net.ipv4.ip_forward to its pre-overlay value"
6800        );
6801
6802        // Tracking sets must be drained by teardown so a re-run starts clean.
6803        assert!(
6804            server.created_veths.is_empty(),
6805            "created_veths should be drained by teardown"
6806        );
6807        assert!(
6808            server.created_bridges.is_empty(),
6809            "created_bridges should be drained by teardown"
6810        );
6811        assert!(
6812            server.created_host_routes.is_empty(),
6813            "created_host_routes should be drained by teardown"
6814        );
6815    }
6816
6817    #[test]
6818    fn build_config_uses_matching_physical_egress_ipv4() {
6819        let server = test_server();
6820        let overlay_ip: IpAddr = "10.200.0.1".parse().unwrap();
6821        let egress: IpAddr = "192.0.2.10".parse().unwrap();
6822        let config = server.build_config(
6823            "priv".to_string(),
6824            "pub".to_string(),
6825            overlay_ip,
6826            16,
6827            51820,
6828            Some(egress),
6829        );
6830        assert_eq!(config.local_endpoint, SocketAddr::new(egress, 51820));
6831    }
6832
6833    #[test]
6834    fn build_config_falls_back_to_unspecified_when_none() {
6835        let server = test_server();
6836        let overlay_ip: IpAddr = "10.200.0.1".parse().unwrap();
6837        let config = server.build_config(
6838            "priv".to_string(),
6839            "pub".to_string(),
6840            overlay_ip,
6841            16,
6842            51820,
6843            None,
6844        );
6845        assert_eq!(
6846            config.local_endpoint,
6847            SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 51820)
6848        );
6849    }
6850
6851    #[test]
6852    fn build_config_falls_back_to_unspecified_on_family_mismatch() {
6853        let server = test_server();
6854        // Overlay is v6 but the resolved physical egress is v4: unusable for
6855        // source selection, so we must fall back to the v6 UNSPECIFIED address.
6856        let overlay_ip: IpAddr = "fd00::1".parse().unwrap();
6857        let egress: IpAddr = "192.0.2.10".parse().unwrap();
6858        let config = server.build_config(
6859            "priv".to_string(),
6860            "pub".to_string(),
6861            overlay_ip,
6862            64,
6863            51820,
6864            Some(egress),
6865        );
6866        assert_eq!(
6867            config.local_endpoint,
6868            SocketAddr::new(IpAddr::V6(Ipv6Addr::UNSPECIFIED), 51820)
6869        );
6870    }
6871
6872    #[test]
6873    fn rootless_forces_unspecified_decision() {
6874        // Rootless mode must force the WG local_endpoint to UNSPECIFIED because
6875        // detect_physical_egress() resolves pasta's in-netns tap IP there.
6876        assert!(rootless_forces_unspecified(true));
6877        // Non-rootless preserves the existing physical-egress selection path.
6878        assert!(!rootless_forces_unspecified(false));
6879    }
6880
6881    #[tokio::test]
6882    async fn transport_for_scope_global_requires_setup() {
6883        let server = test_server();
6884        // No global overlay set up yet -> Global scope errors. (Can't use
6885        // `expect_err` because `&OverlayTransport` is not `Debug`.)
6886        match server.transport_for_scope(&PeerScope::Global) {
6887            Ok(_) => panic!("global overlay should not be set up"),
6888            Err(OverlaydError::Other(m)) => {
6889                assert!(m.contains("global overlay not set up"), "got: {m}");
6890            }
6891            Err(other) => panic!("unexpected error: {other:?}"),
6892        }
6893    }
6894
6895    #[tokio::test]
6896    async fn transport_for_scope_unset_service_errors() {
6897        let server = test_server();
6898        match server.transport_for_scope(&PeerScope::Service {
6899            service: "x".to_string(),
6900        }) {
6901            Ok(_) => panic!("no dedicated overlay should exist for x"),
6902            Err(OverlaydError::Other(m)) => {
6903                assert_eq!(m, "no dedicated overlay for service x");
6904            }
6905            Err(other) => panic!("unexpected error: {other:?}"),
6906        }
6907    }
6908
6909    #[tokio::test]
6910    async fn add_peer_service_scope_before_setup_errors_via_dispatch() {
6911        let mut server = test_server();
6912        let resp = server
6913            .handle(OverlaydRequest::AddPeer {
6914                peer: PeerSpec {
6915                    public_key: "k".to_string(),
6916                    endpoint: "1.2.3.4:51820".to_string(),
6917                    allowed_ips: "10.200.0.2/32".to_string(),
6918                    persistent_keepalive_secs: 0,
6919                    candidates: Vec::new(),
6920                },
6921                scope: PeerScope::Service {
6922                    service: "x".to_string(),
6923                },
6924            })
6925            .await;
6926        match resp {
6927            OverlaydResponse::Err { message } => {
6928                assert_eq!(message, "no dedicated overlay for service x");
6929            }
6930            other => panic!("expected Err response, got {other:?}"),
6931        }
6932    }
6933
6934    /// The host-adapter degrade decision. A `create_interface()` failure is fatal
6935    /// on Linux (the kernel TUN IS the container data path) and degrades to a
6936    /// VM-only overlay on macOS/Windows (containers mesh VM-to-VM, the host
6937    /// utun/Wintun is off the data path). We can't provoke a real utun/Wintun
6938    /// syscall failure from a Linux test box, so we assert the pure `cfg!`-driven
6939    /// classifier instead: on this Linux test runner it must report fatal.
6940    /// (On macOS/Windows the same fn returns `false` — that arm is covered by the
6941    /// cfg, exercised natively, and cannot be asserted here.)
6942    #[test]
6943    fn host_adapter_failure_fatal_decision() {
6944        // Non-mandatory: platform-driven — fatal on Linux, degrade on macOS/Windows.
6945        assert_eq!(
6946            host_adapter_failure_is_fatal(false),
6947            cfg!(target_os = "linux"),
6948            "non-mandatory host-adapter failure is fatal only on Linux (kernel TUN is the data path)"
6949        );
6950        // Mandatory (host-shared macOS nodes where the utun IS the container data
6951        // path): fatal on every platform.
6952        assert!(
6953            host_adapter_failure_is_fatal(true),
6954            "a mandatory host adapter must make failure fatal on every platform"
6955        );
6956    }
6957
6958    /// A VM-only overlay leaves `global_transport == None`. The Global-scope peer
6959    /// dispatch must then WARN-AND-SKIP the on-device install (guests get the
6960    /// peer via guest-config push) rather than erroring — assert the dispatch
6961    /// returns `Ok` and still mirrors the peer into `global_peers`. This is the
6962    /// Linux-runnable proxy for the degraded host-adapter path: it exercises the
6963    /// exact `None`-tolerant branch without needing a real utun/Wintun failure.
6964    #[tokio::test]
6965    async fn add_global_peer_with_no_host_adapter_skips_and_records() {
6966        let mut server = test_server();
6967        assert!(
6968            server.global_transport.is_none(),
6969            "fresh server has no host adapter (VM-only precondition)"
6970        );
6971        let pubkey = "k".to_string();
6972        let resp = server
6973            .handle(OverlaydRequest::AddPeer {
6974                peer: PeerSpec {
6975                    public_key: pubkey.clone(),
6976                    endpoint: "1.2.3.4:51820".to_string(),
6977                    allowed_ips: "10.200.0.2/32".to_string(),
6978                    persistent_keepalive_secs: 0,
6979                    candidates: Vec::new(),
6980                },
6981                scope: PeerScope::Global,
6982            })
6983            .await;
6984        match resp {
6985            OverlaydResponse::Ok => {}
6986            other => panic!("expected Ok (warn-and-skip), got {other:?}"),
6987        }
6988        assert!(
6989            server.global_peers.contains_key(&pubkey),
6990            "Global peer must still be mirrored for guest-config push"
6991        );
6992    }
6993
6994    /// End-to-end Dedicated setup. Needs a real TUN device, so it is ignored by
6995    /// default and only runs on a privileged Linux host (mirrors the crate's
6996    /// other privileged overlay e2e tests).
6997    #[cfg(target_os = "linux")]
6998    #[tokio::test]
6999    #[ignore = "needs CAP_NET_ADMIN; run on a privileged Linux host"]
7000    async fn dedicated_setup_creates_distinct_device_and_routes_service_peer() {
7001        let mut server = test_server();
7002        // Bring up the global overlay first so the cluster CIDR + global device
7003        // exist (the dedicated device must get a distinct port and key).
7004        let global_name = server
7005            .setup_global_overlay(
7006                "dep".to_string(),
7007                "i0".to_string(),
7008                "10.200.0.0/16",
7009                Some("10.200.0.0/28"),
7010                zlayer_core::DEFAULT_WG_PORT,
7011                None,
7012                false,
7013            )
7014            .await
7015            .expect("global overlay up");
7016        assert!(!global_name.is_empty());
7017
7018        // Dedicated service setup.
7019        let info = server
7020            .setup_service_overlay("web", OverlayMode::Dedicated)
7021            .await
7022            .expect("dedicated service overlay up");
7023        assert_eq!(info.mode, OverlayMode::Dedicated);
7024        let port = info.wg_port.expect("dedicated port");
7025        assert_ne!(
7026            port, server.overlay_port,
7027            "dedicated device must not share the global port"
7028        );
7029
7030        let st = server
7031            .service_transports
7032            .get("web")
7033            .expect("service transport recorded");
7034        assert_eq!(st.listen_port, port);
7035        assert_ne!(
7036            st.interface, global_name,
7037            "dedicated interface must differ from global"
7038        );
7039        assert_eq!(
7040            Some(st.public_key.clone()),
7041            info.wg_public_key,
7042            "info pubkey matches recorded transport"
7043        );
7044        assert_ne!(
7045            Some(st.public_key.clone()),
7046            server.transport_public_key,
7047            "dedicated key must differ from global key"
7048        );
7049
7050        // A Service-scoped AddPeer must land on the dedicated device (succeeds),
7051        // proving scope routing targets the per-service transport.
7052        let resp = server
7053            .handle(OverlaydRequest::AddPeer {
7054                peer: PeerSpec {
7055                    public_key: {
7056                        let (_priv, pubk) = OverlayTransport::generate_keys().await.unwrap();
7057                        pubk
7058                    },
7059                    endpoint: "5.6.7.8:51999".to_string(),
7060                    allowed_ips: "10.201.0.2/32".to_string(),
7061                    persistent_keepalive_secs: 25,
7062                    candidates: Vec::new(),
7063                },
7064                scope: PeerScope::Service {
7065                    service: "web".to_string(),
7066                },
7067            })
7068            .await;
7069        assert!(
7070            matches!(resp, OverlaydResponse::Ok),
7071            "service-scoped add_peer should land on the dedicated device, got {resp:?}"
7072        );
7073    }
7074
7075    #[tokio::test]
7076    async fn guest_attach_requires_global_overlay() {
7077        // Without a global overlay (no node public key / transport) a
7078        // guest-managed attach must error rather than allocate anything.
7079        let mut server = test_server();
7080        let resp = server
7081            .handle(OverlaydRequest::AttachContainer {
7082                handle: AttachHandle::GuestManaged {
7083                    id: "vm-1".to_string(),
7084                },
7085                service: "web".to_string(),
7086                join_global: true,
7087                dns_server: None,
7088                dns_domain: None,
7089                ephemeral: false,
7090                isolation_network: None,
7091            })
7092            .await;
7093        match resp {
7094            OverlaydResponse::Err { message } => {
7095                assert!(
7096                    message.contains("global overlay to be set up"),
7097                    "got: {message}"
7098                );
7099            }
7100            other => panic!("expected Err response, got {other:?}"),
7101        }
7102        // Nothing was recorded.
7103        assert!(server.guest_attachments.is_empty());
7104    }
7105
7106    #[tokio::test]
7107    async fn detach_unknown_guest_is_idempotent() {
7108        let mut server = test_server();
7109        // No such guest -> Ok (idempotent), no panic.
7110        server
7111            .detach_container_guest("never-attached")
7112            .await
7113            .expect("detach of unknown guest is a no-op");
7114    }
7115
7116    /// Full guest-managed attach/detach round-trip. Needs a real TUN device (the
7117    /// global overlay must be live so the guest peer can be installed), so it is
7118    /// ignored by default and only runs on a privileged Linux host — mirrors the
7119    /// crate's other privileged overlay e2e tests.
7120    #[cfg(target_os = "linux")]
7121    #[tokio::test]
7122    #[ignore = "needs CAP_NET_ADMIN; run on a privileged Linux host"]
7123    async fn guest_attach_allocates_config_and_detach_releases() {
7124        let mut server = test_server();
7125        server
7126            .setup_global_overlay(
7127                "dep".to_string(),
7128                "i0".to_string(),
7129                "10.200.0.0/16",
7130                Some("10.200.0.0/28"),
7131                zlayer_core::DEFAULT_WG_PORT,
7132                None,
7133                false,
7134            )
7135            .await
7136            .expect("global overlay up");
7137
7138        // Seed a global peer so the guest config carries it through.
7139        let (_p, other_pub) = OverlayTransport::generate_keys().await.unwrap();
7140        let add = server
7141            .handle(OverlaydRequest::AddPeer {
7142                peer: PeerSpec {
7143                    public_key: other_pub.clone(),
7144                    endpoint: "9.9.9.9:51820".to_string(),
7145                    allowed_ips: "10.200.1.0/28".to_string(),
7146                    persistent_keepalive_secs: 25,
7147                    candidates: Vec::new(),
7148                },
7149                scope: PeerScope::Global,
7150            })
7151            .await;
7152        assert!(
7153            matches!(add, OverlaydResponse::Ok),
7154            "seed peer add: {add:?}"
7155        );
7156
7157        let resp = server
7158            .handle(OverlaydRequest::AttachContainer {
7159                handle: AttachHandle::GuestManaged {
7160                    id: "vm-1".to_string(),
7161                },
7162                service: "web".to_string(),
7163                join_global: true,
7164                dns_server: Some("10.200.0.1".parse().unwrap()),
7165                dns_domain: Some("overlay".to_string()),
7166                ephemeral: false,
7167                isolation_network: None,
7168            })
7169            .await;
7170        let config = match resp {
7171            OverlaydResponse::GuestConfig(c) => c,
7172            other => panic!("expected GuestConfig, got {other:?}"),
7173        };
7174        assert!(!config.private_key.is_empty());
7175        assert!(!config.public_key.is_empty());
7176        assert_ne!(config.private_key, config.public_key);
7177        assert_eq!(config.listen_port, server.overlay_port);
7178        assert_eq!(config.dns_server, Some("10.200.0.1".parse().unwrap()));
7179        // Peers = the seeded global peer + this node (self) + nothing else.
7180        assert!(
7181            config.peers.iter().any(|p| p.public_key == other_pub),
7182            "guest must learn the seeded global peer"
7183        );
7184        assert!(
7185            config
7186                .peers
7187                .iter()
7188                .any(|p| Some(&p.public_key) == server.transport_public_key.as_ref()),
7189            "guest must learn THIS node as a peer"
7190        );
7191        // The guest's own key is registered as a global peer (host route).
7192        assert!(server.global_peers.contains_key(&config.public_key));
7193        let info = server
7194            .guest_attachments
7195            .get("vm-1")
7196            .expect("attachment recorded");
7197        assert_eq!(info.overlay_ip, config.overlay_ip);
7198
7199        // Detach releases the peer + IP.
7200        let det = server
7201            .handle(OverlaydRequest::DetachContainer {
7202                handle: AttachHandle::GuestManaged {
7203                    id: "vm-1".to_string(),
7204                },
7205            })
7206            .await;
7207        assert!(matches!(det, OverlaydResponse::Ok), "detach: {det:?}");
7208        assert!(!server.guest_attachments.contains_key("vm-1"));
7209        assert!(!server.global_peers.contains_key(&config.public_key));
7210    }
7211
7212    /// The `setup_service_overlay` dispatch must handle ALL THREE modes —
7213    /// including the default `Auto` — without panicking. `resolve()` is now the
7214    /// identity, so the old `unreachable!("resolve never returns Auto")` arm
7215    /// would panic on the default mode; this proves the arm is gone. Each mode
7216    /// is recorded in `service_modes` BEFORE any netlink/transport work, so we
7217    /// assert on that deterministically regardless of host privilege (the
7218    /// downstream bridge/transport bring-up may succeed or fail depending on
7219    /// `CAP_NET_ADMIN`, but it must never panic).
7220    #[cfg(target_os = "linux")]
7221    #[tokio::test]
7222    async fn dispatch_handles_all_three_modes_without_panic() {
7223        for mode in [
7224            OverlayMode::Auto,
7225            OverlayMode::Shared,
7226            OverlayMode::Dedicated,
7227        ] {
7228            let mut server = test_server();
7229            let service = format!("svc-{mode:?}");
7230            // Must return a Result (Ok or Err) — never panic via `unreachable!`.
7231            let _ = server.setup_service_overlay(&service, mode).await;
7232            // The resolved mode is recorded up front for the attach path.
7233            assert_eq!(
7234                server.service_modes.get(&service).copied(),
7235                Some(mode.resolve()),
7236                "mode {mode:?} must be recorded for the attach path"
7237            );
7238        }
7239    }
7240
7241    /// Two distinct `Shared` services must reuse the SAME node-wide shared
7242    /// bridge (one bridge, not two), while an `Auto` service gets its OWN
7243    /// per-service bridge. Needs `CAP_NET_ADMIN` to create the bridges, so it is
7244    /// ignored by default like the crate's other privileged overlay e2e tests.
7245    #[cfg(target_os = "linux")]
7246    #[tokio::test]
7247    #[ignore = "needs CAP_NET_ADMIN; run on a privileged Linux host"]
7248    async fn shared_services_reuse_one_bridge_auto_gets_its_own() {
7249        let mut server = test_server();
7250        server
7251            .setup_global_overlay(
7252                "dep".to_string(),
7253                "i0".to_string(),
7254                "10.200.0.0/16",
7255                Some("10.200.0.0/26"),
7256                zlayer_core::DEFAULT_WG_PORT,
7257                None,
7258                false,
7259            )
7260            .await
7261            .expect("global overlay up");
7262
7263        // First Shared service creates the shared bridge.
7264        let info_a = server
7265            .setup_service_overlay("web", OverlayMode::Shared)
7266            .await
7267            .expect("shared service web up");
7268        assert_eq!(info_a.mode, OverlayMode::Shared);
7269        let shared_name = server
7270            .shared_bridge
7271            .as_ref()
7272            .expect("shared bridge created")
7273            .name
7274            .clone();
7275        assert_eq!(info_a.name, shared_name);
7276        // Shared services are NOT per-service bridges.
7277        assert!(
7278            !server.service_bridges.contains_key("web"),
7279            "Shared service must not create a per-service bridge"
7280        );
7281
7282        // Second Shared service REUSES the same shared bridge — no new bridge.
7283        let info_b = server
7284            .setup_service_overlay("api", OverlayMode::Shared)
7285            .await
7286            .expect("shared service api up");
7287        assert_eq!(
7288            info_b.name, shared_name,
7289            "a second Shared service must reuse the SAME node-wide bridge"
7290        );
7291        assert!(!server.service_bridges.contains_key("api"));
7292        // Still exactly one shared bridge object.
7293        assert_eq!(
7294            server.shared_bridge.as_ref().map(|b| b.name.clone()),
7295            Some(shared_name.clone())
7296        );
7297
7298        // An Auto service gets its OWN per-service bridge, distinct from the
7299        // shared bridge.
7300        let info_c = server
7301            .setup_service_overlay("batch", OverlayMode::Auto)
7302            .await
7303            .expect("auto service batch up");
7304        assert_eq!(info_c.mode, OverlayMode::Auto);
7305        assert!(
7306            server.service_bridges.contains_key("batch"),
7307            "Auto service must get its own per-service bridge"
7308        );
7309        assert_ne!(
7310            info_c.name, shared_name,
7311            "Auto per-service bridge must differ from the shared bridge"
7312        );
7313
7314        // Both Shared services point their service_interfaces entry at the one
7315        // shared bridge; the Auto service points at its own.
7316        assert_eq!(server.service_interfaces.get("web"), Some(&shared_name));
7317        assert_eq!(server.service_interfaces.get("api"), Some(&shared_name));
7318        assert_ne!(server.service_interfaces.get("batch"), Some(&shared_name));
7319    }
7320
7321    /// A `Shared` service's container attach must draw its IP from the shared
7322    /// bridge pool and must fail cleanly (no panic, clear error) when the shared
7323    /// bridge has not been set up yet. Unprivileged: exercises only the
7324    /// pre-netlink resolution branch.
7325    #[cfg(target_os = "linux")]
7326    #[tokio::test]
7327    async fn attach_shared_without_setup_errors_cleanly() {
7328        let mut server = test_server();
7329        // Mark the service Shared but never set up the shared bridge.
7330        server
7331            .service_modes
7332            .insert("web".to_string(), OverlayMode::Shared);
7333        let err = server
7334            .attach_container_linux(424_242, "web", false, false, None)
7335            .await
7336            .expect_err("attach must fail without a shared bridge");
7337        match err {
7338            OverlaydError::Other(m) => {
7339                assert!(
7340                    m.contains("no shared bridge"),
7341                    "expected shared-bridge error, got: {m}"
7342                );
7343            }
7344            other => panic!("unexpected error variant: {other:?}"),
7345        }
7346    }
7347
7348    /// A container attached on a NAMED isolated network must be recorded in the
7349    /// per-network membership map (`network_members["net-a"]` gains the member's
7350    /// service IP). Needs `CAP_NET_ADMIN` to bring up the bridge + veth, so it is
7351    /// ignored by default like the crate's other privileged overlay e2e tests.
7352    #[cfg(target_os = "linux")]
7353    #[tokio::test]
7354    #[ignore = "needs CAP_NET_ADMIN; run on a privileged Linux host"]
7355    async fn attach_linux_isolated_network_records_membership() {
7356        let mut server = test_server();
7357        server
7358            .setup_global_overlay(
7359                "dep".to_string(),
7360                "i0".to_string(),
7361                "10.200.0.0/16",
7362                Some("10.200.0.0/26"),
7363                zlayer_core::DEFAULT_WG_PORT,
7364                None,
7365                false,
7366            )
7367            .await
7368            .expect("global overlay up");
7369
7370        // An Auto service gives us a real per-service bridge to attach onto.
7371        server
7372            .setup_service_overlay("web", OverlayMode::Auto)
7373            .await
7374            .expect("auto service web up");
7375
7376        // Attach this very process (a live PID with a real netns) onto the named
7377        // isolated network "net-a".
7378        let pid = std::process::id();
7379        let ip = server
7380            .attach_container_linux(pid, "web", false, true, Some("net-a".to_string()))
7381            .await
7382            .expect("attach onto isolated network");
7383
7384        // Membership map gained exactly this member under "net-a".
7385        let members = server
7386            .network_members
7387            .get("net-a")
7388            .expect("net-a membership recorded");
7389        assert!(
7390            members.contains(&ip),
7391            "network_members[net-a] must contain the attached member IP {ip}"
7392        );
7393
7394        // Detach drains the membership and drops the now-empty network entry.
7395        server
7396            .detach_container_linux(pid)
7397            .await
7398            .expect("detach succeeds");
7399        assert!(
7400            !server.network_members.contains_key("net-a"),
7401            "empty isolated network must be dropped from network_members on last detach"
7402        );
7403    }
7404
7405    /// The isolation-network owner key namespace is distinct from the dedicated
7406    /// per-service namespace, so an isolation network and a service of the same
7407    /// name never collide on the same marker/allocator key. Platform-agnostic.
7408    #[test]
7409    fn isolation_owner_key_distinct_from_service_owner_key() {
7410        let iso = crate::network_state::owner_for_isolation_network("alpha");
7411        let svc = crate::network_state::owner_for_service("alpha");
7412        assert_ne!(
7413            iso, svc,
7414            "isolation and service owner keys must not collide for the same name"
7415        );
7416        assert_eq!(iso, "iso:alpha");
7417        assert_eq!(svc, "service:alpha");
7418    }
7419
7420    /// `isolation_network_subnet` is deterministic (same name -> same block so a
7421    /// reused HCN network keeps its subnet across restarts), stays INSIDE the
7422    /// node slice, and lands DIFFERENT isolation networks on DISJOINT sub-blocks
7423    /// (the whole point of L3 isolation — distinct networks must not share an
7424    /// address range). Windows-only (the method is `cfg(windows)`); exercised by
7425    /// `cargo xwin test`.
7426    #[cfg(target_os = "windows")]
7427    #[test]
7428    fn isolation_network_subnet_is_deterministic_disjoint_and_inside_slice() {
7429        let mut server = test_server();
7430        let slice: IpNetwork = "10.200.5.0/26".parse().unwrap();
7431        server.slice_cidr = Some(slice);
7432        let slice_net: ipnet::IpNet = "10.200.5.0/26".parse().unwrap();
7433
7434        // Deterministic: same name -> same block on repeated calls.
7435        let a1 = server.isolation_network_subnet("alpha").unwrap();
7436        let a2 = server.isolation_network_subnet("alpha").unwrap();
7437        assert_eq!(a1, a2, "same isolation network must map to the same subnet");
7438
7439        // Inside the node slice and at the /28 sub-prefix.
7440        assert!(
7441            slice_net.contains(&a1.network()) && slice_net.contains(&a1.broadcast()),
7442            "isolation subnet {a1} must be wholly inside the node slice {slice_net}"
7443        );
7444        assert_eq!(a1.prefix_len(), 28, "expected a /28 isolation sub-block");
7445
7446        // A different network name carving a different /28 block must be disjoint.
7447        // (`beta` and `gamma` hash to different indices than `alpha`; pick whichever
7448        //  of several names lands on a distinct block to assert disjointness.)
7449        let other = ["beta", "gamma", "delta", "omega", "zeta"]
7450            .iter()
7451            .map(|n| server.isolation_network_subnet(n).unwrap())
7452            .find(|s| *s != a1)
7453            .expect("at least one other name must land on a different /28 block");
7454        let overlaps = a1.contains(&other.network()) || other.contains(&a1.network());
7455        assert!(
7456            !overlaps,
7457            "distinct isolation networks must occupy disjoint subnets ({a1} vs {other})"
7458        );
7459    }
7460}