zlayer_overlayd/server.rs
1//! The overlayd server engine.
2//!
3//! [`OverlaydServer`] is a near 1:1 migration of the *mechanics* half of the
4//! agent's `OverlayManager`: it owns the single cluster `WireGuard`
5//! [`OverlayTransport`], the per-service Linux bridges (Linux) / HCN Internal
6//! network + endpoints (Windows), the per-node IP allocator, DNS config, and
7//! NAT traversal. The cluster-brain half (Raft, scheduler, service registry)
8//! stays in the main daemon, which drives this server over the IPC contract in
9//! [`zlayer_types::overlayd`].
10//!
11//! Every [`OverlaydRequest`] maps to a method here via [`OverlaydServer::handle`].
12
13use std::collections::HashMap;
14use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
15#[cfg(target_os = "linux")]
16use std::os::fd::AsFd;
17use std::path::{Path, PathBuf};
18use std::sync::atomic::{AtomicU64, Ordering};
19
20use ipnetwork::IpNetwork;
21use zlayer_overlay::nat::{RelayServerConfig, StunServerConfig, TurnServerConfig};
22use zlayer_overlay::{
23 Candidate, CandidateType, ConnectionType, NatConfig, NatTraversal, OverlayConfig,
24 OverlayTransport, PeerInfo, RelayServer,
25};
26use zlayer_types::overlayd::{
27 AttachHandle, AttachResult, DedicatedServiceStatus, EdgeConfig, EdgePeerStatus,
28 GuestOverlayConfig, NatCandidateWire, NatConfigSpec, NatPeerWire, NatStatusWire, OverlayMode,
29 OverlaydRequest, OverlaydResponse, PeerScope, PeerSpec, PeerStatus, ServiceOverlayInfo,
30 StatusSnapshot, EDGE_CONFIG_VERSION,
31};
32
33use crate::error::OverlaydError;
34use crate::network_state::{
35 owner_for_service, DedicatedPortAllocator, ManagedNetwork, NetworkState,
36};
37
38/// Maximum length for Linux network interface names (IFNAMSIZ - 1 for null terminator).
39const MAX_IFNAME_LEN: usize = 15;
40
41/// Reserved [`zlayer_overlay::allocator::ServiceSubnetRegistry`] key for the
42/// single node-wide shared bridge (`OverlayMode::Shared`). The leading NUL-like
43/// sentinel can never collide with a real service name (service names come from
44/// deployment specs and are DNS-label-shaped), so the shared bridge always gets
45/// exactly one stable subnet distinct from every per-service subnet.
46#[cfg(target_os = "linux")]
47const SHARED_BRIDGE_REGISTRY_KEY: &str = "__zlayer_shared_bridge__";
48
49/// Generate a Linux-safe interface name guaranteed to be <= 15 chars.
50///
51/// Joins the `parts` with `-` after a `"zl-"` prefix and appends `-{suffix}` if
52/// non-empty. When the result exceeds 15 characters, a deterministic hash of all
53/// parts is used instead to keep the name unique and within the kernel limit.
54#[must_use]
55pub fn make_interface_name(parts: &[&str], suffix: &str) -> String {
56 use std::collections::hash_map::DefaultHasher;
57 use std::hash::{Hash, Hasher};
58
59 let base = format!("zl-{}", parts.join("-"));
60 let candidate = if suffix.is_empty() {
61 base
62 } else {
63 format!("{base}-{suffix}")
64 };
65
66 if candidate.len() <= MAX_IFNAME_LEN {
67 return candidate;
68 }
69
70 // Name is too long -- produce a deterministic hash-based name.
71 let mut hasher = DefaultHasher::new();
72 for part in parts {
73 part.hash(&mut hasher);
74 }
75 suffix.hash(&mut hasher);
76 let hash = format!("{:x}", hasher.finish());
77
78 if suffix.is_empty() {
79 // "zl-" (3) + up to 12 hex chars = 15
80 let budget = MAX_IFNAME_LEN - 3;
81 format!("zl-{}", &hash[..budget.min(hash.len())])
82 } else {
83 // "zl-" (3) + hash + "-" (1) + suffix
84 let suffix_cost = 1 + suffix.len(); // "-" + suffix
85 let hash_budget = MAX_IFNAME_LEN.saturating_sub(3 + suffix_cost);
86 if hash_budget == 0 {
87 let budget = MAX_IFNAME_LEN - 3;
88 format!("zl-{}", &hash[..budget.min(hash.len())])
89 } else {
90 format!("zl-{}-{}", &hash[..hash_budget.min(hash.len())], suffix)
91 }
92 }
93}
94
95/// Pure orphan-selection predicate for [`OverlaydServer::prune_orphan_bridges`].
96///
97/// Returns `true` iff `name` is one of OUR per-service bridge (`zl-…-b`) or
98/// dedicated device (`zl-…-d`) interfaces AND is neither in the `live` set (the
99/// names the daemon says SHOULD exist) nor `protected` (the active global `-g`
100/// device, the node-wide `-sh` shared bridge, and any live in-memory service
101/// bridge/device). The `zl-` prefix gate keeps the sweep off unrelated host
102/// links; the `-b`/`-d` suffix gate keeps it off the global/shared interfaces
103/// and the `veth-…`/`vc-…` container-veth namespace (those are reclaimed by the
104/// PID-keyed `sweep_orphan_veths`, never here).
105#[cfg(target_os = "linux")]
106fn is_orphan_service_bridge(
107 name: &str,
108 live: &std::collections::HashSet<&str>,
109 protected: &std::collections::HashSet<String>,
110) -> bool {
111 if !name.starts_with("zl-") {
112 return false;
113 }
114 if !(name.ends_with("-b") || name.ends_with("-d")) {
115 return false;
116 }
117 !live.contains(name) && !protected.contains(name)
118}
119
120/// First usable host address in `subnet`.
121///
122/// For IPv4 this is `network() + 1` (skipping the network address). For IPv6
123/// the same rule applies — the network address is conventionally reserved.
124fn first_usable_ip(subnet: ipnet::IpNet) -> IpAddr {
125 match subnet {
126 ipnet::IpNet::V4(v4) => {
127 let net = u32::from(v4.network());
128 IpAddr::V4(Ipv4Addr::from(net.wrapping_add(1)))
129 }
130 ipnet::IpNet::V6(v6) => {
131 let net = u128::from(v6.network());
132 IpAddr::V6(Ipv6Addr::from(net.wrapping_add(1)))
133 }
134 }
135}
136
137/// Parameters threaded into [`OverlaydServer::attach_to_interface`] when a
138/// container is being attached to a per-service Linux bridge.
139#[cfg(target_os = "linux")]
140#[derive(Debug)]
141struct BridgeAttachParams<'a> {
142 /// Linux bridge name on the host to enslave the host-side veth into.
143 bridge_name: &'a str,
144 /// Bridge's L3 gateway IP. The container's default route is set here.
145 gateway: IpAddr,
146 /// Prefix length of the bridge's subnet.
147 subnet_prefix_len: u8,
148}
149
150/// Tracking info recorded by [`OverlaydServer::attach_container`] for every
151/// container that successfully attaches on Linux (via the per-PID `attached`
152/// map) and for every macOS host-shared container (via the
153/// `host_shared_attachments` map). Used by `detach_container`. Cross-platform
154/// so the host-shared path — which runs on macOS — can reuse the same record.
155#[derive(Debug, Clone)]
156struct AttachInfo {
157 /// IP allocated on the per-service overlay (eth0 inside the container).
158 service_ip: IpAddr,
159 /// Name of the service whose bridge owns `service_ip`.
160 service_name: Option<String>,
161 /// IP allocated on the global overlay (eth1), if the container joined it.
162 /// `Some` iff the container also attached to the global overlay; the
163 /// detach path now deletes `veth-<pid>-g` unconditionally (idempotent), so
164 /// no separate `joined_global` flag is needed.
165 ///
166 /// Linux-only: this is the per-container global/eth1 IP, allocated and read
167 /// solely by the Linux veth attach/detach paths. Host-shared containers
168 /// (macOS/Windows) share the node's single cluster utun and reach the
169 /// global overlay through their node `/32` alias, so they never allocate a
170 /// separate eth1 IP — it is always `None` off Linux and never read there.
171 #[cfg_attr(not(target_os = "linux"), allow(dead_code))]
172 global_ip: Option<IpAddr>,
173 /// True when this attach asked overlayd to reap the per-service bridge
174 /// once the LAST container detaches (ephemeral/per-job networks). False
175 /// for managed services (bridge persists across scale-to-0).
176 ephemeral: bool,
177 /// `Some(network)` when this container joined the named isolated network;
178 /// drives per-network L3 isolation membership cleanup on detach.
179 isolation_network: Option<String>,
180}
181
182/// Tracking info recorded by [`OverlaydServer::attach_container_guest`] for a
183/// guest-managed attach. Platform-agnostic (no netns/veth/HCN): the guest owns
184/// its own `WireGuard` device; the host only allocated the address + registered
185/// the guest's public key as a global peer.
186#[derive(Debug, Clone)]
187struct GuestAttachInfo {
188 /// Overlay IP allocated for the guest (released on detach).
189 overlay_ip: IpAddr,
190 /// Base64 public key registered on the global transport for the guest
191 /// (removed on detach).
192 public_key: String,
193 /// Service whose bridge pool owns `overlay_ip` (Linux service-bridge path);
194 /// `None` when drawn from the node slice. Mirrors `AttachInfo::service_name`
195 /// so detach returns the IP to the right pool.
196 service_name: Option<String>,
197 /// `Some(network)` when this guest joined the named isolated network;
198 /// drives per-network membership cleanup on detach. The guest's own
199 /// enforcement (`WireGuard` `AllowedIPs`) is wired separately — overlayd only
200 /// maintains the membership map here.
201 isolation_network: Option<String>,
202}
203
204/// Bookkeeping for one minted edge peer
205/// ([`OverlaydRequest::MintEdgePeer`]). Like [`GuestAttachInfo`] this is
206/// platform-agnostic — an edge peer is pure IPAM + keygen + a roaming global
207/// `/32` peer plus an optional node-side L3 fence; there is no netns/veth/HCN.
208/// The record carries everything [`OverlaydServer::revoke_edge_peer`] and the
209/// TTL/silence sweep need so they never have to re-derive it.
210#[derive(Debug, Clone)]
211struct EdgeAttachInfo {
212 /// Overlay `/32` allocated for the edge from the node slice (released on
213 /// revoke).
214 overlay_ip: IpAddr,
215 /// Base64 `WireGuard` public key registered on the global transport for the
216 /// edge (removed on revoke).
217 public_key: String,
218 /// Unix-seconds the peer was minted (the boot-grace clock for the sweep).
219 minted_at_unix: u64,
220 /// Unix-seconds the peer expires and is swept regardless of liveness.
221 expires_at_unix: u64,
222 /// The overlay CIDRs the edge was granted at mint time (the resolved
223 /// `--allow` list). Empty for a guest-equivalent (unfenced) edge. Used to
224 /// reproduce the exact fence `peers` set on revoke and to report `allowed`
225 /// in [`EdgePeerStatus`].
226 allowed_targets: Vec<ipnet::IpNet>,
227 /// `Some(network)` when a node-side L3 fence was installed for this edge
228 /// (network id `edge:<name>`); drives fence teardown on revoke. `None` when
229 /// `--allow` was empty (no fence installed).
230 isolation_network: Option<String>,
231}
232
233/// Per-service Linux bridge state. One bridge per service per node; containers
234/// attach to it via veth pairs and cross-node packets ride the single cluster
235/// `OverlayTransport` with the service subnet plumbed into its `AllowedIPs`.
236#[cfg(target_os = "linux")]
237#[derive(Debug)]
238struct ServiceBridge {
239 /// Linux bridge name, kept under IFNAMSIZ-1 by [`make_interface_name`].
240 name: String,
241 /// CIDR of the service's subnet on this node.
242 subnet: ipnet::IpNet,
243 /// Gateway IP within the subnet (first usable address).
244 gateway: IpAddr,
245 /// Per-service IP allocator covering `subnet`.
246 ip_allocator: zlayer_overlay::allocator::IpAllocator,
247}
248
249/// A dedicated per-service `WireGuard` transport (`OverlayMode::Dedicated`).
250///
251/// Unlike Shared mode — where every service subnet is plumbed onto the single
252/// cluster [`OverlayTransport`] via multi-CIDR `AllowedIPs` — a Dedicated
253/// service owns a *second* real `WireGuard` device with its own crypto context,
254/// listen port, overlay IP, and subnet. The device is portable (boringtun
255/// userspace `WireGuard` works on Linux/macOS/Windows), so this struct is
256/// cross-platform; only the bridge/HCN *attachment* of containers onto it is
257/// platform-gated.
258struct ServiceTransport {
259 /// The live dedicated `WireGuard` device. Dropping it tears down the TUN.
260 transport: OverlayTransport,
261 /// Actual interface name (kernel-assigned `utunN` on macOS).
262 interface: String,
263 /// base64 public key of this dedicated device.
264 public_key: String,
265 /// UDP listen port handed out by [`DedicatedPortAllocator`].
266 listen_port: u16,
267 /// This node's overlay IP on the dedicated device.
268 overlay_ip: std::net::IpAddr,
269 /// The service's subnet carried by the dedicated device.
270 subnet: ipnet::IpNet,
271 /// Guest-attach IPAM bounded to `subnet`. VZ-Linux / WSL2 guests that join
272 /// this Dedicated service draw their overlay IP from here so they land on
273 /// the dedicated device's subnet (own crypto) rather than the node slice.
274 /// The node's own `overlay_ip` is reserved at setup so guests never collide
275 /// with it. Unused on Linux, where dedicated containers attach via a
276 /// per-service bridge that owns its own allocator.
277 #[cfg_attr(target_os = "linux", allow(dead_code))]
278 ip_allocator: zlayer_overlay::allocator::IpAllocator,
279}
280
281/// The overlay daemon engine.
282pub struct OverlaydServer {
283 /// Deployment name (used for network naming). Set by `SetupGlobalOverlay`.
284 deployment: String,
285 /// Per-daemon-process disambiguator included in overlay link names. Set by
286 /// `SetupGlobalOverlay`.
287 instance_id: String,
288 /// Root data directory; HCN markers, IPAM state, etc. live under it.
289 data_dir: PathBuf,
290 /// Global overlay interface name.
291 global_interface: Option<String>,
292 /// Global overlay transport (kept alive for the TUN device lifetime). The
293 /// SINGLE cluster-wide `WireGuard` transport; every service subnet is
294 /// plumbed through its `AllowedIPs`.
295 global_transport: Option<OverlayTransport>,
296 /// Service-name -> per-service Linux bridge / placeholder name.
297 service_interfaces: HashMap<String, String>,
298 /// Service-name -> dedicated per-service `WireGuard` transport (Dedicated
299 /// mode). Coexists with `global_transport`. Empty for Shared-only nodes.
300 service_transports: HashMap<String, ServiceTransport>,
301 /// Port allocator for dedicated devices (band above the global WG port).
302 dedicated_ports: DedicatedPortAllocator,
303 /// Per-service bridge state (Linux only).
304 #[cfg(target_os = "linux")]
305 service_bridges: HashMap<String, ServiceBridge>,
306 /// The SINGLE node-wide shared bridge backing every `OverlayMode::Shared`
307 /// service (Linux only). Created once on the first Shared-service setup and
308 /// reused for all subsequent ones; container ports are exposed via the
309 /// userspace free-port L4 proxy (`proxy_manager.rs`), not per-service
310 /// bridges. `None` until the first Shared service is set up.
311 #[cfg(target_os = "linux")]
312 shared_bridge: Option<ServiceBridge>,
313 /// Resolved per-service overlay mode, recorded at `setup_service_overlay_*`
314 /// time so the container ATTACH path knows which data-plane a service uses
315 /// (per-service bridge for `Auto`/`Dedicated` vs the single shared bridge
316 /// for `Shared`) without re-deriving it. Cross-platform.
317 service_modes: HashMap<String, OverlayMode>,
318 /// Local fallback `ServiceSubnetRegistry`. Used by the Linux Shared bridge
319 /// path and by the cross-platform Dedicated path (subnets stay globally
320 /// unique regardless of mode/OS).
321 service_subnet_registry: Option<zlayer_overlay::allocator::ServiceSubnetRegistry>,
322 /// Local raft node id used as the partition key for service-subnet assign.
323 local_node_id: u64,
324 /// Base64 `WireGuard` public key of THIS node's cluster transport, as told
325 /// by the main daemon via `SetLocalWgPubkey` (used for service-subnet
326 /// `AllowedIPs` plumbing).
327 local_wg_pubkey: Option<String>,
328 /// Public key generated for the live global transport, recorded at
329 /// `setup_global_overlay` time so `Status` can surface it (the transport
330 /// itself exposes no public-key accessor).
331 transport_public_key: Option<String>,
332 /// IP allocator for the node's overlay slice.
333 ip_allocator: IpAllocator,
334 /// This node's IP on the global overlay network.
335 node_ip: Option<IpAddr>,
336 /// `WireGuard` listen port for the overlay network.
337 overlay_port: u16,
338 /// Full cluster CIDR (e.g. `10.200.0.0/16`).
339 cluster_cidr: Option<IpNetwork>,
340 /// Per-node slice CIDR.
341 slice_cidr: Option<IpNetwork>,
342 /// Map of HCN namespace GUID -> (`service_name`, `allocated_ip`,
343 /// `isolation_network`) for autoclean. The trailing `isolation_network` lets
344 /// detach drain the per-network membership map for this container.
345 #[cfg(target_os = "windows")]
346 hcn_cleanup: HashMap<windows::core::GUID, (String, std::net::IpAddr, Option<String>)>,
347 /// Per-service container-IP allocators for Windows dedicated services. Each
348 /// is bounded to that service's subnet (not the node slice) so dedicated
349 /// containers draw addresses from their own isolated network. Keyed by
350 /// service name; created lazily on the first dedicated attach.
351 #[cfg(target_os = "windows")]
352 service_ip_allocators: HashMap<String, IpAllocator>,
353 /// Per-PID tracking of overlay attachments on Linux.
354 #[cfg(target_os = "linux")]
355 attached: HashMap<u32, AttachInfo>,
356 /// Per-isolated-network membership: network name -> the set of member
357 /// overlay (service) IPs currently attached to it. Drives per-network L3
358 /// isolation (a member reaches only its own network's members + node +
359 /// egress). Populated on attach, drained on detach, across all platforms.
360 network_members: std::collections::HashMap<String, std::collections::HashSet<IpAddr>>,
361 /// Peers installed on the GLOBAL transport via `AddPeer { Global }`, keyed by
362 /// base64 public key. Tracked here (in wire-safe [`PeerSpec`] form, with the
363 /// keys kept base64 — the boringtun UAPI dump only exposes hex keys) so a
364 /// guest-managed attach can hand the guest the exact peer set the host's own
365 /// global device carries. Platform-agnostic: the guest path runs on macOS.
366 global_peers: HashMap<String, PeerSpec>,
367 /// Guest-managed overlay attachments, keyed by the opaque container `id` from
368 /// [`AttachHandle::GuestManaged`]. Records the allocated overlay IP and the
369 /// generated public key registered in the mesh so `DetachContainer` can
370 /// release the IP and remove the peer.
371 guest_attachments: HashMap<String, GuestAttachInfo>,
372 /// Minted edge peers, keyed by the caller-chosen `name` from
373 /// [`OverlaydRequest::MintEdgePeer`]. Each records the allocated `/32`, the
374 /// generated public key registered on the global transport, the mint/expiry
375 /// clocks, the granted `--allow` targets, and any node-side fence — enough
376 /// for `revoke_edge_peer` (the single teardown path) and the TTL/silence
377 /// sweep to reverse the mint without re-deriving anything. Minted peers die
378 /// with overlayd: this map is never persisted.
379 edge_attachments: HashMap<String, EdgeAttachInfo>,
380 /// Host-shared overlay attachments, keyed by the opaque container `id` from
381 /// [`AttachHandle::HostShared`] (macOS Seatbelt / native-VZ / libkrun
382 /// containers that share the node's host network namespace and its single
383 /// cluster `utun`). Records the distinct overlay `/32` allocated for the
384 /// container so `DetachContainer` can remove the utun alias, drain the
385 /// per-network L3 isolation membership, and release the IP. Cross-platform
386 /// (the host-shared path compiles everywhere; it is exercised on macOS).
387 host_shared_attachments: HashMap<String, AttachInfo>,
388 /// Overlay DNS server listen address, if one was bootstrapped.
389 dns_server_addr: Option<SocketAddr>,
390 /// DNS domain for overlay service discovery.
391 dns_domain: Option<String>,
392 /// Overlay DNS A/AAAA records this node owns (name -> ip).
393 dns_records: HashMap<String, IpAddr>,
394 /// NAT traversal configuration threaded into every `OverlayConfig`.
395 nat_config: Option<NatConfig>,
396 /// Override for `OverlayConfig::uapi_sock_dir`.
397 uapi_sock_dir: Option<PathBuf>,
398 /// Live NAT traversal orchestrator.
399 nat_traversal: Option<NatTraversal>,
400 /// Unix-epoch seconds of the last successful candidate gather / STUN refresh.
401 nat_last_refresh: AtomicU64,
402 /// NAT-traversal candidates each peer advertised, keyed by base64 public
403 /// key. Populated from `AddPeer { Global }` (the join-time candidate
404 /// exchange); the NAT maintenance tick feeds these into
405 /// `NatTraversal::connect_to_peer` to hole-punch / relay toward a peer whose
406 /// direct endpoint has not produced a recent `WireGuard` handshake.
407 peer_candidates: HashMap<String, Vec<Candidate>>,
408 /// The [`ConnectionType`] last negotiated to each peer (keyed by base64
409 /// public key), recorded by the connect loop so `NatStatus` can report
410 /// direct / hole-punched / relayed per peer.
411 peer_connection_type: HashMap<String, ConnectionType>,
412 /// Built-in relay server, started lazily on the first NAT tick when the
413 /// resolved [`NatConfig::relay_server`] is `Some`. Kept alive for the
414 /// daemon's lifetime so its background accept loop keeps running.
415 relay_server: Option<RelayServer>,
416 /// The address the built-in [`Self::relay_server`] actually bound (the real
417 /// port when `listen_port == 0`).
418 relay_bound_addr: Option<SocketAddr>,
419 /// Cluster-shared credential used to derive the built-in relay server's
420 /// `BLAKE2b` auth key. Carried in `NatConfigSpec.relay_server.auth_credential`
421 /// (the main daemon sets it from the cluster HS256 secret) so every node's
422 /// relay client derives the *same* key. `None` when no credential was
423 /// supplied (the relay then derives a key from the empty string — only nodes
424 /// that likewise have no credential can use it).
425 cluster_relay_credential: Option<String>,
426 /// Set when a `Shutdown` request has been received.
427 shutdown_requested: bool,
428 /// IPv4 `net.ipv4.ip_forward` value observed BEFORE the daemon first
429 /// enabled forwarding for an overlay container attach. `Some(prev)` is
430 /// recorded exactly once (the first time we flip it to `1`); teardown
431 /// restores `prev` so a clean shutdown reverts host routing state the
432 /// daemon turned on without clobbering an operator who set it. `None`
433 /// means the daemon never enabled IPv4 forwarding (nothing to revert).
434 #[cfg(target_os = "linux")]
435 prev_ipv4_forward: Option<String>,
436 /// Per-interface IPv6 `net.ipv6.conf.<dev>.forwarding` was enabled on
437 /// these device names for overlay routing. We enable forwarding
438 /// PER-INTERFACE (never `net.ipv6.conf.all.forwarding`, which has the
439 /// documented side effect of forcing `accept_ra=0` + `autoconf=0` on
440 /// every IPv6 interface — including the public NIC — and silently
441 /// dropping the RA-learned default route / path-MTU, which blackholes
442 /// the host's own larger reply packets). Teardown clears forwarding on
443 /// exactly these devices.
444 #[cfg(target_os = "linux")]
445 ipv6_forward_ifaces: std::collections::HashSet<String>,
446 /// Host-side veth device names THIS daemon created (`veth-<pid>-<tag>`),
447 /// recorded right after a successful `create_veth_pair`. A clean global
448 /// teardown deletes each so no host veth half is left dangling once the
449 /// overlay stops. Per-container detach may delete some of these first;
450 /// deletion is idempotent (a missing device is ignored). Only names this
451 /// daemon created are tracked — never a blanket prefix sweep that could
452 /// catch a concurrent overlay's interfaces.
453 #[cfg(target_os = "linux")]
454 created_veths: std::collections::HashSet<String>,
455 /// `zl-*` bridge device names THIS daemon created (per-service and the
456 /// node-wide shared bridge), recorded right after a successful
457 /// `create_bridge` + address + up. Deleting the bridge link on teardown
458 /// also drops its gateway address and up state, so the name alone is enough
459 /// to fully revert it.
460 #[cfg(target_os = "linux")]
461 created_bridges: std::collections::HashSet<String>,
462 /// Host `/32` (`/128`) routes to a container IP via a host-side veth that
463 /// THIS daemon installed via `replace_route_via_dev` (the bridgeless attach
464 /// path). Each entry is `(dest, prefix_len, dev)` — enough to delete the
465 /// exact route on teardown via `delete_route_via_dev`. Deletion is
466 /// idempotent (a route a prior detach already removed is ignored).
467 #[cfg(target_os = "linux")]
468 created_host_routes: Vec<(IpAddr, u8, String)>,
469}
470
471/// Whether rootless mode forces the `WireGuard` `local_endpoint` to UNSPECIFIED.
472///
473/// In rootless mode `detect_physical_egress()` runs inside the daemon netns and
474/// resolves pasta's in-netns tap IP, which is a meaningless WG source/advertised
475/// endpoint to remote peers. Extracted as a pure fn so the decision is testable
476/// without mutating the process-global `ZLAYER_ROOTLESS` env var (env writes race
477/// across parallel tests).
478fn rootless_forces_unspecified(rootless: bool) -> bool {
479 rootless
480}
481
482/// Whether a failure to create the HOST overlay adapter is fatal for the node.
483///
484/// On Linux the host adapter (a kernel TUN brought up via netlink, with the
485/// rootless userns+netns path as a fallback) IS the container data path, so a
486/// creation failure must abort overlay setup. On macOS/Windows, Linux
487/// containers live in a VZ VM / WSL2 distro that creates its OWN overlay device
488/// and meshes VM-to-VM over UDP — the host adapter (utun/Wintun, which needs
489/// root/Administrator) is only the host's own membership in the overlay and is
490/// NOT on the container data path. So on those platforms a host-adapter failure
491/// must DEGRADE to a VM-only overlay (warn + continue) rather than abort.
492///
493/// Extracted as a `cfg!`-driven pure fn so the degrade decision is unit-testable
494/// on Linux without needing to provoke a real utun/Wintun syscall failure.
495fn host_adapter_failure_is_fatal(host_adapter_mandatory: bool) -> bool {
496 cfg!(target_os = "linux") || host_adapter_mandatory
497}
498
499impl OverlaydServer {
500 /// Create a fresh server bound to `data_dir`. The overlay itself is brought
501 /// up lazily by `SetupGlobalOverlay` (which carries the deployment, slice,
502 /// port, and NAT toggle from the main daemon).
503 ///
504 /// # Panics
505 /// Panics only if the compile-time-constant default CIDR `10.200.0.0/16`
506 /// fails to parse (impossible).
507 #[must_use]
508 pub fn new(data_dir: PathBuf) -> Self {
509 // Until SetupGlobalOverlay arrives, the allocator is bounded to the
510 // default cluster /16. SetupGlobalOverlay re-binds it to the node slice.
511 let default_cidr: IpNetwork = "10.200.0.0/16".parse().expect("compile-time constant CIDR");
512 let overlay_port = zlayer_core::DEFAULT_WG_PORT;
513
514 // Rehydrate the dedicated-port allocator from the on-disk marker so a
515 // service that already owns a dedicated overlay re-binds the exact UDP
516 // port it had before this process started.
517 let marker_path = zlayer_paths::ZLayerDirs::new(data_dir.clone()).agent_network_state();
518 let recorded_dedicated_ports: Vec<u16> = NetworkState::load(&marker_path)
519 .networks
520 .iter()
521 .filter(|n| n.owner.starts_with("service:"))
522 .filter_map(|n| n.wg_port)
523 .collect();
524
525 Self {
526 deployment: String::new(),
527 instance_id: String::new(),
528 data_dir,
529 global_interface: None,
530 global_transport: None,
531 service_interfaces: HashMap::new(),
532 service_transports: HashMap::new(),
533 dedicated_ports: DedicatedPortAllocator::new(overlay_port, recorded_dedicated_ports),
534 #[cfg(target_os = "linux")]
535 service_bridges: HashMap::new(),
536 #[cfg(target_os = "linux")]
537 shared_bridge: None,
538 service_modes: HashMap::new(),
539 service_subnet_registry: None,
540 local_node_id: 0,
541 local_wg_pubkey: None,
542 transport_public_key: None,
543 ip_allocator: IpAllocator::new(default_cidr),
544 node_ip: None,
545 overlay_port,
546 cluster_cidr: Some(default_cidr),
547 slice_cidr: None,
548 #[cfg(target_os = "windows")]
549 hcn_cleanup: HashMap::new(),
550 #[cfg(target_os = "windows")]
551 service_ip_allocators: HashMap::new(),
552 #[cfg(target_os = "linux")]
553 attached: HashMap::new(),
554 network_members: std::collections::HashMap::new(),
555 global_peers: HashMap::new(),
556 guest_attachments: HashMap::new(),
557 edge_attachments: HashMap::new(),
558 host_shared_attachments: HashMap::new(),
559 dns_server_addr: None,
560 dns_domain: None,
561 dns_records: HashMap::new(),
562 nat_config: None,
563 uapi_sock_dir: None,
564 nat_traversal: None,
565 nat_last_refresh: AtomicU64::new(0),
566 peer_candidates: HashMap::new(),
567 peer_connection_type: HashMap::new(),
568 relay_server: None,
569 relay_bound_addr: None,
570 cluster_relay_credential: None,
571 shutdown_requested: false,
572 #[cfg(target_os = "linux")]
573 prev_ipv4_forward: None,
574 #[cfg(target_os = "linux")]
575 ipv6_forward_ifaces: std::collections::HashSet::new(),
576 #[cfg(target_os = "linux")]
577 created_veths: std::collections::HashSet::new(),
578 #[cfg(target_os = "linux")]
579 created_bridges: std::collections::HashSet::new(),
580 #[cfg(target_os = "linux")]
581 created_host_routes: Vec::new(),
582 }
583 }
584
585 /// Override the `WireGuard` UAPI socket directory for every overlay
586 /// transport built by this server.
587 #[must_use]
588 pub fn with_uapi_sock_dir(mut self, dir: impl Into<PathBuf>) -> Self {
589 self.uapi_sock_dir = Some(dir.into());
590 self
591 }
592
593 /// Whether a `Shutdown` request has been received.
594 #[must_use]
595 pub fn shutdown_requested(&self) -> bool {
596 self.shutdown_requested
597 }
598
599 /// The root data directory this server was constructed with. Used by the
600 /// uninstall path (`purge_managed_networks`) and for HCN marker resolution.
601 #[must_use]
602 pub fn data_dir(&self) -> &Path {
603 &self.data_dir
604 }
605
606 // -- request dispatch ----------------------------------------------------
607
608 /// Execute one [`OverlaydRequest`], producing the [`OverlaydResponse`] the
609 /// server sends back over IPC. Any internal error is folded into
610 /// [`OverlaydResponse::Err`].
611 pub async fn handle(&mut self, req: OverlaydRequest) -> OverlaydResponse {
612 match self.dispatch(req).await {
613 Ok(resp) => resp,
614 Err(e) => OverlaydResponse::Err {
615 message: e.to_string(),
616 },
617 }
618 }
619
620 #[allow(clippy::too_many_lines)]
621 async fn dispatch(&mut self, req: OverlaydRequest) -> Result<OverlaydResponse, OverlaydError> {
622 match req {
623 OverlaydRequest::SetLocalNodeId { node_id } => {
624 self.local_node_id = node_id;
625 Ok(OverlaydResponse::Ok)
626 }
627 OverlaydRequest::SetLocalWgPubkey { pubkey } => {
628 self.local_wg_pubkey = Some(pubkey);
629 Ok(OverlaydResponse::Ok)
630 }
631 OverlaydRequest::SetupGlobalOverlay {
632 deployment,
633 instance_id,
634 cluster_cidr,
635 slice_cidr,
636 wg_port,
637 nat,
638 host_adapter_mandatory,
639 } => {
640 let name = self
641 .setup_global_overlay(
642 deployment,
643 instance_id,
644 &cluster_cidr,
645 slice_cidr.as_deref(),
646 wg_port,
647 nat,
648 host_adapter_mandatory,
649 )
650 .await?;
651 Ok(OverlaydResponse::BridgeName { name })
652 }
653 OverlaydRequest::TeardownGlobalOverlay => {
654 self.teardown_global_overlay();
655 Ok(OverlaydResponse::Ok)
656 }
657 OverlaydRequest::SetupServiceOverlay { service, mode } => {
658 let info = self.setup_service_overlay(&service, mode).await?;
659 Ok(OverlaydResponse::ServiceOverlay(info))
660 }
661 OverlaydRequest::TeardownServiceOverlay { service } => {
662 self.teardown_service_overlay(&service).await;
663 Ok(OverlaydResponse::Ok)
664 }
665 OverlaydRequest::AllocateIp {
666 service,
667 join_global,
668 } => {
669 let ip = self.allocate_ip(&service, join_global)?;
670 Ok(OverlaydResponse::Ip { ip })
671 }
672 OverlaydRequest::ReleaseIp { ip } => {
673 self.release_ip(ip);
674 Ok(OverlaydResponse::Ok)
675 }
676 OverlaydRequest::AttachContainer {
677 handle,
678 service,
679 join_global,
680 dns_server,
681 dns_domain,
682 ephemeral,
683 isolation_network,
684 } => {
685 // A guest-managed attach takes a wholly separate path: it cannot
686 // build a veth/HCN endpoint (the target is a VM, not a host
687 // process), so it allocates the overlay identity + peer set and
688 // returns it as `GuestConfig`. PID/HCN handles keep the existing
689 // veth/HCN attach and return `Attached`.
690 if let AttachHandle::GuestManaged { id } = handle {
691 // Record the overlay DNS resolver/zone the daemon staged for
692 // this node so the guest config can fall back to them (same
693 // bookkeeping `attach_container` does for the other handles).
694 if let Some(server) = dns_server {
695 self.dns_server_addr = Some(SocketAddr::new(server, 53));
696 }
697 if dns_domain.is_some() {
698 self.dns_domain.clone_from(&dns_domain);
699 }
700 let config = self
701 .attach_container_guest(
702 &id,
703 &service,
704 join_global,
705 dns_server,
706 dns_domain,
707 isolation_network,
708 )
709 .await?;
710 Ok(OverlaydResponse::GuestConfig(config))
711 } else {
712 let result = self
713 .attach_container(
714 handle,
715 &service,
716 join_global,
717 ephemeral,
718 dns_server,
719 dns_domain,
720 isolation_network,
721 )
722 .await?;
723 Ok(OverlaydResponse::Attached(result))
724 }
725 }
726 OverlaydRequest::DetachContainer { handle } => {
727 if let AttachHandle::GuestManaged { id } = handle {
728 self.detach_container_guest(&id).await?;
729 } else {
730 self.detach_container(handle).await?;
731 }
732 Ok(OverlaydResponse::Ok)
733 }
734 // `scope` selects the target device: `Global` (default) = the single
735 // cluster transport; `Service { service }` = that service's
736 // dedicated per-service transport.
737 OverlaydRequest::AddPeer { peer, scope } => {
738 let info = peer_spec_to_info(&peer)?;
739 // VM-only overlay (macOS/Windows host adapter unavailable):
740 // there is no host transport to program for the Global scope, so
741 // WARN-AND-SKIP the on-device install instead of erroring. The
742 // peer is still mirrored into `global_peers` below so guests can
743 // reproduce the global peer set via the separate guest-config
744 // push — the host simply doesn't join. `Some` transports are
745 // unaffected.
746 if matches!(scope, PeerScope::Global) && self.global_transport.is_none() {
747 tracing::warn!(
748 peer = %peer.public_key,
749 "global overlay has no host adapter (VM-only overlay); \
750 skipping host peer install — guests receive this peer via \
751 guest-config push"
752 );
753 } else {
754 let transport = self.transport_for_scope(&scope)?;
755 Self::add_peer_on(transport, &info).await?;
756 }
757 // Record the peer's advertised NAT candidates (if any) so the
758 // NAT maintenance tick can hole-punch / relay toward it. Stored
759 // for both scopes keyed by public key (the cluster transport is
760 // the one carrying packets either way). Empty candidate lists
761 // are dropped from the map so the tick's borrow loop stays cheap.
762 if peer.candidates.is_empty() {
763 self.peer_candidates.remove(&peer.public_key);
764 } else {
765 let parsed: Vec<Candidate> = peer
766 .candidates
767 .iter()
768 .filter_map(wire_to_candidate)
769 .collect();
770 if parsed.is_empty() {
771 self.peer_candidates.remove(&peer.public_key);
772 } else {
773 self.peer_candidates.insert(peer.public_key.clone(), parsed);
774 }
775 }
776 // Mirror Global peers into `global_peers` so a guest-managed
777 // attach can reproduce the host's global peer set for the guest.
778 if matches!(scope, PeerScope::Global) {
779 self.global_peers.insert(peer.public_key.clone(), peer);
780 }
781 Ok(OverlaydResponse::Ok)
782 }
783 OverlaydRequest::RemovePeer { pubkey, scope } => {
784 // VM-only overlay: no host transport for the Global scope, so the
785 // on-device removal is a no-op — just drop it from `global_peers`
786 // below. `Some` transports are unaffected.
787 if matches!(scope, PeerScope::Global) && self.global_transport.is_none() {
788 tracing::warn!(
789 peer = %pubkey,
790 "global overlay has no host adapter (VM-only overlay); \
791 skipping host peer removal"
792 );
793 } else {
794 let transport = self.transport_for_scope(&scope)?;
795 Self::remove_peer_on(transport, &pubkey).await?;
796 }
797 if matches!(scope, PeerScope::Global) {
798 self.global_peers.remove(&pubkey);
799 }
800 self.peer_candidates.remove(&pubkey);
801 self.peer_connection_type.remove(&pubkey);
802 Ok(OverlaydResponse::Ok)
803 }
804 OverlaydRequest::AddAllowedIp {
805 pubkey,
806 cidr,
807 scope,
808 } => {
809 // VM-only overlay: no host device to plumb AllowedIPs into for the
810 // Global scope — warn-and-skip. `Some` transports are unaffected.
811 if matches!(scope, PeerScope::Global) && self.global_transport.is_none() {
812 tracing::warn!(
813 peer = %pubkey,
814 cidr = %cidr,
815 "global overlay has no host adapter (VM-only overlay); \
816 skipping host AllowedIP add"
817 );
818 } else {
819 let transport = self.transport_for_scope(&scope)?;
820 Self::add_allowed_ip_on(transport, &pubkey, &cidr).await?;
821 }
822 Ok(OverlaydResponse::Ok)
823 }
824 OverlaydRequest::RemoveAllowedIp {
825 pubkey,
826 cidr,
827 scope,
828 } => {
829 // VM-only overlay: no host device for the Global scope — the
830 // removal is a no-op. `Some` transports are unaffected.
831 if matches!(scope, PeerScope::Global) && self.global_transport.is_none() {
832 tracing::warn!(
833 peer = %pubkey,
834 cidr = %cidr,
835 "global overlay has no host adapter (VM-only overlay); \
836 skipping host AllowedIP removal"
837 );
838 } else {
839 let transport = self.transport_for_scope(&scope)?;
840 Self::remove_allowed_ip_on(transport, &pubkey, &cidr).await?;
841 }
842 Ok(OverlaydResponse::Ok)
843 }
844 OverlaydRequest::MintEdgePeer {
845 name,
846 ttl_secs,
847 allow,
848 node_endpoint,
849 } => {
850 let config = self
851 .mint_edge_peer(name, ttl_secs, &allow, &node_endpoint)
852 .await?;
853 Ok(OverlaydResponse::EdgeConfig(config))
854 }
855 OverlaydRequest::RevokeEdgePeer { name } => {
856 // The wire has no dedicated revoke response; success (idempotent,
857 // whether or not the name existed) maps to the generic `Ok`.
858 self.revoke_edge_peer(&name).await?;
859 Ok(OverlaydResponse::Ok)
860 }
861 OverlaydRequest::ListEdgePeers => Ok(OverlaydResponse::EdgePeers {
862 peers: self.edge_peer_statuses().await,
863 }),
864 OverlaydRequest::RegisterDns { name, ip } => {
865 self.register_dns(name, ip);
866 Ok(OverlaydResponse::Ok)
867 }
868 OverlaydRequest::UnregisterDns { name } => {
869 self.unregister_dns(&name);
870 Ok(OverlaydResponse::Ok)
871 }
872 OverlaydRequest::WriteScopedResolver {
873 zone,
874 node_ip,
875 port,
876 } => {
877 #[cfg(target_os = "macos")]
878 {
879 zlayer_overlay::dns::write_scoped_resolver(&zone, node_ip, port).map_err(
880 |e| OverlaydError::Overlay(format!("write_scoped_resolver({zone}): {e}")),
881 )?;
882 Ok(OverlaydResponse::Ok)
883 }
884 #[cfg(not(target_os = "macos"))]
885 {
886 let _ = (zone, node_ip, port);
887 Err(OverlaydError::Overlay(
888 "scoped resolver is macOS-only".into(),
889 ))
890 }
891 }
892 OverlaydRequest::RemoveScopedResolver { zone } => {
893 #[cfg(target_os = "macos")]
894 {
895 zlayer_overlay::dns::remove_scoped_resolver(&zone).map_err(|e| {
896 OverlaydError::Overlay(format!("remove_scoped_resolver({zone}): {e}"))
897 })?;
898 Ok(OverlaydResponse::Ok)
899 }
900 #[cfg(not(target_os = "macos"))]
901 {
902 let _ = zone;
903 Err(OverlaydError::Overlay(
904 "scoped resolver is macOS-only".into(),
905 ))
906 }
907 }
908 OverlaydRequest::PruneOrphanBridges { live_bridge_names } => {
909 let reclaimed = self.prune_orphan_bridges(&live_bridge_names).await;
910 Ok(OverlaydResponse::PrunedBridges { reclaimed })
911 }
912 OverlaydRequest::Status => Ok(OverlaydResponse::Status(self.status_snapshot().await)),
913 OverlaydRequest::NatTick => {
914 self.nat_maintenance_tick().await?;
915 Ok(OverlaydResponse::Ok)
916 }
917 OverlaydRequest::NatStatus => Ok(OverlaydResponse::NatStatus(
918 self.nat_status_snapshot().await,
919 )),
920 OverlaydRequest::Shutdown => {
921 self.shutdown_requested = true;
922 self.teardown_global_overlay();
923 Ok(OverlaydResponse::Ok)
924 }
925 }
926 }
927
928 // -- global overlay ------------------------------------------------------
929
930 /// Bring up (or reuse) this node's base/global overlay.
931 ///
932 /// Idempotent: if a global transport is already live, reuse it (recreating
933 /// without this guard could yank the kernel TUN out from under the running
934 /// boringtun worker). Re-binds the IP allocator to `slice_cidr` if one is
935 /// supplied so container IPs never collide across nodes.
936 ///
937 /// # Errors
938 /// Returns an error if key generation or interface creation fails.
939 #[allow(clippy::too_many_lines)]
940 #[allow(clippy::too_many_arguments)]
941 async fn setup_global_overlay(
942 &mut self,
943 deployment: String,
944 instance_id: String,
945 cluster_cidr: &str,
946 slice_cidr: Option<&str>,
947 wg_port: u16,
948 nat: Option<NatConfigSpec>,
949 host_adapter_mandatory: bool,
950 ) -> Result<String, OverlaydError> {
951 self.deployment = deployment;
952 self.instance_id = instance_id;
953 self.overlay_port = wg_port;
954
955 let cluster: IpNetwork = cluster_cidr.parse().map_err(|e| {
956 OverlaydError::Other(format!("invalid cluster CIDR {cluster_cidr}: {e}"))
957 })?;
958 self.cluster_cidr = Some(cluster);
959 if let Some(slice) = slice_cidr {
960 let slice_net: IpNetwork = slice
961 .parse()
962 .map_err(|e| OverlaydError::Other(format!("invalid slice CIDR {slice}: {e}")))?;
963 self.slice_cidr = Some(slice_net);
964 self.ip_allocator = IpAllocator::new(slice_net);
965 }
966 // Thread the full operator-supplied NAT config (STUN/TURN servers,
967 // timeouts, relay-server bind + credential) into overlayd. `None` means
968 // the main daemon supplied no explicit config, so overlayd keeps its
969 // built-in `NatConfig::default()` (NAT enabled, Google STUN). A `Some`
970 // spec is converted verbatim — including the relay credential, stashed
971 // separately so the relay server can be stood up with a cluster-shared
972 // auth key on the first NAT tick.
973 if let Some(spec) = nat {
974 self.cluster_relay_credential = spec
975 .relay_server
976 .as_ref()
977 .and_then(|r| r.auth_credential.clone());
978 self.nat_config = Some(nat_config_spec_to_config(spec));
979 }
980
981 if let Some(name) = self.global_interface.clone() {
982 if self.global_transport.is_some() {
983 tracing::debug!(
984 deployment = %self.deployment,
985 "Global overlay already active, reusing existing transport"
986 );
987 return Ok(name);
988 }
989 }
990
991 let interface_name = make_interface_name(&[&self.deployment, &self.instance_id], "g");
992
993 let (private_key, public_key) = OverlayTransport::generate_keys()
994 .await
995 .map_err(|e| OverlaydError::Overlay(format!("Failed to generate keys: {e}")))?;
996
997 // The node's own overlay IP is the deterministic first-usable host of
998 // its slice (reserved offset 1), NOT a racy `allocate()` that drifts by
999 // allocation order. Containers draw from offset 2 onward, so the node
1000 // IP is stable across restarts and never collides with a container.
1001 let node_ip = self.ip_allocator.node_ip();
1002 self.transport_public_key = Some(public_key.clone());
1003 let physical_egress_ip = match zlayer_overlay::detect_physical_egress().await {
1004 Ok(egress) => Some(egress.ip),
1005 Err(e) => {
1006 tracing::warn!(
1007 error = %e,
1008 "failed to detect physical egress; WireGuard local_endpoint \
1009 will bind UNSPECIFIED for the global overlay"
1010 );
1011 None
1012 }
1013 };
1014 let config = self.build_config(
1015 private_key,
1016 public_key,
1017 node_ip,
1018 16,
1019 self.overlay_port,
1020 physical_egress_ip,
1021 );
1022 // Remove any stale `-g` interface with this (now deterministic) name
1023 // left by a previous daemon instance, so the create below cleanly
1024 // REPLACES it instead of failing "File exists" or orphaning the old
1025 // one. With a stable per-host instance id the name is constant across
1026 // restarts, so exactly one global interface ever exists.
1027 #[cfg(target_os = "linux")]
1028 let _ = crate::netlink::delete_link_by_name(&interface_name).await;
1029 let mut transport = OverlayTransport::new(config, interface_name);
1030
1031 // Creating the host overlay adapter is fatal on Linux (the kernel TUN IS
1032 // the container data path) but only DEGRADES on macOS/Windows: there,
1033 // Linux containers run in a VZ VM / WSL2 distro that creates its own
1034 // overlay device and meshes VM-to-VM over UDP, so the host adapter
1035 // (utun/Wintun, needs root/Administrator) is just the host's own overlay
1036 // membership and is NOT on the container data path. The allocator and
1037 // `node_ip` are already bound above, so guest-config push + IP allocation
1038 // keep working even when the host adapter is unavailable.
1039 // Map the (non-`Send`) `Box<dyn Error>` to an owned `String` BEFORE the
1040 // match so no non-`Send` value is held across the `configure().await`
1041 // below — the daemon's request handler future must stay `Send`.
1042 let create_result = transport
1043 .create_interface()
1044 .await
1045 .map_err(|e| e.to_string());
1046 let actual_name = match create_result {
1047 Ok(()) => {
1048 transport.configure(&[]).await.map_err(|e| {
1049 OverlaydError::Overlay(format!("Failed to configure global overlay: {e}"))
1050 })?;
1051 // Read back the actual interface name (on macOS, the kernel
1052 // assigns utunN).
1053 let actual_name = transport.interface_name().to_string();
1054 self.node_ip = Some(node_ip);
1055 self.global_interface = Some(actual_name.clone());
1056 self.global_transport = Some(transport);
1057 actual_name
1058 }
1059 Err(e) if !host_adapter_failure_is_fatal(host_adapter_mandatory) => {
1060 // macOS / Windows: continue with a VM-only overlay. Leave
1061 // `global_transport == None` (the natural "no host adapter"
1062 // signal), keep `node_ip` so allocation/guest config are
1063 // unaffected, and SKIP `configure` (no device to program).
1064 tracing::warn!(
1065 error = %e,
1066 "host overlay adapter unavailable (needs root/Administrator); \
1067 continuing with VM-only overlay — the host will not join the \
1068 overlay, but containers running in the VM mesh VM-to-VM and IP \
1069 allocation/guest config are unaffected"
1070 );
1071 self.node_ip = Some(node_ip);
1072 self.global_interface = None;
1073 self.global_transport = None;
1074 // No real device exists; return an honest marker so the IPC
1075 // response is a success without implying a live adapter.
1076 "(host-adapter-disabled)".to_string()
1077 }
1078 Err(e) => {
1079 // Linux (and any future fatal-on-failure target): unchanged —
1080 // a host-adapter creation failure aborts overlay setup.
1081 return Err(OverlaydError::Overlay(format!(
1082 "Failed to create global overlay: {e}"
1083 )));
1084 }
1085 };
1086
1087 // In rootless mode the daemon runs in its own network namespace and
1088 // `pasta` provides egress NAT + inbound port forwarding; the host-table
1089 // iptables setup below is at best a no-op inside the netns and at worst
1090 // spurious, so skip it entirely. Otherwise install the host firewall
1091 // rules as usual.
1092 if std::env::var_os("ZLAYER_ROOTLESS").is_none() {
1093 // Stop systemd-networkd / NetworkManager from managing the overlay
1094 // links overlayd just created. With a permissive default match they
1095 // try to bring `zl-*` up / run DHCP and (seen on a CI runner)
1096 // SIGABRT on the networkd watchdog while processing a `zl-*` Link
1097 // UP. Best-effort; reverted in `teardown_global_overlay`.
1098 zlayer_overlay::networkd::mark_overlay_interfaces_unmanaged();
1099
1100 // Allow overlay traffic through the host firewall (UFW / firewalld /
1101 // a bare `iptables -P FORWARD DROP`). Without this, a container's DNS
1102 // query to the node overlay IP — and inter-service overlay traffic —
1103 // is dropped by the host's INPUT/FORWARD policy before it reaches
1104 // ZLayer's resolver. Best-effort: a host without `iptables` logs a
1105 // warning rather than aborting overlay setup.
1106 if let Err(e) =
1107 zlayer_overlay::firewall::ensure_overlay_subnet_rules(&cluster.to_string())
1108 {
1109 tracing::warn!(
1110 error = %e,
1111 cidr = %cluster,
1112 "failed to install overlay firewall allow-rules; service DNS / \
1113 cross-service traffic may be blocked by the host firewall"
1114 );
1115 }
1116
1117 // SNAT overlay-sourced egress so containers can reach the LAN/internet.
1118 // The allow-rules above + `ip_forward` only get the packet *forwarded*
1119 // out the WAN NIC; without masquerade it leaves with a private
1120 // `10.200.0.0/16` source and replies never route back (ENETUNREACH /
1121 // hangs for `wget http://<public-ip>`). Best-effort, same as above.
1122 if let Err(e) =
1123 zlayer_overlay::firewall::ensure_overlay_masquerade(&cluster.to_string())
1124 {
1125 tracing::warn!(
1126 error = %e,
1127 cidr = %cluster,
1128 "failed to install overlay egress masquerade; overlay containers \
1129 may be unable to reach the LAN / internet"
1130 );
1131 }
1132 } else {
1133 tracing::info!(
1134 "rootless mode: skipping host iptables (pasta provides egress + port forwarding)"
1135 );
1136 }
1137
1138 Ok(actual_name)
1139 }
1140
1141 /// Tear down the node's base overlay (e.g. on full uninstall / shutdown).
1142 fn teardown_global_overlay(&mut self) {
1143 if let Some(mut transport) = self.global_transport.take() {
1144 tracing::info!("Shutting down global overlay");
1145 transport.shutdown();
1146 }
1147 self.global_interface = None;
1148 self.transport_public_key = None;
1149
1150 // Revert host network state this daemon mutated so a clean stop
1151 // recovers connectivity WITHOUT requiring a reboot. Forwarding
1152 // sysctls and the overlay iptables chains are otherwise sticky:
1153 // they survive both the daemon stop and an `iptables -F`, so prior
1154 // to this the only way to undo them was a reboot.
1155 #[cfg(target_os = "linux")]
1156 self.revert_forwarding();
1157 zlayer_overlay::firewall::remove_overlay_masquerade();
1158 zlayer_overlay::firewall::remove_overlay_subnet_rules();
1159 // `remove_member_isolation` deliberately leaves the ZLAYER-OVERLAY-ISO
1160 // chain + its FORWARD jump resident (other members may still use them);
1161 // on a full overlay teardown remove the whole chain so nothing leaks.
1162 zlayer_overlay::firewall::remove_overlay_isolation();
1163 // macOS: strip the pf overlay anchor + the two marked `/etc/pf.conf`
1164 // lines this node installs for the cluster/DNS ports. Without this they
1165 // leak past daemon stop (the anchor file and `/etc/pf.conf` refs are
1166 // sticky on disk). Idempotent: a missing anchor / not-root / disabled-pf
1167 // case is treated as a successful no-op by the backend. cfg-gated so
1168 // Linux/Windows teardown behaviour is unchanged.
1169 #[cfg(target_os = "macos")]
1170 if let Err(e) = zlayer_overlay::firewall::remove_overlay_rules() {
1171 tracing::warn!(error = %e, "failed to remove macOS pf overlay rules during teardown");
1172 }
1173 // Remove the systemd-networkd / NetworkManager "unmanaged" drop-ins we
1174 // installed at setup so a clean stop fully reverts host network state.
1175 zlayer_overlay::networkd::unmark_overlay_interfaces_unmanaged();
1176
1177 // Revert the host-side netlink resources this daemon created (veths,
1178 // host /32 routes, bridges). The netlink helpers are async; this fn must
1179 // keep its sync signature, so bridge to the surrounding multi-thread
1180 // tokio runtime via block_in_place + Handle::block_on. Order matters:
1181 // delete routes first (they reference the veth as their oif), then the
1182 // host-side veths, then the bridges (deleting a bridge link drops its
1183 // address + up state). Every delete is best-effort + idempotent: a
1184 // resource a prior per-container detach already removed surfaces as
1185 // NotFound/ESRCH which the helpers treat as success, and a genuine
1186 // failure is logged and skipped so a partial teardown never aborts the
1187 // rest.
1188 #[cfg(target_os = "linux")]
1189 {
1190 let routes: Vec<(IpAddr, u8, String)> = std::mem::take(&mut self.created_host_routes);
1191 let veths: Vec<String> = self.created_veths.drain().collect();
1192 let bridges: Vec<String> = self.created_bridges.drain().collect();
1193
1194 let delete_all = || async {
1195 for (dest, prefix, dev) in &routes {
1196 if let Err(e) = crate::netlink::delete_route_via_dev(*dest, *prefix, dev).await
1197 {
1198 tracing::warn!(
1199 dest = %dest, prefix, dev = %dev, error = %e,
1200 "teardown: failed to delete host route (continuing)"
1201 );
1202 }
1203 }
1204 for veth in &veths {
1205 if let Err(e) = crate::netlink::delete_link_by_name(veth).await {
1206 tracing::warn!(
1207 veth = %veth, error = %e,
1208 "teardown: failed to delete host-side veth (continuing)"
1209 );
1210 }
1211 }
1212 for bridge in &bridges {
1213 if let Err(e) = crate::netlink::delete_link_by_name(bridge).await {
1214 tracing::warn!(
1215 bridge = %bridge, error = %e,
1216 "teardown: failed to delete bridge (continuing)"
1217 );
1218 }
1219 }
1220 };
1221
1222 match tokio::runtime::Handle::try_current() {
1223 Ok(handle) => {
1224 tokio::task::block_in_place(|| handle.block_on(delete_all()));
1225 }
1226 Err(_) => {
1227 // No ambient runtime (e.g. a non-async shutdown path): spin
1228 // up a throwaway current-thread runtime to drive the deletes.
1229 match tokio::runtime::Builder::new_current_thread()
1230 .enable_all()
1231 .build()
1232 {
1233 Ok(rt) => rt.block_on(delete_all()),
1234 Err(e) => tracing::warn!(
1235 error = %e,
1236 "teardown: could not build a runtime to revert netlink \
1237 resources; veths/routes/bridges left in place"
1238 ),
1239 }
1240 }
1241 }
1242 }
1243 }
1244
1245 /// Enable IP forwarding for an overlay container attach, scoped to the
1246 /// address family in use and (for IPv6) to the specific overlay devices.
1247 ///
1248 /// IPv4 has no per-interface forwarding knob that affects routing the way
1249 /// we need, so `net.ipv4.ip_forward` is global — but that is harmless for
1250 /// the host's own INPUT / reply path (it only permits the box to route
1251 /// transit traffic). We snapshot its prior value once so teardown can
1252 /// restore it.
1253 ///
1254 /// IPv6 is the dangerous case: `net.ipv6.conf.all.forwarding=1` forces
1255 /// `accept_ra=0` + `autoconf=0` on EVERY IPv6 interface, which drops the
1256 /// RA-learned default route and path-MTU on the public NIC and blackholes
1257 /// the host's own larger reply packets. We therefore enable forwarding
1258 /// only on the specific overlay device(s) via
1259 /// `net.ipv6.conf.<dev>.forwarding`, which routes overlay traffic without
1260 /// touching the physical NIC's RA / PMTU state.
1261 #[cfg(target_os = "linux")]
1262 fn enable_forwarding_for_attach(
1263 &mut self,
1264 is_v6: bool,
1265 veth_host: &str,
1266 bridge_name: Option<&str>,
1267 ) {
1268 // IPv4 forwarding (global) — required for v4 overlay egress, benign
1269 // for INPUT. Snapshot the prior value exactly once.
1270 if self.prev_ipv4_forward.is_none() {
1271 let prev = crate::netlink::read_sysctl("net.ipv4.ip_forward")
1272 .unwrap_or_else(|_| "0".to_string());
1273 self.prev_ipv4_forward = Some(prev);
1274 }
1275 let _ = crate::netlink::set_sysctl("net.ipv4.ip_forward", "1");
1276
1277 // IPv6 forwarding — PER-INTERFACE only. Enable on the host-side veth
1278 // and (when bridged) the bridge so the overlay routes, without the
1279 // `all.forwarding` RA/PMTU side effect on the physical NIC. The Linux
1280 // sysctl name uses '/' for the interface segment escaped to '.' by
1281 // set_sysctl's dot-translation — so pass the device name with any
1282 // literal dots intact (overlay device names never contain dots).
1283 if is_v6 {
1284 for dev in std::iter::once(veth_host).chain(bridge_name) {
1285 let key = format!("net.ipv6.conf.{dev}.forwarding");
1286 if crate::netlink::set_sysctl(&key, "1").is_ok() {
1287 self.ipv6_forward_ifaces.insert(dev.to_string());
1288 }
1289 }
1290 }
1291 }
1292
1293 /// Revert the forwarding sysctls this daemon enabled (counterpart of
1294 /// [`Self::enable_forwarding_for_attach`]). Restores the snapshotted IPv4
1295 /// value and clears per-interface IPv6 forwarding on exactly the devices
1296 /// we touched. Best-effort: a failed write (device already gone, `/proc`
1297 /// not writable) is ignored — the worst case is the pre-existing sticky
1298 /// state, never a crash on shutdown.
1299 #[cfg(target_os = "linux")]
1300 fn revert_forwarding(&mut self) {
1301 if let Some(prev) = self.prev_ipv4_forward.take() {
1302 let _ = crate::netlink::set_sysctl("net.ipv4.ip_forward", &prev);
1303 }
1304 for dev in self.ipv6_forward_ifaces.drain() {
1305 let key = format!("net.ipv6.conf.{dev}.forwarding");
1306 let _ = crate::netlink::set_sysctl(&key, "0");
1307 }
1308 }
1309
1310 // -- service overlay -----------------------------------------------------
1311
1312 /// Set up the per-service Linux bridge that backs `service` on this node.
1313 ///
1314 /// Returns the bridge name on success.
1315 ///
1316 /// # Errors
1317 /// Returns an error if subnet assignment fails (exhaustion), if the bridge
1318 /// cannot be created, or if the cluster transport rejects the `AllowedIPs`
1319 /// update.
1320 #[cfg(target_os = "linux")]
1321 async fn setup_service_overlay(
1322 &mut self,
1323 service: &str,
1324 mode: OverlayMode,
1325 ) -> Result<ServiceOverlayInfo, OverlaydError> {
1326 // Decision surface is the two predicates on `OverlayMode` (see
1327 // `zlayer_types::overlay`), not an ad-hoc variant match:
1328 // - uses_shared_bridge() -> the single node-wide shared bridge (+ the
1329 // userspace free-port L4 proxy wired in `proxy_manager.rs`).
1330 // - uses_per_service_wg() -> a dedicated per-service WireGuard device.
1331 // - uses_isolation_scope() -> Isolated: Auto topology here; the L3
1332 // fence is applied at ATTACH time via `isolation_network`.
1333 // - otherwise (Auto) -> per-service Linux bridge carried on the
1334 // single cluster-wide WireGuard interface (today's default).
1335 // Record the resolved mode so the container ATTACH path can branch.
1336 let resolved = mode.resolve();
1337 self.service_modes.insert(service.to_string(), resolved);
1338 if resolved.uses_shared_bridge() {
1339 self.setup_service_overlay_shared_bridge(service).await
1340 } else if resolved.uses_per_service_wg() {
1341 self.setup_service_overlay_dedicated(service).await
1342 } else if resolved.uses_isolation_scope() {
1343 // Isolated == Auto topology (per-service bridge on the cluster-wide
1344 // WireGuard); the L3 fence is applied at ATTACH time via
1345 // `isolation_network`, not in segment setup. Same target as the
1346 // default, made explicit so a new mode can't silently fall through.
1347 self.setup_service_overlay_cluster_wg(service).await
1348 } else {
1349 self.setup_service_overlay_cluster_wg(service).await
1350 }
1351 }
1352
1353 /// `Auto`-mode per-service overlay (Linux): a per-service Linux bridge backed
1354 /// by the SINGLE cluster-wide `WireGuard` transport (the service subnet is
1355 /// plumbed onto the cluster device's `AllowedIPs`). This is the original
1356 /// default `setup_service_overlay` body, returning a [`ServiceOverlayInfo`]
1357 /// with the bridge name and all dedicated-device identity fields `None`
1358 /// (`Auto` shares the cluster device).
1359 ///
1360 /// Returns the bridge name on success.
1361 ///
1362 /// # Errors
1363 /// Returns an error if subnet assignment fails (exhaustion), if the bridge
1364 /// cannot be created, or if the cluster transport rejects the `AllowedIPs`
1365 /// update.
1366 #[cfg(target_os = "linux")]
1367 #[allow(clippy::too_many_lines)]
1368 async fn setup_service_overlay_cluster_wg(
1369 &mut self,
1370 service: &str,
1371 ) -> Result<ServiceOverlayInfo, OverlaydError> {
1372 // 1. Idempotency check.
1373 if let Some(existing) = self.service_bridges.get(service) {
1374 let name = existing.name.clone();
1375 tracing::debug!(service = %service, bridge = %name, "Service bridge already active, reusing");
1376 return Ok(cluster_wg_overlay_info(name));
1377 }
1378
1379 // 2. Assign subnet via the (currently local) ServiceSubnetRegistry.
1380 self.ensure_service_subnet_registry()?;
1381 let subnet: ipnet::IpNet = {
1382 let registry = self
1383 .service_subnet_registry
1384 .as_mut()
1385 .expect("ensure_service_subnet_registry leaves Some");
1386 let node_key = self.local_node_id.to_string();
1387 registry.assign(service, &node_key).map_err(|e| {
1388 OverlaydError::Overlay(format!(
1389 "ServiceSubnetRegistry::assign({service}, {node_key}) failed: {e}"
1390 ))
1391 })?
1392 };
1393
1394 // 3+4+6. Create the per-service Linux bridge, assign its gateway, bring
1395 // it up, build the per-service IpAllocator, and record it.
1396 let bridge_name = self.create_service_bridge(service, subnet).await?;
1397
1398 // 5. Plumb subnet into the cluster transport's local AllowedIPs so the
1399 // single cluster device carries this service's cross-node traffic
1400 // (Shared mode shares one crypto context for every service).
1401 if let Some(ref cluster) = self.global_transport {
1402 if let Some(ref pubkey) = self.local_wg_pubkey {
1403 if let Err(e) = cluster.add_allowed_ip(pubkey, subnet).await {
1404 tracing::warn!(
1405 service = %service,
1406 subnet = %subnet,
1407 error = %e,
1408 "Failed to add service subnet to cluster transport AllowedIPs (non-fatal)"
1409 );
1410 }
1411 } else {
1412 tracing::debug!(service = %service, "local_wg_pubkey not yet set; skipping cluster AllowedIPs update");
1413 }
1414 }
1415
1416 Ok(cluster_wg_overlay_info(bridge_name))
1417 }
1418
1419 /// `Shared`-mode per-service overlay (Linux): attach `service` onto the
1420 /// SINGLE node-wide shared Linux bridge (created once, reused by every
1421 /// Shared service on this node), carried on the cluster-wide `WireGuard`
1422 /// interface. There is NO per-service bridge and NO per-service `WireGuard`;
1423 /// container ports are exposed via the userspace free-port L4 proxy
1424 /// (`proxy_manager.rs`). Returns the shared bridge name.
1425 ///
1426 /// Idempotent: the shared bridge is allocated a single subnet and brought up
1427 /// exactly once; subsequent Shared services reuse it. The service is recorded
1428 /// in `service_interfaces` (pointing at the shared bridge) so presence checks
1429 /// and the attach path resolve it.
1430 ///
1431 /// # Errors
1432 /// Returns an error if the one-time shared-subnet assignment fails
1433 /// (exhaustion), if the shared bridge cannot be created, or if the cluster
1434 /// transport rejects the `AllowedIPs` update.
1435 #[cfg(target_os = "linux")]
1436 async fn setup_service_overlay_shared_bridge(
1437 &mut self,
1438 service: &str,
1439 ) -> Result<ServiceOverlayInfo, OverlaydError> {
1440 let bridge_name = self.ensure_shared_bridge().await?;
1441 // Point this service at the shared bridge so presence checks succeed and
1442 // the attach path resolves it to the shared bridge.
1443 self.service_interfaces
1444 .insert(service.to_string(), bridge_name.clone());
1445 tracing::info!(service = %service, bridge = %bridge_name, "Service attached to shared node-wide bridge");
1446 Ok(shared_overlay_info(bridge_name))
1447 }
1448
1449 /// Ensure the single node-wide shared Linux bridge exists, returning its
1450 /// name. Created once with its own subnet (drawn from the same
1451 /// `ServiceSubnetRegistry` every service subnet comes from, under a fixed
1452 /// reserved key so it never collides with a real service) and plumbed onto
1453 /// the cluster transport's `AllowedIPs` so shared containers are
1454 /// mesh-reachable across nodes. Subsequent calls return the existing name.
1455 ///
1456 /// # Errors
1457 /// Returns an error if subnet assignment fails or the bridge cannot be
1458 /// created/addressed/brought up.
1459 #[cfg(target_os = "linux")]
1460 async fn ensure_shared_bridge(&mut self) -> Result<String, OverlaydError> {
1461 use zlayer_overlay::allocator::IpAllocator as OverlayIpAllocator;
1462
1463 if let Some(existing) = self.shared_bridge.as_ref() {
1464 return Ok(existing.name.clone());
1465 }
1466
1467 // One subnet for the whole shared bridge. Use a fixed reserved key in the
1468 // registry (never a real service name) so the shared bridge gets exactly
1469 // one stable subnet, distinct from every per-service subnet.
1470 self.ensure_service_subnet_registry()?;
1471 let subnet: ipnet::IpNet = {
1472 let registry = self
1473 .service_subnet_registry
1474 .as_mut()
1475 .expect("ensure_service_subnet_registry leaves Some");
1476 let node_key = self.local_node_id.to_string();
1477 registry.assign(SHARED_BRIDGE_REGISTRY_KEY, &node_key).map_err(|e| {
1478 OverlaydError::Overlay(format!(
1479 "ServiceSubnetRegistry::assign({SHARED_BRIDGE_REGISTRY_KEY}, {node_key}) failed: {e}"
1480 ))
1481 })?
1482 };
1483
1484 // Deterministic, IFNAMSIZ-safe shared-bridge name (one per node). Use the
1485 // same naming helper as per-service bridges with a fixed key so it stays
1486 // <= 15 chars and is unambiguous (`zl-...-sh`).
1487 let bridge_name =
1488 make_interface_name(&[&self.deployment, &self.instance_id, "shared"], "sh");
1489
1490 if let Err(e) = crate::netlink::create_bridge(&bridge_name).await {
1491 return Err(OverlaydError::Overlay(format!(
1492 "create_bridge({bridge_name}) failed: {e}"
1493 )));
1494 }
1495 if let Err(e) = crate::netlink::set_bridge_stp(&bridge_name, false) {
1496 tracing::warn!(bridge = %bridge_name, error = %e, "set_bridge_stp(off) failed (non-fatal)");
1497 }
1498
1499 // Flush stale addresses first: `create_bridge` is idempotent on EEXIST, so
1500 // a shared bridge that survived a restart would otherwise accumulate a
1501 // second gateway (the same dual-address bug fixed for per-service bridges).
1502 let gateway = first_usable_ip(subnet);
1503 if let Err(e) = crate::netlink::flush_addresses_on_link_by_name(&bridge_name).await {
1504 tracing::warn!(bridge = %bridge_name, error = %e, "flush_addresses_on_link_by_name failed (non-fatal)");
1505 }
1506 if let Err(e) =
1507 crate::netlink::add_address_to_link_by_name(&bridge_name, gateway, subnet.prefix_len())
1508 .await
1509 {
1510 let _ = crate::netlink::delete_bridge(&bridge_name).await;
1511 return Err(OverlaydError::Overlay(format!(
1512 "add_address_to_link_by_name({bridge_name}, {gateway}/{}) failed: {e}",
1513 subnet.prefix_len()
1514 )));
1515 }
1516 if let Err(e) = crate::netlink::set_link_up_by_name(&bridge_name).await {
1517 let _ = crate::netlink::delete_bridge(&bridge_name).await;
1518 return Err(OverlaydError::Overlay(format!(
1519 "set_link_up_by_name({bridge_name}) failed: {e}"
1520 )));
1521 }
1522
1523 // Track the shared bridge for global teardown (deleting the link drops
1524 // its gateway address + up state).
1525 self.created_bridges.insert(bridge_name.clone());
1526
1527 let mut ip_allocator = OverlayIpAllocator::new(&subnet.to_string()).map_err(|e| {
1528 OverlaydError::Overlay(format!("IpAllocator::new({subnet}) failed: {e}"))
1529 })?;
1530 let _ = ip_allocator.allocate_specific(gateway);
1531
1532 // Plumb the shared subnet onto the cluster transport's AllowedIPs so the
1533 // single cluster device carries shared-bridge cross-node traffic (same
1534 // mechanism the cluster-WG per-service path uses).
1535 if let Some(ref cluster) = self.global_transport {
1536 if let Some(ref pubkey) = self.local_wg_pubkey {
1537 if let Err(e) = cluster.add_allowed_ip(pubkey, subnet).await {
1538 tracing::warn!(
1539 subnet = %subnet,
1540 error = %e,
1541 "Failed to add shared-bridge subnet to cluster transport AllowedIPs (non-fatal)"
1542 );
1543 }
1544 } else {
1545 tracing::debug!(
1546 "local_wg_pubkey not yet set; skipping shared-bridge cluster AllowedIPs update"
1547 );
1548 }
1549 }
1550
1551 self.shared_bridge = Some(ServiceBridge {
1552 name: bridge_name.clone(),
1553 subnet,
1554 gateway,
1555 ip_allocator,
1556 });
1557
1558 tracing::info!(bridge = %bridge_name, subnet = %subnet, gateway = %gateway, "Shared node-wide bridge created");
1559 Ok(bridge_name)
1560 }
1561
1562 /// Create the per-service Linux bridge for `service` on `subnet`, assign its
1563 /// gateway, bring it up, build the per-service [`IpAllocator`], and record it
1564 /// in `service_bridges` + `service_interfaces`. Returns the bridge name.
1565 ///
1566 /// Shared and Dedicated mode share this bridge mechanic verbatim — the ONLY
1567 /// difference between the two modes is which `WireGuard` device the service
1568 /// subnet/peers are plumbed onto (the single cluster transport for Shared,
1569 /// the dedicated per-service transport for Dedicated). This helper does NOT
1570 /// touch any transport's `AllowedIPs`; the caller does that against the
1571 /// device it owns.
1572 ///
1573 /// # Errors
1574 /// Returns an error if the bridge cannot be created, addressed, or brought
1575 /// up, or if the per-service `IpAllocator` cannot be built.
1576 #[cfg(target_os = "linux")]
1577 async fn create_service_bridge(
1578 &mut self,
1579 service: &str,
1580 subnet: ipnet::IpNet,
1581 ) -> Result<String, OverlaydError> {
1582 use zlayer_overlay::allocator::IpAllocator as OverlayIpAllocator;
1583
1584 let bridge_name = make_interface_name(&[&self.deployment, &self.instance_id, service], "b");
1585
1586 if let Err(e) = crate::netlink::create_bridge(&bridge_name).await {
1587 return Err(OverlaydError::Overlay(format!(
1588 "create_bridge({bridge_name}) failed: {e}"
1589 )));
1590 }
1591 if let Err(e) = crate::netlink::set_bridge_stp(&bridge_name, false) {
1592 tracing::warn!(bridge = %bridge_name, error = %e, "set_bridge_stp(off) failed (non-fatal)");
1593 }
1594
1595 // Gateway = first usable host in the subnet, assigned to the bridge.
1596 // Flush any pre-existing addresses FIRST: `create_bridge` is idempotent
1597 // on EEXIST, so a bridge that survived a restart would otherwise keep its
1598 // old gateway and we'd stack the new one on top (the observed dual
1599 // /28 + /26 bug). Flushing makes the assignment idempotent and self-heals
1600 // such bridges. Non-fatal: on a brand-new bridge there is nothing to flush.
1601 let gateway = first_usable_ip(subnet);
1602 if let Err(e) = crate::netlink::flush_addresses_on_link_by_name(&bridge_name).await {
1603 tracing::warn!(bridge = %bridge_name, error = %e, "flush_addresses_on_link_by_name failed (non-fatal)");
1604 }
1605 if let Err(e) =
1606 crate::netlink::add_address_to_link_by_name(&bridge_name, gateway, subnet.prefix_len())
1607 .await
1608 {
1609 let _ = crate::netlink::delete_bridge(&bridge_name).await;
1610 return Err(OverlaydError::Overlay(format!(
1611 "add_address_to_link_by_name({bridge_name}, {gateway}/{}) failed: {e}",
1612 subnet.prefix_len()
1613 )));
1614 }
1615 if let Err(e) = crate::netlink::set_link_up_by_name(&bridge_name).await {
1616 let _ = crate::netlink::delete_bridge(&bridge_name).await;
1617 return Err(OverlaydError::Overlay(format!(
1618 "set_link_up_by_name({bridge_name}) failed: {e}"
1619 )));
1620 }
1621
1622 // Track the per-service bridge for global teardown (deleting the link
1623 // drops its gateway address + up state).
1624 self.created_bridges.insert(bridge_name.clone());
1625
1626 // Build per-service IpAllocator, reserve the gateway.
1627 let mut ip_allocator = OverlayIpAllocator::new(&subnet.to_string()).map_err(|e| {
1628 OverlaydError::Overlay(format!("IpAllocator::new({subnet}) failed: {e}"))
1629 })?;
1630 let _ = ip_allocator.allocate_specific(gateway);
1631
1632 self.service_bridges.insert(
1633 service.to_string(),
1634 ServiceBridge {
1635 name: bridge_name.clone(),
1636 subnet,
1637 gateway,
1638 ip_allocator,
1639 },
1640 );
1641 self.service_interfaces
1642 .insert(service.to_string(), bridge_name.clone());
1643
1644 tracing::info!(service = %service, bridge = %bridge_name, subnet = %subnet, gateway = %gateway, "Service bridge created");
1645 Ok(bridge_name)
1646 }
1647
1648 /// Non-Linux variant of `setup_service_overlay`. On Windows the per-service
1649 /// segment is the HCN Internal network created lazily at attach time, and on
1650 /// macOS containers fall through to host networking. Registers the service
1651 /// in `service_interfaces` with a placeholder name so presence checks work.
1652 ///
1653 /// # Errors
1654 /// Infallible on non-Linux; the `Result` is preserved for ABI parity.
1655 #[cfg(not(target_os = "linux"))]
1656 async fn setup_service_overlay(
1657 &mut self,
1658 service: &str,
1659 mode: OverlayMode,
1660 ) -> Result<ServiceOverlayInfo, OverlaydError> {
1661 // Same predicate-driven decision surface as Linux (see
1662 // `zlayer_types::overlay`). The container ATTACH path differentiates the
1663 // modes per-OS; here we only record the resolved mode and register the
1664 // appropriate placeholder/info so presence checks and `Status` work.
1665 //
1666 // - uses_per_service_wg() -> the cross-platform dedicated path (a real
1667 // per-service WireGuard device; on Windows it also stands up a
1668 // per-service HCN Internal network at attach time).
1669 // - otherwise (`Auto` and `Shared`) -> no per-service WireGuard device.
1670 // On macOS both rely on VZ NAT + host-port forwarding (the free-port
1671 // L4 proxy), so they route to the SAME real path — the only honest
1672 // mapping a VZ guest can express (it has no per-service bridge or WG
1673 // to differentiate). On Windows the attach path reads the recorded
1674 // mode to send `Shared` containers onto a shared HCN NAT network and
1675 // `Auto` containers onto the node's base overlay network.
1676 // - uses_isolation_scope() -> Isolated: Auto topology here; the L3
1677 // fence is applied at ATTACH time via `isolation_network`.
1678 let resolved = mode.resolve();
1679 self.service_modes.insert(service.to_string(), resolved);
1680 if resolved.uses_per_service_wg() {
1681 self.setup_service_overlay_dedicated(service).await
1682 } else if resolved.uses_shared_bridge() {
1683 self.setup_service_overlay_shared_bridge(service).await
1684 } else if resolved.uses_isolation_scope() {
1685 // Isolated == Auto topology (per-service bridge on the cluster-wide
1686 // WireGuard); the L3 fence is applied at ATTACH time via
1687 // `isolation_network`, not in segment setup. Same target as the
1688 // default, made explicit so a new mode can't silently fall through.
1689 self.setup_service_overlay_cluster_wg(service).await
1690 } else {
1691 self.setup_service_overlay_cluster_wg(service).await
1692 }
1693 }
1694
1695 /// `Auto`-mode per-service overlay (non-Linux): on Windows the per-service
1696 /// segment is the node's base overlay HCN network used at attach time, and on
1697 /// macOS containers ride VZ NAT. Registers the service in `service_interfaces`
1698 /// with a placeholder name so presence checks work.
1699 ///
1700 /// # Errors
1701 /// Infallible on non-Linux; the `Result` is preserved for ABI parity.
1702 #[cfg(not(target_os = "linux"))]
1703 #[allow(clippy::unused_async)]
1704 async fn setup_service_overlay_cluster_wg(
1705 &mut self,
1706 service: &str,
1707 ) -> Result<ServiceOverlayInfo, OverlaydError> {
1708 let placeholder = make_interface_name(&[&self.deployment, &self.instance_id, service], "b");
1709 self.service_interfaces
1710 .insert(service.to_string(), placeholder.clone());
1711 tracing::debug!(service = %service, "Service overlay bridge setup is Linux-only; using direct networking placeholder");
1712 Ok(cluster_wg_overlay_info(placeholder))
1713 }
1714
1715 /// `Shared`-mode per-service overlay (non-Linux). There is no per-service
1716 /// `WireGuard` device and no per-service bridge:
1717 /// - macOS: the container is a VZ VM behind VZ NAT (a single shared host
1718 /// adapter with host-port forwarding); its ports are exposed by the
1719 /// userspace free-port L4 proxy. Nothing to provision here beyond a
1720 /// placeholder so presence checks succeed.
1721 /// - Windows: containers attach to a SINGLE shared HCN NAT network reused
1722 /// across all Shared services (created lazily at attach time); a placeholder
1723 /// interface is registered here.
1724 ///
1725 /// Registers the service in `service_interfaces` with a placeholder name.
1726 ///
1727 /// # Errors
1728 /// Infallible on non-Linux; the `Result` is preserved for ABI parity.
1729 #[cfg(not(target_os = "linux"))]
1730 #[allow(clippy::unused_async)]
1731 async fn setup_service_overlay_shared_bridge(
1732 &mut self,
1733 service: &str,
1734 ) -> Result<ServiceOverlayInfo, OverlaydError> {
1735 // A single placeholder shared by every Shared service on this node (it
1736 // names the shared data-plane, not a per-service interface).
1737 let placeholder =
1738 make_interface_name(&[&self.deployment, &self.instance_id, "shared"], "sh");
1739 self.service_interfaces
1740 .insert(service.to_string(), placeholder.clone());
1741 tracing::debug!(service = %service, "Shared-mode service uses the node-wide shared data-plane (VZ NAT on macOS / shared HCN NAT on Windows)");
1742 Ok(shared_overlay_info(placeholder))
1743 }
1744
1745 /// Dedicated-mode per-service overlay: stand up a *second* real `WireGuard`
1746 /// device for `service` with its own crypto context, listen port, overlay
1747 /// IP, and subnet — distinct from the single cluster transport.
1748 ///
1749 /// The cross-platform core (identity, subnet assign, transport bring-up,
1750 /// marker persist, status) runs on every OS; only the *attachment* of
1751 /// containers onto the device is platform-gated:
1752 /// - Linux: a per-service bridge (same mechanic as Shared) routed over the
1753 /// dedicated device instead of the cluster device.
1754 /// - Windows: a per-service HCN Internal network (a later task; a clearly
1755 /// marked seam returns an error here for now).
1756 /// - macOS: nothing further — the utun device is the attachment.
1757 ///
1758 /// # Errors
1759 /// Returns an error if port/key/subnet allocation, transport bring-up,
1760 /// marker persistence, or the platform attachment fails.
1761 #[allow(clippy::too_many_lines)]
1762 async fn setup_service_overlay_dedicated(
1763 &mut self,
1764 service: &str,
1765 ) -> Result<ServiceOverlayInfo, OverlaydError> {
1766 // ----- cross-platform core (runs on every OS) -----
1767
1768 // 1. Idempotency: an existing dedicated transport returns its identity.
1769 if let Some(st) = self.service_transports.get(service) {
1770 return Ok(dedicated_overlay_info(
1771 st.interface.clone(),
1772 &st.public_key,
1773 st.listen_port,
1774 st.overlay_ip,
1775 st.subnet,
1776 ));
1777 }
1778
1779 // 2. Identity: reuse a stable identity from the marker if one exists
1780 // (so the device re-binds the same key + port across restarts),
1781 // otherwise mint a fresh port + keypair + interface name.
1782 let marker_path =
1783 zlayer_paths::ZLayerDirs::new(self.data_dir.clone()).agent_network_state();
1784 let recorded = NetworkState::load(&marker_path)
1785 .get(&owner_for_service(service))
1786 .cloned();
1787
1788 let (private_key, public_key, listen_port, iface_hint) = match recorded.as_ref() {
1789 Some(entry)
1790 if entry.wg_private_key.is_some()
1791 && entry.wg_public_key.is_some()
1792 && entry.wg_port.is_some()
1793 && entry.interface.is_some() =>
1794 {
1795 let port = entry.wg_port.expect("checked above");
1796 self.dedicated_ports.reserve(port);
1797 (
1798 entry.wg_private_key.clone().expect("checked above"),
1799 entry.wg_public_key.clone().expect("checked above"),
1800 port,
1801 entry.interface.clone().expect("checked above"),
1802 )
1803 }
1804 _ => {
1805 let port = self.dedicated_ports.allocate()?;
1806 let (priv_key, pub_key) = OverlayTransport::generate_keys()
1807 .await
1808 .map_err(|e| OverlaydError::Overlay(format!("Failed to generate keys: {e}")))?;
1809 let iface =
1810 make_interface_name(&[&self.deployment, &self.instance_id, service], "d");
1811 (priv_key, pub_key, port, iface)
1812 }
1813 };
1814
1815 // 3. Subnet: assign from the same registry Shared uses, so per-service
1816 // subnets stay globally unique regardless of mode.
1817 self.ensure_service_subnet_registry()?;
1818 let subnet: ipnet::IpNet = {
1819 let registry = self
1820 .service_subnet_registry
1821 .as_mut()
1822 .expect("ensure_service_subnet_registry leaves Some");
1823 let node_key = self.local_node_id.to_string();
1824 registry.assign(service, &node_key).map_err(|e| {
1825 OverlaydError::Overlay(format!(
1826 "ServiceSubnetRegistry::assign({service}, {node_key}) failed: {e}"
1827 ))
1828 })?
1829 };
1830 let overlay_ip = first_usable_ip(subnet);
1831
1832 // 4. Build + bring up the dedicated transport. The device's overlay CIDR
1833 // is the service subnet (so boringtun routes that subnet over THIS
1834 // device), and its listen port is the dedicated port.
1835 let physical_egress_ip = match zlayer_overlay::detect_physical_egress().await {
1836 Ok(egress) => Some(egress.ip),
1837 Err(e) => {
1838 tracing::warn!(
1839 error = %e,
1840 service = %service,
1841 "failed to detect physical egress; WireGuard local_endpoint \
1842 will bind UNSPECIFIED for the dedicated overlay"
1843 );
1844 None
1845 }
1846 };
1847 let config = self.build_config(
1848 private_key.clone(),
1849 public_key.clone(),
1850 overlay_ip,
1851 subnet.prefix_len(),
1852 listen_port,
1853 physical_egress_ip,
1854 );
1855 let mut transport = OverlayTransport::new(config, iface_hint);
1856 transport.create_interface().await.map_err(|e| {
1857 OverlaydError::Overlay(format!(
1858 "Failed to create dedicated overlay for {service}: {e}"
1859 ))
1860 })?;
1861 transport.configure(&[]).await.map_err(|e| {
1862 OverlaydError::Overlay(format!(
1863 "Failed to configure dedicated overlay for {service}: {e}"
1864 ))
1865 })?;
1866 let actual_iface = transport.interface_name().to_string();
1867
1868 // 5. Persist the marker so the identity survives restarts. Match the
1869 // base/Shared entry shape (owner/kind/name/id/subnet) plus the
1870 // dedicated WG fields.
1871 let mut marker = NetworkState::load(&marker_path);
1872 marker.upsert(ManagedNetwork {
1873 owner: owner_for_service(service),
1874 kind: "wg-dedicated".to_string(),
1875 name: actual_iface.clone(),
1876 id: public_key.clone(),
1877 subnet: subnet.to_string(),
1878 wg_port: Some(listen_port),
1879 wg_private_key: Some(private_key),
1880 wg_public_key: Some(public_key.clone()),
1881 interface: Some(actual_iface.clone()),
1882 });
1883 if let Err(e) = marker.save(&marker_path) {
1884 tracing::warn!(service = %service, error = %e, path = %marker_path.display(), "failed to persist dedicated-overlay marker (device still live)");
1885 }
1886
1887 // 6. Record the live transport. Build the guest-attach IPAM bounded to
1888 // the service subnet, reserving the node's own dedicated-device IP so
1889 // a joining guest never draws it.
1890 let mut ip_allocator = zlayer_overlay::allocator::IpAllocator::new(&subnet.to_string())
1891 .map_err(|e| {
1892 OverlaydError::Overlay(format!("IpAllocator::new({subnet}) failed: {e}"))
1893 })?;
1894 let _ = ip_allocator.allocate_specific(overlay_ip);
1895 self.service_transports.insert(
1896 service.to_string(),
1897 ServiceTransport {
1898 transport,
1899 interface: actual_iface.clone(),
1900 public_key: public_key.clone(),
1901 listen_port,
1902 overlay_ip,
1903 subnet,
1904 ip_allocator,
1905 },
1906 );
1907
1908 tracing::info!(
1909 service = %service,
1910 interface = %actual_iface,
1911 listen_port,
1912 subnet = %subnet,
1913 overlay_ip = %overlay_ip,
1914 "Dedicated per-service overlay device created"
1915 );
1916
1917 // ----- platform-gated attachment -----
1918 // `name` in the returned info is the container-attach handle: the bridge
1919 // name on Linux, the dedicated interface elsewhere.
1920 let name = self
1921 .attach_dedicated_service(service, subnet, overlay_ip)
1922 .await?;
1923
1924 Ok(dedicated_overlay_info(
1925 name,
1926 &public_key,
1927 listen_port,
1928 overlay_ip,
1929 subnet,
1930 ))
1931 }
1932
1933 /// Linux attachment for a dedicated per-service overlay: create the same
1934 /// per-service bridge Shared uses, but route the service subnet over the
1935 /// DEDICATED device rather than the cluster device.
1936 ///
1937 /// Concretely, the dedicated transport's overlay CIDR already covers
1938 /// `subnet` (set at `build_config` time in the core), so boringtun routes
1939 /// `subnet` out the dedicated TUN; we additionally plumb `subnet` onto this
1940 /// node's own `AllowedIPs` entry on the dedicated device so locally
1941 /// originated packets to the subnet are accepted. Returns the bridge name.
1942 ///
1943 /// # Errors
1944 /// Returns an error if the bridge cannot be created.
1945 #[cfg(target_os = "linux")]
1946 async fn attach_dedicated_service(
1947 &mut self,
1948 service: &str,
1949 subnet: ipnet::IpNet,
1950 overlay_ip: IpAddr,
1951 ) -> Result<String, OverlaydError> {
1952 let _ = overlay_ip;
1953 let bridge_name = self.create_service_bridge(service, subnet).await?;
1954
1955 // Plumb the service subnet onto the DEDICATED device (not the cluster
1956 // device). The dedicated transport's overlay CIDR already routes the
1957 // subnet out its TUN; adding it to our own pubkey's AllowedIPs keeps the
1958 // local-accept side consistent with the Shared path's cluster plumbing.
1959 if let Some(st) = self.service_transports.get(service) {
1960 if let Some(ref pubkey) = self.local_wg_pubkey {
1961 if let Err(e) = st.transport.add_allowed_ip(pubkey, subnet).await {
1962 tracing::warn!(
1963 service = %service,
1964 subnet = %subnet,
1965 error = %e,
1966 "Failed to add service subnet to dedicated transport AllowedIPs (non-fatal)"
1967 );
1968 }
1969 } else {
1970 tracing::debug!(service = %service, "local_wg_pubkey not yet set; skipping dedicated AllowedIPs update");
1971 }
1972 }
1973
1974 Ok(bridge_name)
1975 }
1976
1977 /// Windows attachment for a dedicated per-service overlay.
1978 ///
1979 /// The cross-platform core has already stood up the dedicated Wintun
1980 /// transport (the encrypted node-to-node path for the service subnet). This
1981 /// adds the *container-facing* side: a per-service HCN **Internal** network
1982 /// onto which the agent's containers attach (instead of the node's shared
1983 /// base overlay network), so dedicated-service traffic is isolated at the
1984 /// vSwitch layer. Returns the per-service network's name, which the caller
1985 /// records as the [`ServiceOverlayInfo::name`] attach handle.
1986 ///
1987 /// # Errors
1988 /// Propagates any error from [`Self::ensure_service_network`].
1989 #[cfg(target_os = "windows")]
1990 async fn attach_dedicated_service(
1991 &mut self,
1992 service: &str,
1993 subnet: ipnet::IpNet,
1994 _overlay_ip: IpAddr,
1995 ) -> Result<String, OverlaydError> {
1996 // Create (or reuse) the per-service Internal HCN network. The returned
1997 // GUID is recorded in the marker under `owner_for_service(service)`;
1998 // the `AttachContainer` handler reuses it via the same marker lookup.
1999 let _net_id = self.ensure_service_network(service, subnet).await?;
2000 // The attach handle reported back is the per-service network's name.
2001 let daemon_name = self.deployment_or_default();
2002 Ok(format!(
2003 "{}-svc-{service}",
2004 overlay_network_name(&daemon_name)
2005 ))
2006 }
2007
2008 /// macOS attachment for a dedicated per-service overlay: the cross-platform
2009 /// core already brought up a utun device; there is no bridge, so the
2010 /// interface name itself is the attach handle.
2011 #[cfg(all(not(target_os = "linux"), not(target_os = "windows")))]
2012 #[allow(clippy::unused_async)]
2013 async fn attach_dedicated_service(
2014 &mut self,
2015 service: &str,
2016 _subnet: ipnet::IpNet,
2017 _overlay_ip: IpAddr,
2018 ) -> Result<String, OverlaydError> {
2019 let iface = self
2020 .service_transports
2021 .get(service)
2022 .map(|st| st.interface.clone())
2023 .unwrap_or_default();
2024 Ok(iface)
2025 }
2026
2027 /// Tear down the per-service segment for `service`. Idempotent.
2028 // Only the Linux body awaits (netlink + cluster AllowedIPs); other targets
2029 // are synchronous (transport shutdown is sync) but must keep the async
2030 // signature for the dispatch call.
2031 #[cfg_attr(not(target_os = "linux"), allow(clippy::unused_async))]
2032 async fn teardown_service_overlay(&mut self, service: &str) {
2033 // Drop the recorded mode; a `Shared` service's containers no longer route
2034 // to the shared bridge once it is gone. The node-wide shared bridge
2035 // itself is deliberately NOT torn down here — other Shared services reuse
2036 // it (it is reclaimed only on full overlay teardown / uninstall).
2037 self.service_modes.remove(service);
2038
2039 // Auto-mode segment teardown (per-service bridge on Linux, placeholder
2040 // elsewhere). A Shared-mode service has no per-service bridge, so
2041 // `service_bridges.remove` is a no-op for it (its `service_interfaces`
2042 // placeholder pointing at the shared bridge is removed below).
2043 #[cfg(target_os = "linux")]
2044 {
2045 let removed = self.service_bridges.remove(service);
2046 self.service_interfaces.remove(service);
2047
2048 // Remove the subnet from the cluster AllowedIPs only when we still
2049 // know it (the in-memory entry survived).
2050 if let Some(ref bridge) = removed {
2051 if let Some(ref cluster) = self.global_transport {
2052 if let Some(ref pubkey) = self.local_wg_pubkey {
2053 if let Err(e) = cluster.remove_allowed_ip(pubkey, bridge.subnet).await {
2054 tracing::warn!(
2055 service = %service,
2056 subnet = %bridge.subnet,
2057 error = %e,
2058 "Failed to remove service subnet from cluster AllowedIPs (non-fatal)"
2059 );
2060 }
2061 }
2062 }
2063 }
2064
2065 // Delete the physical bridge by its DETERMINISTIC name, regardless of
2066 // whether the in-memory entry survived. After an overlayd restart the
2067 // `service_bridges` map is empty, so a delete gated on `Some(..)` would
2068 // silently leak the `zl-…-b` link forever (the observed orphan/linkdown
2069 // bridges). `delete_bridge` no-ops on ENODEV, so deleting an absent link
2070 // is safe — and the `-b` suffix never collides with a Shared service's
2071 // shared `-sh` bridge, so this can't tear down the wrong thing.
2072 let bridge_name = removed.as_ref().map_or_else(
2073 || make_interface_name(&[&self.deployment, &self.instance_id, service], "b"),
2074 |b| b.name.clone(),
2075 );
2076 if let Err(e) = crate::netlink::delete_bridge(&bridge_name).await {
2077 tracing::warn!(service = %service, bridge = %bridge_name, error = %e, "delete_bridge failed (non-fatal)");
2078 }
2079
2080 // Release the subnet-registry slot by service name (works whether or
2081 // not the in-memory entry survived).
2082 if let Some(registry) = self.service_subnet_registry.as_mut() {
2083 let node_key = self.local_node_id.to_string();
2084 let _ = registry.release(service, &node_key);
2085 }
2086
2087 if removed.is_some() {
2088 tracing::info!(service = %service, bridge = %bridge_name, "Tore down service bridge");
2089 } else {
2090 tracing::debug!(service = %service, bridge = %bridge_name, "best-effort delete of (possibly absent) service bridge by name");
2091 }
2092 }
2093 #[cfg(not(target_os = "linux"))]
2094 {
2095 if let Some(iface) = self.service_interfaces.remove(service) {
2096 tracing::info!(service = %service, interface = %iface, "Removed service overlay interface (placeholder, non-Linux)");
2097 }
2098 }
2099
2100 // Dedicated-mode teardown (cross-platform): tear down the per-service
2101 // transport, free its port, and drop its marker entry. No-op when the
2102 // service ran in Shared mode (nothing in `service_transports`).
2103 if let Some(mut st) = self.service_transports.remove(service) {
2104 st.transport.shutdown();
2105 self.dedicated_ports.release(st.listen_port);
2106
2107 // Release the subnet assignment (Shared releases it inside the
2108 // Linux block above; the dedicated subnet lives in the same
2109 // registry, so release it here for the dedicated case on every OS).
2110 if let Some(registry) = self.service_subnet_registry.as_mut() {
2111 let node_key = self.local_node_id.to_string();
2112 let _ = registry.release(service, &node_key);
2113 }
2114
2115 let marker_path =
2116 zlayer_paths::ZLayerDirs::new(self.data_dir.clone()).agent_network_state();
2117 let mut marker = NetworkState::load(&marker_path);
2118 let removed_entry = marker.remove(&owner_for_service(service));
2119 if removed_entry.is_some() {
2120 if let Err(e) = marker.save(&marker_path) {
2121 tracing::warn!(service = %service, error = %e, path = %marker_path.display(), "failed to persist dedicated-overlay marker removal");
2122 }
2123 }
2124
2125 // Windows: delete the per-service HCN Internal network this service
2126 // owned. The marker entry's `id` is the bare HCN GUID (set by
2127 // `ensure_service_network`); delete the network so a dedicated
2128 // service tears down cleanly without waiting for a full uninstall.
2129 // Also drop the per-service container-IP allocator.
2130 #[cfg(target_os = "windows")]
2131 {
2132 self.service_ip_allocators.remove(service);
2133 if let Some(entry) = removed_entry.as_ref() {
2134 if entry.kind == "hcn-internal" {
2135 if let Ok(guid) = windows::core::GUID::try_from(entry.id.as_str()) {
2136 match zlayer_hns::network::Network::delete(guid) {
2137 Ok(()) => {
2138 tracing::info!(service = %service, id = %entry.id, "deleted per-service HCN network");
2139 }
2140 Err(e) => {
2141 tracing::warn!(service = %service, id = %entry.id, error = %e, "failed to delete per-service HCN network (may leak until uninstall)");
2142 }
2143 }
2144 } else {
2145 tracing::warn!(service = %service, id = %entry.id, "per-service marker has unparseable HCN GUID; skipping network delete");
2146 }
2147 }
2148 }
2149 }
2150 #[cfg(not(target_os = "windows"))]
2151 drop(removed_entry);
2152
2153 tracing::info!(
2154 service = %service,
2155 interface = %st.interface,
2156 listen_port = st.listen_port,
2157 "Tore down dedicated per-service overlay device"
2158 );
2159 }
2160 }
2161
2162 /// Reclaim orphaned per-service host bridges (and their stale device veths)
2163 /// that no live deployment still owns. `live_bridge_names` is the full set of
2164 /// `zl-…-b` bridge names every currently-restored service SHOULD own,
2165 /// computed by the main daemon from storage.
2166 ///
2167 /// For every host link whose name looks like one of OUR per-service bridge
2168 /// (`…-b`) or dedicated device (`…-d`) interfaces but is NOT in `live` and is
2169 /// NOT the active global (`-g`) or shared (`-sh`) interface, we:
2170 /// 1. delete the link (idempotent — ENODEV is success),
2171 /// 2. release its service-subnet registry assignment + cluster `AllowedIPs`
2172 /// when the `(service, node)` key can be recovered from the registry
2173 /// snapshot by reproducing the deterministic bridge name, and
2174 /// 3. drop any stale in-memory `service_bridges`/`service_interfaces`
2175 /// entries pointing at it.
2176 ///
2177 /// Best-effort + idempotent: a failure on one link is logged and the sweep
2178 /// continues. Returns the names actually reclaimed.
2179 #[cfg(target_os = "linux")]
2180 async fn prune_orphan_bridges(&mut self, live_bridge_names: &[String]) -> Vec<String> {
2181 use std::collections::HashSet;
2182
2183 let links = match crate::netlink::list_all_links().await {
2184 Ok(links) => links,
2185 Err(e) => {
2186 tracing::warn!(error = %e, "prune_orphan_bridges: failed to list host links");
2187 return Vec::new();
2188 }
2189 };
2190
2191 let live: HashSet<&str> = live_bridge_names.iter().map(String::as_str).collect();
2192
2193 // The interfaces we must NEVER reclaim even though they carry the `zl-`
2194 // prefix: the active global transport device and the node-wide shared
2195 // bridge. (Container veths `veth-…`/`vc-…` are handled by the separate
2196 // PID-keyed `sweep_orphan_veths`; here we only target service bridges +
2197 // dedicated device interfaces, which `sweep_orphan_veths` never touches.)
2198 let mut protected: HashSet<String> = HashSet::new();
2199 if let Some(g) = self.global_interface.clone() {
2200 protected.insert(g);
2201 }
2202 if let Some(ref sh) = self.shared_bridge {
2203 protected.insert(sh.name.clone());
2204 }
2205 // Protect every dedicated-service WireGuard transport (`…-d`) by name. A
2206 // `-d` is a WG device, not a bridge — it has no `brif`, so the zero-member
2207 // guard below treats it as 0 members, and the daemon's `live` set only
2208 // carries `…-b` names; without this it would be reaped as a live device.
2209 //
2210 // We deliberately do NOT blanket-protect `service_bridges` (`…-b`) here.
2211 // That map holds BOTH managed-service bridges AND standalone/per-job
2212 // bridges (e.g. a Runner's per-job network), and overlayd cannot tell
2213 // them apart — a standalone container's `DetachContainer` releases the
2214 // veth/IP but never removes the bridge or its `service_bridges` entry, so
2215 // a blanket protect shielded those orphans forever (only a restart, which
2216 // wipes the map, ever cleared them). Managed bridges stay protected by
2217 // being in the daemon's authoritative `live` set; standalone bridges are
2218 // not in storage, so they fall through to the zero-member guard and are
2219 // reclaimed once idle.
2220 for st in self.service_transports.values() {
2221 protected.insert(st.interface.clone());
2222 }
2223
2224 // Snapshot the subnet registry once so we can recover the `(service,
2225 // node)` key for an orphan by reproducing its deterministic bridge/device
2226 // name. The registry has no release-by-subnet API, so we map name ->
2227 // (service, node) here.
2228 let mut name_to_key: HashMap<String, (String, String, ipnet::IpNet)> = HashMap::new();
2229 if let Some(registry) = self.service_subnet_registry.as_ref() {
2230 for ((service, node), subnet) in registry.snapshot().assignments {
2231 let bridge =
2232 make_interface_name(&[&self.deployment, &self.instance_id, &service], "b");
2233 let device =
2234 make_interface_name(&[&self.deployment, &self.instance_id, &service], "d");
2235 name_to_key.insert(bridge, (service.clone(), node.clone(), subnet));
2236 name_to_key.insert(device, (service, node, subnet));
2237 }
2238 }
2239
2240 let mut reclaimed = Vec::new();
2241 for (_index, name) in links {
2242 // Only consider OUR per-service bridge (`-b`) or dedicated device
2243 // (`-d`) interfaces that are neither live nor protected. The pure
2244 // predicate (unit-tested in `orphan_bridge_selection`) keeps us off
2245 // unrelated host links, the global/shared interfaces, and the veth
2246 // namespaces.
2247 if !is_orphan_service_bridge(&name, &live, &protected) {
2248 continue;
2249 }
2250
2251 // Zero-member guard: only reclaim a non-live candidate once it is
2252 // IDLE — no member links. A `-b` bridge with a running container has
2253 // ≥1 veth in its `brif`, so an in-use (or a sub-ms mid-creation,
2254 // pre-attach is the only 0-member window) standalone bridge is left
2255 // alone; an orphan `-d` has no `brif` (0) and is correctly reaped.
2256 // This is what makes dropping the `service_bridges` blanket-protect
2257 // safe — a live managed bridge is already excluded by `live`, and any
2258 // other in-use bridge is excluded here.
2259 if crate::netlink::bridge_member_count(&name).await > 0 {
2260 continue;
2261 }
2262
2263 tracing::info!(link = %name, "prune_orphan_bridges: reclaiming orphan service bridge/device");
2264
2265 // 1. Release the subnet + cluster AllowedIPs when we can recover the
2266 // owning service key from the registry.
2267 if let Some((service, node, subnet)) = name_to_key.get(&name).cloned() {
2268 if let Some(ref cluster) = self.global_transport {
2269 if let Some(ref pubkey) = self.local_wg_pubkey {
2270 if let Err(e) = cluster.remove_allowed_ip(pubkey, subnet).await {
2271 tracing::warn!(
2272 link = %name,
2273 subnet = %subnet,
2274 error = %e,
2275 "prune_orphan_bridges: remove_allowed_ip failed (non-fatal)"
2276 );
2277 }
2278 }
2279 }
2280 if let Some(registry) = self.service_subnet_registry.as_mut() {
2281 let _ = registry.release(&service, &node);
2282 }
2283 }
2284
2285 // 2. Delete the link itself (idempotent).
2286 if let Err(e) = crate::netlink::delete_bridge(&name).await {
2287 tracing::warn!(link = %name, error = %e, "prune_orphan_bridges: delete_bridge failed (non-fatal)");
2288 continue;
2289 }
2290
2291 // 3. Drop any stale in-memory bookkeeping pointing at this link.
2292 self.service_bridges.retain(|_, b| b.name != name);
2293 self.service_interfaces.retain(|_, iface| *iface != name);
2294
2295 reclaimed.push(name);
2296 }
2297
2298 if !reclaimed.is_empty() {
2299 tracing::info!(count = reclaimed.len(), bridges = ?reclaimed, "prune_orphan_bridges: reclaimed orphaned service bridges/devices");
2300 }
2301 reclaimed
2302 }
2303
2304 /// Non-Linux variant: per-service bridges are a Linux-only mechanic (Windows
2305 /// uses HCN networks torn down in `teardown_service_overlay`; macOS rides VZ
2306 /// NAT), so there are no host bridge links to sweep.
2307 #[cfg(not(target_os = "linux"))]
2308 #[allow(clippy::unused_async, clippy::unused_self)]
2309 async fn prune_orphan_bridges(&mut self, _live_bridge_names: &[String]) -> Vec<String> {
2310 Vec::new()
2311 }
2312
2313 /// Initialize the local fallback `ServiceSubnetRegistry` from the configured
2314 /// cluster CIDR. Called on first `setup_service_overlay` use.
2315 ///
2316 /// # Errors
2317 /// Returns an error when no cluster CIDR is configured or the registry
2318 /// cannot be built.
2319 fn ensure_service_subnet_registry(&mut self) -> Result<(), OverlaydError> {
2320 use zlayer_overlay::allocator::ServiceSubnetRegistry;
2321
2322 if self.service_subnet_registry.is_some() {
2323 return Ok(());
2324 }
2325 let cluster_cidr = self.cluster_cidr.ok_or_else(|| {
2326 OverlaydError::Other(
2327 "service subnet registry needs a cluster CIDR (SetupGlobalOverlay first)"
2328 .to_string(),
2329 )
2330 })?;
2331 let cluster_ipnet: ipnet::IpNet = cluster_cidr.to_string().parse().map_err(|e| {
2332 OverlaydError::Other(format!(
2333 "failed to convert cluster CIDR {cluster_cidr} to ipnet::IpNet: {e}"
2334 ))
2335 })?;
2336 // Per-service bridge slice prefix. `/26` (V4) = ~61 usable container
2337 // IPs per service per node — keep in sync with
2338 // `zlayer_scheduler::raft::DEFAULT_SERVICE_SUBNET_SLICE_PREFIX` (the
2339 // canonical default; not imported here to avoid a dependency cycle).
2340 // The older `/28` (13 usable) exhausted under CI churn.
2341 let slice_prefix: u8 = match cluster_ipnet {
2342 ipnet::IpNet::V4(_) => 26,
2343 ipnet::IpNet::V6(_) => 120,
2344 };
2345 let mut registry =
2346 ServiceSubnetRegistry::new(cluster_ipnet, slice_prefix).map_err(|e| {
2347 OverlaydError::Other(format!("failed to build ServiceSubnetRegistry: {e}"))
2348 })?;
2349 // Reserve the node's own overlay IP so no per-service bridge subnet
2350 // overlaps it — the overlay DNS server listens on `<node_ip>:53`, and a
2351 // bridge subnet containing that IP would black-hole its containers' DNS
2352 // (they'd ARP for the node IP on their bridge, where nothing answers).
2353 if let Some(node_ip) = self.node_ip {
2354 registry.reserve_ip(node_ip);
2355 }
2356 self.service_subnet_registry = Some(registry);
2357 Ok(())
2358 }
2359
2360 // -- IP allocation -------------------------------------------------------
2361
2362 /// Allocate an overlay IP from the per-service bridge (Linux) or the node
2363 /// slice (otherwise). `join_global` reserves a second global-overlay IP too,
2364 /// matching the eth1 attach behavior.
2365 ///
2366 /// # Errors
2367 /// Returns an error if the relevant pool is exhausted.
2368 fn allocate_ip(&mut self, service: &str, join_global: bool) -> Result<IpAddr, OverlaydError> {
2369 // `join_global` does not allocate a second IP here: the companion
2370 // global-overlay IP (eth1) is reserved at attach time. `AllocateIp`
2371 // returns only the primary (service / slice) IP the caller asked for.
2372 let _ = join_global;
2373 #[cfg(target_os = "linux")]
2374 {
2375 // A Shared-mode service draws from the single node-wide shared bridge;
2376 // every other mode draws from its own per-service bridge.
2377 let use_shared = self
2378 .service_modes
2379 .get(service)
2380 .copied()
2381 .unwrap_or_default()
2382 .uses_shared_bridge();
2383 if use_shared {
2384 if let Some(bridge) = self.shared_bridge.as_mut() {
2385 return bridge.ip_allocator.allocate().ok_or_else(|| {
2386 OverlaydError::Overlay(format!(
2387 "shared bridge {} subnet {} exhausted",
2388 bridge.name, bridge.subnet
2389 ))
2390 });
2391 }
2392 } else if let Some(bridge) = self.service_bridges.get_mut(service) {
2393 return bridge.ip_allocator.allocate().ok_or_else(|| {
2394 OverlaydError::Overlay(format!(
2395 "service bridge {} subnet {} exhausted",
2396 bridge.name, bridge.subnet
2397 ))
2398 });
2399 }
2400 }
2401 let _ = service;
2402 self.ip_allocator.allocate()
2403 }
2404
2405 /// Return an overlay IP to the allocator (service-bridge pool when known,
2406 /// otherwise the node slice).
2407 fn release_ip(&mut self, ip: IpAddr) {
2408 #[cfg(target_os = "linux")]
2409 {
2410 if let Some(bridge) = self.shared_bridge.as_mut() {
2411 if bridge.subnet.contains(&ip) {
2412 bridge.ip_allocator.release(ip);
2413 return;
2414 }
2415 }
2416 for bridge in self.service_bridges.values_mut() {
2417 if bridge.subnet.contains(&ip) {
2418 bridge.ip_allocator.release(ip);
2419 return;
2420 }
2421 }
2422 }
2423 self.ip_allocator.release(ip);
2424 }
2425
2426 // -- container attach (Linux) -------------------------------------------
2427
2428 /// Wire a container into the overlay and return its [`AttachResult`].
2429 ///
2430 /// # Errors
2431 /// Returns an error if the container cannot be attached.
2432 #[allow(clippy::too_many_arguments)]
2433 async fn attach_container(
2434 &mut self,
2435 handle: AttachHandle,
2436 service: &str,
2437 join_global: bool,
2438 ephemeral: bool,
2439 dns_server: Option<IpAddr>,
2440 dns_domain: Option<String>,
2441 isolation_network: Option<String>,
2442 ) -> Result<AttachResult, OverlaydError> {
2443 // Record the overlay DNS resolver/zone the main daemon staged for this
2444 // node so later attaches (and the Windows HCN endpoint `Dns` schema)
2445 // can fall back to them when a per-attach value isn't supplied.
2446 if let Some(server) = dns_server {
2447 self.dns_server_addr = Some(SocketAddr::new(server, 53));
2448 }
2449 if dns_domain.is_some() {
2450 self.dns_domain.clone_from(&dns_domain);
2451 }
2452 match handle {
2453 AttachHandle::LinuxPid { pid } => {
2454 let ip = self
2455 .attach_container_linux(pid, service, join_global, ephemeral, isolation_network)
2456 .await?;
2457 Ok(AttachResult {
2458 ip,
2459 namespace_guid: None,
2460 })
2461 }
2462 AttachHandle::WindowsContainer { container_id, ip } => {
2463 self.attach_container_windows(
2464 &container_id,
2465 service,
2466 ip,
2467 dns_server,
2468 dns_domain,
2469 isolation_network,
2470 )
2471 .await
2472 }
2473 AttachHandle::HostShared { id } => {
2474 let ip = self
2475 .attach_container_host_shared(&id, service, ephemeral, isolation_network)
2476 .await?;
2477 Ok(AttachResult {
2478 ip,
2479 namespace_guid: None,
2480 })
2481 }
2482 AttachHandle::GuestManaged { .. } => Err(OverlaydError::Other(
2483 "guest-managed attach must go through attach_container_guest, not attach_container"
2484 .to_string(),
2485 )),
2486 }
2487 }
2488
2489 /// Tear down a container's overlay attachment and release its IP.
2490 ///
2491 /// # Errors
2492 /// Returns an error only if a netlink delete fails for a reason other than
2493 /// "link not found".
2494 async fn detach_container(&mut self, handle: AttachHandle) -> Result<(), OverlaydError> {
2495 match handle {
2496 AttachHandle::LinuxPid { pid } => self.detach_container_linux(pid).await,
2497 AttachHandle::WindowsContainer { container_id, .. } => {
2498 self.detach_container_windows(&container_id).await
2499 }
2500 AttachHandle::HostShared { id } => self.detach_container_host_shared(&id).await,
2501 AttachHandle::GuestManaged { .. } => Err(OverlaydError::Other(
2502 "guest-managed detach must go through detach_container_guest, not detach_container"
2503 .to_string(),
2504 )),
2505 }
2506 }
2507
2508 // -- container attach (guest-managed) -----------------------------------
2509
2510 /// Guest-managed overlay attach: allocate the overlay identity for a VM guest
2511 /// that brings up its own kernel `WireGuard` device.
2512 ///
2513 /// overlayd cannot enter the guest's network namespace (it is a VM, not a
2514 /// host process), so instead of a veth/HCN endpoint it:
2515 /// 1. allocates the overlay IP from the SAME pool the Linux attach uses (the
2516 /// per-service bridge pool when one exists, otherwise the node slice) so
2517 /// guest addresses never collide with container addresses;
2518 /// 2. generates a fresh `WireGuard` keypair for the guest;
2519 /// 3. builds the peer set the guest must configure — every GLOBAL peer the
2520 /// host already knows, plus THIS node itself (so the guest can reach the
2521 /// host node over the overlay; carries a keepalive so the guest keeps its
2522 /// NAT mapping open from behind VZ NAT);
2523 /// 4. registers the generated public key as a GLOBAL peer (host route to the
2524 /// guest, roaming endpoint learned from the guest's keepalive) so remote
2525 /// nodes and this node route to it;
2526 /// 5. records the attachment keyed by `id` so `DetachContainer` can release
2527 /// the IP and remove the peer.
2528 ///
2529 /// Platform-agnostic: pure IPAM + keygen + peer bookkeeping (no netns/veth/
2530 /// HCN), so it compiles and runs on macOS (where the overlayd serving a VZ
2531 /// host lives) as well as Linux.
2532 ///
2533 /// # Errors
2534 /// Returns an error if the global overlay is not set up, the IP pool is
2535 /// exhausted, key generation fails, or registering the guest peer fails.
2536 #[allow(clippy::cast_possible_truncation, clippy::too_many_lines)]
2537 async fn attach_container_guest(
2538 &mut self,
2539 id: &str,
2540 service: &str,
2541 join_global: bool,
2542 dns_server: Option<IpAddr>,
2543 dns_domain: Option<String>,
2544 isolation_network: Option<String>,
2545 ) -> Result<GuestOverlayConfig, OverlaydError> {
2546 // The global transport must exist: we both register the guest as a peer
2547 // on it and advertise this node (its public key + listen port) to the
2548 // guest. Resolve both up front so we fail before allocating anything.
2549 let node_public_key = self.transport_public_key.clone().ok_or_else(|| {
2550 OverlaydError::Other(
2551 "guest-managed attach requires the global overlay to be set up first \
2552 (no node WireGuard public key)"
2553 .to_string(),
2554 )
2555 })?;
2556 if self.global_transport.is_none() {
2557 return Err(OverlaydError::Other(
2558 "guest-managed attach requires the global overlay to be set up first \
2559 (no global transport)"
2560 .to_string(),
2561 ));
2562 }
2563
2564 // 1. Allocate the overlay IP from the same pool the Linux attach uses and
2565 // derive the prefix length from that pool's network. On Linux a
2566 // per-service bridge (when present) supplies both the IP and its
2567 // subnet's prefix; otherwise (and on every non-Linux host) the node
2568 // slice / cluster CIDR does.
2569 let (overlay_ip, prefix_len, pool_service, dedicated): (IpAddr, u8, Option<String>, bool) = {
2570 #[cfg(target_os = "linux")]
2571 {
2572 let use_shared = self
2573 .service_modes
2574 .get(service)
2575 .copied()
2576 .unwrap_or_default()
2577 .uses_shared_bridge();
2578 let bridge = if use_shared {
2579 self.shared_bridge.as_mut()
2580 } else {
2581 self.service_bridges.get_mut(service)
2582 };
2583 if let Some(bridge) = bridge {
2584 let ip = bridge.ip_allocator.allocate().ok_or_else(|| {
2585 OverlaydError::Overlay(format!(
2586 "service bridge {} subnet {} exhausted",
2587 bridge.name, bridge.subnet
2588 ))
2589 })?;
2590 let prefix = bridge.subnet.prefix_len();
2591 (ip, prefix, Some(service.to_string()), false)
2592 } else {
2593 let ip = self.ip_allocator.allocate()?;
2594 (ip, self.slice_prefix_len(), None, false)
2595 }
2596 }
2597 #[cfg(not(target_os = "linux"))]
2598 {
2599 // A Dedicated service owns a second WireGuard device (own crypto +
2600 // subnet); its guest draws from that device's allocator and lands
2601 // on the dedicated subnet, not the global cluster mesh. Every other
2602 // mode hairpins through the node slice on the global transport.
2603 let dedicated = self
2604 .service_modes
2605 .get(service)
2606 .copied()
2607 .unwrap_or_default()
2608 .uses_per_service_wg();
2609 if dedicated {
2610 let st = self.service_transports.get_mut(service).ok_or_else(|| {
2611 OverlaydError::Other(format!(
2612 "Dedicated service {service} has no dedicated overlay; \
2613 call setup_service_overlay first"
2614 ))
2615 })?;
2616 let ip = st.ip_allocator.allocate().ok_or_else(|| {
2617 OverlaydError::Overlay(format!(
2618 "dedicated service {service} subnet {} exhausted",
2619 st.subnet
2620 ))
2621 })?;
2622 (ip, st.subnet.prefix_len(), Some(service.to_string()), true)
2623 } else {
2624 let ip = self.ip_allocator.allocate()?;
2625 (ip, self.slice_prefix_len(), None, false)
2626 }
2627 }
2628 };
2629 // `join_global` is informational for a guest-managed attach: the guest's
2630 // single WireGuard device IS its global-overlay endpoint, so there is no
2631 // separate eth1 IP to reserve. Touch it so callers stay consistent with
2632 // the Linux/Windows handles.
2633 let _ = join_global;
2634
2635 // 2. Generate the guest's WireGuard keypair (reuse the transport's
2636 // native x25519 keygen — never reimplement curve25519 here).
2637 let (private_key, public_key) = OverlayTransport::generate_keys().await.map_err(|e| {
2638 // Roll back the IP allocation so a keygen failure leaks nothing.
2639 self.release_guest_ip(overlay_ip, pool_service.as_deref());
2640 OverlaydError::Overlay(format!("failed to generate guest keys: {e}"))
2641 })?;
2642
2643 // 3. Build the peer set. A VZ guest is behind the host's NAT and can only
2644 // reach the LOCAL node (via its NAT gateway) — it cannot dial other
2645 // nodes' or sibling guests' endpoints directly. So it gets exactly ONE
2646 // peer: this node. ALL overlay traffic (including to sibling containers
2647 // and remote nodes) routes through this node, which forwards/hairpins it
2648 // (the node already holds a /32 peer for every container — step 4 — and
2649 // the real inter-node peers). We deliberately do NOT add the per-guest
2650 // /32 peers here: a /32 with no reachable endpoint would win
2651 // longest-prefix routing and black-hole sibling traffic. The endpoint
2652 // returned here is the node's overlay IP as a placeholder; the VZ
2653 // runtime rewrites it to the guest's NAT gateway (the only host address
2654 // the guest can reach) before delivering the config. Keepalive holds the
2655 // guest's NAT mapping open so the node can reach back.
2656 //
2657 // Dedicated mode: the single peer is this node's DEDICATED per-service
2658 // device (its own pubkey + listen port + subnet as AllowedIPs), so the
2659 // guest joins that service's isolated mesh. Every other mode peers with
2660 // the global cluster device, AllowedIPs = the whole cluster CIDR.
2661 let (peer_pubkey, peer_listen_port, peer_allowed) = if dedicated {
2662 let st = self
2663 .service_transports
2664 .get(service)
2665 .expect("dedicated transport allocated above");
2666 (st.public_key.clone(), st.listen_port, st.subnet.to_string())
2667 } else {
2668 let node_allowed = self
2669 .cluster_cidr
2670 .or(self.slice_cidr)
2671 .map_or_else(|| String::from("0.0.0.0/0"), |c| c.to_string());
2672 (node_public_key, self.overlay_port, node_allowed)
2673 };
2674 let node_endpoint = self.node_endpoint_for_guest(peer_listen_port);
2675 let peers: Vec<PeerSpec> = vec![PeerSpec {
2676 public_key: peer_pubkey,
2677 endpoint: node_endpoint,
2678 allowed_ips: peer_allowed,
2679 persistent_keepalive_secs: 25,
2680 // The guest reaches the node via its NAT gateway (the only host
2681 // address it can route to); it does not run the host's ICE-lite
2682 // candidate exchange, so no candidates are advertised here.
2683 candidates: Vec::new(),
2684 }];
2685
2686 // 4. Register the guest's public key as a GLOBAL peer (host route to the
2687 // guest at <overlay_ip>/32, roaming endpoint learned from keepalive).
2688 // Go through the same internal path `AddPeer { Global }` uses.
2689 let host_route = format!(
2690 "{}/{}",
2691 overlay_ip,
2692 if overlay_ip.is_ipv6() { 128 } else { 32 }
2693 );
2694 let guest_peer = PeerSpec {
2695 public_key: public_key.clone(),
2696 // Empty/roaming: the guest is behind NAT; boringtun learns its source
2697 // endpoint from the guest's first keepalive. `0.0.0.0:0` is the
2698 // wire-safe "unset endpoint" sentinel that still parses as a
2699 // SocketAddr (peer_spec_to_info requires a parseable endpoint).
2700 endpoint: "0.0.0.0:0".to_string(),
2701 allowed_ips: host_route,
2702 persistent_keepalive_secs: 0,
2703 // The guest's roaming endpoint is learned from its first keepalive;
2704 // it advertises no NAT candidates (the host learns the source).
2705 candidates: Vec::new(),
2706 };
2707 let guest_peer_info = peer_spec_to_info(&guest_peer)?;
2708 let scope = if dedicated {
2709 PeerScope::Service {
2710 service: service.to_string(),
2711 }
2712 } else {
2713 PeerScope::Global
2714 };
2715 {
2716 let transport = self.transport_for_scope(&scope)?;
2717 if let Err(e) = Self::add_peer_on(transport, &guest_peer_info).await {
2718 self.release_guest_ip(overlay_ip, pool_service.as_deref());
2719 return Err(e);
2720 }
2721 }
2722 // Track it among the global peers (so a *subsequent* guest attach also
2723 // learns about this guest) and record the attachment for detach.
2724 self.global_peers
2725 .insert(public_key.clone(), guest_peer.clone());
2726 // Per-network membership + node-side L3 isolation: record the guest's
2727 // overlay IP in its isolated network's member set, and enforce the
2728 // cross-platform isolation policy on THIS node. A VZ guest hairpins ALL
2729 // its overlay traffic through this node's WireGuard device, so the node
2730 // is the enforcement point: on macOS this dispatches to pf (a per-network
2731 // table + sub-anchor); on Linux it dispatches to iptables (harmless here
2732 // — guests do not run on Linux). The guest's own WireGuard AllowedIPs are
2733 // the in-guest belt; this is the node-side suspenders.
2734 if let Some(ref net) = isolation_network {
2735 let node_ip = self
2736 .node_ip
2737 .unwrap_or(std::net::IpAddr::V4(std::net::Ipv4Addr::new(10, 200, 0, 1)));
2738 let cidr = self
2739 .cluster_cidr
2740 .map_or_else(|| "10.200.0.0/16".to_string(), |c| c.to_string());
2741 // Peers = current members of the network BEFORE inserting this guest.
2742 let peers: Vec<IpAddr> = self
2743 .network_members
2744 .get(net)
2745 .map(|m| m.iter().copied().collect())
2746 .unwrap_or_default();
2747 if let Err(e) = zlayer_overlay::firewall::ensure_member_isolation(
2748 net, overlay_ip, &peers, node_ip, &cidr,
2749 ) {
2750 tracing::warn!(network = %net, member = %overlay_ip, error = %e, "failed to install per-network L3 isolation for guest (non-fatal)");
2751 }
2752 self.network_members
2753 .entry(net.clone())
2754 .or_default()
2755 .insert(overlay_ip);
2756 }
2757 self.guest_attachments.insert(
2758 id.to_string(),
2759 GuestAttachInfo {
2760 overlay_ip,
2761 public_key: public_key.clone(),
2762 service_name: pool_service,
2763 isolation_network,
2764 },
2765 );
2766
2767 // 5. Return the config the caller ships into the guest.
2768 Ok(GuestOverlayConfig {
2769 overlay_ip,
2770 prefix_len,
2771 private_key,
2772 public_key,
2773 // The guest's device listens on the same port as its single in-guest
2774 // peer (the node device it joins): the node's overlay WG port for the
2775 // global mesh, or the dedicated device's listen port in Dedicated mode.
2776 listen_port: peer_listen_port,
2777 peers,
2778 dns_server: dns_server.or_else(|| self.dns_server_addr.map(|s| s.ip())),
2779 dns_domain: dns_domain.or_else(|| self.dns_domain.clone()),
2780 })
2781 }
2782
2783 /// Release a guest-managed attach by `id`: drop the host route + global peer
2784 /// and return the allocated IP to its pool. Idempotent.
2785 ///
2786 /// # Errors
2787 /// Returns an error only if removing the peer from the global transport fails
2788 /// for a reason other than "peer not found".
2789 async fn detach_container_guest(&mut self, id: &str) -> Result<(), OverlaydError> {
2790 let Some(info) = self.guest_attachments.remove(id) else {
2791 return Ok(());
2792 };
2793 // Remove the guest's peer from the same scope it was registered on: a
2794 // Dedicated guest sits on its service's dedicated device, every other
2795 // guest on the global cluster device. Mirror the attach-time scope choice
2796 // so a dedicated guest peer does not leak on teardown.
2797 let scope = match info.service_name.as_deref() {
2798 Some(svc)
2799 if self
2800 .service_modes
2801 .get(svc)
2802 .copied()
2803 .unwrap_or_default()
2804 .uses_per_service_wg() =>
2805 {
2806 PeerScope::Service {
2807 service: svc.to_string(),
2808 }
2809 }
2810 _ => PeerScope::Global,
2811 };
2812 self.global_peers.remove(&info.public_key);
2813 if let Ok(transport) = self.transport_for_scope(&scope) {
2814 if let Err(e) = Self::remove_peer_on(transport, &info.public_key).await {
2815 tracing::warn!(
2816 guest = %id,
2817 pubkey = %info.public_key,
2818 scope = ?scope,
2819 error = %e,
2820 "failed to remove guest peer from its overlay transport"
2821 );
2822 }
2823 }
2824 // Drain the per-network membership set for this guest and tear down the
2825 // node-side L3 isolation rule for it (pf on macOS, iptables on Linux —
2826 // the latter is a no-op for guests, which never run on Linux). Drop the
2827 // network entry once empty.
2828 if let Some(net) = info.isolation_network.as_deref() {
2829 if let Some(set) = self.network_members.get_mut(net) {
2830 set.remove(&info.overlay_ip);
2831 }
2832 let remaining_peers: Vec<IpAddr> = self
2833 .network_members
2834 .get(net)
2835 .map(|m| m.iter().copied().collect())
2836 .unwrap_or_default();
2837 let node_ip = self
2838 .node_ip
2839 .unwrap_or(std::net::IpAddr::V4(std::net::Ipv4Addr::new(10, 200, 0, 1)));
2840 let cidr = self
2841 .cluster_cidr
2842 .map_or_else(|| "10.200.0.0/16".to_string(), |c| c.to_string());
2843 zlayer_overlay::firewall::remove_member_isolation(
2844 net,
2845 info.overlay_ip,
2846 &remaining_peers,
2847 node_ip,
2848 &cidr,
2849 );
2850 if self
2851 .network_members
2852 .get(net)
2853 .is_some_and(std::collections::HashSet::is_empty)
2854 {
2855 self.network_members.remove(net);
2856 }
2857 }
2858 // Return the IP to whichever pool it came from.
2859 self.release_guest_ip(info.overlay_ip, info.service_name.as_deref());
2860 Ok(())
2861 }
2862
2863 // ---- edge peers --------------------------------------------------------
2864 //
2865 // A minted edge peer is a short-lived, unprivileged `WireGuard` endpoint
2866 // (e.g. a remote CI runner) that joins the overlay without any of the
2867 // container plumbing. Mechanically it clones the guest-attach path: allocate
2868 // a node-slice `/32`, generate the peer's OWN keypair, register it as a
2869 // roaming global `/32` peer, and hand back a portable [`EdgeConfig`] whose
2870 // single artifact peer is THIS node. Unlike a guest, an edge peer carries a
2871 // TTL and an optional node-side L3 fence, and is swept when it expires or
2872 // goes silent. Every path is platform-agnostic (no netns/veth/HCN).
2873
2874 /// Grace, in seconds, an edge peer is allowed to complete its first
2875 /// handshake before the silence sweep reaps it. `120` ≈ 3× the artifact's
2876 /// 25s keepalive — enough slack for a remote pod to boot, resolve the node
2877 /// endpoint, and hole-punch — before we assume the mint was abandoned.
2878 const EDGE_CONNECT_GRACE_SECS: u64 = 120;
2879
2880 /// Silence timeout, in seconds: once an edge peer HAS handshaked, it is
2881 /// reaped when no handshake has been seen for this long. Matches
2882 /// `zlayer_overlay::health`'s `HANDSHAKE_TIMEOUT_SECS` (180) so the edge
2883 /// liveness verdict lines up with the rest of the overlay's peer health.
2884 const EDGE_SILENCE_TIMEOUT_SECS: u64 = 180;
2885
2886 /// Mint a short-lived unprivileged edge peer: allocate an overlay `/32`,
2887 /// generate the edge's keypair, register it as a roaming global peer, install
2888 /// the node-side L3 fence when `--allow` is given, and return the portable
2889 /// [`EdgeConfig`] a remote `WireGuard` process needs to join the overlay.
2890 ///
2891 /// Mirrors [`Self::attach_container_guest`]'s keygen/peer bookkeeping but for
2892 /// an off-cluster peer: the EDGE's keypair goes in the config, and the single
2893 /// artifact peer is THIS node (public key from `transport_public_key`).
2894 ///
2895 /// # Errors
2896 /// Returns an error if the name is blank or already live, the global overlay
2897 /// is not up, `node_endpoint` is empty / unspecified / portless, an `--allow`
2898 /// entry resolves to nothing, the slice is exhausted, or keygen / peer
2899 /// registration fails (each of which rolls back any IP already allocated).
2900 async fn mint_edge_peer(
2901 &mut self,
2902 name: String,
2903 ttl_secs: u64,
2904 allow: &[String],
2905 node_endpoint: &str,
2906 ) -> Result<EdgeConfig, OverlaydError> {
2907 let name = name.trim().to_string();
2908 if name.is_empty() {
2909 return Err(OverlaydError::Other(
2910 "edge peer name must not be empty".to_string(),
2911 ));
2912 }
2913 if self.edge_attachments.contains_key(&name) {
2914 return Err(OverlaydError::Other(format!(
2915 "edge peer `{name}` is already minted; revoke it before re-minting"
2916 )));
2917 }
2918 // The global overlay must be up: we register the edge on it AND advertise
2919 // this node's identity to the edge. Resolve both before allocating.
2920 let node_public_key = self.transport_public_key.clone().ok_or_else(|| {
2921 OverlaydError::Other(
2922 "edge mint requires the global overlay to be set up first \
2923 (no node WireGuard public key)"
2924 .to_string(),
2925 )
2926 })?;
2927 if self.global_transport.is_none() {
2928 return Err(OverlaydError::Other(
2929 "edge mint requires the global overlay to be set up first \
2930 (no global transport)"
2931 .to_string(),
2932 ));
2933 }
2934 validate_edge_node_endpoint(node_endpoint)?;
2935 // Resolve `--allow` up front so an unknown service name fails before any
2936 // allocation happens.
2937 let targets = self.resolve_allow_targets(allow)?;
2938
2939 // Allocate the edge's `/32` from the node slice (mirrors the guest path).
2940 let overlay_ip = self.ip_allocator.allocate()?;
2941
2942 // Generate the EDGE's own keypair — never the node's. Roll the IP back on
2943 // any failure below so a partial mint leaks no address.
2944 let (private_key, public_key) = match OverlayTransport::generate_keys().await {
2945 Ok(keys) => keys,
2946 Err(e) => {
2947 self.ip_allocator.release(overlay_ip);
2948 return Err(OverlaydError::Overlay(format!(
2949 "failed to generate edge keys: {e}"
2950 )));
2951 }
2952 };
2953
2954 // Register the edge as a roaming global `/32` peer (endpoint learned from
2955 // its first keepalive; keepalive 0 — mirrors guest registration).
2956 if let Err(e) = self
2957 .register_edge_global_peer(&public_key, overlay_ip)
2958 .await
2959 {
2960 self.ip_allocator.release(overlay_ip);
2961 return Err(e);
2962 }
2963
2964 // Node-side L3 fence: installed ONLY when `--allow` was given. Without
2965 // it, the edge gets guest-equivalent trust (cluster-CIDR AllowedIPs).
2966 let isolation_network = if targets.is_empty() {
2967 None
2968 } else {
2969 Some(self.install_edge_fence(&name, overlay_ip, &targets))
2970 };
2971
2972 // The single artifact peer is THIS node. Its AllowedIPs scope what the
2973 // edge routes to: the resolved targets (+ node + DNS `/32`) when fenced,
2974 // else the whole cluster CIDR.
2975 let dns_ip = self.dns_server_addr.map(|s| s.ip());
2976 let node_allowed = self.edge_node_allowed_ips(&targets, dns_ip);
2977 let node_peer = Self::build_edge_peer_spec(node_public_key, node_endpoint, node_allowed);
2978
2979 let minted_at_unix = now_unix();
2980 let expires_at_unix = minted_at_unix.saturating_add(ttl_secs);
2981
2982 self.edge_attachments.insert(
2983 name.clone(),
2984 EdgeAttachInfo {
2985 overlay_ip,
2986 public_key: public_key.clone(),
2987 minted_at_unix,
2988 expires_at_unix,
2989 allowed_targets: targets,
2990 isolation_network,
2991 },
2992 );
2993
2994 Ok(EdgeConfig {
2995 version: EDGE_CONFIG_VERSION,
2996 name,
2997 overlay_ip,
2998 prefix_len: self.slice_prefix_len(),
2999 private_key,
3000 public_key,
3001 peers: vec![node_peer],
3002 dns_server: dns_ip,
3003 dns_domain: self.dns_domain.clone(),
3004 expires_at_unix,
3005 })
3006 }
3007
3008 /// Register the edge's roaming `/32` on the GLOBAL transport (endpoint
3009 /// `0.0.0.0:0`, keepalive 0 — boringtun learns the source from the edge's
3010 /// first keepalive) and mirror it into `global_peers`. Goes through the same
3011 /// internal path `AddPeer { Global }` uses. On error the caller rolls back the
3012 /// IP allocation.
3013 ///
3014 /// # Errors
3015 /// Returns an error if the endpoint sentinel fails to parse, the global
3016 /// transport is missing, or the UAPI peer write fails.
3017 async fn register_edge_global_peer(
3018 &mut self,
3019 public_key: &str,
3020 overlay_ip: IpAddr,
3021 ) -> Result<(), OverlaydError> {
3022 let edge_peer = PeerSpec {
3023 public_key: public_key.to_string(),
3024 // Roaming: the edge is outside the overlay until its first handshake.
3025 // `0.0.0.0:0` is the wire-safe "unset endpoint" sentinel (still parses
3026 // as a `SocketAddr`, which `peer_spec_to_info` requires).
3027 endpoint: "0.0.0.0:0".to_string(),
3028 allowed_ips: edge_host_route(overlay_ip),
3029 persistent_keepalive_secs: 0,
3030 candidates: Vec::new(),
3031 };
3032 let info = peer_spec_to_info(&edge_peer)?;
3033 {
3034 let transport = self.transport_for_scope(&PeerScope::Global)?;
3035 Self::add_peer_on(transport, &info).await?;
3036 }
3037 self.global_peers.insert(public_key.to_string(), edge_peer);
3038 Ok(())
3039 }
3040
3041 /// Resolve an interior overlay subnet for a service set up on THIS node
3042 /// (used by [`Self::resolve_allow_targets`] for a `--allow <service>` entry):
3043 /// the dedicated per-service device's subnet, or on Linux the per-service /
3044 /// node-wide shared bridge subnet. `None` if `service` is unknown here.
3045 fn edge_service_subnet(&self, service: &str) -> Option<ipnet::IpNet> {
3046 // A Dedicated per-service `WireGuard` device owns its subnet (all OSes).
3047 if let Some(st) = self.service_transports.get(service) {
3048 return Some(st.subnet);
3049 }
3050 #[cfg(target_os = "linux")]
3051 {
3052 if let Some(bridge) = self.service_bridges.get(service) {
3053 return Some(bridge.subnet);
3054 }
3055 // A Shared-mode service is backed by the single node-wide shared
3056 // bridge, keyed by subnet rather than by service name.
3057 if self
3058 .service_modes
3059 .get(service)
3060 .copied()
3061 .unwrap_or_default()
3062 .uses_shared_bridge()
3063 {
3064 if let Some(bridge) = self.shared_bridge.as_ref() {
3065 return Some(bridge.subnet);
3066 }
3067 }
3068 }
3069 None
3070 }
3071
3072 /// Resolve each `--allow` entry to an overlay CIDR: a literal CIDR, a bare
3073 /// overlay IP (→ host `/32` / `/128`), or a service name → its per-node
3074 /// subnet. Blank entries are skipped; an entry that matches none is a hard
3075 /// error naming the CIDR alternative.
3076 ///
3077 /// # Errors
3078 /// Returns [`OverlaydError::Other`] for an entry that is neither a CIDR/IP
3079 /// nor a service set up on this node.
3080 fn resolve_allow_targets(&self, allow: &[String]) -> Result<Vec<ipnet::IpNet>, OverlaydError> {
3081 let mut out: Vec<ipnet::IpNet> = Vec::with_capacity(allow.len());
3082 for raw in allow {
3083 let entry = raw.trim();
3084 if entry.is_empty() {
3085 continue;
3086 }
3087 if let Ok(net) = entry.parse::<ipnet::IpNet>() {
3088 out.push(net);
3089 continue;
3090 }
3091 if let Ok(ip) = entry.parse::<IpAddr>() {
3092 let prefix = if ip.is_ipv6() { 128 } else { 32 };
3093 let net = ipnet::IpNet::new(ip, prefix)
3094 .expect("a host prefix length (32/128) is always in range");
3095 out.push(net);
3096 continue;
3097 }
3098 if let Some(net) = self.edge_service_subnet(entry) {
3099 out.push(net);
3100 continue;
3101 }
3102 return Err(OverlaydError::Other(format!(
3103 "edge --allow target `{entry}` is neither an overlay CIDR/IP nor a service \
3104 set up on this node; pass a CIDR (e.g. `10.200.0.0/16`), an overlay IP, or a \
3105 known service name"
3106 )));
3107 }
3108 Ok(out)
3109 }
3110
3111 /// Build the single artifact peer an edge configures: THIS node, with a 25s
3112 /// keepalive so the edge holds its NAT mapping open from behind its own NAT.
3113 fn build_edge_peer_spec(
3114 node_pubkey: String,
3115 node_endpoint: &str,
3116 allowed_ips_csv: String,
3117 ) -> PeerSpec {
3118 PeerSpec {
3119 public_key: node_pubkey,
3120 endpoint: node_endpoint.to_string(),
3121 allowed_ips: allowed_ips_csv,
3122 persistent_keepalive_secs: 25,
3123 candidates: Vec::new(),
3124 }
3125 }
3126
3127 /// Comma-separated `AllowedIPs` for the artifact's node peer: the resolved
3128 /// `--allow` targets plus the node's own `/32` and the overlay DNS `/32`
3129 /// when fenced, else the whole cluster (guest-equivalent) CIDR.
3130 fn edge_node_allowed_ips(&self, targets: &[ipnet::IpNet], dns_ip: Option<IpAddr>) -> String {
3131 if targets.is_empty() {
3132 return self
3133 .cluster_cidr
3134 .or(self.slice_cidr)
3135 .map_or_else(|| "0.0.0.0/0".to_string(), |c| c.to_string());
3136 }
3137 let mut cidrs: Vec<String> = targets.iter().map(ToString::to_string).collect();
3138 if let Some(node_ip) = self.node_ip {
3139 cidrs.push(edge_host_route(node_ip));
3140 }
3141 if let Some(dns) = dns_ip {
3142 cidrs.push(edge_host_route(dns));
3143 }
3144 cidrs.join(",")
3145 }
3146
3147 /// Install the node-side L3 fence for a fenced edge peer (network id
3148 /// `edge:<name>`): the ACTUAL enforcement of which overlay destinations the
3149 /// edge `/32` may reach. Non-fatal — a firewall hiccup is warned, not
3150 /// propagated, exactly like the guest path — and the network id is always
3151 /// returned so revoke tears the fence down regardless. The caller decides
3152 /// whether a fence exists at all (only when `--allow` is non-empty).
3153 ///
3154 /// `targets` become the fence's permitted `peers` (each target's network
3155 /// address). On Linux those are pairwise ACCEPTs with a catch-all overlay
3156 /// DROP; on macOS `peers` is ignored (the pf table already covers members),
3157 /// so the guarantee is "no stronger than the platform's guest isolation".
3158 fn install_edge_fence(&self, name: &str, edge_ip: IpAddr, targets: &[ipnet::IpNet]) -> String {
3159 let network = format!("edge:{name}");
3160 let node_ip = self
3161 .node_ip
3162 .unwrap_or(IpAddr::V4(Ipv4Addr::new(10, 200, 0, 1)));
3163 let cidr = self
3164 .cluster_cidr
3165 .map_or_else(|| "10.200.0.0/16".to_string(), |c| c.to_string());
3166 let peers: Vec<IpAddr> = targets.iter().map(ipnet::IpNet::network).collect();
3167 if let Err(e) = zlayer_overlay::firewall::ensure_member_isolation(
3168 &network, edge_ip, &peers, node_ip, &cidr,
3169 ) {
3170 tracing::warn!(
3171 network = %network,
3172 member = %edge_ip,
3173 error = %e,
3174 "failed to install per-edge L3 isolation fence (non-fatal)"
3175 );
3176 }
3177 network
3178 }
3179
3180 /// Revoke a minted edge peer by name — the SINGLE teardown path (the sweep
3181 /// reuses it). Idempotent: `Ok(false)` when the name is absent. Removes the
3182 /// global `/32` peer, tears down the fence if one was installed, and returns
3183 /// the `/32` to the node slice.
3184 ///
3185 /// # Errors
3186 /// Never returns `Err` in practice: peer removal is best-effort (warned, not
3187 /// propagated) so a transport hiccup cannot strand the allocation. The
3188 /// `Result` is kept for symmetry with the other teardown methods.
3189 async fn revoke_edge_peer(&mut self, name: &str) -> Result<bool, OverlaydError> {
3190 let Some(info) = self.edge_attachments.remove(name) else {
3191 return Ok(false);
3192 };
3193 // Drop the edge's `/32` peer from the global transport (and the mirror).
3194 self.global_peers.remove(&info.public_key);
3195 if let Ok(transport) = self.transport_for_scope(&PeerScope::Global) {
3196 if let Err(e) = Self::remove_peer_on(transport, &info.public_key).await {
3197 tracing::warn!(
3198 edge = %name,
3199 pubkey = %info.public_key,
3200 error = %e,
3201 "failed to remove edge peer from the global transport (non-fatal)"
3202 );
3203 }
3204 }
3205 // Tear down the node-side L3 fence, if one was installed. Reproduce the
3206 // exact `peers` set from the stored targets so the removal matches.
3207 if let Some(network) = info.isolation_network.as_deref() {
3208 let node_ip = self
3209 .node_ip
3210 .unwrap_or(IpAddr::V4(Ipv4Addr::new(10, 200, 0, 1)));
3211 let cidr = self
3212 .cluster_cidr
3213 .map_or_else(|| "10.200.0.0/16".to_string(), |c| c.to_string());
3214 let peers: Vec<IpAddr> = info
3215 .allowed_targets
3216 .iter()
3217 .map(ipnet::IpNet::network)
3218 .collect();
3219 zlayer_overlay::firewall::remove_member_isolation(
3220 network,
3221 info.overlay_ip,
3222 &peers,
3223 node_ip,
3224 &cidr,
3225 );
3226 }
3227 // Return the `/32` to the node-slice pool.
3228 self.ip_allocator.release(info.overlay_ip);
3229 Ok(true)
3230 }
3231
3232 /// Pure, unit-testable reap predicate for one edge peer. Reaped when: the TTL
3233 /// has elapsed; OR it never handshaked and the boot-grace window is up; OR it
3234 /// handshaked once and has been silent past the timeout.
3235 fn edge_reap_due(
3236 info: &EdgeAttachInfo,
3237 last_handshake_unix: Option<i64>,
3238 now_unix: u64,
3239 ) -> bool {
3240 // Hard TTL expiry wins regardless of liveness.
3241 if now_unix >= info.expires_at_unix {
3242 return true;
3243 }
3244 match last_handshake_unix.filter(|&h| h > 0) {
3245 // A real handshake was seen: reap after the silence window.
3246 Some(h) => {
3247 let last = u64::try_from(h).unwrap_or(0);
3248 now_unix.saturating_sub(last) >= Self::EDGE_SILENCE_TIMEOUT_SECS
3249 }
3250 // Never handshaked (None / 0 / negative): reap once boot-grace is up.
3251 None => now_unix.saturating_sub(info.minted_at_unix) >= Self::EDGE_CONNECT_GRACE_SECS,
3252 }
3253 }
3254
3255 /// Snapshot the global transport's per-peer last-handshake once, indexed by
3256 /// HEX public key (`parse_peer_status` reports hex; edge keys are base64, so
3257 /// callers convert via [`zlayer_overlay::nat::pubkey_b64_to_hex`]). A missing
3258 /// transport / failed dump yields an empty map (every edge then reads as
3259 /// never-handshaked).
3260 async fn edge_handshake_map(&self) -> HashMap<String, i64> {
3261 let mut map: HashMap<String, i64> = HashMap::new();
3262 if let Some(transport) = self.global_transport.as_ref() {
3263 if let Ok(dump) = transport.status().await {
3264 for p in parse_peer_status(&dump) {
3265 map.insert(p.public_key, p.last_handshake_unix_secs);
3266 }
3267 }
3268 }
3269 map
3270 }
3271
3272 /// Sweep expired / silent edge peers: snapshot handshakes ONCE, collect the
3273 /// due names, then revoke each (the borrow-safe collect-then-revoke order
3274 /// avoids mutating `edge_attachments` while iterating it).
3275 ///
3276 /// Public because the daemon's serve loop drives it on a timer (the edge
3277 /// sweep task) rather than through a wire request — there is no
3278 /// `SweepEdgePeers` [`OverlaydRequest`]; TTL/silence reaping is overlayd's
3279 /// own background responsibility, like the NAT maintenance tick.
3280 pub async fn sweep_edge_peers(&mut self, now_unix: u64) {
3281 if self.edge_attachments.is_empty() {
3282 return;
3283 }
3284 let handshakes = self.edge_handshake_map().await;
3285 let due: Vec<String> = self
3286 .edge_attachments
3287 .iter()
3288 .filter(|(_, info)| {
3289 let hs = zlayer_overlay::nat::pubkey_b64_to_hex(&info.public_key)
3290 .and_then(|hex| handshakes.get(&hex).copied());
3291 Self::edge_reap_due(info, hs, now_unix)
3292 })
3293 .map(|(name, _)| name.clone())
3294 .collect();
3295 for name in due {
3296 match self.revoke_edge_peer(&name).await {
3297 Ok(true) => tracing::info!(edge = %name, "swept expired/silent edge peer"),
3298 Ok(false) => {}
3299 Err(e) => {
3300 tracing::warn!(edge = %name, error = %e, "failed to sweep edge peer");
3301 }
3302 }
3303 }
3304 }
3305
3306 /// Join every live edge peer with its last-handshake from the global
3307 /// transport for [`OverlaydRequest::ListEdgePeers`]. A non-positive handshake
3308 /// (never seen) is reported as `None`.
3309 async fn edge_peer_statuses(&self) -> Vec<EdgePeerStatus> {
3310 let handshakes = self.edge_handshake_map().await;
3311 self.edge_attachments
3312 .iter()
3313 .map(|(name, info)| {
3314 let last_handshake_unix_secs =
3315 zlayer_overlay::nat::pubkey_b64_to_hex(&info.public_key)
3316 .and_then(|hex| handshakes.get(&hex).copied())
3317 .filter(|&h| h > 0);
3318 EdgePeerStatus {
3319 name: name.clone(),
3320 overlay_ip: info.overlay_ip,
3321 public_key: info.public_key.clone(),
3322 minted_at_unix: info.minted_at_unix,
3323 expires_at_unix: info.expires_at_unix,
3324 last_handshake_unix_secs,
3325 allowed: info
3326 .allowed_targets
3327 .iter()
3328 .map(ToString::to_string)
3329 .collect(),
3330 }
3331 })
3332 .collect()
3333 }
3334
3335 // -- container attach (macOS host-shared) -------------------------------
3336
3337 /// Host-shared overlay attach: give a macOS host-shared container
3338 /// ([`AttachHandle::HostShared`] — Seatbelt / native-VZ / libkrun) its own
3339 /// first-class L3 overlay membership.
3340 ///
3341 /// A host-shared container shares the node's host network namespace and its
3342 /// single cluster `utun`; it cannot get its own netns/veth (Seatbelt) or its
3343 /// own kernel `WireGuard` device (no guest VM to run one). So instead of a
3344 /// veth/HCN endpoint or a per-guest WG keypair, this:
3345 /// 1. allocates a DISTINCT overlay `/32` from the node slice (never the node
3346 /// IP — `IpAllocator` reserves offset 1 — and never `None`). The node
3347 /// slice is already advertised cluster-wide as this node's `AllowedIPs`,
3348 /// so the `/32` auto-routes to this node with no peer reconfiguration;
3349 /// 2. adds that `/32` as an alias on the node's overlay `utun` so the kernel
3350 /// delivers inbound overlay packets for it locally (boringtun decrypts
3351 /// and writes the plaintext packet to the utun, which only delivers to a
3352 /// configured local address);
3353 /// 3. records per-network membership + installs node-side L3 isolation when
3354 /// `isolation_network` is set (pf on macOS), exactly like the guest path;
3355 /// 4. records the attachment keyed by `id` so `DetachContainer` can remove
3356 /// the alias, drain the membership, and release the IP.
3357 ///
3358 /// HONEST CONSTRAINT: host-shared containers share the node's single cluster
3359 /// `utun`, so `OverlayMode::Dedicated`'s per-service `WireGuard` CRYPTO
3360 /// isolation cannot apply to them — there is no per-container WG device
3361 /// without a netns or a guest VM to host one. They still get a distinct
3362 /// overlay IP + L3 isolation (per-network membership / pf) + overlay DNS,
3363 /// which is full first-class L3 overlay membership. This is a real OS
3364 /// constraint of host-shared execution, not a stub.
3365 ///
3366 /// # Errors
3367 /// Returns an error if the node slice is exhausted, or if the global overlay
3368 /// interface is not set up (so there is no `utun` to alias the `/32` on).
3369 async fn attach_container_host_shared(
3370 &mut self,
3371 id: &str,
3372 service: &str,
3373 ephemeral: bool,
3374 isolation_network: Option<String>,
3375 ) -> Result<IpAddr, OverlaydError> {
3376 // 1. Allocate a distinct /32 from the node slice. Never the node IP
3377 // (reserved at offset 1), never None — exhaustion maps to the same
3378 // `OverlaydError::Overlay` the other attach paths surface.
3379 let ip = self.ip_allocator.allocate()?;
3380 let prefix_len: u8 = if ip.is_ipv6() { 128 } else { 32 };
3381
3382 // 2. Make the /32 locally deliverable on the node's overlay utun via an
3383 // alias on the single cluster transport's interface. Roll the IP
3384 // allocation back on any failure so nothing leaks.
3385 let alias_res = if let Some(transport) = self.global_transport.as_ref() {
3386 transport
3387 .add_alias(ip, prefix_len)
3388 .await
3389 .map_err(|e| OverlaydError::Overlay(e.to_string()))
3390 } else {
3391 Err(OverlaydError::Other(
3392 "host-shared attach requires the global overlay to be set up first \
3393 (no utun to alias the container /32 on)"
3394 .to_string(),
3395 ))
3396 };
3397 if let Err(e) = alias_res {
3398 self.ip_allocator.release(ip);
3399 return Err(e);
3400 }
3401
3402 // 3. Per-network membership + node-side L3 isolation (mirror the guest
3403 // path). The host-shared container hairpins all overlay traffic
3404 // through this node's WireGuard device, so the node is the
3405 // enforcement point (pf on macOS).
3406 if let Some(ref net) = isolation_network {
3407 let node_ip = self
3408 .node_ip
3409 .unwrap_or(std::net::IpAddr::V4(std::net::Ipv4Addr::new(10, 200, 0, 1)));
3410 let cidr = self
3411 .cluster_cidr
3412 .map_or_else(|| "10.200.0.0/16".to_string(), |c| c.to_string());
3413 // Peers = current members of the network BEFORE inserting this one.
3414 let peers: Vec<IpAddr> = self
3415 .network_members
3416 .get(net)
3417 .map(|m| m.iter().copied().collect())
3418 .unwrap_or_default();
3419 if let Err(e) =
3420 zlayer_overlay::firewall::ensure_member_isolation(net, ip, &peers, node_ip, &cidr)
3421 {
3422 tracing::warn!(network = %net, member = %ip, error = %e, "failed to install per-network L3 isolation for host-shared container (non-fatal)");
3423 }
3424 self.network_members
3425 .entry(net.clone())
3426 .or_default()
3427 .insert(ip);
3428 }
3429
3430 // 4. Record the attachment so detach can reverse all of the above.
3431 self.host_shared_attachments.insert(
3432 id.to_string(),
3433 AttachInfo {
3434 service_ip: ip,
3435 service_name: Some(service.to_string()),
3436 // No separate global/eth1 IP: a host-shared container reaches the
3437 // global overlay through the SAME /32 aliased on the node utun.
3438 global_ip: None,
3439 ephemeral,
3440 isolation_network,
3441 },
3442 );
3443
3444 Ok(ip)
3445 }
3446
3447 /// Release a host-shared attach by `id`: remove the utun `/32` alias, drain
3448 /// its per-network L3 isolation membership, and return the IP to the node
3449 /// slice. Idempotent. Mirrors [`Self::detach_container_guest`].
3450 ///
3451 /// # Errors
3452 /// Returns `Ok` even when removing the alias fails (best-effort, logged) —
3453 /// the IP is always returned to the pool so it can never leak.
3454 async fn detach_container_host_shared(&mut self, id: &str) -> Result<(), OverlaydError> {
3455 let Some(info) = self.host_shared_attachments.remove(id) else {
3456 return Ok(());
3457 };
3458 // Drain the per-network membership set and tear down the node-side L3
3459 // isolation rule for this container; drop the network entry once empty.
3460 if let Some(net) = info.isolation_network.as_deref() {
3461 if let Some(set) = self.network_members.get_mut(net) {
3462 set.remove(&info.service_ip);
3463 }
3464 let remaining_peers: Vec<IpAddr> = self
3465 .network_members
3466 .get(net)
3467 .map(|m| m.iter().copied().collect())
3468 .unwrap_or_default();
3469 let node_ip = self
3470 .node_ip
3471 .unwrap_or(std::net::IpAddr::V4(std::net::Ipv4Addr::new(10, 200, 0, 1)));
3472 let cidr = self
3473 .cluster_cidr
3474 .map_or_else(|| "10.200.0.0/16".to_string(), |c| c.to_string());
3475 zlayer_overlay::firewall::remove_member_isolation(
3476 net,
3477 info.service_ip,
3478 &remaining_peers,
3479 node_ip,
3480 &cidr,
3481 );
3482 if self
3483 .network_members
3484 .get(net)
3485 .is_some_and(std::collections::HashSet::is_empty)
3486 {
3487 self.network_members.remove(net);
3488 }
3489 }
3490 // Remove the utun /32 alias (best-effort: a failed removal must not
3491 // strand the IP, so we log and still release below).
3492 let prefix_len: u8 = if info.service_ip.is_ipv6() { 128 } else { 32 };
3493 if let Some(transport) = self.global_transport.as_ref() {
3494 if let Err(e) = transport.remove_alias(info.service_ip, prefix_len).await {
3495 tracing::warn!(
3496 container = %id,
3497 ip = %info.service_ip,
3498 error = %e,
3499 "failed to remove host-shared overlay /32 alias from utun (non-fatal)"
3500 );
3501 }
3502 }
3503 // Return the IP to the node slice.
3504 self.ip_allocator.release(info.service_ip);
3505
3506 // Per-job segment lifecycle observability. Unlike the Linux veth path —
3507 // which reaps a per-service BRIDGE on the last ephemeral detach — a
3508 // host-shared container shares the node's single cluster utun and owns
3509 // no per-service bridge or dedicated WG device to tear down (see the
3510 // HONEST CONSTRAINT note on `attach_container_host_shared`). The only
3511 // per-segment state is its overlay `/32` + per-network membership, both
3512 // already reversed above. So `ephemeral` and `service_name` drive the
3513 // last-leaver TRACE here (mirroring the Linux ephemeral-teardown log)
3514 // rather than a bridge teardown: an ephemeral (per-job) segment's IP
3515 // return is logged at info level for reclamation traceability, a
3516 // managed service's at debug.
3517 let service = info.service_name.as_deref().unwrap_or("<none>");
3518 if info.ephemeral {
3519 tracing::info!(
3520 container = %id,
3521 service = %service,
3522 ip = %info.service_ip,
3523 "ephemeral host-shared overlay member detached — per-job segment /32 returned to node slice"
3524 );
3525 } else {
3526 tracing::debug!(
3527 container = %id,
3528 service = %service,
3529 ip = %info.service_ip,
3530 "host-shared overlay member detached — /32 returned to node slice"
3531 );
3532 }
3533 Ok(())
3534 }
3535
3536 /// Release a guest overlay IP back to the pool it was drawn from: the named
3537 /// service bridge's allocator (Linux) when `service` is set and the bridge
3538 /// still exists, otherwise the node slice allocator.
3539 fn release_guest_ip(&mut self, ip: IpAddr, service: Option<&str>) {
3540 #[cfg(target_os = "linux")]
3541 {
3542 // A Shared-mode service drew from the single node-wide shared bridge,
3543 // which is keyed by subnet, not by service name. Try it first.
3544 if let Some(bridge) = self.shared_bridge.as_mut() {
3545 if bridge.subnet.contains(&ip) {
3546 bridge.ip_allocator.release(ip);
3547 return;
3548 }
3549 }
3550 if let Some(svc) = service {
3551 if let Some(bridge) = self.service_bridges.get_mut(svc) {
3552 bridge.ip_allocator.release(ip);
3553 return;
3554 }
3555 }
3556 }
3557 #[cfg(not(target_os = "linux"))]
3558 {
3559 // A Dedicated-mode guest drew its IP from the per-service transport's
3560 // allocator (keyed by service name); return it there so the dedicated
3561 // subnet does not leak addresses across guest churn.
3562 if let Some(svc) = service {
3563 if let Some(st) = self.service_transports.get_mut(svc) {
3564 st.ip_allocator.release(ip);
3565 return;
3566 }
3567 }
3568 }
3569 let _ = service;
3570 self.ip_allocator.release(ip);
3571 }
3572
3573 /// Prefix length of the address pool guest IPs are drawn from when not using
3574 /// a per-service bridge: the node slice if assigned, else the cluster CIDR.
3575 fn slice_prefix_len(&self) -> u8 {
3576 self.slice_cidr.or(self.cluster_cidr).map_or(
3577 if self.node_ip.is_some_and(|ip| ip.is_ipv6()) {
3578 64
3579 } else {
3580 24
3581 },
3582 |c| c.prefix(),
3583 )
3584 }
3585
3586 /// Reachable `WireGuard` endpoint for THIS node, advertised to a guest as a
3587 /// peer on `listen_port` (the node's global overlay port, or a Dedicated
3588 /// service's per-service device port). overlayd has no public reflexive
3589 /// address at this layer, so it uses the node's overlay-listen identity
3590 /// (`node_ip:listen_port`); the caller (the VZ runtime that ships the config
3591 /// into the guest) rewrites it to the concrete VZ-NAT gateway endpoint the
3592 /// guest can dial. Falls back to the unspecified address when no node IP is
3593 /// assigned yet.
3594 fn node_endpoint_for_guest(&self, listen_port: u16) -> String {
3595 let ip = self.node_ip.unwrap_or(IpAddr::V4(Ipv4Addr::UNSPECIFIED));
3596 SocketAddr::new(ip, listen_port).to_string()
3597 }
3598
3599 /// Linux veth/netns attach. On non-Linux this returns the node's overlay IP
3600 /// (host networking) and is never wired for a `LinuxPid` handle in practice.
3601 #[cfg(target_os = "linux")]
3602 #[allow(clippy::too_many_lines)]
3603 async fn attach_container_linux(
3604 &mut self,
3605 container_pid: u32,
3606 service: &str,
3607 join_global: bool,
3608 ephemeral: bool,
3609 isolation_network: Option<String>,
3610 ) -> Result<IpAddr, OverlaydError> {
3611 // Resolve which bridge backs this service. A `Shared`-mode service
3612 // attaches onto the SINGLE node-wide shared bridge; every other mode
3613 // (`Auto`, `Dedicated`) attaches onto its own per-service bridge. The
3614 // mode was recorded at `setup_service_overlay` time.
3615 let use_shared = self
3616 .service_modes
3617 .get(service)
3618 .copied()
3619 .unwrap_or_default()
3620 .uses_shared_bridge();
3621
3622 let (bridge_name, bridge_subnet, bridge_gateway, container_ip) = if use_shared {
3623 let bridge = self.shared_bridge.as_mut().ok_or_else(|| {
3624 OverlaydError::Other(format!(
3625 "no shared bridge for Shared-mode service {service}; call setup_service_overlay() first"
3626 ))
3627 })?;
3628 let ip = bridge.ip_allocator.allocate().ok_or_else(|| {
3629 OverlaydError::Overlay(format!(
3630 "shared bridge {} subnet {} exhausted",
3631 bridge.name, bridge.subnet
3632 ))
3633 })?;
3634 (bridge.name.clone(), bridge.subnet, bridge.gateway, ip)
3635 } else {
3636 let bridge = self.service_bridges.get_mut(service).ok_or_else(|| {
3637 OverlaydError::Other(format!(
3638 "no service bridge for service {service}; call setup_service_overlay() first"
3639 ))
3640 })?;
3641 let ip = bridge.ip_allocator.allocate().ok_or_else(|| {
3642 OverlaydError::Overlay(format!(
3643 "service bridge {} subnet {} exhausted",
3644 bridge.name, bridge.subnet
3645 ))
3646 })?;
3647 (bridge.name.clone(), bridge.subnet, bridge.gateway, ip)
3648 };
3649
3650 let bridge_params = BridgeAttachParams {
3651 bridge_name: &bridge_name,
3652 gateway: bridge_gateway,
3653 subnet_prefix_len: bridge_subnet.prefix_len(),
3654 };
3655 if let Err(e) = self
3656 .attach_to_interface(
3657 container_pid,
3658 container_ip,
3659 "s",
3660 "eth0",
3661 Some(&bridge_params),
3662 )
3663 .await
3664 {
3665 if use_shared {
3666 if let Some(bridge) = self.shared_bridge.as_mut() {
3667 bridge.ip_allocator.release(container_ip);
3668 }
3669 } else if let Some(bridge) = self.service_bridges.get_mut(service) {
3670 bridge.ip_allocator.release(container_ip);
3671 }
3672 return Err(e);
3673 }
3674
3675 let mut global_ip: Option<IpAddr> = None;
3676 if join_global && self.global_interface.is_some() {
3677 let g_ip = self.ip_allocator.allocate()?;
3678 self.attach_to_interface(container_pid, g_ip, "g", "eth1", None)
3679 .await?;
3680 global_ip = Some(g_ip);
3681 }
3682
3683 // Per-network L3 isolation: when this attach joins a named isolated
3684 // network, install the Docker-style iptables rules pinning this member
3685 // to its own network's members + node + egress, then record it in the
3686 // membership map. Non-fatal: a host without iptables logs and proceeds.
3687 if let Some(ref net) = isolation_network {
3688 let node_ip = self
3689 .node_ip
3690 .unwrap_or(std::net::IpAddr::V4(std::net::Ipv4Addr::new(10, 200, 0, 1)));
3691 let cidr = self
3692 .cluster_cidr
3693 .map_or_else(|| "10.200.0.0/16".to_string(), |c| c.to_string());
3694 let peers: Vec<IpAddr> = self
3695 .network_members
3696 .get(net)
3697 .map(|m| m.iter().copied().collect())
3698 .unwrap_or_default();
3699 if let Err(e) = zlayer_overlay::firewall::ensure_member_isolation(
3700 net,
3701 container_ip,
3702 &peers,
3703 node_ip,
3704 &cidr,
3705 ) {
3706 tracing::warn!(network = %net, member = %container_ip, error = %e, "failed to install per-network L3 isolation (non-fatal)");
3707 }
3708 self.network_members
3709 .entry(net.clone())
3710 .or_default()
3711 .insert(container_ip);
3712 }
3713
3714 self.attached.insert(
3715 container_pid,
3716 AttachInfo {
3717 service_ip: container_ip,
3718 service_name: Some(service.to_string()),
3719 global_ip,
3720 ephemeral,
3721 isolation_network,
3722 },
3723 );
3724
3725 Ok(container_ip)
3726 }
3727
3728 /// Non-Linux fallback: containers share the host network, so return the
3729 /// node's overlay IP (or loopback).
3730 #[cfg(not(target_os = "linux"))]
3731 #[allow(clippy::unused_async)]
3732 async fn attach_container_linux(
3733 &mut self,
3734 _container_pid: u32,
3735 service: &str,
3736 _join_global: bool,
3737 _ephemeral: bool,
3738 _isolation_network: Option<String>,
3739 ) -> Result<IpAddr, OverlaydError> {
3740 tracing::debug!(service = %service, "LinuxPid attach is a no-op off Linux; using node overlay IP");
3741 Ok(self.node_ip.unwrap_or(IpAddr::V4(Ipv4Addr::LOCALHOST)))
3742 }
3743
3744 /// Release the overlay resources held by a Linux container PID. Idempotent.
3745 #[cfg(target_os = "linux")]
3746 async fn detach_container_linux(&mut self, pid: u32) -> Result<(), OverlaydError> {
3747 // "Process id or not, kill the adapter": the host-side veth name is
3748 // deterministic (`veth-<pid>-{s,g}`), so delete it UNCONDITIONALLY by
3749 // name — even when no attach record survives (a previous daemon crashed
3750 // before recording it, or it was already reaped). Without this, a missing
3751 // record left the host veth orphaned until the PID-keyed periodic sweep
3752 // (which only fires once the PID is dead). The deletes are idempotent
3753 // (ENODEV = success), so the always-on `-g` delete is harmless when the
3754 // container never joined the global overlay.
3755 let info = self.attached.remove(&pid);
3756
3757 let veth_s = format!("veth-{pid}-s");
3758 if let Err(e) = crate::netlink::delete_link_by_name(&veth_s).await {
3759 tracing::warn!(link = %veth_s, pid, error = %e, "Failed to delete service veth");
3760 }
3761 let veth_g = format!("veth-{pid}-g");
3762 if let Err(e) = crate::netlink::delete_link_by_name(&veth_g).await {
3763 tracing::warn!(link = %veth_g, pid, error = %e, "Failed to delete global veth");
3764 }
3765
3766 // No attach record -> nothing more to release (IP/registry bookkeeping
3767 // is keyed off the record). The veths above are already gone.
3768 let Some(info) = info else {
3769 return Ok(());
3770 };
3771
3772 // Return the service IP to whichever pool owns it. A Shared-mode service
3773 // drew its IP from the single node-wide shared bridge (no per-service
3774 // bridge exists for it), so try the shared bridge by subnet containment
3775 // before the named per-service bridge.
3776 if self.shared_bridge.as_mut().is_some_and(|b| {
3777 b.subnet.contains(&info.service_ip) && b.ip_allocator.release(info.service_ip)
3778 }) {
3779 // released into the shared bridge
3780 } else if let Some(svc) = info.service_name.as_deref() {
3781 if let Some(bridge) = self.service_bridges.get_mut(svc) {
3782 bridge.ip_allocator.release(info.service_ip);
3783 } else {
3784 tracing::debug!(service = %svc, ip = %info.service_ip, "detach: service bridge already torn down; dropping service IP release");
3785 }
3786 } else {
3787 self.ip_allocator.release(info.service_ip);
3788 }
3789 if let Some(g) = info.global_ip {
3790 self.ip_allocator.release(g);
3791 }
3792
3793 // Per-network L3 isolation drain: remove this member from its isolated
3794 // network's membership set and tear down its iptables rules against the
3795 // remaining members. Drop the network entry once empty.
3796 if let Some(net) = info.isolation_network.as_deref() {
3797 if let Some(set) = self.network_members.get_mut(net) {
3798 set.remove(&info.service_ip);
3799 }
3800 let still: Vec<IpAddr> = self
3801 .network_members
3802 .get(net)
3803 .map(|m| m.iter().copied().collect())
3804 .unwrap_or_default();
3805 let node_ip = self
3806 .node_ip
3807 .unwrap_or(std::net::IpAddr::V4(std::net::Ipv4Addr::new(10, 200, 0, 1)));
3808 let cidr = self
3809 .cluster_cidr
3810 .map_or_else(|| "10.200.0.0/16".to_string(), |c| c.to_string());
3811 zlayer_overlay::firewall::remove_member_isolation(
3812 net,
3813 info.service_ip,
3814 &still,
3815 node_ip,
3816 &cidr,
3817 );
3818 if self
3819 .network_members
3820 .get(net)
3821 .is_some_and(std::collections::HashSet::is_empty)
3822 {
3823 self.network_members.remove(net);
3824 }
3825 }
3826
3827 // Ephemeral last-leaver teardown: a standalone/per-job bridge is reclaimed
3828 // the moment its LAST container leaves (the periodic prune is only the
3829 // ~300s backstop). Managed attaches use ephemeral=false so their bridge
3830 // persists across scale-to-0. Route through teardown_service_overlay so
3831 // overlayd's in-memory state stays synced — never a hand `ip link del`.
3832 // This container's veth is already removed above, so a 0 member count
3833 // means no containers remain on the bridge.
3834 if info.ephemeral {
3835 if let Some(svc) = info.service_name.clone() {
3836 if let Some(bridge_name) = self.service_bridges.get(&svc).map(|b| b.name.clone()) {
3837 if crate::netlink::bridge_member_count(&bridge_name).await == 0 {
3838 tracing::info!(service = %svc, bridge = %bridge_name, "ephemeral overlay bridge idle after last detach — tearing down");
3839 self.teardown_service_overlay(&svc).await;
3840 }
3841 }
3842 }
3843 }
3844 Ok(())
3845 }
3846
3847 /// Non-Linux fallback: nothing to detach (host networking).
3848 #[cfg(not(target_os = "linux"))]
3849 #[allow(clippy::unused_async)]
3850 async fn detach_container_linux(&mut self, _pid: u32) -> Result<(), OverlaydError> {
3851 Ok(())
3852 }
3853
3854 /// Best-effort sweep of orphan veth endpoints whose owning container PID is
3855 /// no longer alive. Names matching `veth-<pid>-*` / `vc-<pid>-*` where
3856 /// `/proc/<pid>` does not exist are deleted.
3857 #[cfg(target_os = "linux")]
3858 async fn sweep_orphan_veths() {
3859 let links = match crate::netlink::list_all_links().await {
3860 Ok(links) => links,
3861 Err(e) => {
3862 tracing::warn!(error = %e, "Failed to list links for orphan sweep");
3863 return;
3864 }
3865 };
3866 for (_index, name) in links {
3867 let remainder = if let Some(r) = name.strip_prefix("veth-") {
3868 r
3869 } else if let Some(r) = name.strip_prefix("vc-") {
3870 r
3871 } else {
3872 continue;
3873 };
3874 let Some(pid_str) = remainder.split('-').next() else {
3875 continue;
3876 };
3877 let pid: u32 = match pid_str.parse() {
3878 Ok(p) => p,
3879 Err(_) => continue,
3880 };
3881 if Path::new(&format!("/proc/{pid}")).exists() {
3882 continue;
3883 }
3884 tracing::info!(link = %name, pid = pid, "Deleting orphan veth");
3885 if let Err(e) = crate::netlink::delete_link_by_name(&name).await {
3886 tracing::warn!(link = %name, error = %e, "Failed to delete orphan veth");
3887 }
3888 }
3889 }
3890
3891 #[cfg(target_os = "linux")]
3892 #[allow(clippy::too_many_lines)]
3893 async fn attach_to_interface(
3894 &mut self,
3895 container_pid: u32,
3896 ip: IpAddr,
3897 tag: &str,
3898 container_iface: &str,
3899 bridge: Option<&BridgeAttachParams<'_>>,
3900 ) -> Result<(), OverlaydError> {
3901 // Best-effort cleanup of orphan veths left by a previous daemon crash.
3902 Self::sweep_orphan_veths().await;
3903
3904 let is_v6 = ip.is_ipv6();
3905 let prefix_len: u8 = if let Some(b) = bridge {
3906 b.subnet_prefix_len
3907 } else if is_v6 {
3908 64
3909 } else {
3910 24
3911 };
3912 let host_prefix: u8 = if is_v6 { 128 } else { 32 };
3913
3914 let veth_host = format!("veth-{container_pid}-{tag}");
3915 let veth_pending = format!("vc-{container_pid}-{tag}");
3916 let veth_container = container_iface.to_string();
3917
3918 let container_ns_fd = std::os::fd::OwnedFd::from(
3919 std::fs::File::open(format!("/proc/{container_pid}/ns/net")).map_err(|e| {
3920 OverlaydError::Overlay(format!("Failed to open /proc/{container_pid}/ns/net: {e}"))
3921 })?,
3922 );
3923
3924 crate::netlink::delete_link_by_name(&veth_host)
3925 .await
3926 .map_err(|e| OverlaydError::Overlay(format!("pre-cleanup delete {veth_host}: {e}")))?;
3927 crate::netlink::delete_link_by_name(&veth_pending)
3928 .await
3929 .map_err(|e| {
3930 OverlaydError::Overlay(format!("pre-cleanup delete {veth_pending}: {e}"))
3931 })?;
3932
3933 let bridge_gateway: Option<IpAddr> = bridge.map(|b| b.gateway);
3934 let bridge_name: Option<String> = bridge.map(|b| b.bridge_name.to_string());
3935 let node_ip = self.node_ip;
3936
3937 let result: Result<(), OverlaydError> = async {
3938 crate::netlink::create_veth_pair(&veth_host, &veth_pending)
3939 .await
3940 .map_err(|e| OverlaydError::Overlay(format!("create veth pair: {e}")))?;
3941
3942 crate::netlink::move_link_into_netns_fd_and_rename(
3943 &veth_pending,
3944 AsFd::as_fd(&container_ns_fd),
3945 &veth_container,
3946 )
3947 .map_err(|e| OverlaydError::Overlay(format!("move veth into netns: {e}")))?;
3948
3949 let vc = veth_container.clone();
3950 let bridge_gateway_for_netns = bridge_gateway;
3951 tokio::task::spawn_blocking(move || {
3952 crate::netlink::with_netns_fd_async(container_ns_fd, move || async move {
3953 crate::netlink::add_address_to_link_by_name(&vc, ip, prefix_len).await?;
3954 crate::netlink::set_link_up_by_name(&vc).await?;
3955 crate::netlink::set_link_up_by_name("lo").await?;
3956 if let Some(gw) = bridge_gateway_for_netns {
3957 crate::netlink::add_default_route_via_gateway(gw).await?;
3958 }
3959 Ok(())
3960 })
3961 })
3962 .await
3963 .map_err(|e| OverlaydError::Overlay(format!("container netns task panicked: {e}")))?
3964 .map_err(|e| OverlaydError::Overlay(format!("container netns ops: {e}")))?;
3965
3966 crate::netlink::set_link_up_by_name(&veth_host)
3967 .await
3968 .map_err(|e| OverlaydError::Overlay(format!("set {veth_host} up: {e}")))?;
3969
3970 if let Some(bname) = bridge_name.as_deref() {
3971 crate::netlink::add_link_to_bridge(&veth_host, bname)
3972 .await
3973 .map_err(|e| {
3974 OverlaydError::Overlay(format!(
3975 "enslave {veth_host} to bridge {bname}: {e}"
3976 ))
3977 })?;
3978 } else {
3979 crate::netlink::replace_route_via_dev(ip, host_prefix, &veth_host, node_ip)
3980 .await
3981 .map_err(|e| {
3982 OverlaydError::Overlay(format!("host route for {ip}/{host_prefix}: {e}"))
3983 })?;
3984 }
3985
3986 Ok(())
3987 }
3988 .await;
3989
3990 // Enable IP forwarding so the host routes between the overlay device(s)
3991 // and the egress NIC. CRITICAL: this is scoped to the address family
3992 // actually in use and (for IPv6) to the specific overlay devices —
3993 // NEVER `net.ipv6.conf.all.forwarding`, whose documented kernel side
3994 // effect is to force `accept_ra=0` + `autoconf=0` on every IPv6
3995 // interface (including the public NIC), dropping the RA-learned default
3996 // route / path-MTU and blackholing the host's own larger reply packets
3997 // (e.g. inbound SSH stalls after key exchange). Done outside the
3998 // attach `result` block so a forwarding-sysctl failure can never roll
3999 // back a successful veth attach. Tracked so teardown reverts it.
4000 if result.is_ok() {
4001 self.enable_forwarding_for_attach(is_v6, &veth_host, bridge_name.as_deref());
4002
4003 // Track the host-side resources this attach created so a clean
4004 // global teardown reverts every host mutation. The host-side veth
4005 // half exists in both the bridged and bridgeless paths; the host
4006 // `/32`(`/128`) route is installed ONLY on the bridgeless path
4007 // (`replace_route_via_dev` above), so record it only when there was
4008 // no bridge to enslave into. All deletions are idempotent, so a
4009 // resource a later per-container detach removes first is harmless.
4010 self.created_veths.insert(veth_host.clone());
4011 if bridge_name.is_none() {
4012 self.created_host_routes
4013 .push((ip, host_prefix, veth_host.clone()));
4014 }
4015 }
4016
4017 if result.is_err() {
4018 let _ = crate::netlink::delete_link_by_name(&veth_host).await;
4019 let _ = crate::netlink::delete_link_by_name(&veth_pending).await;
4020 }
4021 result
4022 }
4023
4024 // -- container attach (Windows HCN) -------------------------------------
4025
4026 /// Windows attach: ensure the overlay HCN Internal network exists, allocate
4027 /// or validate the IP, create the per-container HCN endpoint + namespace,
4028 /// and return the bare-lowercase namespace GUID for the agent to embed in
4029 /// the compute-system document.
4030 ///
4031 /// # Errors
4032 /// Returns an error if the network/endpoint cannot be created or the slice
4033 /// is exhausted.
4034 #[cfg(target_os = "windows")]
4035 #[allow(clippy::too_many_lines)]
4036 async fn attach_container_windows(
4037 &mut self,
4038 container_id: &str,
4039 service: &str,
4040 ip_override: Option<IpAddr>,
4041 dns_server: Option<IpAddr>,
4042 dns_domain: Option<String>,
4043 isolation_network: Option<String>,
4044 ) -> Result<AttachResult, OverlaydError> {
4045 // Resolve whether THIS service has a dedicated per-service overlay. It
4046 // does iff a live dedicated transport exists OR a `hcn-internal` marker
4047 // entry is recorded under `owner_for_service(service)` (the network
4048 // survives daemon restarts even if the transport map is empty mid-init).
4049 // Dedicated services attach onto their OWN per-service Internal network
4050 // and draw IPs from the service subnet; everything else uses the node's
4051 // base/shared overlay network and the node slice.
4052 let dedicated_subnet = self.dedicated_service_subnet(service);
4053 // A `Shared`-mode service attaches onto the SINGLE shared HCN NAT network
4054 // reused across all Shared services (container ports are exposed via the
4055 // userspace free-port L4 proxy). The mode was recorded at setup time.
4056 let use_shared_nat = self
4057 .service_modes
4058 .get(service)
4059 .copied()
4060 .unwrap_or_default()
4061 .uses_shared_bridge();
4062
4063 let (net_id, ip, prefix_length) = if let Some(net) = isolation_network.as_deref() {
4064 // ----- per-isolation-network Internal HCN network path -----
4065 //
4066 // An "isolated" ZLayer network routes its members onto a dedicated
4067 // HCN Internal vSwitch keyed by the isolation-network NAME (not the
4068 // service). HCN Internal vSwitches are mutually isolated by default,
4069 // so same-network members share one vSwitch (reach each other +
4070 // egress via the network gateway + the node), while different
4071 // isolation networks land on separate vSwitches and cannot reach
4072 // each other — L3 isolation with NO ACLs and NO per-member churn.
4073 // This mirrors the Dedicated per-service branch below, but keyed by
4074 // the isolation-network name and drawing IPs from a per-network
4075 // subnet carved deterministically from the node slice.
4076 let iso_subnet = self.isolation_network_subnet(net)?;
4077 let net_id = self.ensure_isolation_network(net, iso_subnet).await?;
4078
4079 // Per-network container IPs come from the isolation network's own
4080 // subnet (never the node slice), via a lazily-created allocator
4081 // bounded to that subnet. The allocator is keyed by the isolation
4082 // network's owner key so it never collides with a same-named
4083 // dedicated service's allocator. An `ip_override` is honored only
4084 // when it falls inside the isolation subnet.
4085 let iso_ipnetwork: IpNetwork = iso_subnet.to_string().parse().map_err(|e| {
4086 OverlaydError::Other(format!(
4087 "failed to parse isolation subnet {iso_subnet}: {e}"
4088 ))
4089 })?;
4090 let alloc_key = crate::network_state::owner_for_isolation_network(net);
4091 let allocator = self
4092 .service_ip_allocators
4093 .entry(alloc_key)
4094 .or_insert_with(|| IpAllocator::new(iso_ipnetwork));
4095 let ip = match ip_override {
4096 Some(ip) if iso_subnet.contains(&ip) => ip,
4097 Some(ip) => {
4098 return Err(OverlaydError::Other(format!(
4099 "overridden IP {ip} is not inside isolation network subnet {iso_subnet} for network {net}"
4100 )));
4101 }
4102 None => allocator.allocate()?,
4103 };
4104 (net_id, ip, iso_subnet.prefix_len())
4105 } else if use_shared_nat {
4106 // ----- shared HCN NAT network path -----
4107 let slice = self.slice_cidr.ok_or_else(|| {
4108 OverlaydError::Other(
4109 "no node slice assigned yet (SetupGlobalOverlay with slice_cidr first)"
4110 .to_string(),
4111 )
4112 })?;
4113 let slice_ipnet: ipnet::IpNet = slice.to_string().parse().map_err(|e| {
4114 OverlaydError::Other(format!("failed to parse slice CIDR {slice}: {e}"))
4115 })?;
4116 let net_id = self.ensure_shared_nat_network(slice_ipnet).await?;
4117 let ip = match ip_override {
4118 Some(ip) => ip,
4119 None => self.ip_allocator.allocate()?,
4120 };
4121 (net_id, ip, slice_ipnet.prefix_len())
4122 } else if let Some(svc_subnet) = dedicated_subnet {
4123 // ----- dedicated per-service network path -----
4124 let net_id = self.ensure_service_network(service, svc_subnet).await?;
4125
4126 // Allocate (or validate) the IP from the SERVICE subnet, not the
4127 // node slice. A per-service allocator is created lazily and bounded
4128 // to the service subnet so addresses stay inside the dedicated
4129 // network. An `ip_override` inside the service subnet is honored;
4130 // one outside it is rejected so a slice-allocated IP can't leak onto
4131 // the dedicated network.
4132 let svc_ipnetwork: IpNetwork = svc_subnet.to_string().parse().map_err(|e| {
4133 OverlaydError::Other(format!("failed to parse service subnet {svc_subnet}: {e}"))
4134 })?;
4135 let allocator = self
4136 .service_ip_allocators
4137 .entry(service.to_string())
4138 .or_insert_with(|| IpAllocator::new(svc_ipnetwork));
4139 let ip = match ip_override {
4140 Some(ip) if svc_subnet.contains(&ip) => ip,
4141 Some(ip) => {
4142 return Err(OverlaydError::Other(format!(
4143 "overridden IP {ip} is not inside dedicated service subnet {svc_subnet} for service {service}"
4144 )));
4145 }
4146 None => allocator.allocate()?,
4147 };
4148 (net_id, ip, svc_subnet.prefix_len())
4149 } else {
4150 // ----- shared base overlay network path (unchanged) -----
4151 let slice = self.slice_cidr.ok_or_else(|| {
4152 OverlaydError::Other(
4153 "no node slice assigned yet (SetupGlobalOverlay with slice_cidr first)"
4154 .to_string(),
4155 )
4156 })?;
4157 let slice_ipnet: ipnet::IpNet = slice.to_string().parse().map_err(|e| {
4158 OverlaydError::Other(format!("failed to parse slice CIDR {slice}: {e}"))
4159 })?;
4160 let net_id = self.ensure_overlay_network(slice_ipnet).await?;
4161 let ip = match ip_override {
4162 Some(ip) => ip,
4163 None => self.ip_allocator.allocate()?,
4164 };
4165 (net_id, ip, slice_ipnet.prefix_len())
4166 };
4167
4168 // 3. Create the endpoint + per-container namespace on the network.
4169 let dns_server_eff = dns_server.or_else(|| self.dns_server_addr.map(|a| a.ip()));
4170 let dns_domain_for_attach = dns_domain.or_else(|| self.dns_domain.clone());
4171 let cluster_cidr = self.cluster_cidr.map(|c| c.to_string()).unwrap_or_default();
4172 let owner_tag = owner_tag(&self.deployment_or_default());
4173 let cid = container_id.to_string();
4174
4175 let attachment = tokio::task::spawn_blocking(move || {
4176 zlayer_hns::attach::EndpointAttachment::create_overlay(
4177 net_id,
4178 &owner_tag,
4179 cid.as_str(),
4180 ip,
4181 prefix_length,
4182 &cluster_cidr,
4183 dns_server_eff,
4184 dns_domain_for_attach.as_deref(),
4185 )
4186 })
4187 .await
4188 .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?
4189 .map_err(|e| OverlaydError::Overlay(format!("HCN overlay endpoint attach failed: {e}")))?;
4190
4191 let namespace_id = attachment.namespace_id();
4192 let bare_guid = format_guid_bare(namespace_id);
4193
4194 // Per-network membership: record the container's IP in its isolated
4195 // network's member set. Windows enforcement is an HCN ACL — a
4196 // Linux-incompatible mechanism wired separately; overlayd only maintains
4197 // the membership map here and does NOT call the iptables firewall helper.
4198 if let Some(ref net) = isolation_network {
4199 self.network_members
4200 .entry(net.clone())
4201 .or_default()
4202 .insert(ip);
4203 }
4204
4205 // Record for autoclean keyed by namespace GUID.
4206 self.hcn_cleanup
4207 .insert(namespace_id, (service.to_string(), ip, isolation_network));
4208
4209 tracing::info!(
4210 ns = %bare_guid,
4211 service = %service,
4212 ip = %ip,
4213 "Attached container to HCN overlay"
4214 );
4215
4216 Ok(AttachResult {
4217 ip,
4218 namespace_guid: Some(bare_guid),
4219 })
4220 }
4221
4222 /// Non-Windows path: a `WindowsContainer` handle has no meaning off Windows.
4223 #[cfg(not(target_os = "windows"))]
4224 #[allow(clippy::unused_async)]
4225 async fn attach_container_windows(
4226 &mut self,
4227 _container_id: &str,
4228 _service: &str,
4229 _ip_override: Option<IpAddr>,
4230 _dns_server: Option<IpAddr>,
4231 _dns_domain: Option<String>,
4232 _isolation_network: Option<String>,
4233 ) -> Result<AttachResult, OverlaydError> {
4234 Err(OverlaydError::Other(
4235 "WindowsContainer attach is only supported on Windows".to_string(),
4236 ))
4237 }
4238
4239 /// Detach a Windows container by its bare namespace GUID and release its IP.
4240 /// Idempotent: unknown ids are a no-op.
4241 #[cfg(target_os = "windows")]
4242 async fn detach_container_windows(
4243 &mut self,
4244 namespace_guid: &str,
4245 ) -> Result<(), OverlaydError> {
4246 use windows::core::GUID;
4247
4248 let Ok(guid) = GUID::try_from(namespace_guid) else {
4249 tracing::warn!(ns = %namespace_guid, "detach: unparseable namespace GUID");
4250 return Ok(());
4251 };
4252 if let Some((service, ip, isolation_network)) = self.hcn_cleanup.remove(&guid) {
4253 // Release the IP into the pool it was drawn from. An isolation-network
4254 // member drew from the per-network allocator (keyed by the isolation
4255 // owner key), NOT the node slice; release it there so the isolation
4256 // subnet doesn't leak addresses. Everything else came from the node
4257 // slice.
4258 if let Some(net) = isolation_network.as_deref() {
4259 let alloc_key = crate::network_state::owner_for_isolation_network(net);
4260 if let Some(allocator) = self.service_ip_allocators.get_mut(&alloc_key) {
4261 allocator.release(ip);
4262 } else {
4263 self.ip_allocator.release(ip);
4264 }
4265 } else {
4266 self.ip_allocator.release(ip);
4267 }
4268 // Drain the per-network membership set.
4269 let mut net_now_empty: Option<String> = None;
4270 if let Some(net) = isolation_network.as_deref() {
4271 if let Some(set) = self.network_members.get_mut(net) {
4272 set.remove(&ip);
4273 }
4274 if self
4275 .network_members
4276 .get(net)
4277 .is_some_and(std::collections::HashSet::is_empty)
4278 {
4279 self.network_members.remove(net);
4280 net_now_empty = Some(net.to_string());
4281 }
4282 }
4283 tracing::info!(ns = %namespace_guid, service = %service, ip = %ip, "Released HCN overlay attachment");
4284
4285 // Last-member teardown: when the final member of an isolation network
4286 // leaves, reclaim its per-network HCN Internal network (mirroring the
4287 // per-service network teardown in `teardown_service_overlay`) so we
4288 // don't leak an HCN vSwitch until the next full uninstall. Drop the
4289 // per-network IP allocator and the marker entry too.
4290 if let Some(net) = net_now_empty {
4291 self.teardown_isolation_network(&net).await;
4292 }
4293 }
4294 Ok(())
4295 }
4296
4297 /// Reclaim the per-isolation-network HCN Internal network for `net`: delete
4298 /// the HCN network by the GUID recorded in the marker, drop its marker entry,
4299 /// and discard the per-network IP allocator. Best-effort and idempotent —
4300 /// called once the last member of the isolation network detaches. Mirrors the
4301 /// per-service network teardown in [`Self::teardown_service_overlay`].
4302 #[cfg(target_os = "windows")]
4303 async fn teardown_isolation_network(&mut self, net: &str) {
4304 let owner = crate::network_state::owner_for_isolation_network(net);
4305
4306 // Drop the per-network container-IP allocator.
4307 self.service_ip_allocators.remove(&owner);
4308
4309 let marker_path =
4310 zlayer_paths::ZLayerDirs::new(self.data_dir.clone()).agent_network_state();
4311 let mut marker = crate::network_state::NetworkState::load(&marker_path);
4312 let removed_entry = marker.remove(&owner);
4313 if removed_entry.is_some() {
4314 if let Err(e) = marker.save(&marker_path) {
4315 tracing::warn!(network = %net, error = %e, path = %marker_path.display(), "failed to persist isolation-network marker removal");
4316 }
4317 }
4318
4319 if let Some(entry) = removed_entry {
4320 if entry.kind == "hcn-internal" {
4321 match windows::core::GUID::try_from(entry.id.as_str()) {
4322 Ok(guid) => {
4323 let id_str = entry.id.clone();
4324 let net_owned = net.to_string();
4325 let delete = tokio::task::spawn_blocking(move || {
4326 zlayer_hns::network::Network::delete(guid)
4327 })
4328 .await;
4329 match delete {
4330 Ok(Ok(())) => {
4331 tracing::info!(network = %net_owned, id = %id_str, "deleted per-isolation-network HCN network on last detach");
4332 }
4333 Ok(Err(e)) => {
4334 tracing::warn!(network = %net_owned, id = %id_str, error = %e, "failed to delete isolation-network HCN network (may leak until uninstall)");
4335 }
4336 Err(e) => {
4337 tracing::warn!(network = %net_owned, id = %id_str, error = %e, "spawn_blocking join failed deleting isolation-network HCN network");
4338 }
4339 }
4340 }
4341 Err(_) => {
4342 tracing::warn!(network = %net, id = %entry.id, "isolation-network marker has unparseable HCN GUID; skipping network delete");
4343 }
4344 }
4345 }
4346 }
4347 }
4348
4349 /// Non-Windows path.
4350 #[cfg(not(target_os = "windows"))]
4351 #[allow(clippy::unused_async)]
4352 async fn detach_container_windows(
4353 &mut self,
4354 _namespace_guid: &str,
4355 ) -> Result<(), OverlaydError> {
4356 Ok(())
4357 }
4358
4359 /// Ensure the per-daemon HCN overlay (Internal vSwitch, no physical-NIC
4360 /// binding) exists on the host, reusing one recorded in the
4361 /// `{data_dir}/agent_network.json` marker or discoverable by name, and
4362 /// recording it in the marker on create.
4363 ///
4364 /// # Errors
4365 /// Propagates the underlying `zlayer_hns` error on create failure.
4366 #[cfg(target_os = "windows")]
4367 #[allow(clippy::too_many_lines)]
4368 async fn ensure_overlay_network(
4369 &mut self,
4370 slice_cidr: ipnet::IpNet,
4371 ) -> Result<windows::core::GUID, OverlaydError> {
4372 use windows::core::GUID;
4373
4374 let daemon_name = self.deployment_or_default();
4375 let net_name = overlay_network_name(&daemon_name);
4376 let marker_path =
4377 zlayer_paths::ZLayerDirs::new(self.data_dir.clone()).agent_network_state();
4378
4379 // Fast path: marker names a network GUID that still exists; reopen it.
4380 if let Some(recorded_id) = crate::network_state::NetworkState::load(&marker_path)
4381 .get(crate::network_state::OWNER_BASE)
4382 .and_then(|entry| GUID::try_from(entry.id.as_str()).ok())
4383 {
4384 let reopened = tokio::task::spawn_blocking(move || {
4385 zlayer_hns::network::Network::open(recorded_id).ok()
4386 })
4387 .await
4388 .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?;
4389 if reopened.is_some() {
4390 tracing::info!(name = %net_name, "reusing HCN overlay network from marker");
4391 return Ok(recorded_id);
4392 }
4393 }
4394
4395 // Idempotency: reuse a host network whose queried name matches ours.
4396 let target_name = net_name.clone();
4397 let existing = tokio::task::spawn_blocking(move || -> Option<GUID> {
4398 let guids = zlayer_hns::network::list("{}").ok()?;
4399 for guid in guids {
4400 let Ok(network) = zlayer_hns::network::Network::open(guid) else {
4401 continue;
4402 };
4403 if matches!(network.query("{}"), Ok(props) if props.name == target_name) {
4404 return Some(guid);
4405 }
4406 }
4407 None
4408 })
4409 .await
4410 .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?;
4411
4412 if let Some(existing_id) = existing {
4413 tracing::info!(name = %net_name, "reusing existing HCN overlay network");
4414 return Ok(existing_id);
4415 }
4416
4417 let net_id = GUID::new()
4418 .map_err(|e| OverlaydError::Other(format!("GUID::new for overlay network: {e}")))?;
4419 let subnet_str = slice_cidr.to_string();
4420
4421 // Default: an HCN Internal network — an internal vSwitch with NO
4422 // physical-NIC binding — so container traffic never touches the
4423 // operator's gateway adapter. Setting ZLAYER_HCN_UPLINK_ADAPTER opts
4424 // into the legacy Transparent model bound to that named uplink.
4425 let use_transparent = std::env::var(zlayer_hns::adapter::ZLAYER_UPLINK_ENV)
4426 .ok()
4427 .is_some_and(|v| !v.trim().is_empty());
4428
4429 let net_name_for_create = net_name.clone();
4430 let subnet_for_create = subnet_str.clone();
4431 if use_transparent {
4432 let uplink = zlayer_hns::adapter::find_primary_adapter()
4433 .map_err(|e| OverlaydError::Other(format!("find_primary_adapter: {e}")))?;
4434 tracing::warn!(uplink = %uplink, "ZLAYER_HCN_UPLINK_ADAPTER set: creating HCN *Transparent* overlay bound to a physical NIC");
4435 tokio::task::spawn_blocking(move || {
4436 zlayer_hns::network::Network::create_transparent(
4437 net_id,
4438 &net_name_for_create,
4439 &subnet_for_create,
4440 &uplink,
4441 )
4442 })
4443 .await
4444 .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?
4445 .map_err(|e| {
4446 OverlaydError::Overlay(format!("HcnCreateNetwork transparent ({net_name}): {e}"))
4447 })?;
4448 } else {
4449 tokio::task::spawn_blocking(move || {
4450 zlayer_hns::network::Network::create_internal(
4451 net_id,
4452 &net_name_for_create,
4453 &subnet_for_create,
4454 )
4455 })
4456 .await
4457 .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?
4458 .map_err(|e| {
4459 OverlaydError::Overlay(format!("HcnCreateNetwork internal ({net_name}): {e}"))
4460 })?;
4461 }
4462
4463 // HCN's Static IPAM needs ~1-2s after network create to settle its
4464 // address pool; without this the first endpoint frequently fails with
4465 // HCN_E_ADDR_INVALID_OR_RESERVED.
4466 tokio::time::sleep(std::time::Duration::from_secs(2)).await;
4467
4468 tracing::info!(
4469 subnet = %subnet_str,
4470 mode = if use_transparent { "Transparent" } else { "Internal" },
4471 "created HCN overlay network"
4472 );
4473
4474 // Persist the marker so subsequent runs reuse this network by GUID and a
4475 // full uninstall knows to delete it. Best-effort.
4476 let mut marker = crate::network_state::NetworkState::load(&marker_path);
4477 marker.upsert(crate::network_state::ManagedNetwork {
4478 owner: crate::network_state::OWNER_BASE.to_string(),
4479 kind: if use_transparent {
4480 "hcn-transparent"
4481 } else {
4482 "hcn-internal"
4483 }
4484 .to_string(),
4485 name: net_name.clone(),
4486 id: format_guid_bare(net_id),
4487 subnet: subnet_str.clone(),
4488 // Base/Shared HCN network: no dedicated WireGuard identity.
4489 wg_port: None,
4490 wg_private_key: None,
4491 wg_public_key: None,
4492 interface: None,
4493 });
4494 if let Err(e) = marker.save(&marker_path) {
4495 tracing::warn!(error = %e, path = %marker_path.display(), "failed to persist agent network marker (network still reusable by name)");
4496 }
4497
4498 Ok(net_id)
4499 }
4500
4501 /// Ensure the SINGLE shared HCN **NAT** network exists on the host, reusing
4502 /// one recorded under the [`OWNER_SHARED_NAT`] marker owner (or discoverable
4503 /// by its derived name) and recording it on create. Reused across every
4504 /// `OverlayMode::Shared` service on this node.
4505 ///
4506 /// NAT gives Shared containers outbound connectivity and lets the userspace
4507 /// free-port L4 proxy (`proxy_manager.rs`) forward `host:FREEPORT` ->
4508 /// `container_ip:port` without a per-service vSwitch — the Windows analogue
4509 /// of the Linux node-wide shared bridge. Modeled on
4510 /// [`Self::ensure_overlay_network`] but keyed on [`OWNER_SHARED_NAT`] and
4511 /// forced to the NAT network type.
4512 ///
4513 /// Returns the network GUID.
4514 ///
4515 /// # Errors
4516 /// Propagates the underlying `zlayer_hns` error on create failure, or an
4517 /// error if the slice CIDR has no usable gateway host.
4518 #[cfg(target_os = "windows")]
4519 #[allow(clippy::too_many_lines)]
4520 async fn ensure_shared_nat_network(
4521 &mut self,
4522 slice_cidr: ipnet::IpNet,
4523 ) -> Result<windows::core::GUID, OverlaydError> {
4524 use windows::core::GUID;
4525
4526 let daemon_name = self.deployment_or_default();
4527 // Shared NAT network name: `<base overlay name>-shared` so it is
4528 // unambiguously distinct from the base network and per-service networks.
4529 let net_name = format!("{}-shared", overlay_network_name(&daemon_name));
4530 let owner = crate::network_state::OWNER_SHARED_NAT.to_string();
4531 let marker_path =
4532 zlayer_paths::ZLayerDirs::new(self.data_dir.clone()).agent_network_state();
4533
4534 // Fast path: marker names a network GUID that still exists; reopen it.
4535 let recorded_id = crate::network_state::NetworkState::load(&marker_path)
4536 .get(&owner)
4537 .filter(|entry| entry.kind == "hcn-nat")
4538 .and_then(|entry| GUID::try_from(entry.id.as_str()).ok());
4539 if let Some(recorded_id) = recorded_id {
4540 let reopened = tokio::task::spawn_blocking(move || {
4541 zlayer_hns::network::Network::open(recorded_id).ok()
4542 })
4543 .await
4544 .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?;
4545 if reopened.is_some() {
4546 tracing::info!(name = %net_name, "reusing shared HCN NAT network from marker");
4547 return Ok(recorded_id);
4548 }
4549 }
4550
4551 // Idempotency: reuse a host network whose queried name matches ours.
4552 let target_name = net_name.clone();
4553 let existing = tokio::task::spawn_blocking(move || -> Option<GUID> {
4554 let guids = zlayer_hns::network::list("{}").ok()?;
4555 for guid in guids {
4556 let Ok(network) = zlayer_hns::network::Network::open(guid) else {
4557 continue;
4558 };
4559 if matches!(network.query("{}"), Ok(props) if props.name == target_name) {
4560 return Some(guid);
4561 }
4562 }
4563 None
4564 })
4565 .await
4566 .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?;
4567
4568 if let Some(existing_id) = existing {
4569 tracing::info!(name = %net_name, "reusing existing shared HCN NAT network");
4570 return Ok(existing_id);
4571 }
4572
4573 let net_id = GUID::new()
4574 .map_err(|e| OverlaydError::Other(format!("GUID::new for shared NAT network: {e}")))?;
4575 let subnet_str = slice_cidr.to_string();
4576 let settings = shared_nat_settings(&net_name, &subnet_str).ok_or_else(|| {
4577 OverlaydError::Other(format!(
4578 "shared NAT network: slice CIDR '{subnet_str}' has no usable gateway host"
4579 ))
4580 })?;
4581
4582 let net_name_for_create = net_name.clone();
4583 tokio::task::spawn_blocking(move || {
4584 zlayer_hns::network::Network::create(net_id, &settings)
4585 })
4586 .await
4587 .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?
4588 .map_err(|e| OverlaydError::Overlay(format!("HcnCreateNetwork NAT ({net_name}): {e}")))?;
4589 let _ = net_name_for_create;
4590
4591 // HCN's IPAM needs ~1-2s after network create to settle its address pool
4592 // (same wait as the base/Internal networks).
4593 tokio::time::sleep(std::time::Duration::from_secs(2)).await;
4594
4595 tracing::info!(subnet = %subnet_str, "created shared HCN NAT network");
4596
4597 let mut marker = crate::network_state::NetworkState::load(&marker_path);
4598 marker.upsert(crate::network_state::ManagedNetwork {
4599 owner,
4600 kind: "hcn-nat".to_string(),
4601 name: net_name.clone(),
4602 id: format_guid_bare(net_id),
4603 subnet: subnet_str.clone(),
4604 wg_port: None,
4605 wg_private_key: None,
4606 wg_public_key: None,
4607 interface: None,
4608 });
4609 if let Err(e) = marker.save(&marker_path) {
4610 tracing::warn!(error = %e, path = %marker_path.display(), "failed to persist shared NAT network marker (network still reusable by name)");
4611 }
4612
4613 Ok(net_id)
4614 }
4615
4616 /// Ensure the per-service HCN **Internal** network for `service` exists on
4617 /// the host, reusing one recorded under the `service:<name>` marker owner
4618 /// (or discoverable by its derived name) and recording it on create.
4619 ///
4620 /// This is the Windows analogue of the Linux per-service bridge: a
4621 /// dedicated (`OverlayMode::Dedicated`) service gets its OWN isolated HCN
4622 /// Internal network — an internal vSwitch with NO physical-NIC binding —
4623 /// distinct from the node's shared base overlay network. Containers attach
4624 /// to it (rather than the base network) so dedicated-service traffic is
4625 /// segregated at the vSwitch layer. Modeled on [`Self::ensure_overlay_network`]
4626 /// but keyed on [`owner_for_service`] and forced to the Internal type (never
4627 /// Transparent — the on-box test asserts zero external vSwitches for
4628 /// dedicated services).
4629 ///
4630 /// Returns the network GUID.
4631 ///
4632 /// # Errors
4633 /// Propagates the underlying `zlayer_hns` error on create failure.
4634 #[cfg(target_os = "windows")]
4635 #[allow(clippy::too_many_lines)]
4636 async fn ensure_service_network(
4637 &mut self,
4638 service: &str,
4639 subnet: ipnet::IpNet,
4640 ) -> Result<windows::core::GUID, OverlaydError> {
4641 use windows::core::GUID;
4642
4643 let daemon_name = self.deployment_or_default();
4644 // Per-service network name: `<base overlay name>-svc-<service>` so it is
4645 // unambiguously distinct from the base network and from other services.
4646 let net_name = format!("{}-svc-{service}", overlay_network_name(&daemon_name));
4647 let owner = owner_for_service(service);
4648 let marker_path =
4649 zlayer_paths::ZLayerDirs::new(self.data_dir.clone()).agent_network_state();
4650
4651 // Fast path: marker names a network GUID that still exists; reopen it.
4652 // Only honor the recorded id when it belongs to an HCN-internal entry —
4653 // a Dedicated WireGuard marker (`kind == "wg-dedicated"`) stores the
4654 // transport public key in `id`, NOT an HCN GUID, so it must be ignored
4655 // for HCN reuse.
4656 let recorded_hcn_id = crate::network_state::NetworkState::load(&marker_path)
4657 .get(&owner)
4658 .filter(|entry| entry.kind == "hcn-internal")
4659 .and_then(|entry| GUID::try_from(entry.id.as_str()).ok());
4660 if let Some(recorded_id) = recorded_hcn_id {
4661 let reopened = tokio::task::spawn_blocking(move || {
4662 zlayer_hns::network::Network::open(recorded_id).ok()
4663 })
4664 .await
4665 .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?;
4666 if reopened.is_some() {
4667 tracing::info!(name = %net_name, service = %service, "reusing per-service HCN network from marker");
4668 return Ok(recorded_id);
4669 }
4670 }
4671
4672 // Idempotency: reuse a host network whose queried name matches ours.
4673 let target_name = net_name.clone();
4674 let existing = tokio::task::spawn_blocking(move || -> Option<GUID> {
4675 let guids = zlayer_hns::network::list("{}").ok()?;
4676 for guid in guids {
4677 let Ok(network) = zlayer_hns::network::Network::open(guid) else {
4678 continue;
4679 };
4680 if matches!(network.query("{}"), Ok(props) if props.name == target_name) {
4681 return Some(guid);
4682 }
4683 }
4684 None
4685 })
4686 .await
4687 .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?;
4688
4689 if let Some(existing_id) = existing {
4690 tracing::info!(name = %net_name, service = %service, "reusing existing per-service HCN network");
4691 return Ok(existing_id);
4692 }
4693
4694 let net_id = GUID::new()
4695 .map_err(|e| OverlaydError::Other(format!("GUID::new for per-service network: {e}")))?;
4696 let subnet_str = subnet.to_string();
4697
4698 // ALWAYS Internal for a dedicated service — never Transparent. The
4699 // dedicated requirement is isolation; an Internal network binds NO
4700 // physical NIC (no external vSwitch), which is what the on-box test
4701 // asserts.
4702 let net_name_for_create = net_name.clone();
4703 let subnet_for_create = subnet_str.clone();
4704 tokio::task::spawn_blocking(move || {
4705 zlayer_hns::network::Network::create_internal(
4706 net_id,
4707 &net_name_for_create,
4708 &subnet_for_create,
4709 )
4710 })
4711 .await
4712 .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?
4713 .map_err(|e| {
4714 OverlaydError::Overlay(format!("HcnCreateNetwork internal ({net_name}): {e}"))
4715 })?;
4716
4717 // HCN's Static IPAM needs ~1-2s after network create to settle its
4718 // address pool; without this the first endpoint frequently fails with
4719 // HCN_E_ADDR_INVALID_OR_RESERVED (same wait as the base network).
4720 tokio::time::sleep(std::time::Duration::from_secs(2)).await;
4721
4722 tracing::info!(
4723 service = %service,
4724 subnet = %subnet_str,
4725 "created per-service HCN Internal network"
4726 );
4727
4728 // Persist the marker (owner = `service:<name>`, kind = `hcn-internal`)
4729 // so subsequent runs reuse this network by GUID and a full uninstall
4730 // (`purge_managed_networks`, which sweeps every `kind` starting with
4731 // `hcn`) deletes it. Best-effort.
4732 //
4733 // A dedicated Windows service shares the SAME owner key for two facts:
4734 // the dedicated WireGuard identity (written by the cross-platform core
4735 // in `setup_service_overlay_dedicated`, kind `wg-dedicated`) and this
4736 // HCN network's GUID. The marker is keyed by owner, so carry the WG
4737 // identity fields over when we rewrite the entry to `hcn-internal` — the
4738 // single entry then holds both the HCN GUID (in `id`) and the WG
4739 // identity (in the `wg_*`/`interface` fields), and the WG private key
4740 // survives restarts. (The core re-asserts the `wg-dedicated` shape on
4741 // the next setup; this path re-asserts `hcn-internal` again right after
4742 // — both are self-healing because the network is also reusable by name.)
4743 let mut marker = crate::network_state::NetworkState::load(&marker_path);
4744 let carried = marker.get(&owner).cloned();
4745 marker.upsert(crate::network_state::ManagedNetwork {
4746 owner,
4747 kind: "hcn-internal".to_string(),
4748 name: net_name.clone(),
4749 id: format_guid_bare(net_id),
4750 subnet: subnet_str.clone(),
4751 wg_port: carried.as_ref().and_then(|c| c.wg_port),
4752 wg_private_key: carried.as_ref().and_then(|c| c.wg_private_key.clone()),
4753 wg_public_key: carried.as_ref().and_then(|c| c.wg_public_key.clone()),
4754 interface: carried.as_ref().and_then(|c| c.interface.clone()),
4755 });
4756 if let Err(e) = marker.save(&marker_path) {
4757 tracing::warn!(service = %service, error = %e, path = %marker_path.display(), "failed to persist per-service network marker (network still reusable by name)");
4758 }
4759
4760 Ok(net_id)
4761 }
4762
4763 /// Resolve the per-isolation-network subnet for `net`, carving a fixed-size
4764 /// sub-block out of the node slice deterministically by name hash.
4765 ///
4766 /// Isolation networks attach onto a dedicated HCN Internal vSwitch and need
4767 /// their OWN address pool (never the node slice's shared pool) so a member's
4768 /// IP is on-link with its network's gateway. Unlike dedicated services,
4769 /// isolation networks aren't registered in the cluster's
4770 /// [`ServiceSubnetRegistry`] (a standalone isolated container may use the
4771 /// base overlay, where no `SetupServiceOverlay` ran), so the subnet is
4772 /// derived locally and deterministically: the node slice is split into
4773 /// `/<sub_prefix>` blocks and the network name selects one by hash. The
4774 /// derivation is stable across restarts (same name -> same block) so a
4775 /// reused HCN network keeps the same subnet.
4776 ///
4777 /// # Errors
4778 /// Returns an error if no node slice is assigned yet, the slice CIDR is
4779 /// unparseable, or the slice cannot be subnetted (e.g. already at the host
4780 /// prefix).
4781 #[cfg(target_os = "windows")]
4782 fn isolation_network_subnet(&self, net: &str) -> Result<ipnet::IpNet, OverlaydError> {
4783 use std::hash::{Hash, Hasher};
4784
4785 let slice = self.slice_cidr.ok_or_else(|| {
4786 OverlaydError::Other(
4787 "no node slice assigned yet (SetupGlobalOverlay with slice_cidr first)".to_string(),
4788 )
4789 })?;
4790 let slice_ipnet: ipnet::IpNet = slice.to_string().parse().map_err(|e| {
4791 OverlaydError::Other(format!("failed to parse slice CIDR {slice}: {e}"))
4792 })?;
4793
4794 // Carve the slice into /<sub_prefix> blocks. A `/28` (V4) gives ~13
4795 // usable container IPs per isolation network per node — enough for the
4796 // isolated-container use case — while leaving room for several distinct
4797 // isolation networks inside one node slice. Clamp to the slice prefix so
4798 // a slice already more specific than the target just yields itself.
4799 let sub_prefix: u8 = match slice_ipnet {
4800 ipnet::IpNet::V4(_) => 28u8.max(slice_ipnet.prefix_len()),
4801 ipnet::IpNet::V6(_) => 124u8.max(slice_ipnet.prefix_len()),
4802 };
4803
4804 let blocks: Vec<ipnet::IpNet> = slice_ipnet
4805 .subnets(sub_prefix)
4806 .map_err(|e| {
4807 OverlaydError::Other(format!(
4808 "failed to subnet slice {slice_ipnet} into /{sub_prefix} blocks: {e}"
4809 ))
4810 })?
4811 .collect();
4812 if blocks.is_empty() {
4813 return Err(OverlaydError::Other(format!(
4814 "slice {slice_ipnet} yielded no /{sub_prefix} blocks for isolation network {net}"
4815 )));
4816 }
4817
4818 let mut hasher = std::collections::hash_map::DefaultHasher::new();
4819 net.hash(&mut hasher);
4820 // `% blocks.len()` is always < blocks.len() <= usize::MAX, so this never
4821 // truncates; `try_from` keeps clippy happy without an unchecked cast.
4822 let idx = usize::try_from(hasher.finish() % blocks.len() as u64).unwrap_or(0);
4823 Ok(blocks[idx])
4824 }
4825
4826 /// Ensure the per-isolation-network HCN **Internal** network for `net` exists
4827 /// on the host, reusing one recorded under the
4828 /// [`owner_for_isolation_network`] marker owner (or discoverable by its
4829 /// derived name) and recording it on create.
4830 ///
4831 /// This is the Windows mechanism for per-network L3 isolation: every
4832 /// `ZLayer` "isolated" network gets its OWN HCN Internal vSwitch — an
4833 /// internal vSwitch with NO physical-NIC binding. HCN Internal vSwitches are
4834 /// mutually isolated by default, so same-network members (sharing this
4835 /// vSwitch) reach each other + egress + the node, while members of a
4836 /// different isolation network land on a different vSwitch and cannot reach
4837 /// them. No ACLs, no per-member churn. Modeled on
4838 /// [`Self::ensure_service_network`] but keyed on
4839 /// [`owner_for_isolation_network`] and named `<overlay>-iso-<net>`.
4840 ///
4841 /// Returns the network GUID.
4842 ///
4843 /// # Errors
4844 /// Propagates the underlying `zlayer_hns` error on create failure.
4845 #[cfg(target_os = "windows")]
4846 async fn ensure_isolation_network(
4847 &mut self,
4848 net: &str,
4849 subnet: ipnet::IpNet,
4850 ) -> Result<windows::core::GUID, OverlaydError> {
4851 use windows::core::GUID;
4852
4853 let daemon_name = self.deployment_or_default();
4854 // Per-isolation-network name: `<base overlay name>-iso-<net>` so it is
4855 // unambiguously distinct from the base network and per-service networks.
4856 let net_name = format!("{}-iso-{net}", overlay_network_name(&daemon_name));
4857 let owner = crate::network_state::owner_for_isolation_network(net);
4858 let marker_path =
4859 zlayer_paths::ZLayerDirs::new(self.data_dir.clone()).agent_network_state();
4860
4861 // Fast path: marker names a network GUID that still exists; reopen it.
4862 let recorded_hcn_id = crate::network_state::NetworkState::load(&marker_path)
4863 .get(&owner)
4864 .filter(|entry| entry.kind == "hcn-internal")
4865 .and_then(|entry| GUID::try_from(entry.id.as_str()).ok());
4866 if let Some(recorded_id) = recorded_hcn_id {
4867 let reopened = tokio::task::spawn_blocking(move || {
4868 zlayer_hns::network::Network::open(recorded_id).ok()
4869 })
4870 .await
4871 .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?;
4872 if reopened.is_some() {
4873 tracing::info!(name = %net_name, network = %net, "reusing per-isolation-network HCN network from marker");
4874 return Ok(recorded_id);
4875 }
4876 }
4877
4878 // Idempotency: reuse a host network whose queried name matches ours.
4879 let target_name = net_name.clone();
4880 let existing = tokio::task::spawn_blocking(move || -> Option<GUID> {
4881 let guids = zlayer_hns::network::list("{}").ok()?;
4882 for guid in guids {
4883 let Ok(network) = zlayer_hns::network::Network::open(guid) else {
4884 continue;
4885 };
4886 if matches!(network.query("{}"), Ok(props) if props.name == target_name) {
4887 return Some(guid);
4888 }
4889 }
4890 None
4891 })
4892 .await
4893 .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?;
4894
4895 if let Some(existing_id) = existing {
4896 tracing::info!(name = %net_name, network = %net, "reusing existing per-isolation-network HCN network");
4897 return Ok(existing_id);
4898 }
4899
4900 let net_id = GUID::new().map_err(|e| {
4901 OverlaydError::Other(format!("GUID::new for per-isolation-network network: {e}"))
4902 })?;
4903 let subnet_str = subnet.to_string();
4904
4905 // ALWAYS Internal for an isolation network — never Transparent. The
4906 // isolation requirement is exactly the Internal-vSwitch property: no
4907 // physical-NIC binding, mutually isolated from other Internal vSwitches.
4908 let net_name_for_create = net_name.clone();
4909 let subnet_for_create = subnet_str.clone();
4910 tokio::task::spawn_blocking(move || {
4911 zlayer_hns::network::Network::create_internal(
4912 net_id,
4913 &net_name_for_create,
4914 &subnet_for_create,
4915 )
4916 })
4917 .await
4918 .map_err(|e| OverlaydError::Other(format!("spawn_blocking join failed: {e}")))?
4919 .map_err(|e| {
4920 OverlaydError::Overlay(format!("HcnCreateNetwork internal ({net_name}): {e}"))
4921 })?;
4922
4923 // HCN's Static IPAM needs ~1-2s after network create to settle its
4924 // address pool; without this the first endpoint frequently fails with
4925 // HCN_E_ADDR_INVALID_OR_RESERVED (same wait as the per-service network).
4926 tokio::time::sleep(std::time::Duration::from_secs(2)).await;
4927
4928 tracing::info!(
4929 network = %net,
4930 subnet = %subnet_str,
4931 "created per-isolation-network HCN Internal network"
4932 );
4933
4934 // Persist the marker (owner = `iso:<net>`, kind = `hcn-internal`) so
4935 // subsequent runs reuse this network by GUID and a full uninstall
4936 // (`purge_managed_networks`, which sweeps every `kind` starting with
4937 // `hcn`) deletes it. Best-effort.
4938 let mut marker = crate::network_state::NetworkState::load(&marker_path);
4939 marker.upsert(crate::network_state::ManagedNetwork {
4940 owner,
4941 kind: "hcn-internal".to_string(),
4942 name: net_name.clone(),
4943 id: format_guid_bare(net_id),
4944 subnet: subnet_str.clone(),
4945 // Isolation HCN network: no dedicated WireGuard identity.
4946 wg_port: None,
4947 wg_private_key: None,
4948 wg_public_key: None,
4949 interface: None,
4950 });
4951 if let Err(e) = marker.save(&marker_path) {
4952 tracing::warn!(network = %net, error = %e, path = %marker_path.display(), "failed to persist per-isolation-network marker (network still reusable by name)");
4953 }
4954
4955 Ok(net_id)
4956 }
4957
4958 /// Resolve the dedicated per-service subnet for `service`, if the service
4959 /// runs in `OverlayMode::Dedicated` on this node.
4960 ///
4961 /// Source of truth, in order:
4962 /// 1. The live [`ServiceTransport`] in `service_transports` (the normal
4963 /// case once `SetupServiceOverlay` has run this process).
4964 /// 2. A persisted `hcn-internal` marker entry under
4965 /// [`owner_for_service`]`(service)` — covers the window where the HCN
4966 /// network exists from a prior run but the transport map is still empty.
4967 ///
4968 /// Returns `None` for Shared-mode services (attach onto the base network).
4969 #[cfg(target_os = "windows")]
4970 fn dedicated_service_subnet(&self, service: &str) -> Option<ipnet::IpNet> {
4971 if let Some(st) = self.service_transports.get(service) {
4972 return Some(st.subnet);
4973 }
4974 let marker_path =
4975 zlayer_paths::ZLayerDirs::new(self.data_dir.clone()).agent_network_state();
4976 crate::network_state::NetworkState::load(&marker_path)
4977 .get(&owner_for_service(service))
4978 .filter(|entry| entry.kind == "hcn-internal")
4979 .and_then(|entry| entry.subnet.parse::<ipnet::IpNet>().ok())
4980 }
4981
4982 /// The daemon name used for HCN network/owner naming, defaulting to
4983 /// `"zlayer"` when no deployment has been set yet.
4984 #[cfg(target_os = "windows")]
4985 fn deployment_or_default(&self) -> String {
4986 if self.deployment.is_empty() {
4987 "zlayer".to_string()
4988 } else {
4989 self.deployment.clone()
4990 }
4991 }
4992
4993 // -- peers ---------------------------------------------------------------
4994
4995 /// Resolve a [`PeerScope`] to the live [`OverlayTransport`] its ops target.
4996 ///
4997 /// `Global` -> the single cluster transport; `Service { service }` -> that
4998 /// service's dedicated per-service transport (Dedicated mode only).
4999 ///
5000 /// # Errors
5001 /// Returns an error if the global overlay is not up (for `Global`) or no
5002 /// dedicated overlay exists for the named service (for `Service`).
5003 fn transport_for_scope(&self, scope: &PeerScope) -> Result<&OverlayTransport, OverlaydError> {
5004 match scope {
5005 PeerScope::Global => self
5006 .global_transport
5007 .as_ref()
5008 .ok_or_else(|| OverlaydError::Other("global overlay not set up".into())),
5009 PeerScope::Service { service } => self
5010 .service_transports
5011 .get(service)
5012 .map(|s| &s.transport)
5013 .ok_or_else(|| {
5014 OverlaydError::Other(format!("no dedicated overlay for service {service}"))
5015 }),
5016 }
5017 }
5018
5019 /// Add a peer to a resolved transport.
5020 ///
5021 /// # Errors
5022 /// Wraps the underlying transport error.
5023 async fn add_peer_on(
5024 transport: &OverlayTransport,
5025 peer: &PeerInfo,
5026 ) -> Result<(), OverlaydError> {
5027 transport
5028 .add_peer(peer)
5029 .await
5030 .map_err(|e| OverlaydError::Overlay(format!("add_peer failed: {e}")))
5031 }
5032
5033 /// Remove a peer (by base64 public key) from a resolved transport.
5034 ///
5035 /// # Errors
5036 /// Wraps the underlying transport error.
5037 async fn remove_peer_on(
5038 transport: &OverlayTransport,
5039 pubkey: &str,
5040 ) -> Result<(), OverlaydError> {
5041 transport
5042 .remove_peer(pubkey)
5043 .await
5044 .map_err(|e| OverlaydError::Overlay(format!("remove_peer failed: {e}")))
5045 }
5046
5047 /// Plumb a CIDR into a peer's `AllowedIPs` on a resolved transport.
5048 ///
5049 /// # Errors
5050 /// Returns an error when the CIDR is invalid or the UAPI write fails.
5051 async fn add_allowed_ip_on(
5052 transport: &OverlayTransport,
5053 pubkey: &str,
5054 cidr: &str,
5055 ) -> Result<(), OverlaydError> {
5056 let net: ipnet::IpNet = cidr
5057 .parse()
5058 .map_err(|e| OverlaydError::Other(format!("invalid CIDR {cidr}: {e}")))?;
5059 transport
5060 .add_allowed_ip(pubkey, net)
5061 .await
5062 .map_err(|e| OverlaydError::Overlay(format!("add_allowed_ip failed: {e}")))
5063 }
5064
5065 /// Remove a CIDR from a peer's `AllowedIPs` on a resolved transport.
5066 ///
5067 /// # Errors
5068 /// Returns an error when the CIDR is invalid or the UAPI write fails.
5069 async fn remove_allowed_ip_on(
5070 transport: &OverlayTransport,
5071 pubkey: &str,
5072 cidr: &str,
5073 ) -> Result<(), OverlaydError> {
5074 let net: ipnet::IpNet = cidr
5075 .parse()
5076 .map_err(|e| OverlaydError::Other(format!("invalid CIDR {cidr}: {e}")))?;
5077 transport
5078 .remove_allowed_ip(pubkey, net)
5079 .await
5080 .map_err(|e| OverlaydError::Overlay(format!("remove_allowed_ip failed: {e}")))
5081 }
5082
5083 // -- DNS -----------------------------------------------------------------
5084
5085 /// Register an overlay DNS A/AAAA record.
5086 fn register_dns(&mut self, name: String, ip: IpAddr) {
5087 self.dns_records.insert(name, ip);
5088 }
5089
5090 /// Remove an overlay DNS record.
5091 fn unregister_dns(&mut self, name: &str) {
5092 self.dns_records.remove(name);
5093 }
5094
5095 // -- NAT -----------------------------------------------------------------
5096
5097 /// Periodic NAT traversal maintenance: lazily start NAT (and the built-in
5098 /// relay server), re-probe STUN, refresh relays, and run the connect-half —
5099 /// hole-punching / relaying toward every peer whose direct endpoint has not
5100 /// produced a recent `WireGuard` handshake.
5101 ///
5102 /// No-op when NAT traversal is disabled in the resolved [`NatConfig`].
5103 ///
5104 /// # Errors
5105 /// Returns an error when the underlying STUN refresh fails.
5106 async fn nat_maintenance_tick(&mut self) -> Result<(), OverlaydError> {
5107 // Lazily start NAT traversal on the first tick if a config asks for it.
5108 if self.nat_traversal.is_none() {
5109 let config = self.nat_config.clone().unwrap_or_default();
5110 if config.enabled {
5111 // Stand up the built-in relay server here (once) when the
5112 // resolved config carries a `relay_server`. The auth credential
5113 // MUST be cluster-wide-shared (every node's relay *client*
5114 // derives the same BLAKE2b key via `derive_auth_key`), so it
5115 // comes from `cluster_relay_credential` — the cluster HS256
5116 // secret the main daemon stamped into
5117 // `NatConfigSpec.relay_server.auth_credential`, NOT the node's
5118 // per-node WireGuard key. When no credential was supplied the
5119 // relay derives a key from the empty string (only same-config
5120 // nodes can use it).
5121 if let Some(relay_cfg) = config.relay_server.clone() {
5122 if self.relay_server.is_none() {
5123 let credential = self.cluster_relay_credential.clone().unwrap_or_default();
5124 let relay = RelayServer::new(&relay_cfg, &credential);
5125 match relay.start().await {
5126 Ok(bound) => {
5127 tracing::info!(
5128 bound = %bound,
5129 external = %relay_cfg.external_addr,
5130 "Built-in relay server started"
5131 );
5132 self.relay_bound_addr = Some(bound);
5133 self.relay_server = Some(relay);
5134 }
5135 Err(e) => {
5136 tracing::warn!(error = %e, "Built-in relay server failed to start");
5137 }
5138 }
5139 }
5140 }
5141
5142 let mut nat = NatTraversal::new(config, self.overlay_port);
5143 match nat.gather_candidates().await {
5144 Ok(candidates) => {
5145 tracing::info!(count = candidates.len(), "Gathered NAT candidates");
5146 self.nat_last_refresh.store(now_unix(), Ordering::SeqCst);
5147 self.nat_traversal = Some(nat);
5148 }
5149 Err(e) => {
5150 tracing::warn!(error = %e, "NAT candidate gathering failed");
5151 return Ok(());
5152 }
5153 }
5154 // First-tick connect: try to establish toward every already-known
5155 // peer (peers added before NAT came up).
5156 self.nat_connect_known_peers().await;
5157 } else {
5158 return Ok(());
5159 }
5160 }
5161
5162 // Refresh STUN/relay state, then run the connect-half for peers that
5163 // still lack a recent handshake.
5164 if let Some(nat) = self.nat_traversal.as_mut() {
5165 match nat.refresh().await {
5166 Ok(changed) => {
5167 if changed {
5168 tracing::info!("NAT reflexive address changed during refresh");
5169 }
5170 self.nat_last_refresh.store(now_unix(), Ordering::SeqCst);
5171 }
5172 Err(e) => {
5173 return Err(OverlaydError::Overlay(format!(
5174 "NAT maintenance tick failed: {e}"
5175 )));
5176 }
5177 }
5178 }
5179 self.nat_connect_known_peers().await;
5180 Ok(())
5181 }
5182
5183 /// The NAT connect-half: for every peer with advertised candidates that has
5184 /// no recent `WireGuard` handshake, call [`NatTraversal::connect_to_peer`]
5185 /// (which itself updates the live device's peer endpoint) and record the
5186 /// resulting [`ConnectionType`].
5187 ///
5188 /// Best-effort: a peer with no live global transport, no candidates, or a
5189 /// failed traversal is left untouched (its persistent direct endpoint keeps
5190 /// retrying). Candidate sets are collected into a local `Vec` first so the
5191 /// borrow of `self.nat_traversal` / `self.global_transport` does not overlap
5192 /// the mutable borrow of `self.peer_connection_type`.
5193 async fn nat_connect_known_peers(&mut self) {
5194 // No host transport (VM-only overlay) or no NAT orchestrator → nothing
5195 // to connect on this node.
5196 let (Some(_), Some(_)) = (self.global_transport.as_ref(), self.nat_traversal.as_ref())
5197 else {
5198 return;
5199 };
5200 if self.peer_candidates.is_empty() {
5201 return;
5202 }
5203
5204 // Peers whose handshake is older than this cutoff (or never seen) are
5205 // candidates for a (re)connect attempt. WireGuard's default keepalive is
5206 // 25s; 3× that is a generous "the direct endpoint is clearly not
5207 // establishing" threshold that avoids churning healthy peers.
5208 let cutoff = now_unix().saturating_sub(75);
5209
5210 // Snapshot the (pubkey, candidates) work set up front to satisfy the
5211 // borrow checker (we borrow self.transport + self.nat below).
5212 let work: Vec<(String, Vec<Candidate>)> = self
5213 .peer_candidates
5214 .iter()
5215 .map(|(k, v)| (k.clone(), v.clone()))
5216 .collect();
5217
5218 let transport = self.global_transport.as_ref().expect("checked above");
5219 let nat = self.nat_traversal.as_ref().expect("checked above");
5220 let mut results: Vec<(String, ConnectionType)> = Vec::new();
5221
5222 for (pubkey, candidates) in &work {
5223 // Skip peers that already have a fresh handshake on the live device.
5224 match transport.check_peer_handshake(pubkey, cutoff).await {
5225 Ok(true) => continue,
5226 Ok(false) => {}
5227 Err(e) => {
5228 tracing::debug!(peer = %pubkey, error = %e, "handshake check failed; attempting connect anyway");
5229 }
5230 }
5231 match nat.connect_to_peer(transport, pubkey, candidates).await {
5232 Ok(connection_type) => {
5233 tracing::info!(
5234 peer = %pubkey,
5235 connection = %connection_type,
5236 "NAT traversal established connection to peer"
5237 );
5238 results.push((pubkey.clone(), connection_type));
5239 }
5240 Err(e) => {
5241 tracing::debug!(peer = %pubkey, error = %e, "NAT traversal could not connect to peer this tick");
5242 }
5243 }
5244 }
5245
5246 for (pubkey, ct) in results {
5247 self.peer_connection_type.insert(pubkey, ct);
5248 }
5249 }
5250
5251 /// Build a [`NatStatusWire`] from the live NAT orchestrator: this node's
5252 /// local candidates, the per-peer connection types recorded by the connect
5253 /// loop (with each peer's current remote endpoint parsed from the UAPI
5254 /// status dump), and the last STUN-refresh timestamp.
5255 async fn nat_status_snapshot(&self) -> NatStatusWire {
5256 let candidates = self
5257 .nat_traversal
5258 .as_ref()
5259 .map(|n| n.local_candidates().iter().map(candidate_to_wire).collect())
5260 .unwrap_or_default();
5261
5262 // Map hex-pubkey -> current remote endpoint from the live device's UAPI
5263 // dump. The dump keys peers by hex; `peer_connection_type` keys by
5264 // base64, so the join below converts each base64 key to hex.
5265 let mut endpoints: HashMap<String, String> = HashMap::new();
5266 if let Some(transport) = self.global_transport.as_ref() {
5267 if let Ok(dump) = transport.status().await {
5268 for p in parse_peer_status(&dump) {
5269 if !p.endpoint.is_empty() {
5270 endpoints.insert(p.public_key, p.endpoint);
5271 }
5272 }
5273 }
5274 }
5275
5276 let peers = self
5277 .peer_connection_type
5278 .iter()
5279 .map(|(pubkey, ct)| {
5280 let remote_endpoint = zlayer_overlay::nat::pubkey_b64_to_hex(pubkey)
5281 .and_then(|hex| endpoints.get(&hex).cloned());
5282 NatPeerWire {
5283 node_id: pubkey.clone(),
5284 connection_type: ct.to_string(),
5285 remote_endpoint,
5286 }
5287 })
5288 .collect();
5289
5290 NatStatusWire {
5291 candidates,
5292 peers,
5293 last_refresh: self.nat_last_refresh.load(Ordering::SeqCst),
5294 }
5295 }
5296
5297 // -- status --------------------------------------------------------------
5298
5299 /// Build a [`StatusSnapshot`] from current overlay state.
5300 async fn status_snapshot(&self) -> StatusSnapshot {
5301 let mut peers: Vec<PeerStatus> = Vec::new();
5302 let public_key = self.transport_public_key.clone();
5303
5304 if let Some(transport) = self.global_transport.as_ref() {
5305 // Parse the UAPI dump for per-peer state. Best-effort: a parse
5306 // failure leaves the peer list empty rather than failing Status.
5307 if let Ok(dump) = transport.status().await {
5308 peers = parse_peer_status(&dump);
5309 }
5310 }
5311
5312 let service_count = u32::try_from(self.service_count()).unwrap_or(u32::MAX);
5313 let peer_count = u32::try_from(peers.len()).unwrap_or(u32::MAX);
5314
5315 // Per dedicated per-service overlay device: count its peers the same
5316 // way the global status does (parse the UAPI/status dump).
5317 let mut dedicated_services: Vec<DedicatedServiceStatus> = Vec::new();
5318 for (svc, st) in &self.service_transports {
5319 let peer_count = match st.transport.status().await {
5320 Ok(dump) => u32::try_from(parse_peer_status(&dump).len()).unwrap_or(u32::MAX),
5321 Err(_) => 0,
5322 };
5323 dedicated_services.push(DedicatedServiceStatus {
5324 service: svc.clone(),
5325 interface: st.interface.clone(),
5326 public_key: st.public_key.clone(),
5327 listen_port: st.listen_port,
5328 overlay_ip: st.overlay_ip,
5329 subnet: st.subnet.to_string(),
5330 peer_count,
5331 });
5332 }
5333
5334 StatusSnapshot {
5335 interface: self.global_interface.clone(),
5336 node_ip: self.node_ip,
5337 public_key,
5338 overlay_cidr: self.cluster_cidr.map(|c| c.to_string()),
5339 slice_cidr: self.slice_cidr.map(|c| c.to_string()),
5340 peer_count,
5341 service_count,
5342 peers,
5343 dedicated_services,
5344 }
5345 }
5346
5347 /// Number of per-service overlays set up on this node (Shared bridges /
5348 /// placeholders plus any Dedicated transports not already counted there).
5349 fn service_count(&self) -> usize {
5350 let extra_dedicated = self
5351 .service_transports
5352 .keys()
5353 .filter(|svc| !self.service_interfaces.contains_key(*svc))
5354 .count();
5355 self.service_interfaces.len() + extra_dedicated
5356 }
5357
5358 // -- config helper -------------------------------------------------------
5359
5360 fn build_config(
5361 &self,
5362 private_key: String,
5363 public_key: String,
5364 ip: IpAddr,
5365 mask: u8,
5366 listen_port: u16,
5367 physical_egress_ip: Option<IpAddr>,
5368 ) -> OverlayConfig {
5369 // Pick the source/advertised address for the WireGuard endpoint.
5370 //
5371 // Default is the family-matched UNSPECIFIED (`0.0.0.0` / `::`), which lets
5372 // the kernel pick a source per outgoing packet. When the caller resolved a
5373 // physical-egress IP (see `detect_physical_egress`) *and* its family
5374 // matches the overlay IP's family, we pin `local_endpoint` to that IP so
5375 // boringtun's data socket sources from — and advertises — the real NIC
5376 // rather than whatever the default route (possibly a VPN mesh) would pick.
5377 //
5378 // Family mismatch (e.g. physical egress is v4 but this overlay is v6) is
5379 // unusable for source selection, so we warn and fall back to UNSPECIFIED.
5380 //
5381 // boringtun limitation: boringtun 0.7's `DeviceConfig` exposes no way to
5382 // inject or pin the WireGuard DATA socket (its `uapi_fd` is the UAPI
5383 // CONTROL socket only), so `SO_BINDTODEVICE` on the data socket is
5384 // impossible today. Setting `local_endpoint` to the physical IP governs
5385 // source-address selection and the advertised endpoint, which is the
5386 // realistic scope of control we have.
5387 let unspecified = match ip {
5388 IpAddr::V4(_) => IpAddr::V4(Ipv4Addr::UNSPECIFIED),
5389 IpAddr::V6(_) => IpAddr::V6(Ipv6Addr::UNSPECIFIED),
5390 };
5391 let local_addr =
5392 if rootless_forces_unspecified(std::env::var_os("ZLAYER_ROOTLESS").is_some()) {
5393 // Rootless: detect_physical_egress() resolves pasta's in-netns tap IP
5394 // (e.g. 192.168.68.x), which is useless as a WG source/advertised
5395 // endpoint to remote peers. Force UNSPECIFIED; the kernel picks the
5396 // source per packet and the real reachable endpoint comes from the
5397 // advertise_addr path + pasta forwarding.
5398 unspecified
5399 } else {
5400 match physical_egress_ip {
5401 Some(egress) if egress.is_ipv4() == ip.is_ipv4() => egress,
5402 Some(egress) => {
5403 tracing::warn!(
5404 physical_egress_ip = %egress,
5405 overlay_ip = %ip,
5406 "physical egress IP family does not match overlay IP family; \
5407 falling back to UNSPECIFIED for WireGuard local_endpoint"
5408 );
5409 unspecified
5410 }
5411 None => unspecified,
5412 }
5413 };
5414 let mut config = OverlayConfig {
5415 local_endpoint: SocketAddr::new(local_addr, listen_port),
5416 private_key,
5417 public_key,
5418 overlay_cidr: format!("{ip}/{mask}"),
5419 ..OverlayConfig::default()
5420 };
5421 if let Some(nat) = self.nat_config.clone() {
5422 config.nat = nat;
5423 }
5424 if let Some(dir) = self.uapi_sock_dir.clone() {
5425 config.uapi_sock_dir = dir;
5426 }
5427 config
5428 }
5429}
5430
5431/// Build an `Auto`-mode [`ServiceOverlayInfo`]: the per-service bridge/placeholder
5432/// name with every dedicated-device identity field left `None` (`Auto` carries
5433/// the service subnet on the single cluster-wide `WireGuard` device).
5434fn cluster_wg_overlay_info(name: String) -> ServiceOverlayInfo {
5435 ServiceOverlayInfo {
5436 name,
5437 mode: OverlayMode::Auto,
5438 wg_public_key: None,
5439 wg_port: None,
5440 overlay_ip: None,
5441 subnet: None,
5442 }
5443}
5444
5445/// Build a `Shared`-mode [`ServiceOverlayInfo`]: the shared node-wide
5446/// bridge/placeholder name with every dedicated-device identity field left
5447/// `None` (Shared mode shares the single cluster device and the node-wide
5448/// bridge; ports are exposed by the userspace free-port L4 proxy).
5449fn shared_overlay_info(name: String) -> ServiceOverlayInfo {
5450 ServiceOverlayInfo {
5451 name,
5452 mode: OverlayMode::Shared,
5453 wg_public_key: None,
5454 wg_port: None,
5455 overlay_ip: None,
5456 subnet: None,
5457 }
5458}
5459
5460/// Build a Dedicated-mode [`ServiceOverlayInfo`] from a dedicated device's
5461/// identity. `name` is the container-attach handle (bridge name on Linux, the
5462/// dedicated interface elsewhere).
5463fn dedicated_overlay_info(
5464 name: String,
5465 public_key: &str,
5466 listen_port: u16,
5467 overlay_ip: IpAddr,
5468 subnet: ipnet::IpNet,
5469) -> ServiceOverlayInfo {
5470 ServiceOverlayInfo {
5471 name,
5472 mode: OverlayMode::Dedicated,
5473 wg_public_key: Some(public_key.to_string()),
5474 wg_port: Some(listen_port),
5475 overlay_ip: Some(overlay_ip),
5476 subnet: Some(subnet.to_string()),
5477 }
5478}
5479
5480/// Convert a wire [`PeerSpec`] into a `zlayer_overlay::PeerInfo`.
5481///
5482/// # Errors
5483/// Returns an error if `endpoint` cannot be parsed as a `host:port`
5484/// [`SocketAddr`].
5485pub fn peer_spec_to_info(spec: &PeerSpec) -> Result<PeerInfo, OverlaydError> {
5486 let endpoint: SocketAddr = spec.endpoint.parse().map_err(|e| {
5487 OverlaydError::Other(format!("invalid peer endpoint {}: {e}", spec.endpoint))
5488 })?;
5489 Ok(PeerInfo::new(
5490 spec.public_key.clone(),
5491 endpoint,
5492 &spec.allowed_ips,
5493 std::time::Duration::from_secs(spec.persistent_keepalive_secs),
5494 ))
5495}
5496
5497/// Parse a `wg`-style UAPI/`status` dump into [`PeerStatus`] entries.
5498///
5499/// The dump is a series of `key=value` lines; each `public_key=` line starts a
5500/// new peer block, and subsequent `endpoint=` / `allowed_ip=` /
5501/// `latest_handshake=` lines belong to it.
5502fn parse_peer_status(dump: &str) -> Vec<PeerStatus> {
5503 let mut peers: Vec<PeerStatus> = Vec::new();
5504 let mut current: Option<PeerStatus> = None;
5505 let mut allowed: Vec<String> = Vec::new();
5506
5507 let flush = |peers: &mut Vec<PeerStatus>,
5508 current: &mut Option<PeerStatus>,
5509 allowed: &mut Vec<String>| {
5510 if let Some(mut p) = current.take() {
5511 p.allowed_ips = allowed.join(",");
5512 peers.push(p);
5513 }
5514 allowed.clear();
5515 };
5516
5517 for line in dump.lines() {
5518 let line = line.trim();
5519 let Some((key, value)) = line.split_once('=') else {
5520 continue;
5521 };
5522 match key.trim() {
5523 "public_key" | "peer" => {
5524 flush(&mut peers, &mut current, &mut allowed);
5525 current = Some(PeerStatus {
5526 public_key: value.trim().to_string(),
5527 endpoint: String::new(),
5528 allowed_ips: String::new(),
5529 last_handshake_unix_secs: 0,
5530 });
5531 }
5532 "endpoint" => {
5533 if let Some(p) = current.as_mut() {
5534 p.endpoint = value.trim().to_string();
5535 }
5536 }
5537 "allowed_ip" | "allowed_ips" if current.is_some() => {
5538 allowed.push(value.trim().to_string());
5539 }
5540 "latest_handshake" | "last_handshake_time_sec" => {
5541 if let Some(p) = current.as_mut() {
5542 p.last_handshake_unix_secs = value.trim().parse().unwrap_or(0);
5543 }
5544 }
5545 _ => {}
5546 }
5547 }
5548 flush(&mut peers, &mut current, &mut allowed);
5549 peers
5550}
5551
5552/// Convert a wire [`NatConfigSpec`] into the live [`NatConfig`] overlayd drives.
5553///
5554/// Sub-fields left at their zero value in the spec fall back to
5555/// [`NatConfig::default`]'s value (so a sparsely-populated spec still gets sane
5556/// STUN servers / timeouts). The `relay_server`'s `auth_credential` is stripped
5557/// here — it is carried separately on the server (`cluster_relay_credential`)
5558/// because `RelayServerConfig` has no credential field; this conversion only
5559/// produces the bind/external/max-sessions triple it does carry.
5560fn nat_config_spec_to_config(spec: NatConfigSpec) -> NatConfig {
5561 let defaults = NatConfig::default();
5562 NatConfig {
5563 enabled: spec.enabled,
5564 stun_servers: if spec.stun_servers.is_empty() {
5565 defaults.stun_servers
5566 } else {
5567 spec.stun_servers
5568 .into_iter()
5569 .map(|address| StunServerConfig {
5570 address,
5571 label: None,
5572 })
5573 .collect()
5574 },
5575 turn_servers: spec
5576 .turn_servers
5577 .into_iter()
5578 .map(|t| TurnServerConfig {
5579 address: t.addr,
5580 username: t.username,
5581 credential: t.credential,
5582 region: None,
5583 })
5584 .collect(),
5585 hole_punch_timeout_secs: if spec.hole_punch_timeout_secs == 0 {
5586 defaults.hole_punch_timeout_secs
5587 } else {
5588 spec.hole_punch_timeout_secs
5589 },
5590 stun_refresh_interval_secs: if spec.stun_refresh_interval_secs == 0 {
5591 defaults.stun_refresh_interval_secs
5592 } else {
5593 spec.stun_refresh_interval_secs
5594 },
5595 max_candidate_pairs: if spec.max_candidate_pairs == 0 {
5596 defaults.max_candidate_pairs
5597 } else {
5598 spec.max_candidate_pairs
5599 },
5600 relay_server: spec.relay_server.map(|r| RelayServerConfig {
5601 listen_port: r.listen_port,
5602 external_addr: r.external_addr,
5603 max_sessions: if r.max_sessions == 0 {
5604 default_max_relay_sessions()
5605 } else {
5606 r.max_sessions
5607 },
5608 }),
5609 }
5610}
5611
5612/// Default relay `max_sessions` used when a spec leaves it at `0`. Mirrors
5613/// `zlayer_overlay::nat::config`'s private `default_max_relay_sessions` (100).
5614const fn default_max_relay_sessions() -> usize {
5615 100
5616}
5617
5618/// Parse a wire [`NatCandidateWire`] into a live [`Candidate`].
5619///
5620/// Returns `None` when the address does not parse as a `host:port` socket
5621/// address or the type string is unrecognized. Priority is taken verbatim from
5622/// the wire (the advertiser already computed it) so the receiver honors the
5623/// peer's own preference ordering.
5624fn wire_to_candidate(w: &NatCandidateWire) -> Option<Candidate> {
5625 let address: SocketAddr = w.address.parse().ok()?;
5626 let candidate_type = match w.candidate_type.as_str() {
5627 "host" => CandidateType::Host,
5628 "server-reflexive" => CandidateType::ServerReflexive,
5629 "relay" => CandidateType::Relay,
5630 _ => return None,
5631 };
5632 let mut c = Candidate::new(candidate_type, address);
5633 c.priority = w.priority;
5634 Some(c)
5635}
5636
5637/// Convert a live [`Candidate`] into its wire [`NatCandidateWire`] form for a
5638/// `NatStatus` response.
5639fn candidate_to_wire(c: &Candidate) -> NatCandidateWire {
5640 let candidate_type = match c.candidate_type {
5641 CandidateType::Host => "host",
5642 CandidateType::ServerReflexive => "server-reflexive",
5643 CandidateType::Relay => "relay",
5644 };
5645 NatCandidateWire {
5646 candidate_type: candidate_type.to_string(),
5647 address: c.address.to_string(),
5648 priority: c.priority,
5649 }
5650}
5651
5652/// Current Unix time in whole seconds.
5653fn now_unix() -> u64 {
5654 std::time::SystemTime::now()
5655 .duration_since(std::time::UNIX_EPOCH)
5656 .unwrap_or_default()
5657 .as_secs()
5658}
5659
5660/// Format an overlay address as a host route (`<ip>/32` or `<ip>/128`) — the
5661/// `AllowedIPs` form for a single overlay endpoint (an edge `/32`, the node, or
5662/// the DNS server).
5663fn edge_host_route(ip: IpAddr) -> String {
5664 format!("{ip}/{}", if ip.is_ipv6() { 128 } else { 32 })
5665}
5666
5667/// Validate the `node_endpoint` an edge peer will dial: it must be non-empty,
5668/// carry an explicit non-zero port, and not be the unspecified address (the
5669/// edge sits outside the overlay until its first handshake, so `0.0.0.0`/`::`
5670/// is meaningless as its bootstrap endpoint). A literal `host:port` /
5671/// `[v6]:port` is fully validated; a `hostname:port` (kept textual so a DNS
5672/// name survives the wire) is accepted once its trailing port parses.
5673///
5674/// # Errors
5675/// Returns [`OverlaydError::Other`] for an empty, unspecified, or portless
5676/// endpoint.
5677fn validate_edge_node_endpoint(endpoint: &str) -> Result<(), OverlaydError> {
5678 let ep = endpoint.trim();
5679 if ep.is_empty() {
5680 return Err(OverlaydError::Other(
5681 "edge node_endpoint must not be empty".to_string(),
5682 ));
5683 }
5684 // Preferred path: a literal socket address (IPv4/IPv6 + port).
5685 if let Ok(sa) = ep.parse::<SocketAddr>() {
5686 if sa.ip().is_unspecified() {
5687 return Err(OverlaydError::Other(format!(
5688 "edge node_endpoint `{ep}` must be a reachable address, not the \
5689 unspecified `0.0.0.0`/`::`"
5690 )));
5691 }
5692 if sa.port() == 0 {
5693 return Err(OverlaydError::Other(format!(
5694 "edge node_endpoint `{ep}` must include a non-zero port"
5695 )));
5696 }
5697 return Ok(());
5698 }
5699 // Fallback: a `hostname:port` endpoint (textual, so a DNS name survives).
5700 // Require a trailing `:port` that parses to a non-zero port.
5701 let Some((host, port)) = ep.rsplit_once(':') else {
5702 return Err(OverlaydError::Other(format!(
5703 "edge node_endpoint `{ep}` must be `host:port`"
5704 )));
5705 };
5706 if host.is_empty() {
5707 return Err(OverlaydError::Other(format!(
5708 "edge node_endpoint `{ep}` is missing a host"
5709 )));
5710 }
5711 match port.parse::<u16>() {
5712 Ok(0) | Err(_) => Err(OverlaydError::Other(format!(
5713 "edge node_endpoint `{ep}` must include a non-zero port"
5714 ))),
5715 Ok(_) => Ok(()),
5716 }
5717}
5718
5719/// Offset (relative to the slice's network address) reserved for the node's
5720/// own overlay IP. Offset 1 is always the first usable host of the slice, so
5721/// the node IP is deterministic (`base + 1`) regardless of allocation order.
5722const NODE_RESERVED_OFFSET: u64 = 1;
5723
5724/// Simple IP address allocator supporting both IPv4 and IPv6, bounded to a
5725/// specific CIDR (typically a per-node `/28` slice). Allocations past the last
5726/// usable host return an exhaustion error.
5727///
5728/// Offset [`NODE_RESERVED_OFFSET`] (the first usable host) is reserved for the
5729/// node's own overlay IP and is never handed out by [`IpAllocator::allocate`],
5730/// so the node IP stays deterministic across restarts and immune to container
5731/// allocation order. Use [`IpAllocator::node_ip`] to read it.
5732struct IpAllocator {
5733 /// CIDR the allocator is bounded to.
5734 cidr: IpNetwork,
5735 /// Base (network) address of the CIDR.
5736 base: IpAddr,
5737 /// Monotonic counter for the next allocation offset relative to `base`.
5738 /// Starts at [`NODE_RESERVED_OFFSET`] + 1 so the node's reserved IP is
5739 /// never returned to a container.
5740 next_offset: AtomicU64,
5741 /// IPs returned by `release(...)`. `allocate()` drains this first before
5742 /// incrementing `next_offset`.
5743 released: parking_lot::Mutex<Vec<IpAddr>>,
5744}
5745
5746impl IpAllocator {
5747 fn new(cidr: IpNetwork) -> Self {
5748 Self {
5749 base: cidr.network(),
5750 cidr,
5751 // Reserve offset 1 for the node's own overlay IP; container
5752 // allocation starts at offset 2.
5753 next_offset: AtomicU64::new(NODE_RESERVED_OFFSET + 1),
5754 released: parking_lot::Mutex::new(Vec::new()),
5755 }
5756 }
5757
5758 /// The node's own overlay IP for this slice: the first usable host
5759 /// (`base + 1`), reserved so no container ever receives it. Deterministic
5760 /// for a given slice CIDR, independent of allocation order or restarts.
5761 fn node_ip(&self) -> IpAddr {
5762 self.compute_addr(NODE_RESERVED_OFFSET)
5763 }
5764
5765 #[allow(clippy::cast_possible_truncation)]
5766 fn compute_addr(&self, offset: u64) -> IpAddr {
5767 match self.base {
5768 IpAddr::V4(base_v4) => {
5769 let base_u32 = u32::from_be_bytes(base_v4.octets());
5770 let addr = base_u32.wrapping_add(offset as u32);
5771 IpAddr::V4(Ipv4Addr::from(addr.to_be_bytes()))
5772 }
5773 IpAddr::V6(base_v6) => {
5774 let base_u128 = u128::from(base_v6);
5775 let addr = base_u128.wrapping_add(u128::from(offset));
5776 IpAddr::V6(Ipv6Addr::from(addr))
5777 }
5778 }
5779 }
5780
5781 /// Allocate the next IP in the slice, reusing released IPs first.
5782 ///
5783 /// # Errors
5784 /// Returns [`OverlaydError::Overlay`] when the CIDR is exhausted.
5785 fn allocate(&self) -> Result<IpAddr, OverlaydError> {
5786 if let Some(ip) = self.released.lock().pop() {
5787 return Ok(ip);
5788 }
5789 let offset = self.next_offset.fetch_add(1, Ordering::SeqCst);
5790 let addr = self.compute_addr(offset);
5791
5792 let in_cidr = self.cidr.contains(addr);
5793 let is_v4_broadcast = matches!(
5794 (&self.cidr, &addr),
5795 (IpNetwork::V4(v4), IpAddr::V4(a)) if *a == v4.broadcast()
5796 );
5797 if !in_cidr || is_v4_broadcast {
5798 return Err(OverlaydError::Overlay(format!(
5799 "IP allocator exhausted: next address {addr} is outside slice {}",
5800 self.cidr
5801 )));
5802 }
5803 Ok(addr)
5804 }
5805
5806 /// Return an IP to the free pool. Idempotent. The node's reserved IP is
5807 /// never accepted back into the pool so it can never be handed to a
5808 /// container by a later `allocate()`.
5809 fn release(&self, ip: IpAddr) {
5810 if ip == self.node_ip() {
5811 return;
5812 }
5813 let mut released = self.released.lock();
5814 if !released.contains(&ip) {
5815 released.push(ip);
5816 }
5817 }
5818}
5819
5820// -- Windows HCN helpers (ported from the agent's hcs runtime) --------------
5821
5822/// Owner tag stamped onto every HCN endpoint this server creates. The legacy
5823/// single-instance value is `"zlayer"`; any other name is used verbatim so two
5824/// daemons running side-by-side never sweep each other's endpoints.
5825#[cfg(target_os = "windows")]
5826fn owner_tag(daemon_name: &str) -> String {
5827 if daemon_name == "zlayer" {
5828 "zlayer".to_string()
5829 } else {
5830 daemon_name.to_string()
5831 }
5832}
5833
5834/// Name of the per-daemon HCN overlay network on the host. Legacy
5835/// single-instance value is `"zlayer-overlay"`; any other name becomes
5836/// `"<daemon_name>-overlay"`.
5837#[cfg(target_os = "windows")]
5838fn overlay_network_name(daemon_name: &str) -> String {
5839 if daemon_name == "zlayer" {
5840 "zlayer-overlay".to_string()
5841 } else {
5842 format!("{daemon_name}-overlay")
5843 }
5844}
5845
5846/// Build the [`zlayer_hns::schema::HostComputeNetwork`] document for the single
5847/// shared HCN **NAT** network. A NAT network gives every attached container
5848/// outbound connectivity and host-port forwarding (driven by the userspace
5849/// free-port L4 proxy), without a per-service vSwitch — the Windows analogue of
5850/// the Linux node-wide shared bridge. The Static IPAM declares a default route
5851/// to the subnet gateway so HCN reserves only the gateway (same
5852/// `HCN_E_ADDR_INVALID_OR_RESERVED` avoidance the Internal/Transparent paths
5853/// use). Returns `None` when `subnet` has no usable gateway host.
5854#[cfg(target_os = "windows")]
5855fn shared_nat_settings(name: &str, subnet: &str) -> Option<zlayer_hns::schema::HostComputeNetwork> {
5856 use zlayer_hns::schema::{HostComputeNetwork, Ipam, NetworkType, Route, SchemaVersion, Subnet};
5857
5858 let net: ipnet::IpNet = subnet.parse().ok()?;
5859 let ipnet::IpNet::V4(v4) = net else {
5860 // HCN's NAT IPAM is IPv4 in the current schema.
5861 return None;
5862 };
5863 if v4.prefix_len() >= 31 {
5864 return None;
5865 }
5866 let gateway = std::net::Ipv4Addr::from(u32::from(v4.network()).checked_add(1)?).to_string();
5867
5868 Some(HostComputeNetwork {
5869 id: None,
5870 name: name.to_string(),
5871 ty: NetworkType::Nat,
5872 policies: Vec::new(),
5873 mac_pool: None,
5874 dns: None,
5875 ipams: vec![Ipam {
5876 ty: "Static".to_string(),
5877 subnets: vec![Subnet {
5878 ip_address_prefix: subnet.to_string(),
5879 routes: vec![Route {
5880 next_hop: gateway,
5881 destination_prefix: "0.0.0.0/0".to_string(),
5882 metric: None,
5883 }],
5884 policies: Vec::new(),
5885 }],
5886 }],
5887 flags: 0,
5888 schema_version: SchemaVersion::default(),
5889 })
5890}
5891
5892/// Format a GUID as the bare, lowercase, un-braced string HCN/HCS use to
5893/// identify a namespace inside a compute-system document's
5894/// `Container.Networking.Namespace` field (e.g. `aabbccdd-eeff-...`).
5895#[cfg(target_os = "windows")]
5896fn format_guid_bare(id: windows::core::GUID) -> String {
5897 format!("{id:?}")
5898 .trim_matches(|c: char| c == '{' || c == '}')
5899 .to_ascii_lowercase()
5900}
5901
5902/// Delete every host-level HCN network this server created for `daemon_name` and
5903/// clear the persistent marker. Called on a full uninstall — never on a routine
5904/// stop/restart. Best-effort throughout. Synchronous (HCN calls are blocking).
5905#[cfg(target_os = "windows")]
5906pub fn purge_managed_networks(data_dir: &Path, daemon_name: &str) {
5907 use windows::core::GUID;
5908
5909 let marker_path = zlayer_paths::ZLayerDirs::new(data_dir.to_path_buf()).agent_network_state();
5910 let state = crate::network_state::NetworkState::load(&marker_path);
5911
5912 // Pass 1: delete recorded HCN networks by GUID.
5913 for entry in &state.networks {
5914 if !entry.kind.starts_with("hcn") {
5915 continue;
5916 }
5917 match GUID::try_from(entry.id.as_str()) {
5918 Ok(guid) => match zlayer_hns::network::Network::delete(guid) {
5919 Ok(()) => {
5920 tracing::info!(name = %entry.name, id = %entry.id, "deleted managed HCN network");
5921 }
5922 Err(e) => {
5923 tracing::warn!(name = %entry.name, id = %entry.id, error = %e, "failed to delete managed HCN network");
5924 }
5925 },
5926 Err(e) => {
5927 tracing::warn!(id = %entry.id, error = %e, "managed network marker has unparseable GUID");
5928 }
5929 }
5930 }
5931
5932 // Pass 2: name-sweep fallback for an overlay network whose marker entry was
5933 // lost (crash between create and marker write).
5934 let overlay_name = overlay_network_name(daemon_name);
5935 if let Ok(guids) = zlayer_hns::network::list("{}") {
5936 for guid in guids {
5937 let Ok(network) = zlayer_hns::network::Network::open(guid) else {
5938 continue;
5939 };
5940 let is_ours = matches!(network.query("{}"), Ok(props) if props.name == overlay_name);
5941 drop(network);
5942 if is_ours {
5943 match zlayer_hns::network::Network::delete(guid) {
5944 Ok(()) => {
5945 tracing::info!(name = %overlay_name, "deleted overlay HCN network (name sweep)");
5946 }
5947 Err(e) => {
5948 tracing::warn!(name = %overlay_name, error = %e, "failed to delete overlay network (name sweep)");
5949 }
5950 }
5951 }
5952 }
5953 }
5954
5955 if marker_path.exists() {
5956 if let Err(e) = std::fs::remove_file(&marker_path) {
5957 tracing::warn!(error = %e, path = %marker_path.display(), "failed to remove agent network marker");
5958 }
5959 }
5960}
5961
5962#[cfg(test)]
5963mod tests {
5964 use super::*;
5965
5966 // -- edge peers ----------------------------------------------------------
5967
5968 /// An [`EdgeAttachInfo`] with the given mint/expiry clocks; the other fields
5969 /// are irrelevant to [`OverlaydServer::edge_reap_due`] (a pure predicate over
5970 /// `minted_at_unix` / `expires_at_unix` + the handshake arg).
5971 fn edge_info(minted_at_unix: u64, expires_at_unix: u64) -> EdgeAttachInfo {
5972 EdgeAttachInfo {
5973 overlay_ip: "10.200.0.9".parse().unwrap(),
5974 public_key: "edge-pub-b64".to_string(),
5975 minted_at_unix,
5976 expires_at_unix,
5977 allowed_targets: Vec::new(),
5978 isolation_network: None,
5979 }
5980 }
5981
5982 #[test]
5983 fn edge_reap_due_truth_table() {
5984 // Minted at t=1000 with a 1h TTL (expires at 4600).
5985 let info = edge_info(1000, 4600);
5986
5987 // Fresh, never handshaked, within the 120s boot grace -> retained.
5988 assert!(!OverlaydServer::edge_reap_due(&info, None, 1060));
5989 // Never handshaked, boot grace elapsed -> reaped.
5990 assert!(OverlaydServer::edge_reap_due(&info, None, 1200));
5991 // A `0` (and a nonsensical negative) handshake reads as "never".
5992 assert!(!OverlaydServer::edge_reap_due(&info, Some(0), 1060));
5993 assert!(OverlaydServer::edge_reap_due(&info, Some(0), 1200));
5994 assert!(OverlaydServer::edge_reap_due(&info, Some(-5), 1200));
5995
5996 // Handshaked at t=2000, still fresh (silent 100s < 180) -> retained.
5997 assert!(!OverlaydServer::edge_reap_due(&info, Some(2000), 2100));
5998 // Handshaked then silent past the 180s timeout -> reaped.
5999 assert!(OverlaydServer::edge_reap_due(&info, Some(2000), 2300));
6000
6001 // TTL expiry wins regardless of a healthy recent handshake.
6002 assert!(OverlaydServer::edge_reap_due(&info, Some(4590), 4600));
6003 }
6004
6005 #[test]
6006 fn edge_reap_due_boundaries() {
6007 let info = edge_info(1000, 4600);
6008
6009 // Boot-grace boundary: reaped at exactly +120s, retained at +119s.
6010 assert!(!OverlaydServer::edge_reap_due(&info, None, 1000 + 119));
6011 assert!(OverlaydServer::edge_reap_due(&info, None, 1000 + 120));
6012
6013 // Silence boundary: reaped at exactly +180s after handshake, retained at
6014 // +179s.
6015 assert!(!OverlaydServer::edge_reap_due(
6016 &info,
6017 Some(2000),
6018 2000 + 179
6019 ));
6020 assert!(OverlaydServer::edge_reap_due(&info, Some(2000), 2000 + 180));
6021
6022 // TTL boundary: reaped at exactly expires_at, retained one second before
6023 // (with a fresh handshake so only the TTL can trigger it).
6024 assert!(!OverlaydServer::edge_reap_due(&info, Some(4599), 4599));
6025 assert!(OverlaydServer::edge_reap_due(&info, Some(4599), 4600));
6026 }
6027
6028 #[test]
6029 fn edge_node_endpoint_validation() {
6030 // Valid literal socket addresses (v4 + v6) and a hostname:port.
6031 assert!(validate_edge_node_endpoint("203.0.113.7:51820").is_ok());
6032 assert!(validate_edge_node_endpoint("[2001:db8::1]:51820").is_ok());
6033 assert!(validate_edge_node_endpoint("node.example.com:51820").is_ok());
6034
6035 // Empty / whitespace-only.
6036 assert!(validate_edge_node_endpoint("").is_err());
6037 assert!(validate_edge_node_endpoint(" ").is_err());
6038 // Unspecified address (a literal 0.0.0.0 / ::).
6039 assert!(validate_edge_node_endpoint("0.0.0.0:51820").is_err());
6040 assert!(validate_edge_node_endpoint("[::]:51820").is_err());
6041 // Portless / zero-port.
6042 assert!(validate_edge_node_endpoint("203.0.113.7").is_err());
6043 assert!(validate_edge_node_endpoint("203.0.113.7:0").is_err());
6044 assert!(validate_edge_node_endpoint("node.example.com").is_err());
6045 }
6046
6047 #[test]
6048 fn edge_host_route_formats_v4_and_v6() {
6049 assert_eq!(
6050 edge_host_route("10.200.0.9".parse().unwrap()),
6051 "10.200.0.9/32"
6052 );
6053 assert_eq!(edge_host_route("fd00::9".parse().unwrap()), "fd00::9/128");
6054 }
6055
6056 // -- edge peer lifecycle on a REAL WireGuard transport (A3 proof) ---------
6057 //
6058 // The tests below are the transport-touching half of the edge proof: they
6059 // stand up a live global overlay (a real boringtun `WireGuard` device) and
6060 // assert that `mint_edge_peer` REGISTERS the peer's `/32` on the device
6061 // (visible in the UAPI `status()` dump), that `revoke_edge_peer` and the
6062 // TTL/silence sweep REMOVE it, and that the freed overlay IP is reusable.
6063 //
6064 // A real device needs `CAP_NET_ADMIN` (Linux, via `sudo`) or root (macOS),
6065 // so every test carries `#[ignore]`. They are deliberately NOT
6066 // `#[cfg(target_os = "linux")]`-gated (unlike the crate's older privileged
6067 // e2e tests): the entire edge code path — mint/revoke/sweep, IPAM, keygen,
6068 // the roaming `/32` peer — is platform-agnostic (boringtun userspace
6069 // `WireGuard` runs on Linux/macOS/Windows), so leaving them un-gated lets
6070 // the suite COMPILE-check them on any dev host while they only EXECUTE on a
6071 // privileged host under `--ignored`. Run them with:
6072 //
6073 // ```sh
6074 // # Linux runner (overlayd has no cargo features — do NOT pass --features):
6075 // sudo -E cargo test -p zlayer-overlayd -- --ignored --nocapture edge_lifecycle
6076 // # macOS (as root):
6077 // sudo -E cargo test -p zlayer-overlayd -- --ignored --nocapture edge_lifecycle
6078 // ```
6079
6080 /// Stand up a fresh [`OverlaydServer`] with a LIVE global overlay for an
6081 /// edge-lifecycle test. Each call takes a UNIQUE deployment name, `WireGuard`
6082 /// listen port, and UAPI socket dir so the `#[ignore]`d tests can run in
6083 /// parallel (cargo's default) without clobbering each other's kernel
6084 /// interface, UDP port, or control socket. Requires a real device:
6085 /// `CAP_NET_ADMIN` on Linux (`sudo`) or root on macOS.
6086 async fn edge_lifecycle_server(deployment: &str, wg_port: u16) -> OverlaydServer {
6087 let base = std::env::temp_dir().join(format!(
6088 "zlayer-overlayd-edge-{deployment}-{}-{}",
6089 std::process::id(),
6090 now_unix()
6091 ));
6092 let uapi = base.join("uapi");
6093 std::fs::create_dir_all(&uapi).expect("create per-test UAPI socket dir");
6094
6095 let mut server = OverlaydServer::new(base).with_uapi_sock_dir(uapi);
6096 server
6097 .setup_global_overlay(
6098 deployment.to_string(),
6099 "e".to_string(),
6100 "10.200.0.0/16",
6101 Some("10.200.0.0/28"),
6102 wg_port,
6103 None,
6104 false,
6105 )
6106 .await
6107 .expect("global overlay up (requires a real WireGuard device)");
6108 assert!(
6109 server.global_transport.is_some(),
6110 "edge lifecycle tests require a REAL WireGuard device (a live \
6111 `global_transport`). Run under sudo on Linux (CAP_NET_ADMIN) or as \
6112 root on macOS; a degraded VM-only overlay (host adapter unavailable) \
6113 leaves global_transport == None and cannot exercise the transport."
6114 );
6115 server
6116 }
6117
6118 /// Does the live UAPI `status()` dump carry the edge peer keyed by
6119 /// `edge_pub_b64` (base64, as [`OverlayTransport::generate_keys`] returns it)
6120 /// with `overlay_ip`'s host route in its `AllowedIPs`?
6121 ///
6122 /// The live device reports peer keys in HEX; [`parse_peer_status`] surfaces
6123 /// them verbatim. The edge's key is base64, so we convert it via
6124 /// [`zlayer_overlay::nat::pubkey_b64_to_hex`] — exactly as
6125 /// [`OverlaydServer::edge_handshake_map`] does — before the (case-insensitive)
6126 /// lookup.
6127 fn dump_has_edge_slash32(dump: &str, edge_pub_b64: &str, overlay_ip: IpAddr) -> bool {
6128 let Some(hex) = zlayer_overlay::nat::pubkey_b64_to_hex(edge_pub_b64) else {
6129 return false;
6130 };
6131 let want = edge_host_route(overlay_ip);
6132 parse_peer_status(dump)
6133 .into_iter()
6134 .filter(|p| p.public_key.eq_ignore_ascii_case(&hex))
6135 .any(|p| p.allowed_ips.split(',').any(|cidr| cidr == want))
6136 }
6137
6138 /// Read the live global transport's UAPI dump (the test asserts against a
6139 /// real device, so a failed dump is a hard error, not an empty fallback).
6140 async fn transport_dump(server: &OverlaydServer) -> String {
6141 server
6142 .global_transport
6143 .as_ref()
6144 .expect("global transport present")
6145 .status()
6146 .await
6147 .expect("WireGuard UAPI status dump")
6148 }
6149
6150 /// A3 proof, mint→revoke: a minted edge registers a `/32` on the real WG
6151 /// transport (visible in the UAPI dump under its hex-converted pubkey), and
6152 /// revoke removes it from both the device and the bookkeeping (idempotently).
6153 #[tokio::test]
6154 #[ignore = "requires CAP_NET_ADMIN to create a real WireGuard device"]
6155 async fn edge_lifecycle_mint_registers_slash32_peer_then_revoke_removes_it() {
6156 let mut server = edge_lifecycle_server("edgl1", 53710).await;
6157
6158 let config = server
6159 .mint_edge_peer("t1".to_string(), 3600, &[], "203.0.113.1:51820")
6160 .await
6161 .expect("mint edge peer t1");
6162
6163 // Returned config shape.
6164 assert_eq!(config.version, EDGE_CONFIG_VERSION);
6165 assert_eq!(config.name, "t1");
6166 assert_eq!(
6167 config.peers.len(),
6168 1,
6169 "the edge's single artifact peer is THIS node"
6170 );
6171 let slice: ipnet::IpNet = "10.200.0.0/28".parse().unwrap();
6172 assert!(
6173 slice.contains(&config.overlay_ip),
6174 "edge overlay_ip {} must fall inside the node slice {slice}",
6175 config.overlay_ip
6176 );
6177
6178 // Bookkeeping recorded.
6179 assert!(
6180 server.edge_attachments.contains_key("t1"),
6181 "mint must record the edge attachment"
6182 );
6183
6184 // The minted /32 must be LIVE on the real transport.
6185 let dump = transport_dump(&server).await;
6186 assert!(
6187 dump_has_edge_slash32(&dump, &config.public_key, config.overlay_ip),
6188 "minted edge pubkey (hex) with {} must appear in the WireGuard UAPI \
6189 dump after mint:\n{dump}",
6190 edge_host_route(config.overlay_ip)
6191 );
6192
6193 // Revoke drops it from the device AND the bookkeeping.
6194 let removed = server.revoke_edge_peer("t1").await.expect("revoke t1");
6195 assert!(removed, "revoke of a live edge returns Ok(true)");
6196 assert!(
6197 server.edge_attachments.is_empty(),
6198 "revoke must drop the edge attachment"
6199 );
6200 let dump = transport_dump(&server).await;
6201 assert!(
6202 !dump_has_edge_slash32(&dump, &config.public_key, config.overlay_ip),
6203 "revoked edge pubkey must be GONE from the WireGuard UAPI dump:\n{dump}"
6204 );
6205
6206 // Idempotent: a second revoke reports nothing removed.
6207 let removed_again = server
6208 .revoke_edge_peer("t1")
6209 .await
6210 .expect("second revoke does not error");
6211 assert!(
6212 !removed_again,
6213 "second revoke of an absent edge is Ok(false)"
6214 );
6215 }
6216
6217 /// A3 proof, TTL sweep: a peer minted with a 1s TTL is reaped by
6218 /// `sweep_edge_peers(now)` for any `now` past its expiry — removed from the
6219 /// device and the bookkeeping. `now_unix` is injected, so no real waiting.
6220 #[tokio::test]
6221 #[ignore = "requires CAP_NET_ADMIN to create a real WireGuard device"]
6222 async fn edge_lifecycle_ttl_sweep_reaps_expired_peer() {
6223 let mut server = edge_lifecycle_server("edgl2", 53711).await;
6224
6225 let config = server
6226 .mint_edge_peer("ttl".to_string(), 1, &[], "203.0.113.1:51820")
6227 .await
6228 .expect("mint edge peer ttl");
6229
6230 // Present on the device right after mint.
6231 let dump = transport_dump(&server).await;
6232 assert!(dump_has_edge_slash32(
6233 &dump,
6234 &config.public_key,
6235 config.overlay_ip
6236 ));
6237
6238 // Sweep at a clock strictly past the 1s TTL (minted_at + 2).
6239 let minted_at = server
6240 .edge_attachments
6241 .get("ttl")
6242 .expect("attachment recorded")
6243 .minted_at_unix;
6244 server.sweep_edge_peers(minted_at + 2).await;
6245
6246 assert!(
6247 server.edge_attachments.is_empty(),
6248 "TTL-expired edge must be swept from the bookkeeping"
6249 );
6250 let dump = transport_dump(&server).await;
6251 assert!(
6252 !dump_has_edge_slash32(&dump, &config.public_key, config.overlay_ip),
6253 "TTL-expired edge pubkey must be GONE from the WireGuard UAPI dump:\n{dump}"
6254 );
6255 }
6256
6257 /// A3 proof, silence sweep: an edge that never handshakes (no real client
6258 /// connects) is retained through its boot-grace window and reaped once the
6259 /// grace elapses — even though its TTL is far in the future, so ONLY the
6260 /// silence path can trigger it. `now_unix` is injected.
6261 #[tokio::test]
6262 #[ignore = "requires CAP_NET_ADMIN to create a real WireGuard device"]
6263 async fn edge_lifecycle_silence_sweep_reaps_never_handshaked_peer() {
6264 let mut server = edge_lifecycle_server("edgl3", 53712).await;
6265
6266 // Long TTL so the TTL path can never be what reaps it.
6267 let config = server
6268 .mint_edge_peer("silent".to_string(), 3600, &[], "203.0.113.1:51820")
6269 .await
6270 .expect("mint edge peer silent");
6271 let minted_at = server
6272 .edge_attachments
6273 .get("silent")
6274 .expect("attachment recorded")
6275 .minted_at_unix;
6276 let grace = OverlaydServer::EDGE_CONNECT_GRACE_SECS;
6277
6278 // Control: one second BEFORE the boot grace elapses, the never-handshaked
6279 // edge is still retained (nothing to reap yet).
6280 server.sweep_edge_peers(minted_at + grace - 1).await;
6281 assert!(
6282 server.edge_attachments.contains_key("silent"),
6283 "edge within its boot-grace window must survive the sweep"
6284 );
6285 let dump = transport_dump(&server).await;
6286 assert!(
6287 dump_has_edge_slash32(&dump, &config.public_key, config.overlay_ip),
6288 "edge within its boot-grace window must still be on the device"
6289 );
6290
6291 // One second AFTER the grace: the never-handshaked edge is reaped.
6292 server.sweep_edge_peers(minted_at + grace + 1).await;
6293 assert!(
6294 server.edge_attachments.is_empty(),
6295 "silent edge past its boot-grace window must be swept"
6296 );
6297 let dump = transport_dump(&server).await;
6298 assert!(
6299 !dump_has_edge_slash32(&dump, &config.public_key, config.overlay_ip),
6300 "swept silent edge pubkey must be GONE from the WireGuard UAPI dump:\n{dump}"
6301 );
6302 }
6303
6304 /// A3 proof, IP reuse: the `/32` freed by revoking one edge is reused by the
6305 /// next mint (the allocator drains its released free-list before advancing
6306 /// the monotonic counter), and the new peer is live under a fresh key.
6307 #[tokio::test]
6308 #[ignore = "requires CAP_NET_ADMIN to create a real WireGuard device"]
6309 async fn edge_lifecycle_freed_ip_is_reusable() {
6310 let mut server = edge_lifecycle_server("edgl4", 53713).await;
6311
6312 let a = server
6313 .mint_edge_peer("a".to_string(), 3600, &[], "203.0.113.1:51820")
6314 .await
6315 .expect("mint a");
6316 let ip_a = a.overlay_ip;
6317
6318 let removed = server.revoke_edge_peer("a").await.expect("revoke a");
6319 assert!(removed, "revoke of the live edge `a` returns Ok(true)");
6320
6321 let b = server
6322 .mint_edge_peer("b".to_string(), 3600, &[], "203.0.113.2:51820")
6323 .await
6324 .expect("mint b");
6325
6326 assert_eq!(
6327 b.overlay_ip, ip_a,
6328 "the /32 freed by revoking `a` must be reused by the next mint `b`"
6329 );
6330 assert_ne!(
6331 a.public_key, b.public_key,
6332 "each mint generates its own fresh keypair"
6333 );
6334
6335 // The reused /32 is live on the device under b's (new) key.
6336 let dump = transport_dump(&server).await;
6337 assert!(
6338 dump_has_edge_slash32(&dump, &b.public_key, b.overlay_ip),
6339 "the reused /32 must be live under b's fresh pubkey:\n{dump}"
6340 );
6341 }
6342
6343 #[cfg(target_os = "linux")]
6344 #[test]
6345 fn orphan_bridge_selection() {
6346 use std::collections::HashSet;
6347
6348 // Two live per-service bridges the daemon says SHOULD exist.
6349 let live: HashSet<&str> = ["zl-prod-0-web-b", "zl-prod-0-api-b"].into_iter().collect();
6350 // The active global device and node-wide shared bridge are protected,
6351 // plus a live in-memory dedicated device.
6352 let protected: HashSet<String> = ["zl-prod-0-g", "zl-prod-0-shared-sh", "zl-prod-0-db-d"]
6353 .into_iter()
6354 .map(String::from)
6355 .collect();
6356
6357 // The full set of host links the kernel would report.
6358 let host_links = [
6359 // Live -> keep.
6360 "zl-prod-0-web-b",
6361 "zl-prod-0-api-b",
6362 // Protected global / shared / live dedicated device -> keep.
6363 "zl-prod-0-g",
6364 "zl-prod-0-shared-sh",
6365 "zl-prod-0-db-d",
6366 // Orphan bridges (the user's observed leaks) -> reclaim.
6367 "zl-1ca4568944-b",
6368 "zl-81c6bc17c7-b",
6369 // Orphan dedicated device -> reclaim.
6370 "zl-prod-0-gone-d",
6371 // Container veths owned by the PID-keyed sweep, never here -> skip.
6372 "veth-4242-s",
6373 "vc-4242-g",
6374 // Unrelated host links -> skip.
6375 "eth0",
6376 "lo",
6377 "docker0",
6378 "zl-not-a-bridge",
6379 ];
6380
6381 let orphans: Vec<&str> = host_links
6382 .into_iter()
6383 .filter(|n| is_orphan_service_bridge(n, &live, &protected))
6384 .collect();
6385
6386 assert_eq!(
6387 orphans,
6388 vec!["zl-1ca4568944-b", "zl-81c6bc17c7-b", "zl-prod-0-gone-d"],
6389 "only orphaned -b/-d service bridges/devices are selected; \
6390 live, protected (-g/-sh/live -d), veth, and unrelated links are excluded"
6391 );
6392 }
6393
6394 #[test]
6395 fn peer_spec_to_info_parses_endpoint_and_keepalive() {
6396 let spec = PeerSpec {
6397 public_key: "base64key".to_string(),
6398 endpoint: "1.2.3.4:51820".to_string(),
6399 allowed_ips: "10.200.0.5/32,10.200.1.0/24".to_string(),
6400 persistent_keepalive_secs: 25,
6401 candidates: Vec::new(),
6402 };
6403 let info = peer_spec_to_info(&spec).expect("valid spec");
6404 assert_eq!(info.public_key, "base64key");
6405 assert_eq!(info.endpoint, "1.2.3.4:51820".parse().unwrap());
6406 assert_eq!(info.allowed_ips, "10.200.0.5/32,10.200.1.0/24");
6407 assert_eq!(
6408 info.persistent_keepalive_interval,
6409 std::time::Duration::from_secs(25)
6410 );
6411 }
6412
6413 #[test]
6414 fn peer_spec_to_info_rejects_bad_endpoint() {
6415 let spec = PeerSpec {
6416 public_key: "k".to_string(),
6417 endpoint: "not-a-socket-addr".to_string(),
6418 allowed_ips: String::new(),
6419 persistent_keepalive_secs: 0,
6420 candidates: Vec::new(),
6421 };
6422 assert!(peer_spec_to_info(&spec).is_err());
6423 }
6424
6425 #[test]
6426 fn interface_name_never_exceeds_limit() {
6427 let cases: Vec<(&[&str], &str)> = vec![
6428 (&["a"], "g"),
6429 (&["zlayer-manager"], "g"),
6430 (&["my-very-long-deployment-name-that-goes-on-and-on"], "g"),
6431 (&["zlayer", "manager"], "s"),
6432 (
6433 &["abcdefghijklmnopqrstuvwxyz", "abcdefghijklmnopqrstuvwxyz"],
6434 "s",
6435 ),
6436 (&["x"], ""),
6437 ];
6438 for (parts, suffix) in &cases {
6439 let name = make_interface_name(parts, suffix);
6440 assert!(name.len() <= MAX_IFNAME_LEN, "Name '{name}' too long");
6441 assert!(name.starts_with("zl-"));
6442 }
6443 }
6444
6445 #[test]
6446 fn node_ip_is_first_usable_and_reserved() {
6447 let cidr: IpNetwork = "10.200.0.0/26".parse().unwrap();
6448 let alloc = IpAllocator::new(cidr);
6449
6450 // The node IP is the deterministic first-usable host of the slice.
6451 let expected_node_ip: IpAddr = "10.200.0.1".parse().unwrap();
6452 assert_eq!(alloc.node_ip(), expected_node_ip);
6453
6454 // Several container allocations must NEVER hand out the node IP, and
6455 // the node IP stays put regardless of allocation order.
6456 let mut handed_out = Vec::new();
6457 for _ in 0..10 {
6458 let ip = alloc.allocate().expect("slice not exhausted");
6459 assert_ne!(
6460 ip, expected_node_ip,
6461 "allocate() returned the reserved node IP"
6462 );
6463 handed_out.push(ip);
6464 }
6465 // Reservation holds after the allocations.
6466 assert_eq!(alloc.node_ip(), expected_node_ip);
6467
6468 // First container allocation is offset 2 (base + 2), proving offset 1
6469 // (the node) was reserved and skipped.
6470 assert_eq!(handed_out[0], "10.200.0.2".parse::<IpAddr>().unwrap());
6471
6472 // Releasing the node IP must not pollute the free pool with it.
6473 alloc.release(expected_node_ip);
6474 let next = alloc.allocate().expect("slice not exhausted");
6475 assert_ne!(
6476 next, expected_node_ip,
6477 "node IP leaked back into the pool via release()"
6478 );
6479 }
6480
6481 #[test]
6482 fn node_ip_ipv6_is_first_usable() {
6483 let cidr: IpNetwork = "fd00:200::/64".parse().unwrap();
6484 let alloc = IpAllocator::new(cidr);
6485 let expected: IpAddr = "fd00:200::1".parse().unwrap();
6486 assert_eq!(alloc.node_ip(), expected);
6487 for _ in 0..5 {
6488 assert_ne!(alloc.allocate().unwrap(), expected);
6489 }
6490 assert_eq!(alloc.node_ip(), expected);
6491 }
6492
6493 #[test]
6494 fn interface_name_is_deterministic() {
6495 assert_eq!(
6496 make_interface_name(&["zlayer-manager"], "g"),
6497 make_interface_name(&["zlayer-manager"], "g")
6498 );
6499 }
6500
6501 #[test]
6502 fn parse_peer_status_splits_blocks() {
6503 let dump = "\
6504public_key=AAA
6505endpoint=1.2.3.4:51820
6506allowed_ip=10.200.0.2/32
6507allowed_ip=10.200.1.0/24
6508latest_handshake=1700000000
6509public_key=BBB
6510endpoint=5.6.7.8:51820
6511allowed_ip=10.200.0.3/32
6512latest_handshake=0
6513";
6514 let peers = parse_peer_status(dump);
6515 assert_eq!(peers.len(), 2);
6516 assert_eq!(peers[0].public_key, "AAA");
6517 assert_eq!(peers[0].endpoint, "1.2.3.4:51820");
6518 assert_eq!(peers[0].allowed_ips, "10.200.0.2/32,10.200.1.0/24");
6519 assert_eq!(peers[0].last_handshake_unix_secs, 1_700_000_000);
6520 assert_eq!(peers[1].public_key, "BBB");
6521 assert_eq!(peers[1].last_handshake_unix_secs, 0);
6522 }
6523
6524 #[tokio::test]
6525 async fn status_snapshot_before_setup_is_empty() {
6526 let server = OverlaydServer::new(std::path::PathBuf::from("/tmp/zlayer-overlayd-test"));
6527 let snap = server.status_snapshot().await;
6528 assert!(snap.interface.is_none());
6529 assert!(snap.node_ip.is_none());
6530 assert!(snap.public_key.is_none());
6531 assert_eq!(snap.peer_count, 0);
6532 assert_eq!(snap.service_count, 0);
6533 assert!(snap.peers.is_empty());
6534 }
6535
6536 #[tokio::test]
6537 async fn allocate_and_release_ip_round_trip() {
6538 let mut server = OverlaydServer::new(std::path::PathBuf::from("/tmp/zlayer-overlayd-test"));
6539 let a = server.allocate_ip("svc", false).expect("alloc a");
6540 let b = server.allocate_ip("svc", false).expect("alloc b");
6541 assert_ne!(a, b);
6542 server.release_ip(a);
6543 // Released IP is handed back before the monotonic counter advances.
6544 let c = server.allocate_ip("svc", false).expect("alloc c");
6545 assert_eq!(c, a);
6546 }
6547
6548 /// Build a throwaway server bound to a unique temp data dir so the marker
6549 /// file (rehydrated in `new`) never collides between tests.
6550 fn test_server() -> OverlaydServer {
6551 let dir = std::env::temp_dir().join(format!(
6552 "zlayer-overlayd-scope-{}-{}",
6553 std::process::id(),
6554 now_unix()
6555 ));
6556 OverlaydServer::new(dir)
6557 }
6558
6559 /// `nat_config_spec_to_config` fills sparse fields from `NatConfig::default`
6560 /// and copies populated ones verbatim (the Step-0 wire-config threading).
6561 #[test]
6562 fn nat_config_spec_to_config_fills_defaults_and_copies() {
6563 // Empty spec → defaults (default STUN servers, default timeouts).
6564 let cfg = nat_config_spec_to_config(NatConfigSpec::default());
6565 let d = NatConfig::default();
6566 assert_eq!(cfg.stun_servers.len(), d.stun_servers.len());
6567 assert_eq!(cfg.hole_punch_timeout_secs, d.hole_punch_timeout_secs);
6568 assert_eq!(cfg.max_candidate_pairs, d.max_candidate_pairs);
6569 assert!(cfg.relay_server.is_none());
6570
6571 // Populated spec → copied verbatim; relay credential is NOT on the
6572 // produced RelayServerConfig (it is carried separately on the server).
6573 let spec = NatConfigSpec {
6574 enabled: true,
6575 stun_servers: vec!["stun.example:3478".to_string()],
6576 turn_servers: vec![zlayer_types::nat_wire::TurnServerSpec {
6577 addr: "turn.example:3478".to_string(),
6578 username: "u".to_string(),
6579 credential: "p".to_string(),
6580 }],
6581 hole_punch_timeout_secs: 9,
6582 stun_refresh_interval_secs: 40,
6583 max_candidate_pairs: 3,
6584 relay_server: Some(zlayer_types::nat_wire::RelayServerSpec {
6585 listen_port: 3478,
6586 external_addr: "1.2.3.4:3478".to_string(),
6587 max_sessions: 7,
6588 auth_credential: Some("cluster-secret".to_string()),
6589 }),
6590 };
6591 let cfg = nat_config_spec_to_config(spec);
6592 assert_eq!(cfg.stun_servers.len(), 1);
6593 assert_eq!(cfg.stun_servers[0].address, "stun.example:3478");
6594 assert_eq!(cfg.turn_servers.len(), 1);
6595 assert_eq!(cfg.hole_punch_timeout_secs, 9);
6596 assert_eq!(cfg.max_candidate_pairs, 3);
6597 let relay = cfg.relay_server.expect("relay present");
6598 assert_eq!(relay.listen_port, 3478);
6599 assert_eq!(relay.max_sessions, 7);
6600 }
6601
6602 /// `wire_to_candidate` parses valid candidates and rejects bad ones;
6603 /// `candidate_to_wire` is its inverse for the type/address/priority triple.
6604 #[test]
6605 fn candidate_wire_conversions_round_trip() {
6606 let w = NatCandidateWire {
6607 candidate_type: "server-reflexive".to_string(),
6608 address: "203.0.113.5:51820".to_string(),
6609 priority: 50,
6610 };
6611 let c = wire_to_candidate(&w).expect("valid candidate");
6612 assert_eq!(c.candidate_type, CandidateType::ServerReflexive);
6613 assert_eq!(c.priority, 50);
6614 let back = candidate_to_wire(&c);
6615 assert_eq!(back, w);
6616
6617 // Bad address / type → None.
6618 assert!(wire_to_candidate(&NatCandidateWire {
6619 candidate_type: "host".to_string(),
6620 address: "not-an-addr".to_string(),
6621 priority: 1,
6622 })
6623 .is_none());
6624 assert!(wire_to_candidate(&NatCandidateWire {
6625 candidate_type: "bogus".to_string(),
6626 address: "1.2.3.4:5".to_string(),
6627 priority: 1,
6628 })
6629 .is_none());
6630 }
6631
6632 /// `AddPeer` carrying candidates records them in `peer_candidates`; a
6633 /// candidate-free add (or one with only-invalid candidates) leaves no entry,
6634 /// and `RemovePeer` clears them.
6635 #[tokio::test]
6636 async fn add_peer_records_candidates_and_remove_clears_them() {
6637 let mut server = test_server();
6638 let pubkey = "base64key".to_string();
6639 let resp = server
6640 .handle(OverlaydRequest::AddPeer {
6641 peer: PeerSpec {
6642 public_key: pubkey.clone(),
6643 endpoint: "1.2.3.4:51820".to_string(),
6644 allowed_ips: "10.200.0.2/32".to_string(),
6645 persistent_keepalive_secs: 25,
6646 candidates: vec![NatCandidateWire {
6647 candidate_type: "host".to_string(),
6648 address: "192.168.1.5:51820".to_string(),
6649 priority: 100,
6650 }],
6651 },
6652 scope: PeerScope::Global,
6653 })
6654 .await;
6655 assert!(matches!(resp, OverlaydResponse::Ok));
6656 assert_eq!(
6657 server.peer_candidates.get(&pubkey).map(Vec::len),
6658 Some(1),
6659 "candidates must be recorded"
6660 );
6661
6662 // Remove clears the candidate + connection-type bookkeeping.
6663 let resp = server
6664 .handle(OverlaydRequest::RemovePeer {
6665 pubkey: pubkey.clone(),
6666 scope: PeerScope::Global,
6667 })
6668 .await;
6669 assert!(matches!(resp, OverlaydResponse::Ok));
6670 assert!(!server.peer_candidates.contains_key(&pubkey));
6671 }
6672
6673 /// `NatStatus` returns a `NatStatusWire` (empty before any tick) — proving
6674 /// the new IPC pair is wired through `dispatch`.
6675 #[tokio::test]
6676 async fn nat_status_request_returns_wire_snapshot() {
6677 let mut server = test_server();
6678 let resp = server.handle(OverlaydRequest::NatStatus).await;
6679 match resp {
6680 OverlaydResponse::NatStatus(wire) => {
6681 assert!(wire.candidates.is_empty());
6682 assert!(wire.peers.is_empty());
6683 }
6684 other => panic!("expected NatStatus response, got {other:?}"),
6685 }
6686 }
6687
6688 /// True when the process can mutate netlink + `/proc/sys` (root). The
6689 /// teardown-completeness test below is `#[ignore]`d and additionally skips
6690 /// (not fails) when run via `--ignored` without privileges, matching the
6691 /// crate's "skip gracefully when not root" convention.
6692 #[cfg(target_os = "linux")]
6693 fn is_root() -> bool {
6694 // SAFETY: `geteuid` is a pure read of the caller's effective uid.
6695 #[allow(unsafe_code)]
6696 let euid = unsafe { libc::geteuid() };
6697 euid == 0
6698 }
6699
6700 /// End-to-end teardown completeness: populate the server's
6701 /// `created_veths` / `created_bridges` / `created_host_routes` tracking sets
6702 /// with REAL host resources created via netlink, snapshot
6703 /// `net.ipv4.ip_forward`, force it to `1` (recording the prior value in
6704 /// `prev_ipv4_forward` exactly as `enable_forwarding_for_attach` does), then
6705 /// drive the same teardown the `Shutdown` request triggers
6706 /// (`handle(OverlaydRequest::Shutdown)`), and assert: every tracked veth /
6707 /// bridge / route is gone at the kernel level AND `ip_forward` is restored to
6708 /// the snapshot.
6709 ///
6710 /// This is the regression for the full teardown fix (revert routes + veths +
6711 /// bridges + forwarding sysctl on shutdown). Names are unique and <=15 chars;
6712 /// a belt-and-braces cleanup runs before the asserts so a failed assertion
6713 /// still leaves the host clean. Skips (returns) when not root.
6714 #[cfg(target_os = "linux")]
6715 #[tokio::test(flavor = "multi_thread")]
6716 #[ignore = "needs CAP_NET_ADMIN + /proc/sys write; run on a privileged Linux host"]
6717 async fn shutdown_teardown_reverts_resources_and_ip_forward() {
6718 if !is_root() {
6719 eprintln!("skipping shutdown_teardown_reverts_resources_and_ip_forward: requires root");
6720 return;
6721 }
6722
6723 let suffix = format!("{:x}", now_unix() & 0xff_ffff);
6724 let veth_host = format!("vh-{suffix}");
6725 let veth_peer = format!("vp-{suffix}");
6726 let bridge = format!("zlb-{suffix}");
6727 assert!(veth_host.len() <= 15, "veth host name exceeds IFNAMSIZ");
6728 assert!(veth_peer.len() <= 15, "veth peer name exceeds IFNAMSIZ");
6729 assert!(bridge.len() <= 15, "bridge name exceeds IFNAMSIZ");
6730
6731 let dest = IpAddr::V4(Ipv4Addr::new(10, 233, 0, 9));
6732 let prefix: u8 = 32;
6733
6734 // --- create real host resources and register them with the server's
6735 // teardown-tracking sets, exactly as the attach paths do. ---
6736 crate::netlink::create_veth_pair(&veth_host, &veth_peer)
6737 .await
6738 .expect("create_veth_pair");
6739 crate::netlink::create_bridge(&bridge)
6740 .await
6741 .expect("create_bridge");
6742 crate::netlink::replace_route_via_dev(dest, prefix, &veth_host, None)
6743 .await
6744 .expect("replace_route_via_dev");
6745
6746 let mut server = test_server();
6747 server.created_veths.insert(veth_host.clone());
6748 server.created_bridges.insert(bridge.clone());
6749 server
6750 .created_host_routes
6751 .push((dest, prefix, veth_host.clone()));
6752
6753 // Snapshot ip_forward, then flip it to 1 and record the prior value the
6754 // way enable_forwarding_for_attach does so revert_forwarding restores it.
6755 let snapshot =
6756 crate::netlink::read_sysctl("net.ipv4.ip_forward").unwrap_or_else(|_| "0".to_string());
6757 server.prev_ipv4_forward = Some(snapshot.clone());
6758 crate::netlink::set_sysctl("net.ipv4.ip_forward", "1").expect("set ip_forward=1");
6759
6760 // --- drive teardown via the real Shutdown dispatch path ---
6761 let resp = server.handle(OverlaydRequest::Shutdown).await;
6762 assert!(
6763 matches!(resp, OverlaydResponse::Ok),
6764 "Shutdown should return Ok, got {resp:?}"
6765 );
6766
6767 // Snapshot kernel state AFTER teardown.
6768 let veth_gone = !std::path::Path::new(&format!("/sys/class/net/{veth_host}")).exists();
6769 let bridge_gone = !std::path::Path::new(&format!("/sys/class/net/{bridge}")).exists();
6770 let route_gone = {
6771 let target = format!("10.233.0.9/{prefix}");
6772 std::process::Command::new("ip")
6773 .args(["route", "show", &target, "dev", &veth_host])
6774 .output()
6775 .map_or(true, |o| !o.status.success() || o.stdout.is_empty())
6776 };
6777 let ip_forward_after = crate::netlink::read_sysctl("net.ipv4.ip_forward")
6778 .unwrap_or_else(|_| "unknown".to_string());
6779
6780 // Belt-and-braces cleanup before asserting so the host stays clean even
6781 // if an assertion fails (teardown should have done all of this already).
6782 let _ = crate::netlink::delete_route_via_dev(dest, prefix, &veth_host).await;
6783 let _ = crate::netlink::delete_link_by_name(&veth_host).await;
6784 let _ = crate::netlink::delete_link_by_name(&veth_peer).await;
6785 let _ = crate::netlink::delete_link_by_name(&bridge).await;
6786 // Restore ip_forward to the snapshot regardless of teardown outcome.
6787 let _ = crate::netlink::set_sysctl("net.ipv4.ip_forward", &snapshot);
6788
6789 // --- assertions ---
6790 assert!(veth_gone, "teardown should delete the tracked host veth");
6791 assert!(bridge_gone, "teardown should delete the tracked bridge");
6792 assert!(
6793 route_gone,
6794 "teardown should delete the tracked /32 host route"
6795 );
6796 assert_eq!(
6797 ip_forward_after.trim(),
6798 snapshot.trim(),
6799 "teardown should restore net.ipv4.ip_forward to its pre-overlay value"
6800 );
6801
6802 // Tracking sets must be drained by teardown so a re-run starts clean.
6803 assert!(
6804 server.created_veths.is_empty(),
6805 "created_veths should be drained by teardown"
6806 );
6807 assert!(
6808 server.created_bridges.is_empty(),
6809 "created_bridges should be drained by teardown"
6810 );
6811 assert!(
6812 server.created_host_routes.is_empty(),
6813 "created_host_routes should be drained by teardown"
6814 );
6815 }
6816
6817 #[test]
6818 fn build_config_uses_matching_physical_egress_ipv4() {
6819 let server = test_server();
6820 let overlay_ip: IpAddr = "10.200.0.1".parse().unwrap();
6821 let egress: IpAddr = "192.0.2.10".parse().unwrap();
6822 let config = server.build_config(
6823 "priv".to_string(),
6824 "pub".to_string(),
6825 overlay_ip,
6826 16,
6827 51820,
6828 Some(egress),
6829 );
6830 assert_eq!(config.local_endpoint, SocketAddr::new(egress, 51820));
6831 }
6832
6833 #[test]
6834 fn build_config_falls_back_to_unspecified_when_none() {
6835 let server = test_server();
6836 let overlay_ip: IpAddr = "10.200.0.1".parse().unwrap();
6837 let config = server.build_config(
6838 "priv".to_string(),
6839 "pub".to_string(),
6840 overlay_ip,
6841 16,
6842 51820,
6843 None,
6844 );
6845 assert_eq!(
6846 config.local_endpoint,
6847 SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 51820)
6848 );
6849 }
6850
6851 #[test]
6852 fn build_config_falls_back_to_unspecified_on_family_mismatch() {
6853 let server = test_server();
6854 // Overlay is v6 but the resolved physical egress is v4: unusable for
6855 // source selection, so we must fall back to the v6 UNSPECIFIED address.
6856 let overlay_ip: IpAddr = "fd00::1".parse().unwrap();
6857 let egress: IpAddr = "192.0.2.10".parse().unwrap();
6858 let config = server.build_config(
6859 "priv".to_string(),
6860 "pub".to_string(),
6861 overlay_ip,
6862 64,
6863 51820,
6864 Some(egress),
6865 );
6866 assert_eq!(
6867 config.local_endpoint,
6868 SocketAddr::new(IpAddr::V6(Ipv6Addr::UNSPECIFIED), 51820)
6869 );
6870 }
6871
6872 #[test]
6873 fn rootless_forces_unspecified_decision() {
6874 // Rootless mode must force the WG local_endpoint to UNSPECIFIED because
6875 // detect_physical_egress() resolves pasta's in-netns tap IP there.
6876 assert!(rootless_forces_unspecified(true));
6877 // Non-rootless preserves the existing physical-egress selection path.
6878 assert!(!rootless_forces_unspecified(false));
6879 }
6880
6881 #[tokio::test]
6882 async fn transport_for_scope_global_requires_setup() {
6883 let server = test_server();
6884 // No global overlay set up yet -> Global scope errors. (Can't use
6885 // `expect_err` because `&OverlayTransport` is not `Debug`.)
6886 match server.transport_for_scope(&PeerScope::Global) {
6887 Ok(_) => panic!("global overlay should not be set up"),
6888 Err(OverlaydError::Other(m)) => {
6889 assert!(m.contains("global overlay not set up"), "got: {m}");
6890 }
6891 Err(other) => panic!("unexpected error: {other:?}"),
6892 }
6893 }
6894
6895 #[tokio::test]
6896 async fn transport_for_scope_unset_service_errors() {
6897 let server = test_server();
6898 match server.transport_for_scope(&PeerScope::Service {
6899 service: "x".to_string(),
6900 }) {
6901 Ok(_) => panic!("no dedicated overlay should exist for x"),
6902 Err(OverlaydError::Other(m)) => {
6903 assert_eq!(m, "no dedicated overlay for service x");
6904 }
6905 Err(other) => panic!("unexpected error: {other:?}"),
6906 }
6907 }
6908
6909 #[tokio::test]
6910 async fn add_peer_service_scope_before_setup_errors_via_dispatch() {
6911 let mut server = test_server();
6912 let resp = server
6913 .handle(OverlaydRequest::AddPeer {
6914 peer: PeerSpec {
6915 public_key: "k".to_string(),
6916 endpoint: "1.2.3.4:51820".to_string(),
6917 allowed_ips: "10.200.0.2/32".to_string(),
6918 persistent_keepalive_secs: 0,
6919 candidates: Vec::new(),
6920 },
6921 scope: PeerScope::Service {
6922 service: "x".to_string(),
6923 },
6924 })
6925 .await;
6926 match resp {
6927 OverlaydResponse::Err { message } => {
6928 assert_eq!(message, "no dedicated overlay for service x");
6929 }
6930 other => panic!("expected Err response, got {other:?}"),
6931 }
6932 }
6933
6934 /// The host-adapter degrade decision. A `create_interface()` failure is fatal
6935 /// on Linux (the kernel TUN IS the container data path) and degrades to a
6936 /// VM-only overlay on macOS/Windows (containers mesh VM-to-VM, the host
6937 /// utun/Wintun is off the data path). We can't provoke a real utun/Wintun
6938 /// syscall failure from a Linux test box, so we assert the pure `cfg!`-driven
6939 /// classifier instead: on this Linux test runner it must report fatal.
6940 /// (On macOS/Windows the same fn returns `false` — that arm is covered by the
6941 /// cfg, exercised natively, and cannot be asserted here.)
6942 #[test]
6943 fn host_adapter_failure_fatal_decision() {
6944 // Non-mandatory: platform-driven — fatal on Linux, degrade on macOS/Windows.
6945 assert_eq!(
6946 host_adapter_failure_is_fatal(false),
6947 cfg!(target_os = "linux"),
6948 "non-mandatory host-adapter failure is fatal only on Linux (kernel TUN is the data path)"
6949 );
6950 // Mandatory (host-shared macOS nodes where the utun IS the container data
6951 // path): fatal on every platform.
6952 assert!(
6953 host_adapter_failure_is_fatal(true),
6954 "a mandatory host adapter must make failure fatal on every platform"
6955 );
6956 }
6957
6958 /// A VM-only overlay leaves `global_transport == None`. The Global-scope peer
6959 /// dispatch must then WARN-AND-SKIP the on-device install (guests get the
6960 /// peer via guest-config push) rather than erroring — assert the dispatch
6961 /// returns `Ok` and still mirrors the peer into `global_peers`. This is the
6962 /// Linux-runnable proxy for the degraded host-adapter path: it exercises the
6963 /// exact `None`-tolerant branch without needing a real utun/Wintun failure.
6964 #[tokio::test]
6965 async fn add_global_peer_with_no_host_adapter_skips_and_records() {
6966 let mut server = test_server();
6967 assert!(
6968 server.global_transport.is_none(),
6969 "fresh server has no host adapter (VM-only precondition)"
6970 );
6971 let pubkey = "k".to_string();
6972 let resp = server
6973 .handle(OverlaydRequest::AddPeer {
6974 peer: PeerSpec {
6975 public_key: pubkey.clone(),
6976 endpoint: "1.2.3.4:51820".to_string(),
6977 allowed_ips: "10.200.0.2/32".to_string(),
6978 persistent_keepalive_secs: 0,
6979 candidates: Vec::new(),
6980 },
6981 scope: PeerScope::Global,
6982 })
6983 .await;
6984 match resp {
6985 OverlaydResponse::Ok => {}
6986 other => panic!("expected Ok (warn-and-skip), got {other:?}"),
6987 }
6988 assert!(
6989 server.global_peers.contains_key(&pubkey),
6990 "Global peer must still be mirrored for guest-config push"
6991 );
6992 }
6993
6994 /// End-to-end Dedicated setup. Needs a real TUN device, so it is ignored by
6995 /// default and only runs on a privileged Linux host (mirrors the crate's
6996 /// other privileged overlay e2e tests).
6997 #[cfg(target_os = "linux")]
6998 #[tokio::test]
6999 #[ignore = "needs CAP_NET_ADMIN; run on a privileged Linux host"]
7000 async fn dedicated_setup_creates_distinct_device_and_routes_service_peer() {
7001 let mut server = test_server();
7002 // Bring up the global overlay first so the cluster CIDR + global device
7003 // exist (the dedicated device must get a distinct port and key).
7004 let global_name = server
7005 .setup_global_overlay(
7006 "dep".to_string(),
7007 "i0".to_string(),
7008 "10.200.0.0/16",
7009 Some("10.200.0.0/28"),
7010 zlayer_core::DEFAULT_WG_PORT,
7011 None,
7012 false,
7013 )
7014 .await
7015 .expect("global overlay up");
7016 assert!(!global_name.is_empty());
7017
7018 // Dedicated service setup.
7019 let info = server
7020 .setup_service_overlay("web", OverlayMode::Dedicated)
7021 .await
7022 .expect("dedicated service overlay up");
7023 assert_eq!(info.mode, OverlayMode::Dedicated);
7024 let port = info.wg_port.expect("dedicated port");
7025 assert_ne!(
7026 port, server.overlay_port,
7027 "dedicated device must not share the global port"
7028 );
7029
7030 let st = server
7031 .service_transports
7032 .get("web")
7033 .expect("service transport recorded");
7034 assert_eq!(st.listen_port, port);
7035 assert_ne!(
7036 st.interface, global_name,
7037 "dedicated interface must differ from global"
7038 );
7039 assert_eq!(
7040 Some(st.public_key.clone()),
7041 info.wg_public_key,
7042 "info pubkey matches recorded transport"
7043 );
7044 assert_ne!(
7045 Some(st.public_key.clone()),
7046 server.transport_public_key,
7047 "dedicated key must differ from global key"
7048 );
7049
7050 // A Service-scoped AddPeer must land on the dedicated device (succeeds),
7051 // proving scope routing targets the per-service transport.
7052 let resp = server
7053 .handle(OverlaydRequest::AddPeer {
7054 peer: PeerSpec {
7055 public_key: {
7056 let (_priv, pubk) = OverlayTransport::generate_keys().await.unwrap();
7057 pubk
7058 },
7059 endpoint: "5.6.7.8:51999".to_string(),
7060 allowed_ips: "10.201.0.2/32".to_string(),
7061 persistent_keepalive_secs: 25,
7062 candidates: Vec::new(),
7063 },
7064 scope: PeerScope::Service {
7065 service: "web".to_string(),
7066 },
7067 })
7068 .await;
7069 assert!(
7070 matches!(resp, OverlaydResponse::Ok),
7071 "service-scoped add_peer should land on the dedicated device, got {resp:?}"
7072 );
7073 }
7074
7075 #[tokio::test]
7076 async fn guest_attach_requires_global_overlay() {
7077 // Without a global overlay (no node public key / transport) a
7078 // guest-managed attach must error rather than allocate anything.
7079 let mut server = test_server();
7080 let resp = server
7081 .handle(OverlaydRequest::AttachContainer {
7082 handle: AttachHandle::GuestManaged {
7083 id: "vm-1".to_string(),
7084 },
7085 service: "web".to_string(),
7086 join_global: true,
7087 dns_server: None,
7088 dns_domain: None,
7089 ephemeral: false,
7090 isolation_network: None,
7091 })
7092 .await;
7093 match resp {
7094 OverlaydResponse::Err { message } => {
7095 assert!(
7096 message.contains("global overlay to be set up"),
7097 "got: {message}"
7098 );
7099 }
7100 other => panic!("expected Err response, got {other:?}"),
7101 }
7102 // Nothing was recorded.
7103 assert!(server.guest_attachments.is_empty());
7104 }
7105
7106 #[tokio::test]
7107 async fn detach_unknown_guest_is_idempotent() {
7108 let mut server = test_server();
7109 // No such guest -> Ok (idempotent), no panic.
7110 server
7111 .detach_container_guest("never-attached")
7112 .await
7113 .expect("detach of unknown guest is a no-op");
7114 }
7115
7116 /// Full guest-managed attach/detach round-trip. Needs a real TUN device (the
7117 /// global overlay must be live so the guest peer can be installed), so it is
7118 /// ignored by default and only runs on a privileged Linux host — mirrors the
7119 /// crate's other privileged overlay e2e tests.
7120 #[cfg(target_os = "linux")]
7121 #[tokio::test]
7122 #[ignore = "needs CAP_NET_ADMIN; run on a privileged Linux host"]
7123 async fn guest_attach_allocates_config_and_detach_releases() {
7124 let mut server = test_server();
7125 server
7126 .setup_global_overlay(
7127 "dep".to_string(),
7128 "i0".to_string(),
7129 "10.200.0.0/16",
7130 Some("10.200.0.0/28"),
7131 zlayer_core::DEFAULT_WG_PORT,
7132 None,
7133 false,
7134 )
7135 .await
7136 .expect("global overlay up");
7137
7138 // Seed a global peer so the guest config carries it through.
7139 let (_p, other_pub) = OverlayTransport::generate_keys().await.unwrap();
7140 let add = server
7141 .handle(OverlaydRequest::AddPeer {
7142 peer: PeerSpec {
7143 public_key: other_pub.clone(),
7144 endpoint: "9.9.9.9:51820".to_string(),
7145 allowed_ips: "10.200.1.0/28".to_string(),
7146 persistent_keepalive_secs: 25,
7147 candidates: Vec::new(),
7148 },
7149 scope: PeerScope::Global,
7150 })
7151 .await;
7152 assert!(
7153 matches!(add, OverlaydResponse::Ok),
7154 "seed peer add: {add:?}"
7155 );
7156
7157 let resp = server
7158 .handle(OverlaydRequest::AttachContainer {
7159 handle: AttachHandle::GuestManaged {
7160 id: "vm-1".to_string(),
7161 },
7162 service: "web".to_string(),
7163 join_global: true,
7164 dns_server: Some("10.200.0.1".parse().unwrap()),
7165 dns_domain: Some("overlay".to_string()),
7166 ephemeral: false,
7167 isolation_network: None,
7168 })
7169 .await;
7170 let config = match resp {
7171 OverlaydResponse::GuestConfig(c) => c,
7172 other => panic!("expected GuestConfig, got {other:?}"),
7173 };
7174 assert!(!config.private_key.is_empty());
7175 assert!(!config.public_key.is_empty());
7176 assert_ne!(config.private_key, config.public_key);
7177 assert_eq!(config.listen_port, server.overlay_port);
7178 assert_eq!(config.dns_server, Some("10.200.0.1".parse().unwrap()));
7179 // Peers = the seeded global peer + this node (self) + nothing else.
7180 assert!(
7181 config.peers.iter().any(|p| p.public_key == other_pub),
7182 "guest must learn the seeded global peer"
7183 );
7184 assert!(
7185 config
7186 .peers
7187 .iter()
7188 .any(|p| Some(&p.public_key) == server.transport_public_key.as_ref()),
7189 "guest must learn THIS node as a peer"
7190 );
7191 // The guest's own key is registered as a global peer (host route).
7192 assert!(server.global_peers.contains_key(&config.public_key));
7193 let info = server
7194 .guest_attachments
7195 .get("vm-1")
7196 .expect("attachment recorded");
7197 assert_eq!(info.overlay_ip, config.overlay_ip);
7198
7199 // Detach releases the peer + IP.
7200 let det = server
7201 .handle(OverlaydRequest::DetachContainer {
7202 handle: AttachHandle::GuestManaged {
7203 id: "vm-1".to_string(),
7204 },
7205 })
7206 .await;
7207 assert!(matches!(det, OverlaydResponse::Ok), "detach: {det:?}");
7208 assert!(!server.guest_attachments.contains_key("vm-1"));
7209 assert!(!server.global_peers.contains_key(&config.public_key));
7210 }
7211
7212 /// The `setup_service_overlay` dispatch must handle ALL THREE modes —
7213 /// including the default `Auto` — without panicking. `resolve()` is now the
7214 /// identity, so the old `unreachable!("resolve never returns Auto")` arm
7215 /// would panic on the default mode; this proves the arm is gone. Each mode
7216 /// is recorded in `service_modes` BEFORE any netlink/transport work, so we
7217 /// assert on that deterministically regardless of host privilege (the
7218 /// downstream bridge/transport bring-up may succeed or fail depending on
7219 /// `CAP_NET_ADMIN`, but it must never panic).
7220 #[cfg(target_os = "linux")]
7221 #[tokio::test]
7222 async fn dispatch_handles_all_three_modes_without_panic() {
7223 for mode in [
7224 OverlayMode::Auto,
7225 OverlayMode::Shared,
7226 OverlayMode::Dedicated,
7227 ] {
7228 let mut server = test_server();
7229 let service = format!("svc-{mode:?}");
7230 // Must return a Result (Ok or Err) — never panic via `unreachable!`.
7231 let _ = server.setup_service_overlay(&service, mode).await;
7232 // The resolved mode is recorded up front for the attach path.
7233 assert_eq!(
7234 server.service_modes.get(&service).copied(),
7235 Some(mode.resolve()),
7236 "mode {mode:?} must be recorded for the attach path"
7237 );
7238 }
7239 }
7240
7241 /// Two distinct `Shared` services must reuse the SAME node-wide shared
7242 /// bridge (one bridge, not two), while an `Auto` service gets its OWN
7243 /// per-service bridge. Needs `CAP_NET_ADMIN` to create the bridges, so it is
7244 /// ignored by default like the crate's other privileged overlay e2e tests.
7245 #[cfg(target_os = "linux")]
7246 #[tokio::test]
7247 #[ignore = "needs CAP_NET_ADMIN; run on a privileged Linux host"]
7248 async fn shared_services_reuse_one_bridge_auto_gets_its_own() {
7249 let mut server = test_server();
7250 server
7251 .setup_global_overlay(
7252 "dep".to_string(),
7253 "i0".to_string(),
7254 "10.200.0.0/16",
7255 Some("10.200.0.0/26"),
7256 zlayer_core::DEFAULT_WG_PORT,
7257 None,
7258 false,
7259 )
7260 .await
7261 .expect("global overlay up");
7262
7263 // First Shared service creates the shared bridge.
7264 let info_a = server
7265 .setup_service_overlay("web", OverlayMode::Shared)
7266 .await
7267 .expect("shared service web up");
7268 assert_eq!(info_a.mode, OverlayMode::Shared);
7269 let shared_name = server
7270 .shared_bridge
7271 .as_ref()
7272 .expect("shared bridge created")
7273 .name
7274 .clone();
7275 assert_eq!(info_a.name, shared_name);
7276 // Shared services are NOT per-service bridges.
7277 assert!(
7278 !server.service_bridges.contains_key("web"),
7279 "Shared service must not create a per-service bridge"
7280 );
7281
7282 // Second Shared service REUSES the same shared bridge — no new bridge.
7283 let info_b = server
7284 .setup_service_overlay("api", OverlayMode::Shared)
7285 .await
7286 .expect("shared service api up");
7287 assert_eq!(
7288 info_b.name, shared_name,
7289 "a second Shared service must reuse the SAME node-wide bridge"
7290 );
7291 assert!(!server.service_bridges.contains_key("api"));
7292 // Still exactly one shared bridge object.
7293 assert_eq!(
7294 server.shared_bridge.as_ref().map(|b| b.name.clone()),
7295 Some(shared_name.clone())
7296 );
7297
7298 // An Auto service gets its OWN per-service bridge, distinct from the
7299 // shared bridge.
7300 let info_c = server
7301 .setup_service_overlay("batch", OverlayMode::Auto)
7302 .await
7303 .expect("auto service batch up");
7304 assert_eq!(info_c.mode, OverlayMode::Auto);
7305 assert!(
7306 server.service_bridges.contains_key("batch"),
7307 "Auto service must get its own per-service bridge"
7308 );
7309 assert_ne!(
7310 info_c.name, shared_name,
7311 "Auto per-service bridge must differ from the shared bridge"
7312 );
7313
7314 // Both Shared services point their service_interfaces entry at the one
7315 // shared bridge; the Auto service points at its own.
7316 assert_eq!(server.service_interfaces.get("web"), Some(&shared_name));
7317 assert_eq!(server.service_interfaces.get("api"), Some(&shared_name));
7318 assert_ne!(server.service_interfaces.get("batch"), Some(&shared_name));
7319 }
7320
7321 /// A `Shared` service's container attach must draw its IP from the shared
7322 /// bridge pool and must fail cleanly (no panic, clear error) when the shared
7323 /// bridge has not been set up yet. Unprivileged: exercises only the
7324 /// pre-netlink resolution branch.
7325 #[cfg(target_os = "linux")]
7326 #[tokio::test]
7327 async fn attach_shared_without_setup_errors_cleanly() {
7328 let mut server = test_server();
7329 // Mark the service Shared but never set up the shared bridge.
7330 server
7331 .service_modes
7332 .insert("web".to_string(), OverlayMode::Shared);
7333 let err = server
7334 .attach_container_linux(424_242, "web", false, false, None)
7335 .await
7336 .expect_err("attach must fail without a shared bridge");
7337 match err {
7338 OverlaydError::Other(m) => {
7339 assert!(
7340 m.contains("no shared bridge"),
7341 "expected shared-bridge error, got: {m}"
7342 );
7343 }
7344 other => panic!("unexpected error variant: {other:?}"),
7345 }
7346 }
7347
7348 /// A container attached on a NAMED isolated network must be recorded in the
7349 /// per-network membership map (`network_members["net-a"]` gains the member's
7350 /// service IP). Needs `CAP_NET_ADMIN` to bring up the bridge + veth, so it is
7351 /// ignored by default like the crate's other privileged overlay e2e tests.
7352 #[cfg(target_os = "linux")]
7353 #[tokio::test]
7354 #[ignore = "needs CAP_NET_ADMIN; run on a privileged Linux host"]
7355 async fn attach_linux_isolated_network_records_membership() {
7356 let mut server = test_server();
7357 server
7358 .setup_global_overlay(
7359 "dep".to_string(),
7360 "i0".to_string(),
7361 "10.200.0.0/16",
7362 Some("10.200.0.0/26"),
7363 zlayer_core::DEFAULT_WG_PORT,
7364 None,
7365 false,
7366 )
7367 .await
7368 .expect("global overlay up");
7369
7370 // An Auto service gives us a real per-service bridge to attach onto.
7371 server
7372 .setup_service_overlay("web", OverlayMode::Auto)
7373 .await
7374 .expect("auto service web up");
7375
7376 // Attach this very process (a live PID with a real netns) onto the named
7377 // isolated network "net-a".
7378 let pid = std::process::id();
7379 let ip = server
7380 .attach_container_linux(pid, "web", false, true, Some("net-a".to_string()))
7381 .await
7382 .expect("attach onto isolated network");
7383
7384 // Membership map gained exactly this member under "net-a".
7385 let members = server
7386 .network_members
7387 .get("net-a")
7388 .expect("net-a membership recorded");
7389 assert!(
7390 members.contains(&ip),
7391 "network_members[net-a] must contain the attached member IP {ip}"
7392 );
7393
7394 // Detach drains the membership and drops the now-empty network entry.
7395 server
7396 .detach_container_linux(pid)
7397 .await
7398 .expect("detach succeeds");
7399 assert!(
7400 !server.network_members.contains_key("net-a"),
7401 "empty isolated network must be dropped from network_members on last detach"
7402 );
7403 }
7404
7405 /// The isolation-network owner key namespace is distinct from the dedicated
7406 /// per-service namespace, so an isolation network and a service of the same
7407 /// name never collide on the same marker/allocator key. Platform-agnostic.
7408 #[test]
7409 fn isolation_owner_key_distinct_from_service_owner_key() {
7410 let iso = crate::network_state::owner_for_isolation_network("alpha");
7411 let svc = crate::network_state::owner_for_service("alpha");
7412 assert_ne!(
7413 iso, svc,
7414 "isolation and service owner keys must not collide for the same name"
7415 );
7416 assert_eq!(iso, "iso:alpha");
7417 assert_eq!(svc, "service:alpha");
7418 }
7419
7420 /// `isolation_network_subnet` is deterministic (same name -> same block so a
7421 /// reused HCN network keeps its subnet across restarts), stays INSIDE the
7422 /// node slice, and lands DIFFERENT isolation networks on DISJOINT sub-blocks
7423 /// (the whole point of L3 isolation — distinct networks must not share an
7424 /// address range). Windows-only (the method is `cfg(windows)`); exercised by
7425 /// `cargo xwin test`.
7426 #[cfg(target_os = "windows")]
7427 #[test]
7428 fn isolation_network_subnet_is_deterministic_disjoint_and_inside_slice() {
7429 let mut server = test_server();
7430 let slice: IpNetwork = "10.200.5.0/26".parse().unwrap();
7431 server.slice_cidr = Some(slice);
7432 let slice_net: ipnet::IpNet = "10.200.5.0/26".parse().unwrap();
7433
7434 // Deterministic: same name -> same block on repeated calls.
7435 let a1 = server.isolation_network_subnet("alpha").unwrap();
7436 let a2 = server.isolation_network_subnet("alpha").unwrap();
7437 assert_eq!(a1, a2, "same isolation network must map to the same subnet");
7438
7439 // Inside the node slice and at the /28 sub-prefix.
7440 assert!(
7441 slice_net.contains(&a1.network()) && slice_net.contains(&a1.broadcast()),
7442 "isolation subnet {a1} must be wholly inside the node slice {slice_net}"
7443 );
7444 assert_eq!(a1.prefix_len(), 28, "expected a /28 isolation sub-block");
7445
7446 // A different network name carving a different /28 block must be disjoint.
7447 // (`beta` and `gamma` hash to different indices than `alpha`; pick whichever
7448 // of several names lands on a distinct block to assert disjointness.)
7449 let other = ["beta", "gamma", "delta", "omega", "zeta"]
7450 .iter()
7451 .map(|n| server.isolation_network_subnet(n).unwrap())
7452 .find(|s| *s != a1)
7453 .expect("at least one other name must land on a different /28 block");
7454 let overlaps = a1.contains(&other.network()) || other.contains(&a1.network());
7455 assert!(
7456 !overlaps,
7457 "distinct isolation networks must occupy disjoint subnets ({a1} vs {other})"
7458 );
7459 }
7460}