zlayer-overlay 0.13.0

Encrypted overlay networking for containers using boringtun userspace WireGuard
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
//! Inbound firewall-rule management for the overlay + API + Raft ports.
//!
//! On Windows this module installs three inbound-allow rules in Windows
//! Defender Firewall via the `INetFwPolicy2` COM API:
//!
//! - `ZLayer Overlay (UDP)` — the Wintun/boringtun listen port
//! - `ZLayer API (TCP)`     — the daemon HTTP/gRPC port
//! - `ZLayer Raft (TCP)`    — the scheduler Raft port
//!
//! Rules are scoped to the **Private + Domain** profiles only. Public profile
//! is intentionally excluded so laptops on untrusted networks (coffee-shop
//! Wi-Fi, airport, etc.) do not start accepting inbound cluster traffic.
//!
//! [`ensure_overlay_rules`] is idempotent: if a rule with the same name
//! already exists it is left in place rather than duplicated.
//!
//! On non-Windows targets both functions are no-ops that return `Ok(())`.
//! Linux nodes are expected to manage their own `iptables`/`nftables` or
//! `firewalld` state out-of-band, and macOS has its own model (`pfctl` /
//! Application Firewall) that isn't in scope for this phase.

use std::net::IpAddr;

use thiserror::Error;

/// Errors produced while installing or removing Windows firewall rules.
#[derive(Error, Debug)]
pub enum FirewallError {
    /// A COM call failed. Includes the underlying `HRESULT` message.
    #[error("Windows COM call failed: {0}")]
    Com(String),

    /// `CoInitializeEx` returned a failure status.
    #[error("CoInitializeEx failed: {0}")]
    ComInit(String),

    /// The `INetFwPolicy2` interface could not be instantiated.
    #[error("INetFwPolicy2 not available: {0}")]
    PolicyUnavailable(String),

    /// Adding a firewall rule failed. Includes the rule name.
    #[error("Failed to add firewall rule '{name}': {reason}")]
    AddRule {
        /// Display name of the rule that could not be created.
        name: String,
        /// Underlying error message from the Windows API.
        reason: String,
    },

    /// Removing a firewall rule failed. Includes the rule name.
    #[error("Failed to remove firewall rule '{name}': {reason}")]
    RemoveRule {
        /// Display name of the rule that could not be removed.
        name: String,
        /// Underlying error message from the Windows API.
        reason: String,
    },

    /// A string could not be converted to the `BSTR` / wide-string form
    /// required by the Windows COM API.
    #[error("String conversion failed: {0}")]
    StringConversion(String),
}

#[cfg(windows)]
mod windows;

#[cfg(target_os = "linux")]
mod linux;

#[cfg(target_os = "macos")]
mod macos;

/// Display name of the inbound overlay (`WireGuard` UDP) firewall rule.
pub const OVERLAY_RULE_NAME: &str = "ZLayer Overlay (UDP)";

/// Display name of the inbound API (HTTP/gRPC TCP) firewall rule.
pub const API_RULE_NAME: &str = "ZLayer API (TCP)";

/// Display name of the inbound Raft (TCP) firewall rule.
pub const RAFT_RULE_NAME: &str = "ZLayer Raft (TCP)";

/// All three rule names that this module manages, in the order they are
/// installed / removed.
pub const MANAGED_RULE_NAMES: &[&str] = &[OVERLAY_RULE_NAME, API_RULE_NAME, RAFT_RULE_NAME];

/// Ensure the three inbound allow-rules exist in Windows Defender Firewall
/// for the overlay UDP, API TCP, and Raft TCP ports.
///
/// Idempotent: if a rule with the expected name already exists it is left
/// untouched. Rules are scoped to the Private + Domain profiles only.
///
/// On non-Windows targets this is a no-op that returns `Ok(())`.
///
/// # Arguments
///
/// * `wg_port`   — UDP inbound port for the overlay (boringtun)
/// * `api_port`  — TCP inbound port for the daemon API
/// * `raft_port` — TCP inbound port for the Raft scheduler
///
/// # Errors
///
/// Returns a [`FirewallError`] if COM initialization fails, the
/// `INetFwPolicy2` service is unavailable, or the Windows Firewall API
/// rejects a rule creation (typically because the process lacks
/// administrator privileges). On non-Windows targets this cannot fail.
pub fn ensure_overlay_rules(
    wg_port: u16,
    api_port: u16,
    raft_port: u16,
) -> Result<(), FirewallError> {
    #[cfg(target_os = "linux")]
    {
        self::linux::ensure_overlay_rules(wg_port, api_port, raft_port)
    }
    #[cfg(target_os = "macos")]
    {
        self::macos::ensure_overlay_rules(wg_port, api_port, raft_port)
    }
    #[cfg(windows)]
    {
        self::windows::ensure_overlay_rules(wg_port, api_port, raft_port)
    }
    #[cfg(not(any(target_os = "linux", target_os = "macos", windows)))]
    {
        let _ = (wg_port, api_port, raft_port);
        Ok(())
    }
}

/// Ensure a single dynamically-published host port is allowed inbound.
///
/// Used by the L4 proxy when an `OverlayMode::Shared` service publishes a
/// free host port (`host:FREEPORT -> container_ip:port`): on a default-deny
/// host the published port must be opened so peers on other nodes can reach
/// the proxied service. `udp` selects the transport (UDP when true, else TCP).
///
/// Idempotent and best-effort, with the same non-fatal contract as
/// [`ensure_overlay_rules`]. No-op on targets without a firewall backend.
///
/// # Errors
///
/// Returns a [`FirewallError`] only when the platform backend reports an
/// unexpected failure (see the per-OS modules); a missing-privilege case is
/// downgraded to a warning by the backend.
pub fn ensure_published_port(port: u16, udp: bool) -> Result<(), FirewallError> {
    #[cfg(target_os = "linux")]
    {
        self::linux::ensure_published_port(port, udp)
    }
    #[cfg(target_os = "macos")]
    {
        self::macos::ensure_published_port(port, udp)
    }
    #[cfg(windows)]
    {
        self::windows::ensure_published_port(port, udp)
    }
    #[cfg(not(any(target_os = "linux", target_os = "macos", windows)))]
    {
        let _ = (port, udp);
        Ok(())
    }
}

/// Remove the inbound allow-rule for a previously-published host port (the
/// counterpart of [`ensure_published_port`]). Safe to call when nothing is
/// installed; every backend tolerates a missing rule. No-op on targets without
/// a firewall backend.
pub fn remove_published_port(port: u16, udp: bool) {
    #[cfg(target_os = "linux")]
    {
        self::linux::remove_published_port(port, udp);
    }
    #[cfg(target_os = "macos")]
    {
        self::macos::remove_published_port(port, udp);
    }
    #[cfg(windows)]
    {
        self::windows::remove_published_port(port, udp);
    }
    #[cfg(not(any(target_os = "linux", target_os = "macos", windows)))]
    {
        let _ = (port, udp);
    }
}

/// Remove any ZLayer-managed inbound firewall rules that this module would
/// otherwise install.
///
/// Safe to call when the rules do not exist — missing rules are treated as
/// a successful no-op. On non-Windows targets this is a no-op that returns
/// `Ok(())`.
///
/// # Errors
///
/// Returns a [`FirewallError`] if COM initialization fails, the
/// `INetFwPolicy2` service is unavailable, or the Windows Firewall API
/// rejects the remove call. "Rule not found" is not treated as an error.
/// On non-Windows targets this cannot fail.
pub fn remove_overlay_rules() -> Result<(), FirewallError> {
    #[cfg(target_os = "linux")]
    {
        self::linux::remove_overlay_rules();
        Ok(())
    }
    #[cfg(target_os = "macos")]
    {
        self::macos::remove_overlay_rules()
    }
    #[cfg(windows)]
    {
        self::windows::remove_overlay_rules()
    }
    #[cfg(not(any(target_os = "linux", target_os = "macos", windows)))]
    {
        Ok(())
    }
}

/// Ensure the host firewall permits overlay traffic to/from `overlay_cidr`
/// (e.g. the cluster CIDR `10.200.0.0/16`).
///
/// On a default-deny Linux host (UFW / firewalld / `iptables -P FORWARD DROP`),
/// a container's DNS query to the node overlay IP — and inter-service overlay
/// traffic — is dropped before it reaches `ZLayer`'s resolver. This installs a
/// dedicated `ZLAYER-OVERLAY` chain (jumped from the top of `INPUT`/`FORWARD`)
/// that ACCEPTs the overlay CIDR, so `ZLayer`'s own DNS and service-to-service
/// networking work without the operator hand-authorising it. Idempotent.
///
/// On non-Linux targets this is a no-op that returns `Ok(())` (Windows manages
/// per-port inbound rules via [`ensure_overlay_rules`]; macOS is out of scope).
///
/// # Errors
///
/// Returns a [`FirewallError`] only when the `iptables`/`ip6tables` binary
/// cannot be spawned or rejects a rule. Callers should treat a failure as
/// non-fatal (log + continue) — a restricted environment without `iptables`
/// must not abort overlay setup.
pub fn ensure_overlay_subnet_rules(overlay_cidr: &str) -> Result<(), FirewallError> {
    #[cfg(target_os = "linux")]
    {
        self::linux::ensure_overlay_subnet_rules(overlay_cidr)
    }
    #[cfg(not(target_os = "linux"))]
    {
        let _ = overlay_cidr;
        Ok(())
    }
}

/// Remove the `ZLayer`-managed overlay-subnet firewall chain (the counterpart
/// of [`ensure_overlay_subnet_rules`]). Safe to call when nothing is installed;
/// missing rules are tolerated. No-op on non-Linux targets.
pub fn remove_overlay_subnet_rules() {
    #[cfg(target_os = "linux")]
    self::linux::remove_overlay_subnet_rules();
}

/// Ensure overlay-sourced traffic is SNAT'd (masqueraded) when it egresses a
/// non-overlay interface toward the LAN/internet, for the cluster `overlay_cidr`
/// (e.g. `10.200.0.0/16`).
///
/// The host filter-table ACCEPTs (see [`ensure_overlay_subnet_rules`]) plus
/// `ip_forward=1` get an overlay packet *forwarded* out the WAN NIC — but with
/// its private overlay source address, so replies can never route back and every
/// outbound connection from an overlay container hangs. This installs a
/// `nat`-table masquerade (a dedicated `ZLAYER-OVERLAY-NAT` chain jumped from
/// `POSTROUTING`) that rewrites the source to the host address for traffic
/// leaving the overlay, while leaving intra-overlay (`zl-*`) forwarding alone.
/// Idempotent.
///
/// On non-Linux targets this is a no-op that returns `Ok(())` (macOS/Windows use
/// their own NAT models, out of scope here).
///
/// # Errors
///
/// Returns a [`FirewallError`] only when the `iptables`/`ip6tables` binary
/// cannot be spawned or rejects a rule. Callers should treat a failure as
/// non-fatal (log + continue) — a restricted environment without `iptables` must
/// not abort overlay setup.
pub fn ensure_overlay_masquerade(overlay_cidr: &str) -> Result<(), FirewallError> {
    #[cfg(target_os = "linux")]
    {
        self::linux::ensure_overlay_masquerade(overlay_cidr)
    }
    #[cfg(not(target_os = "linux"))]
    {
        let _ = overlay_cidr;
        Ok(())
    }
}

/// Remove the `ZLayer`-managed overlay egress masquerade (the counterpart of
/// [`ensure_overlay_masquerade`]). Safe to call when nothing is installed;
/// missing rules are tolerated. No-op on non-Linux targets.
pub fn remove_overlay_masquerade() {
    #[cfg(target_os = "linux")]
    self::linux::remove_overlay_masquerade();
}

/// Remove the `ZLayer`-managed per-member L3-isolation chain
/// (`ZLAYER-OVERLAY-ISO`) and its `FORWARD` jump — the global-teardown
/// counterpart of [`ensure_member_isolation`].
///
/// [`remove_member_isolation`] intentionally leaves the chain and jump resident
/// because other members may still rely on them; this removes the whole chain on
/// a full overlay teardown so nothing leaks. Safe to call when nothing is
/// installed; missing rules are tolerated. No-op on non-Linux targets (macOS uses
/// a node-side pf sub-anchor and Windows a per-network HCN vSwitch, each torn
/// down by their own member-removal path).
pub fn remove_overlay_isolation() {
    #[cfg(target_os = "linux")]
    self::linux::remove_overlay_isolation();
}

/// Install Docker-style per-network L3 isolation for one overlay member.
///
/// `member_ip` may reach each address in `peers` (bidirectionally), egress
/// (LAN/internet), and the daemon `node_ip` — but NOT other networks' members or
/// arbitrary cluster IPs within `overlay_cidr` (e.g. `10.200.0.0/16`). On Linux
/// this is enforced via a dedicated `ZLAYER-OVERLAY-ISO` filter chain jumped from
/// the TOP of `FORWARD` (above the blanket overlay accept), holding top-inserted
/// `RETURN` allows for the peers/node and an appended `-d <overlay_cidr> -j DROP`
/// catch-all. On macOS the node enforces the same policy on hairpinned VZ-guest
/// traffic via a per-`network` `pf` table + sub-anchor.
///
/// `network` is the overlay network name: it keys the macOS `pf` table/anchor
/// (one per isolated network) and is unused by the Linux backend (the pairwise
/// rules already isolate). Idempotent.
///
/// On targets other than Linux/macOS this is a no-op that returns `Ok(())`
/// (Windows uses its own networking model, out of scope here).
///
/// # Errors
///
/// Returns a [`FirewallError`] only when the `iptables` / `pfctl` binary cannot
/// be spawned or rejects a rule. Callers should treat a failure as non-fatal
/// (log and continue) — a restricted environment without the firewall binary
/// must not abort overlay setup.
pub fn ensure_member_isolation(
    network: &str,
    member_ip: IpAddr,
    peers: &[IpAddr],
    node_ip: IpAddr,
    overlay_cidr: &str,
) -> Result<(), FirewallError> {
    #[cfg(target_os = "linux")]
    {
        self::linux::ensure_member_isolation(network, member_ip, peers, node_ip, overlay_cidr)
    }
    #[cfg(target_os = "macos")]
    {
        self::macos::ensure_member_isolation(network, member_ip, peers, node_ip, overlay_cidr)
    }
    #[cfg(not(any(target_os = "linux", target_os = "macos")))]
    {
        let _ = (network, member_ip, peers, node_ip, overlay_cidr);
        Ok(())
    }
}

/// Remove the per-member L3-isolation rules installed by
/// [`ensure_member_isolation`] for `member_ip` on `network`. Best-effort and safe
/// to call when nothing is installed; missing rules are tolerated. No-op on
/// targets other than Linux/macOS.
pub fn remove_member_isolation(
    network: &str,
    member_ip: IpAddr,
    peers: &[IpAddr],
    node_ip: IpAddr,
    overlay_cidr: &str,
) {
    #[cfg(target_os = "linux")]
    {
        self::linux::remove_member_isolation(network, member_ip, peers, node_ip, overlay_cidr);
    }
    #[cfg(target_os = "macos")]
    {
        self::macos::remove_member_isolation(network, member_ip, peers, node_ip, overlay_cidr);
    }
    #[cfg(not(any(target_os = "linux", target_os = "macos")))]
    {
        let _ = (network, member_ip, peers, node_ip, overlay_cidr);
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    /// The cross-platform [`remove_overlay_isolation`] wrapper is invoked
    /// unconditionally on a full overlay teardown (see
    /// `OverlaydServer::teardown_global_overlay`). It must be safe to call when
    /// no isolation state was ever installed: a no-op on non-Linux targets, and
    /// a tolerant best-effort cleanup on Linux (each `iptables` step swallows its
    /// exit status). Always-run, non-root: on a box lacking `iptables` or
    /// privileges the Linux path simply fails each step silently and returns. The
    /// regression this guards: teardown must never panic or abort because the ISO
    /// chain is absent, and it must be idempotent across repeated calls.
    #[test]
    fn remove_overlay_isolation_wrapper_is_idempotent_and_panic_free() {
        remove_overlay_isolation();
        remove_overlay_isolation();
    }

    /// [`remove_member_isolation`] is the per-member counterpart, also called on
    /// the detach/teardown path. Removing isolation for a member whose rules were
    /// never installed must be a tolerant no-op (missing-rule exit codes are
    /// swallowed), never a panic — exercised here with throwaway addresses on a
    /// throwaway network name so no production chain is touched.
    #[test]
    fn remove_member_isolation_no_state_is_panic_free() {
        let member: IpAddr = "10.200.99.2".parse().expect("valid member ip");
        let node: IpAddr = "10.200.0.1".parse().expect("valid node ip");
        let peers: [IpAddr; 1] = ["10.200.99.3".parse().expect("valid peer ip")];
        // No ensure_member_isolation was ever called for this (network, member),
        // so every probe/delete misses — must stay a clean no-op, twice over.
        remove_member_isolation(
            "zl-test-never-installed",
            member,
            &peers,
            node,
            "10.200.0.0/16",
        );
        remove_member_isolation(
            "zl-test-never-installed",
            member,
            &peers,
            node,
            "10.200.0.0/16",
        );
    }
}