isb_core/egress/
plumb.rs

1//! The incus side of a sandbox's egress policy.
2//!
3//! Each sandbox with an `egress:` policy gets a bridge network of its own
4//! (`isbbrx<hash>`) and a network ACL, and its NIC is attached to both:
5//!
6//! - the bridge does not NAT or route, so the host forwards nothing for it;
7//! - the ACL's default egress action is `drop`, and its one allow rule is
8//!   TCP to the bridge's own address on the ports the policy uses, where the
9//!   egress proxy listens;
10//! - the bridge's dnsmasq has no upstream and answers only the allowed
11//!   names (with the bridge address, so the guest connects to the proxy);
12//!   with `egress: none` it does no DNS at all.
13//!
14//! Nothing here needs root: incusd owns the bridge, the ACL and dnsmasq.
15
16use std::collections::BTreeMap;
17
18use serde_json::{Value, json};
19
20use super::policy::Policy;
21use crate::client::{Client, encode_segment};
22use crate::error::{Error, Result};
23
24/// Egress bridges are named `isbbrx<8 hex>`, under the `isbbr+` pattern
25/// `isb host setup` opens DHCP and DNS for, and the `isbbrx+` pattern it
26/// opens the proxy's ports on.
27pub const NET_PREFIX: &str = "isbbrx";
28/// Marks the bridge, ACL and instance as one sandbox's egress: `project/instance`.
29pub const KEY_FOR: &str = "user.isb.egress-for";
30/// The policy as JSON, on the bridge (what the proxy enforces) and the instance.
31pub const KEY_POLICY: &str = "user.isb.egress";
32/// When the bridge was made (unix seconds): garbage collection leaves a young one alone.
33pub const KEY_CREATED: &str = "user.isb.egress-created";
34/// The fingerprint of the CA the guest trusts, once it does.
35pub const KEY_CA_INSTALLED: &str = "user.isb.egress-ca";
36
37pub type Props = BTreeMap<String, String>;
38
39fn digest8(project: &str, instance: &str) -> String {
40    let d = ring::digest::digest(
41        &ring::digest::SHA256,
42        format!("isb-egress\0{project}\0{instance}").as_bytes(),
43    );
44    d.as_ref()[..4].iter().map(|b| format!("{b:02x}")).collect()
45}
46
47/// The bridge for sandbox `instance` in incus project `project`.
48pub fn network_name(project: &str, instance: &str) -> String {
49    format!("{NET_PREFIX}{}", digest8(project, instance))
50}
51
52/// The ACL for the same sandbox.
53pub fn acl_name(project: &str, instance: &str) -> String {
54    format!("isbx-{}", digest8(project, instance))
55}
56
57/// Everything about one sandbox's egress plumbing.
58#[derive(Debug, Clone, PartialEq)]
59pub struct Plumbing {
60    pub project: String,
61    pub instance: String,
62    pub network: String,
63    pub acl: String,
64    pub policy: Policy,
65}
66
67impl Plumbing {
68    pub fn new(project: &str, instance: &str, policy: Policy) -> Plumbing {
69        Plumbing {
70            project: project.to_string(),
71            instance: instance.to_string(),
72            network: network_name(project, instance),
73            acl: acl_name(project, instance),
74            policy,
75        }
76    }
77
78    /// `project/instance`, the value of [`KEY_FOR`].
79    pub fn owner(&self) -> String {
80        format!("{}/{}", self.project, self.instance)
81    }
82
83    /// The instance's NIC: the egress bridge, behind the ACL.
84    pub fn nic(&self) -> Props {
85        [
86            ("type", "nic"),
87            ("name", "eth0"),
88            ("network", self.network.as_str()),
89            ("security.acls", self.acl.as_str()),
90            ("security.acls.default.egress.action", "drop"),
91            ("security.acls.default.ingress.action", "allow"),
92        ]
93        .into_iter()
94        .map(|(k, v)| (k.to_string(), v.to_string()))
95        .collect()
96    }
97
98    /// The policy as stored in `user.isb.egress`.
99    pub fn policy_json(&self) -> String {
100        let mut v = serde_json::to_value(&self.policy).expect("a policy serializes");
101        v["project"] = json!(self.project);
102        v["instance"] = json!(self.instance);
103        v.to_string()
104    }
105
106    /// dnsmasq's extra configuration: no upstream, no hosts file, and an
107    /// answer (the bridge address) only for the allowed names. Everything
108    /// else is NXDOMAIN, and nothing is ever forwarded.
109    pub fn dnsmasq(&self, ip: &str) -> String {
110        if self.policy.is_none() {
111            return String::new();
112        }
113        let mut lines = vec![
114            "no-resolv".to_string(),
115            "no-hosts".into(),
116            "address=/#/".into(),
117        ];
118        lines.extend(
119            self.policy
120                .dns_names()
121                .into_iter()
122                .map(|n| format!("address=/{n}/{ip}")),
123        );
124        lines.join("\n")
125    }
126
127    /// The bridge's config. `ip` is the address incus picked for it. With
128    /// no hosts allowed the bridge has no address at all: nothing for the
129    /// guest to reach, not even a DNS port.
130    pub fn network_config(&self, ip: Option<&str>, created: u64) -> Props {
131        let mut c: Props = [
132            (
133                "ipv4.address",
134                if self.policy.is_none() {
135                    "none"
136                } else {
137                    "auto"
138                },
139            ),
140            ("ipv4.nat", "false"),
141            ("ipv4.routing", "false"),
142            ("ipv6.address", "none"),
143        ]
144        .into_iter()
145        .map(|(k, v)| (k.to_string(), v.to_string()))
146        .collect();
147        c.insert(KEY_FOR.into(), self.owner());
148        c.insert(KEY_POLICY.into(), self.policy_json());
149        c.insert(KEY_CREATED.into(), created.to_string());
150        if let (false, Some(ip)) = (self.policy.is_none(), ip) {
151            c.insert("raw.dnsmasq".into(), self.dnsmasq(ip));
152        }
153        c
154    }
155
156    /// The ACL body: TCP to the bridge address on the policy's ports, and
157    /// nothing else (the default action drops the rest).
158    pub fn acl_body(&self, ip: &str) -> Value {
159        let ports: Vec<String> = self.policy.ports().iter().map(u16::to_string).collect();
160        let egress = if ports.is_empty() {
161            json!([])
162        } else {
163            json!([{
164                "action": "allow",
165                "state": "enabled",
166                "protocol": "tcp",
167                "destination": format!("{ip}/32"),
168                "destination_port": ports.join(","),
169                "description": "the egress proxy",
170            }])
171        };
172        json!({
173            "description": format!("isb egress for {}", self.owner()),
174            "egress": egress,
175            "ingress": [],
176            "config": {KEY_FOR: self.owner()},
177        })
178    }
179}
180
181fn host(base: &Client) -> Client {
182    base.clone().project("default")
183}
184
185/// The address of a bridge, from its `ipv4.address` (`10.1.2.1/24`).
186pub fn bridge_ip(net: &Value) -> Option<String> {
187    net["config"]["ipv4.address"]
188        .as_str()
189        .and_then(|a| a.split('/').next())
190        .filter(|a| a.parse::<std::net::Ipv4Addr>().is_ok())
191        .map(String::from)
192}
193
194fn now() -> u64 {
195    std::time::SystemTime::now()
196        .duration_since(std::time::UNIX_EPOCH)
197        .map(|d| d.as_secs())
198        .unwrap_or(0)
199}
200
201/// Make the bridge and the ACL, or bring existing ones to `p`. Idempotent;
202/// the instance's NIC can refer to them once this returns. Returns the
203/// bridge address the proxy listens on.
204pub fn prepare(client: &Client, p: &Plumbing, report: &mut dyn FnMut(&str)) -> Result<String> {
205    let h = host(client);
206    let t = h.get_timeouts().other;
207    let net_path = format!("/1.0/networks/{}", encode_segment(&p.network));
208    let mut net = h.get_opt(&net_path)?;
209    if let Some(n) = &net {
210        let owner = n["config"][KEY_FOR].as_str().unwrap_or_default();
211        if owner != p.owner() {
212            return Err(Error::invalid(format!(
213                "network {} exists and is not the egress network of {}",
214                p.network,
215                p.owner()
216            )));
217        }
218    } else {
219        report(&format!(
220            "{}: creating egress network {}",
221            p.instance, p.network
222        ));
223        let body = json!({
224            "name": p.network,
225            "type": "bridge",
226            "description": format!("isb egress for {}", p.owner()),
227            "config": p.network_config(None, now()),
228        });
229        h.mutate(
230            "POST",
231            "/1.0/networks",
232            Some(&body),
233            &format!("create network {}", p.network),
234            t,
235        )?;
236        net = Some(h.get(&net_path)?);
237    }
238    let mut net = net.expect("set above");
239    let none = p.policy.is_none();
240    if !none && bridge_ip(&net).is_none() {
241        // It was `egress: none`, with no address: give the bridge one.
242        h.mutate(
243            "PATCH",
244            &net_path,
245            Some(&json!({"config": {"ipv4.address": "auto"}})),
246            &format!("give network {} an address", p.network),
247            t,
248        )?;
249        net = h.get(&net_path)?;
250    }
251    let ip = match bridge_ip(&net) {
252        Some(ip) => ip,
253        None if none => String::new(),
254        None => {
255            return Err(Error::invalid(format!(
256                "network {} has no IPv4 address",
257                p.network
258            )));
259        }
260    };
261    // Bring the config to the policy: only keys that differ are written.
262    let mut want = p.network_config(Some(&ip), now());
263    want.remove(KEY_CREATED);
264    if !none {
265        want.remove("ipv4.address");
266    }
267    let have = &net["config"];
268    let mut cfg = have.clone();
269    let mut changed = false;
270    for (k, v) in &want {
271        if have[k].as_str() != Some(v.as_str()) {
272            cfg[k] = json!(v);
273            changed = true;
274        }
275    }
276    if !want.contains_key("raw.dnsmasq") && have.get("raw.dnsmasq").is_some() {
277        cfg.as_object_mut()
278            .expect("config is a map")
279            .remove("raw.dnsmasq");
280        changed = true;
281    }
282    if changed {
283        report(&format!(
284            "{}: updating egress network {}",
285            p.instance, p.network
286        ));
287        h.mutate(
288            "PUT",
289            &net_path,
290            Some(&json!({"description": net["description"], "config": cfg})),
291            &format!("update network {}", p.network),
292            t,
293        )?;
294    }
295    put_acl(&h, p, &ip)?;
296    allow_in_project(client, &p.network, true)?;
297    super::ca::kick();
298    Ok(ip)
299}
300
301fn put_acl(h: &Client, p: &Plumbing, ip: &str) -> Result<()> {
302    let t = h.get_timeouts().other;
303    let path = format!("/1.0/network-acls/{}", encode_segment(&p.acl));
304    let body = p.acl_body(ip);
305    if h.get_opt(&path)?.is_some() {
306        h.mutate(
307            "PUT",
308            &path,
309            Some(&body),
310            &format!("update ACL {}", p.acl),
311            t,
312        )?;
313    } else {
314        let mut b = body;
315        b["name"] = json!(p.acl);
316        h.mutate(
317            "POST",
318            "/1.0/network-acls",
319            Some(&b),
320            &format!("create ACL {}", p.acl),
321            t,
322        )?;
323    }
324    Ok(())
325}
326
327/// A restricted project (an org's) lists the networks its instances may
328/// use; the egress bridge has to be on that list. A project that is not
329/// restricted needs nothing.
330fn allow_in_project(client: &Client, network: &str, add: bool) -> Result<()> {
331    let h = host(client);
332    let path = format!("/1.0/projects/{}", encode_segment(client.project_name()));
333    let Some(p) = h.get_opt(&path)? else {
334        return Ok(());
335    };
336    let cfg = &p["config"];
337    if cfg["restricted"].as_str() != Some("true") {
338        return Ok(());
339    }
340    let list = cfg["restricted.networks.access"]
341        .as_str()
342        .unwrap_or_default();
343    let mut names: Vec<&str> = list
344        .split(',')
345        .map(str::trim)
346        .filter(|s| !s.is_empty())
347        .collect();
348    if add == names.contains(&network) {
349        return Ok(());
350    }
351    if add {
352        names.push(network);
353    } else {
354        names.retain(|n| *n != network);
355    }
356    let mut cfg = cfg.clone();
357    cfg["restricted.networks.access"] = json!(names.join(","));
358    h.mutate(
359        "PUT",
360        &path,
361        Some(&json!({"description": p["description"], "config": cfg})),
362        &format!(
363            "allow network {network} in project {}",
364            client.project_name()
365        ),
366        h.get_timeouts().other,
367    )?;
368    Ok(())
369}
370
371/// Delete a sandbox's egress bridge, ACL and CA, once its instance is gone.
372/// Missing pieces are fine.
373pub fn teardown(client: &Client, instance: &str) -> Result<()> {
374    let project = client.project_name();
375    let h = host(client);
376    let t = h.get_timeouts().other;
377    let (net, acl) = (network_name(project, instance), acl_name(project, instance));
378    let net_path = format!("/1.0/networks/{}", encode_segment(&net));
379    if let Some(n) = h.get_opt(&net_path)? {
380        // Never delete what is not this sandbox's.
381        if n["config"][KEY_FOR].as_str() != Some(format!("{project}/{instance}").as_str()) {
382            return Ok(());
383        }
384        allow_in_project(client, &net, false)?;
385        h.mutate(
386            "DELETE",
387            &net_path,
388            None,
389            &format!("delete network {net}"),
390            t,
391        )?;
392    }
393    let acl_path = format!("/1.0/network-acls/{}", encode_segment(&acl));
394    if h.get_opt(&acl_path)?.is_some() {
395        h.mutate("DELETE", &acl_path, None, &format!("delete ACL {acl}"), t)?;
396    }
397    super::ca::forget(&net);
398    Ok(())
399}
400
401/// The egress bridges incus holds, each with its policy: what the proxy
402/// manager reconciles against.
403pub fn list(client: &Client) -> Result<Vec<Value>> {
404    let v = host(client).get("/1.0/networks?recursion=1")?;
405    Ok(v.as_array()
406        .map(|a| {
407            a.iter()
408                .filter(|n| n["config"][KEY_FOR].is_string())
409                .cloned()
410                .collect()
411        })
412        .unwrap_or_default())
413}
414
415#[cfg(test)]
416mod tests {
417    use super::*;
418    use crate::egress::policy::EgressSpec;
419
420    fn plumbing(spec: &EgressSpec) -> Plumbing {
421        let n = network_name("default", "plugin");
422        Plumbing::new("default", "plugin", Policy::from_spec(spec, &n).unwrap())
423    }
424
425    #[test]
426    fn names_are_stable_short_and_per_sandbox() {
427        let n = network_name("default", "plugin");
428        assert!(n.starts_with("isbbrx") && n.len() == 14 && n.len() <= 15);
429        assert_eq!(n, network_name("default", "plugin"));
430        assert_ne!(n, network_name("isb-acme", "plugin"));
431        assert_ne!(n, network_name("default", "other"));
432        assert!(acl_name("default", "plugin").starts_with("isbx-"));
433    }
434
435    #[test]
436    fn dnsmasq_answers_only_allowed_names() {
437        let p = plumbing(&EgressSpec::allow(["api.example.com", "*.cdn.net:8443"]));
438        assert_eq!(
439            p.dnsmasq("10.9.8.1"),
440            "no-resolv\nno-hosts\naddress=/#/\naddress=/api.example.com/10.9.8.1\naddress=/cdn.net/10.9.8.1"
441        );
442        assert_eq!(plumbing(&EgressSpec::none()).dnsmasq("10.9.8.1"), "");
443    }
444
445    #[test]
446    fn none_leaves_the_bridge_without_an_address_and_a_list_turns_dns_on() {
447        let none = plumbing(&EgressSpec::none()).network_config(None, 1);
448        assert_eq!(none["ipv4.address"], "none");
449        assert!(!none.contains_key("raw.dnsmasq"));
450        assert!(!none.contains_key("dns.mode"));
451        let some =
452            plumbing(&EgressSpec::allow(["a.example.com"])).network_config(Some("10.9.8.1"), 1);
453        assert!(some["raw.dnsmasq"].contains("address=/a.example.com/10.9.8.1"));
454        assert_eq!(some["ipv4.address"], "auto");
455        for c in [&none, &some] {
456            assert_eq!(c["ipv4.nat"], "false");
457            assert_eq!(c["ipv4.routing"], "false");
458            assert_eq!(c["ipv6.address"], "none");
459            assert_eq!(c[KEY_FOR], "default/plugin");
460        }
461    }
462
463    #[test]
464    fn the_acl_allows_only_the_proxy_ports_on_the_bridge_address() {
465        let p = plumbing(&EgressSpec::allow([
466            "a.example.com",
467            "b.example.com:8443",
468            "c.example.com:80",
469        ]));
470        let acl = p.acl_body("10.9.8.1");
471        let rules = acl["egress"].as_array().unwrap();
472        assert_eq!(rules.len(), 1);
473        assert_eq!(rules[0]["action"], "allow");
474        assert_eq!(rules[0]["protocol"], "tcp");
475        assert_eq!(rules[0]["destination"], "10.9.8.1/32");
476        assert_eq!(rules[0]["destination_port"], "80,443,8443");
477        assert_eq!(acl["ingress"].as_array().unwrap().len(), 0);
478        let none = plumbing(&EgressSpec::none()).acl_body("10.9.8.1");
479        assert_eq!(none["egress"].as_array().unwrap().len(), 0);
480    }
481
482    #[test]
483    fn the_nic_drops_by_default() {
484        let nic = plumbing(&EgressSpec::none()).nic();
485        assert_eq!(nic["type"], "nic");
486        assert_eq!(nic["network"], network_name("default", "plugin"));
487        assert_eq!(nic["security.acls.default.egress.action"], "drop");
488        assert_eq!(nic["name"], "eth0");
489    }
490
491    #[test]
492    fn the_stored_policy_names_the_sandbox() {
493        let p = plumbing(&EgressSpec::allow(["a.example.com"]));
494        let v: Value = serde_json::from_str(&p.policy_json()).unwrap();
495        assert_eq!(v["project"], "default");
496        assert_eq!(v["instance"], "plugin");
497        assert_eq!(v["entries"][0], "a.example.com:443");
498    }
499
500    #[test]
501    fn a_bridge_address_is_read_without_its_prefix() {
502        let n = json!({"config": {"ipv4.address": "10.41.184.1/24"}});
503        assert_eq!(bridge_ip(&n).as_deref(), Some("10.41.184.1"));
504        assert_eq!(
505            bridge_ip(&json!({"config": {"ipv4.address": "none"}})),
506            None
507        );
508    }
509}
isb_core/egress/plumb.rs

isb_core/egress/
plumb.rs
