isb_server/servers/
vm.rs

1//! Dedicated VMs: an org of its own kernel, one click away.
2//!
3//! `org_create` with `placement: {vm: {...}}` (`isb org create acme --vm`)
4//! makes an incus VM on the control plane's own host, in the `isb-system`
5//! project, installs incus and isb in it through the incus API (file push
6//! and exec, no SSH), runs `isb serve --agent` there with a certificate from
7//! the control plane's CA, registers it as server `vm-<org>` and places the
8//! org on it. From then on it is an ordinary server; the control plane just
9//! made it itself, and deleting the org can delete the VM with it.
10//!
11//! The VM sits on the host's managed bridge (`incusbr0`, as builder VMs do):
12//! it reaches the internet through NAT, and its firewall (ufw) lets only the
13//! host's address on that bridge reach the agent port, which still takes
14//! only the control plane's client certificate. Its address is pinned on the
15//! NIC so it survives restarts.
16
17use std::time::{Duration, Instant};
18
19use serde::{Deserialize, Serialize};
20use serde_json::{Value, json};
21
22use super::provision::Provision;
23use crate::client::{Client, encode_segment};
24use crate::error::{Error, Result};
25use crate::exec::{ExecEvent, ExecOptions, Stdin};
26use crate::org::OrgId;
27use crate::sandbox::Sandbox;
28
29/// Where dedicated VMs live: isb's own project, never an org's.
30pub const PROJECT: &str = crate::registry::PROJECT;
31/// The guest: Ubuntu 24.04, which Zabbly's incus packages support.
32pub const IMAGE: &str = "images:ubuntu/24.04";
33pub const DEFAULT_CPUS: u32 = 2;
34pub const DEFAULT_MEMORY: &str = "4GiB";
35pub const DEFAULT_DISK: &str = "40GiB";
36const MIN_MEMORY: u64 = 2 << 30;
37const MIN_DISK: u64 = 10 << 30;
38const MAX_CPUS: u32 = 256;
39/// Marks an instance as a dedicated VM, with its org as the value.
40pub const KEY_ORG: &str = "user.isb.dedicated-vm";
41/// The server name it is registered under.
42pub const KEY_SERVER: &str = "user.isb.server";
43
44/// The VM's size as asked (`placement: {vm: {cpus, memory, disk}}`).
45#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
46#[serde(deny_unknown_fields)]
47pub struct VmOptions {
48    #[serde(default)]
49    pub cpus: Option<u32>,
50    #[serde(default)]
51    pub memory: Option<String>,
52    #[serde(default)]
53    pub disk: Option<String>,
54}
55
56/// The VM's size, defaults filled in and checked.
57#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
58pub struct VmSize {
59    pub cpus: u32,
60    pub memory: String,
61    pub disk: String,
62}
63
64impl VmOptions {
65    pub fn resolve(&self) -> Result<VmSize> {
66        let cpus = self.cpus.unwrap_or(DEFAULT_CPUS);
67        if cpus == 0 || cpus > MAX_CPUS {
68            return Err(Error::invalid(format!(
69                "VM cpus {cpus}: between 1 and {MAX_CPUS}"
70            )));
71        }
72        let size = |what: &str, v: &Option<String>, def: &str, min: u64| -> Result<String> {
73            let v = v.as_deref().map(str::trim).unwrap_or(def).to_string();
74            let b = parse_bytes(&v).ok_or_else(|| {
75                Error::invalid(format!("VM {what} {v:?}: a size such as 4GiB or 40GiB"))
76            })?;
77            if b < min {
78                return Err(Error::invalid(format!(
79                    "VM {what} {v}: at least {} GiB",
80                    min >> 30
81                )));
82            }
83            Ok(v)
84        };
85        Ok(VmSize {
86            cpus,
87            memory: size("memory", &self.memory, DEFAULT_MEMORY, MIN_MEMORY)?,
88            disk: size("disk", &self.disk, DEFAULT_DISK, MIN_DISK)?,
89        })
90    }
91}
92
93/// A size incus takes, in bytes: digits and a unit (`4GiB`, `512MB`).
94pub fn parse_bytes(s: &str) -> Option<u64> {
95    let digits = s.chars().take_while(char::is_ascii_digit).count();
96    if digits == 0 {
97        return None;
98    }
99    let n: u64 = s[..digits].parse().ok()?;
100    let mult: u64 = match &s[digits..] {
101        "" | "B" => 1,
102        "kB" => 1_000,
103        "MB" => 1_000_000,
104        "GB" => 1_000_000_000,
105        "TB" => 1_000_000_000_000,
106        "KiB" => 1 << 10,
107        "MiB" => 1 << 20,
108        "GiB" => 1 << 30,
109        "TiB" => 1 << 40,
110        _ => return None,
111    };
112    n.checked_mul(mult)
113}
114
115/// Where an org is to run, from `org_create`'s arguments: `placement` (`"local"`,
116/// `{"server": NAME}` or `{"vm": {...}}`), or the older `server` field.
117#[derive(Debug, Clone, PartialEq, Eq)]
118pub enum Placement {
119    Local,
120    Server(String),
121    Vm(VmSize),
122}
123
124pub fn placement(a: &Value) -> Result<Placement> {
125    #[derive(Deserialize)]
126    #[serde(rename_all = "snake_case", deny_unknown_fields)]
127    enum P {
128        Local,
129        Server(String),
130        Vm(VmOptions),
131    }
132    let server = a.get("server").and_then(Value::as_str);
133    let p = match a.get("placement") {
134        None | Some(Value::Null) => None,
135        Some(v) => Some(serde_json::from_value::<P>(v.clone()).map_err(|e| {
136            Error::invalid(format!(
137                "placement: \"local\", {{\"server\": NAME}} or {{\"vm\": {{\"cpus\", \"memory\", \"disk\"}}}} ({e})"
138            ))
139        })?),
140    };
141    let p = match (p, server) {
142        (Some(_), Some(_)) => {
143            return Err(Error::invalid("give placement or server, not both"));
144        }
145        (Some(p), None) => p,
146        (None, Some(s)) => P::Server(s.to_string()),
147        (None, None) => P::Local,
148    };
149    Ok(match p {
150        P::Local => Placement::Local,
151        P::Server(s) if s == "local" => Placement::Local,
152        P::Server(s) => Placement::Server(s),
153        P::Vm(o) => Placement::Vm(o.resolve()?),
154    })
155}
156
157/// The server, and the instance, a dedicated VM for `org` is.
158pub fn server_name(org: &OrgId) -> String {
159    format!("vm-{org}")
160}
161
162/// Whether this host can run VMs, and why not.
163pub fn support(client: &Client) -> std::result::Result<(), String> {
164    support_from(
165        client.server_info().map_err(|e| e.to_string())?,
166        std::path::Path::new("/dev/kvm").exists(),
167    )
168}
169
170fn support_from(info: Value, kvm: bool) -> std::result::Result<(), String> {
171    let driver = info["environment"]["driver"].as_str().unwrap_or("");
172    if !driver.split('|').any(|d| d.trim() == "qemu") {
173        return Err(format!(
174            "incus on this host runs no VMs (its drivers: {}){}",
175            if driver.is_empty() { "unknown" } else { driver },
176            if kvm {
177                ""
178            } else {
179                "; there is no /dev/kvm, as on cloud VMs without nested virtualization"
180            }
181        ));
182    }
183    if !kvm {
184        return Err("this host has no /dev/kvm (a cloud VM without nested virtualization?)".into());
185    }
186    Ok(())
187}
188
189/// `POST /1.0/instances` for the VM.
190pub fn instance_body(org: &OrgId, size: &VmSize, pool: &str, network: &str) -> Result<Value> {
191    let name = server_name(org);
192    let src = crate::plan::ImageSource::parse(IMAGE)?;
193    Ok(json!({
194        "name": name,
195        "type": "virtual-machine",
196        "description": format!("isb: dedicated VM for org {org}"),
197        "source": src.to_api(None),
198        "config": {
199            "limits.cpu": size.cpus.to_string(),
200            "limits.memory": size.memory,
201            KEY_ORG: org.as_str(),
202            KEY_SERVER: name,
203            "user.owner": "isb",
204        },
205        "devices": {
206            "root": {"type": "disk", "path": "/", "pool": pool, "size": size.disk},
207            "eth0": {"type": "nic", "name": "eth0", "network": network},
208        },
209        "profiles": ["default"],
210    }))
211}
212
213/// The project dedicated VMs go in, made if missing (as `isb registry
214/// setup` makes it: its own profiles and volumes, the host's images and
215/// networks).
216fn ensure_project(host: &Client) -> Result<()> {
217    if host.get_opt(&format!("/1.0/projects/{PROJECT}"))?.is_some() {
218        return Ok(());
219    }
220    match host.mutate(
221        "POST",
222        "/1.0/projects",
223        Some(&json!({
224            "name": PROJECT,
225            "description": "isb system services (not an org)",
226            "config": {
227                "features.images": "false",
228                "features.profiles": "true",
229                "features.storage.volumes": "true",
230                "features.networks": "false",
231            },
232        })),
233        &format!("create project {PROJECT}"),
234        host.get_timeouts().other,
235    ) {
236        Ok(_) => Ok(()),
237        Err(e) if e.is_conflict() => Ok(()),
238        Err(e) => Err(e),
239    }
240}
241
242/// The host's managed bridge a VM goes on (`incusbr0` if there is one,
243/// never an org's), and the host's address on it.
244fn bridge(host: &Client) -> Result<(String, String)> {
245    let nets = host.get("/1.0/networks?recursion=1")?;
246    let managed: Vec<&Value> = nets
247        .as_array()
248        .into_iter()
249        .flatten()
250        .filter(|n| n["managed"].as_bool() == Some(true) && n["type"] == "bridge")
251        .filter(|n| !n["name"].as_str().unwrap_or("").starts_with("isbbr"))
252        .collect();
253    let n = managed
254        .iter()
255        .find(|n| n["name"] == "incusbr0")
256        .or(managed.first())
257        .ok_or_else(|| Error::invalid("no managed bridge on this host for a dedicated VM"))?;
258    let addr = n["config"]["ipv4.address"]
259        .as_str()
260        .and_then(|a| a.split('/').next())
261        .filter(|a| a.parse::<std::net::Ipv4Addr>().is_ok())
262        .ok_or_else(|| {
263            Error::invalid(format!(
264                "bridge {} has no IPv4 address for the VM's agent to be reached on",
265                n["name"].as_str().unwrap_or("")
266            ))
267        })?;
268    Ok((
269        n["name"].as_str().unwrap_or("").to_string(),
270        addr.to_string(),
271    ))
272}
273
274/// A VM that is up: where the control plane dials it, and the host's
275/// address its firewall lets in.
276#[derive(Debug, Clone)]
277pub struct Booted {
278    pub address: String,
279    pub host_address: String,
280}
281
282/// Make (or find) the VM for `org`, start it and wait until it runs
283/// commands and has an address. Idempotent: an existing VM of this org is
284/// reused; an instance of that name that is not one is refused.
285pub fn boot(client: &Client, org: &OrgId, size: &VmSize, p: &Provision) -> Result<Booted> {
286    let host = client.clone().project("default");
287    let sys = client.clone().project(PROJECT);
288    let name = server_name(org);
289    p.step("support");
290    support(client).map_err(|e| Error::invalid(format!("dedicated VM: {e}")))?;
291    p.step("vm");
292    ensure_project(&host)?;
293    let (network, host_address) = bridge(&host)?;
294    let path = format!("/1.0/instances/{}", encode_segment(&name));
295    match sys.get_opt(&path)? {
296        Some(i) => {
297            if i["config"][KEY_ORG].as_str() != Some(org.as_str()) {
298                return Err(Error::AlreadyExists(format!(
299                    "instance {name} in project {PROJECT} is not org {org}'s dedicated VM"
300                )));
301            }
302            p.log(&format!(
303                "VM {name} exists ({})",
304                i["status"].as_str().unwrap_or("")
305            ));
306        }
307        None => {
308            let pool = crate::sandbox::host_facts(&host)?.pick_pool(None)?;
309            p.log(&format!(
310                "creating VM {name} in project {PROJECT}: {} CPUs, {} memory, {} disk on {pool}, network {network}",
311                size.cpus, size.memory, size.disk
312            ));
313            sys.mutate(
314                "POST",
315                "/1.0/instances",
316                Some(&instance_body(org, size, &pool, &network)?),
317                &format!("create VM {name}"),
318                Duration::from_secs(1200),
319            )?;
320        }
321    }
322    let state = sys.get(&format!("{path}/state"))?;
323    if state["status"].as_str() != Some("Running") {
324        p.log(&format!("starting VM {name}"));
325        sys.mutate(
326            "PUT",
327            &format!("{path}/state"),
328            Some(&json!({"action": "start", "timeout": 60})),
329            &format!("start VM {name}"),
330            Duration::from_secs(300),
331        )?;
332    }
333    p.step("boot");
334    p.log("waiting for the VM's agent and an address");
335    let sb = Sandbox::get(&sys, &name)?;
336    let deadline = Instant::now() + Duration::from_secs(600);
337    let mut last: String;
338    loop {
339        let ok = sb
340            .exec_stream(
341                ["/bin/true"],
342                ExecOptions::default().timeout(Duration::from_secs(20)),
343            )
344            .and_then(|s| s.collect_output())
345            .map(|o| o.success());
346        match ok {
347            Ok(true) => {
348                if let Some(ip) = address(&sys, &path)? {
349                    pin(&sys, &path, &ip, p);
350                    p.log(&format!("VM {name} is up at {ip}"));
351                    return Ok(Booted {
352                        address: ip,
353                        host_address,
354                    });
355                }
356                last = format!(
357                    "no IPv4 address yet on {network} (behind a default-deny firewall such as ufw, run `sudo isb host setup`: it lets {network} through)"
358                );
359            }
360            Ok(false) => last = "the guest agent answered with an error".into(),
361            Err(e) => last = e.to_string(),
362        }
363        if Instant::now() >= deadline {
364            return Err(Error::OperationFailed {
365                step: format!("boot VM {name}"),
366                message: last,
367            });
368        }
369        std::thread::sleep(Duration::from_secs(2));
370    }
371}
372
373/// The VM's global IPv4 address on the NIC isb gave it, once it has one.
374/// A guest names the interface itself (`enp5s0`), and its own incus adds
375/// bridges, so the NIC is found by its MAC address.
376fn address(sys: &Client, path: &str) -> Result<Option<String>> {
377    let i = sys.get(path)?;
378    let st = sys.get(&format!("{path}/state"))?;
379    Ok(nic_address(
380        &st,
381        i["config"]["volatile.eth0.hwaddr"].as_str(),
382    ))
383}
384
385fn nic_address(state: &Value, hwaddr: Option<&str>) -> Option<String> {
386    let hwaddr = hwaddr?.to_ascii_lowercase();
387    state["network"]
388        .as_object()?
389        .values()
390        .filter(|n| n["hwaddr"].as_str().map(str::to_ascii_lowercase) == Some(hwaddr.clone()))
391        .flat_map(|n| n["addresses"].as_array().cloned().unwrap_or_default())
392        .find(|a| a["family"] == "inet" && a["scope"] == "global")
393        .and_then(|a| a["address"].as_str().map(str::to_string))
394}
395
396/// Reserve `ip` for the VM on the bridge, so it keeps it across restarts
397/// (the control plane dials it, and its certificate names it).
398fn pin(sys: &Client, path: &str, ip: &str, p: &Provision) {
399    let r = (|| -> Result<()> {
400        let i = sys.get(path)?;
401        let mut eth0 = i["devices"]["eth0"].clone();
402        if eth0["ipv4.address"].as_str() == Some(ip) {
403            return Ok(());
404        }
405        eth0["ipv4.address"] = json!(ip);
406        sys.mutate(
407            "PATCH",
408            path,
409            Some(&json!({"devices": {"eth0": eth0}})),
410            "pin the VM's address",
411            Duration::from_secs(60),
412        )?;
413        Ok(())
414    })();
415    if let Err(e) = r {
416        p.log(&format!(
417            "could not pin {ip} on the VM's NIC ({e}); it keeps its DHCP lease"
418        ));
419    }
420}
421
422/// Copy the isb binary and run the bootstrap script in the VM, its output
423/// going to `p`'s log. The script (it holds the agent's key) goes on stdin
424/// and is never written to the guest's disk.
425pub fn install(
426    client: &Client,
427    org: &OrgId,
428    binary: &[u8],
429    script: &str,
430    upload: &str,
431    p: &Provision,
432) -> Result<String> {
433    let sys = client.clone().project(PROJECT);
434    let name = server_name(org);
435    p.step("upload");
436    p.log(&format!(
437        "copying isb into the VM ({} MiB)",
438        binary.len() >> 20
439    ));
440    sys.push_file(&name, upload, binary, 0, 0, 0o700)?;
441    p.step("install");
442    p.log("installing incus, isb, the agent's unit and the firewall (a few minutes)");
443    let sb = Sandbox::get(&sys, &name)?;
444    let mut s = sb.exec_stream(
445        ["bash", "-s"],
446        ExecOptions::default()
447            .env(
448                "PATH",
449                "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
450            )
451            .env("DEBIAN_FRONTEND", "noninteractive")
452            .timeout(Duration::from_secs(30 * 60))
453            .stdin(Stdin::Bytes(script.as_bytes().to_vec())),
454    )?;
455    let mut buf = Vec::new();
456    let mut last = String::new();
457    let mut emit = |chunk: Vec<u8>, last: &mut String| {
458        buf.extend(chunk);
459        while let Some(i) = buf.iter().position(|b| *b == b'\n') {
460            let line: Vec<u8> = buf.drain(..=i).collect();
461            let text = String::from_utf8_lossy(&line[..line.len() - 1]);
462            let text = text.trim_end_matches('\r');
463            if !text.trim().is_empty() {
464                p.log(text);
465                *last = text.to_string();
466            }
467        }
468    };
469    while let Some(ev) = s.next_event() {
470        match ev {
471            ExecEvent::Stdout(b) | ExecEvent::Stderr(b) => emit(b, &mut last),
472        }
473    }
474    let code = s.wait()?;
475    if code != 0 {
476        return Err(Error::OperationFailed {
477            step: format!("install the agent in VM {name}"),
478            message: format!("exit {code}: {last}"),
479        });
480    }
481    Ok(last)
482}
483
484/// Delete org `org`'s dedicated VM (stopped first); gone already is fine.
485pub fn delete(client: &Client, project: &str, instance: &str) -> Result<()> {
486    let c = client.clone().project(project);
487    match c.get_opt(&format!("/1.0/instances/{}", encode_segment(instance)))? {
488        None => Ok(()),
489        Some(i) if i["config"][KEY_ORG].as_str().is_none() => Err(Error::invalid(format!(
490            "instance {instance} in {project} is not a dedicated VM; not deleting it"
491        ))),
492        Some(_) => match Sandbox::remove(&c, instance, true) {
493            Err(e) if e.is_not_found() => Ok(()),
494            r => r,
495        },
496    }
497}
498
499#[cfg(test)]
500mod tests {
501    use super::*;
502
503    #[test]
504    fn placements_parse_and_check() {
505        assert_eq!(placement(&json!({"org": "a"})).unwrap(), Placement::Local);
506        assert_eq!(
507            placement(&json!({"placement": "local"})).unwrap(),
508            Placement::Local
509        );
510        assert_eq!(
511            placement(&json!({"server": "local"})).unwrap(),
512            Placement::Local
513        );
514        assert_eq!(
515            placement(&json!({"server": "hel-1"})).unwrap(),
516            Placement::Server("hel-1".into())
517        );
518        assert_eq!(
519            placement(&json!({"placement": {"server": "hel-1"}})).unwrap(),
520            Placement::Server("hel-1".into())
521        );
522        assert_eq!(
523            placement(&json!({"placement": {"vm": {}}})).unwrap(),
524            Placement::Vm(VmSize {
525                cpus: 2,
526                memory: "4GiB".into(),
527                disk: "40GiB".into()
528            })
529        );
530        assert_eq!(
531            placement(
532                &json!({"placement": {"vm": {"cpus": 8, "memory": "16GiB", "disk": "200GiB"}}})
533            )
534            .unwrap(),
535            Placement::Vm(VmSize {
536                cpus: 8,
537                memory: "16GiB".into(),
538                disk: "200GiB".into()
539            })
540        );
541        for bad in [
542            json!({"placement": {"vm": {}}, "server": "x"}),
543            json!({"placement": "vm"}),
544            json!({"placement": {"vm": {"cpus": 0}}}),
545            json!({"placement": {"vm": {"memory": "1GiB"}}}),
546            json!({"placement": {"vm": {"memory": "lots"}}}),
547            json!({"placement": {"vm": {"disk": "5GiB"}}}),
548            json!({"placement": {"vm": {"gpus": 1}}}),
549            json!({"placement": {"cloud": "x"}}),
550        ] {
551            assert!(placement(&bad).is_err(), "{bad}");
552        }
553    }
554
555    #[test]
556    fn the_address_is_the_nics_whatever_the_guest_calls_it() {
557        let st = json!({"network": {
558            "lo": {"hwaddr": "", "addresses": [{"family": "inet", "address": "127.0.0.1", "scope": "local"}]},
559            "incusbr0": {"hwaddr": "10:66:6a:00:00:01", "addresses": [{"family": "inet", "address": "10.9.9.1", "scope": "global"}]},
560            "enp5s0": {"hwaddr": "10:66:6a:38:c0:17", "addresses": [
561                {"family": "inet6", "address": "fd42::1", "scope": "global"},
562                {"family": "inet", "address": "10.180.0.64", "scope": "global"}
563            ]}
564        }});
565        assert_eq!(
566            nic_address(&st, Some("10:66:6A:38:C0:17")).as_deref(),
567            Some("10.180.0.64")
568        );
569        assert_eq!(nic_address(&st, None), None);
570        assert_eq!(nic_address(&st, Some("10:66:6a:ff:ff:ff")), None);
571    }
572
573    #[test]
574    fn sizes() {
575        assert_eq!(parse_bytes("4GiB"), Some(4 << 30));
576        assert_eq!(parse_bytes("512MB"), Some(512_000_000));
577        assert_eq!(parse_bytes("1024"), Some(1024));
578        for bad in ["", "GiB", "4 GiB", "4gib", "-1GiB"] {
579            assert_eq!(parse_bytes(bad), None, "{bad}");
580        }
581    }
582
583    #[test]
584    fn vm_support_needs_qemu_and_kvm() {
585        let qemu = json!({"environment": {"driver": "lxc | qemu"}});
586        assert!(support_from(qemu.clone(), true).is_ok());
587        let e = support_from(qemu, false).unwrap_err();
588        assert!(e.contains("/dev/kvm"), "{e}");
589        let e = support_from(json!({"environment": {"driver": "lxc"}}), false).unwrap_err();
590        assert!(e.contains("runs no VMs") && e.contains("nested"), "{e}");
591    }
592
593    #[test]
594    fn the_vm_is_labelled_sized_and_on_the_hosts_bridge() {
595        let org = OrgId::new("acme").unwrap();
596        let size = VmOptions::default().resolve().unwrap();
597        let b = instance_body(&org, &size, "default", "incusbr0").unwrap();
598        assert_eq!(b["name"], "vm-acme");
599        assert_eq!(b["type"], "virtual-machine");
600        assert_eq!(b["config"]["limits.cpu"], "2");
601        assert_eq!(b["config"]["limits.memory"], "4GiB");
602        assert_eq!(b["config"][KEY_ORG], "acme");
603        assert_eq!(b["config"][KEY_SERVER], "vm-acme");
604        assert_eq!(b["devices"]["root"]["size"], "40GiB");
605        assert_eq!(b["devices"]["eth0"]["network"], "incusbr0");
606        assert_eq!(b["source"]["alias"], "ubuntu/24.04");
607        assert_eq!(server_name(&OrgId::new("a".repeat(31)).unwrap()).len(), 34);
608        super::super::bootstrap::validate_name(&server_name(&org)).unwrap();
609    }
610}
isb_server/servers/vm.rs

isb_server/servers/
vm.rs
