isb_server/servers/
bootstrap.rs

1//! `isb server add`: make a fresh Linux box an isb agent over SSH.
2//!
3//! The SSH key is used for this and nothing after: it installs incus (from
4//! Zabbly's stable channel, as `isb machine` does in its VM), the isb binary
5//! (checksum checked on both ends), runs `isb host setup`, writes the
6//! agent's TLS material and a systemd unit for `isb serve --agent`, and
7//! optionally closes the box's firewall to everything but SSH and the agent
8//! port from the control plane. From then on the control plane speaks only
9//! mTLS to the agent.
10
11use std::io::Write;
12use std::path::{Path, PathBuf};
13use std::process::{Command, Stdio};
14
15use super::pki::Leaf;
16use super::provision::Provision;
17use crate::error::{Error, Result};
18
19/// Where things go on the box.
20pub const AGENT_USER: &str = "isb";
21pub const AGENT_HOME: &str = "/var/lib/isb";
22pub const AGENT_TLS_DIR: &str = "/etc/isb-agent";
23pub const AGENT_UNIT: &str = "isb-agent.service";
24pub const DEFAULT_AGENT_PORT: u16 = 7443;
25
26/// What `isb server add` was given.
27#[derive(Debug, Clone)]
28pub struct AddOptions {
29    pub name: String,
30    /// `user@host`; the user is root or has passwordless sudo.
31    pub ssh: String,
32    pub ssh_port: u16,
33    pub key: PathBuf,
34    /// What the control plane dials, if not the SSH host.
35    pub address: Option<String>,
36    pub agent_port: u16,
37    /// Firewall the box to SSH plus the agent port from these (ufw).
38    pub allow_from: Vec<String>,
39    /// A Linux isb binary to install; default the release of this version.
40    pub isb_binary: Option<PathBuf>,
41    pub version: Option<String>,
42    /// Install this control plane's own executable (it must be a Linux
43    /// build for the box's architecture).
44    pub self_binary: bool,
45    /// Serve the box's orgs' domains on its own 80 and 443 (`isb host
46    /// setup --public-ingress`, the agent's `--ingress-http/https`).
47    pub public_ingress: bool,
48}
49
50/// Server names: `[a-z0-9-]`, a letter first, at most 40 (so `vm-` and
51/// the longest org name fit).
52pub fn validate_name(name: &str) -> Result<()> {
53    let ok = !name.is_empty()
54        && name.len() <= 40
55        && name.starts_with(|c: char| c.is_ascii_lowercase())
56        && name
57            .bytes()
58            .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-')
59        && name != "local";
60    if !ok {
61        return Err(Error::invalid(format!(
62            "server name {name:?}: [a-z0-9-], starting with a letter, at most 40 characters, not \"local\""
63        )));
64    }
65    Ok(())
66}
67
68/// The host part of `user@host`.
69pub fn ssh_host(ssh: &str) -> Result<(&str, &str)> {
70    let (u, h) = ssh
71        .split_once('@')
72        .ok_or_else(|| Error::invalid(format!("--ssh {ssh:?}: expected user@host")))?;
73    if !host_ok(u) || !host_ok(h) {
74        return Err(Error::invalid(format!("--ssh {ssh:?}: not a user@host")));
75    }
76    Ok((u, h))
77}
78
79/// A user or host name, or an address: nothing a shell or ssh would read
80/// as more.
81fn host_ok(s: &str) -> bool {
82    !s.is_empty()
83        && s.len() <= 253
84        && !s.starts_with('-')
85        && s.bytes()
86            .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'-' | b'_' | b':'))
87}
88
89/// What the control plane dials: a host name or an address.
90pub fn check_address(s: &str) -> Result<()> {
91    if !host_ok(s) || s.contains('_') {
92        return Err(Error::invalid(format!(
93            "--address {s:?}: a host name or an IP address"
94        )));
95    }
96    Ok(())
97}
98
99/// A CIDR or address for `ufw allow from`.
100pub fn check_cidr(s: &str) -> Result<()> {
101    let (ip, len) = s.split_once('/').unwrap_or((s, ""));
102    let ok = ip.parse::<std::net::IpAddr>().is_ok()
103        && (len.is_empty() || len.parse::<u8>().is_ok_and(|n| n <= 128));
104    if !ok {
105        return Err(Error::invalid(format!(
106            "--allow-from {s:?}: an address or CIDR"
107        )));
108    }
109    Ok(())
110}
111
112/// Runs commands on the box over SSH with exactly the given key: no agent,
113/// no user ssh config, a known_hosts file of its own.
114pub struct Ssh {
115    target: String,
116    port: u16,
117    key: PathBuf,
118    known_hosts: PathBuf,
119}
120
121impl Ssh {
122    pub fn new(target: &str, port: u16, key: &Path, known_hosts: &Path) -> Ssh {
123        Ssh {
124            target: target.into(),
125            port,
126            key: key.into(),
127            known_hosts: known_hosts.into(),
128        }
129    }
130
131    fn command(&self, remote: &str) -> Command {
132        let mut c = Command::new("ssh");
133        c.env_remove("SSH_AUTH_SOCK")
134            .args(["-F", "/dev/null", "-i"])
135            .arg(&self.key)
136            .args([
137                "-o",
138                "IdentitiesOnly=yes",
139                "-o",
140                "IdentityAgent=none",
141                "-o",
142                "BatchMode=yes",
143                "-o",
144                "StrictHostKeyChecking=accept-new",
145                "-o",
146                "ConnectTimeout=15",
147                "-o",
148                "ServerAliveInterval=15",
149                "-o",
150                "ServerAliveCountMax=4",
151                "-o",
152            ])
153            .arg(format!("UserKnownHostsFile={}", self.known_hosts.display()))
154            .arg("-p")
155            .arg(self.port.to_string())
156            .arg(&self.target)
157            .arg("--")
158            .arg(remote);
159        c
160    }
161
162    /// Run `remote` with `input` on its stdin; its stdout, or the step's
163    /// failure with its stderr's tail.
164    pub fn run(&self, step: &str, remote: &str, input: &[u8]) -> Result<String> {
165        let mut c = self.command(remote);
166        c.stdin(Stdio::piped())
167            .stdout(Stdio::piped())
168            .stderr(Stdio::piped());
169        let mut child = c.spawn().map_err(|e| Error::OperationFailed {
170            step: step.into(),
171            message: format!("ssh: {e}"),
172        })?;
173        let mut stdin = child.stdin.take();
174        let data = input.to_vec();
175        let feeder = std::thread::spawn(move || {
176            if let Some(s) = stdin.as_mut() {
177                let _ = s.write_all(&data);
178            }
179            drop(stdin);
180        });
181        let out = child.wait_with_output()?;
182        let _ = feeder.join();
183        if !out.status.success() {
184            let err = String::from_utf8_lossy(&out.stderr);
185            let tail: Vec<&str> = err.lines().rev().take(15).collect();
186            return Err(Error::OperationFailed {
187                step: step.into(),
188                message: format!(
189                    "exit {}: {}",
190                    out.status.code().unwrap_or(-1),
191                    tail.into_iter().rev().collect::<Vec<_>>().join("\n")
192                ),
193            });
194        }
195        Ok(String::from_utf8_lossy(&out.stdout).into_owned())
196    }
197}
198
199/// `x86_64` or `aarch64` from an ELF header, if it is a Linux executable.
200pub fn elf_arch(b: &[u8]) -> Option<&'static str> {
201    if b.len() < 20 || &b[..4] != b"\x7fELF" {
202        return None;
203    }
204    match u16::from_le_bytes([b[18], b[19]]) {
205        0x3e => Some("x86_64"),
206        0xb7 => Some("aarch64"),
207        _ => None,
208    }
209}
210
211fn shell_quote(s: &str) -> String {
212    format!("'{}'", s.replace('\'', r"'\''"))
213}
214
215/// The agent's unit.
216pub fn render_unit(port: u16, public_ingress: bool) -> String {
217    let ingress = if public_ingress {
218        " --ingress-http 0.0.0.0:80 --ingress-https 0.0.0.0:443"
219    } else {
220        ""
221    };
222    format!(
223        "[Unit]
224Description=isb agent (isb serve --agent, managed by an isb control plane)
225After=network-online.target incus.socket
226Wants=network-online.target incus.socket
227
228[Service]
229Type=simple
230User={AGENT_USER}
231SupplementaryGroups=incus-admin
232WorkingDirectory={AGENT_HOME}
233Environment=HOME={AGENT_HOME}
234Environment=ISB_SERVE_SOCKET=/run/isb/serve.sock
235RuntimeDirectory=isb
236RuntimeDirectoryMode=0700
237ExecStart=/usr/local/bin/isb serve --agent --agent-listen 0.0.0.0:{port} --agent-tls {AGENT_TLS_DIR} --state-dir {AGENT_HOME}/state{ingress}
238Restart=always
239RestartSec=2
240
241[Install]
242WantedBy=multi-user.target
243"
244    )
245}
246
247/// The root script: everything idempotent, so a second `server add` (or a
248/// rerun after a failure) converges. `ssh_port` stays open in the
249/// firewall; a dedicated VM, bootstrapped through incus, has none.
250#[expect(clippy::too_many_arguments)]
251pub fn render_script(
252    upload: &str,
253    sha256: &str,
254    ca_pem: &str,
255    leaf: &Leaf,
256    port: u16,
257    allow_from: &[String],
258    ssh_port: Option<u16>,
259    public_ingress: bool,
260) -> String {
261    let mut fw = String::new();
262    if !allow_from.is_empty() {
263        fw.push_str(
264            "command -v ufw >/dev/null 2>&1 || { apt-get update; apt-get install -y --no-install-recommends ufw; }\n",
265        );
266        if let Some(p) = ssh_port {
267            fw.push_str(&format!("ufw allow {p}/tcp\n"));
268        }
269        for c in allow_from {
270            fw.push_str(&format!(
271                "ufw allow proto tcp from {} to any port {port}\n",
272                shell_quote(c)
273            ));
274        }
275        fw.push_str("ufw --force enable\n");
276    }
277    format!(
278        r#"set -euo pipefail
279export DEBIAN_FRONTEND=noninteractive
280echo "{sha256}  {upload}" | sha256sum -c --quiet -
281id -u {user} >/dev/null 2>&1 || useradd --system --create-home --home-dir {home} --shell /usr/sbin/nologin {user}
282uid=$(id -u {user})
283# Containers get their own range, and incus may map the agent's uid 1:1
284# (restricted.idmap.uid in each org's project).
285changed=0
286for l in root:1000000:1000000000 "root:$uid:1"; do
287  for f in /etc/subuid /etc/subgid; do
288    grep -qx "$l" "$f" || {{ echo "$l" >> "$f"; changed=1; }}
289  done
290done
291if ! command -v incus >/dev/null 2>&1; then
292  apt-get update
293  apt-get install -y --no-install-recommends curl ca-certificates
294  install -d -m 0755 /etc/apt/keyrings
295  curl -fsSL https://pkgs.zabbly.com/key.asc -o /etc/apt/keyrings/zabbly.asc
296  cat > /etc/apt/sources.list.d/zabbly-incus-stable.sources <<EOF
297Enabled: yes
298Types: deb
299URIs: https://pkgs.zabbly.com/incus/stable
300Suites: $(. /etc/os-release && echo "$VERSION_CODENAME")
301Components: main
302Architectures: $(dpkg --print-architecture)
303Signed-By: /etc/apt/keyrings/zabbly.asc
304EOF
305  apt-get update
306  apt-get install -y --no-install-recommends incus
307elif [ "$changed" = 1 ]; then
308  systemctl restart incus.service || true
309fi
310if [ -z "$(incus storage list --format csv 2>/dev/null)" ]; then
311  incus admin init --auto
312fi
313usermod -aG incus-admin {user}
314install -m 0755 {upload_q} /usr/local/bin/isb
315rm -f {upload_q}
316{fw}# The local registry, for builds; then host setup installs its CA.
317runuser -u {user} -- env HOME={home} /usr/local/bin/isb registry setup --state-dir {home}/state \
318  || echo "isb: the local registry could not be set up; builds on this server will fail" >&2
319/usr/local/bin/isb host setup --user {user}{host_ingress}
320install -d -o {user} -g {user} -m 0700 {tls}
321cat > {tls}/ca.crt <<'ISB_EOF'
322{ca}ISB_EOF
323umask 077
324cat > {tls}/tls.crt.new <<'ISB_EOF'
325{cert}ISB_EOF
326cat > {tls}/tls.key.new <<'ISB_EOF'
327{key}ISB_EOF
328mv {tls}/tls.crt.new {tls}/tls.crt
329mv {tls}/tls.key.new {tls}/tls.key
330chown {user}:{user} {tls}/ca.crt {tls}/tls.crt {tls}/tls.key
331chmod 0644 {tls}/ca.crt
332umask 022
333cat > /etc/systemd/system/{unit} <<'ISB_EOF'
334{unit_text}ISB_EOF
335# The upgrade helper: `isb server upgrade` stages a binary, this installs it.
336{helper}systemctl daemon-reload
337systemctl enable {unit} >/dev/null 2>&1
338systemctl restart {unit}
339echo "incus $(incus version 2>/dev/null | tail -n1 | awk '{{print $NF}}')"
340"#,
341        upload_q = shell_quote(upload),
342        user = AGENT_USER,
343        home = AGENT_HOME,
344        tls = AGENT_TLS_DIR,
345        ca = ca_pem,
346        cert = leaf.cert,
347        key = leaf.key,
348        unit = AGENT_UNIT,
349        unit_text = render_unit(port, public_ingress),
350        helper = super::upgrade::helper_install(),
351        host_ingress = if public_ingress {
352            " --public-ingress"
353        } else {
354            ""
355        },
356    )
357}
358
359/// Run the bootstrap, reporting each step to `p`.
360pub fn run(
361    o: &AddOptions,
362    ssh: &Ssh,
363    ca_pem: &str,
364    leaf: &Leaf,
365    scratch: &Path,
366    p: &Provision,
367) -> Result<String> {
368    let (user, _) = ssh_host(&o.ssh)?;
369    p.step("check");
370    p.log(&format!("checking the box: ssh {}:{}", o.ssh, o.ssh_port));
371    let uname = ssh.run("check the box", "uname -sm", b"")?;
372    let arch = match uname.split_whitespace().collect::<Vec<_>>().as_slice() {
373        ["Linux", "x86_64"] => "x86_64",
374        ["Linux", "aarch64"] => "aarch64",
375        _ => {
376            return Err(Error::invalid(format!(
377                "server {}: isb agents run on Linux x86_64 or aarch64, not {}",
378                o.name,
379                uname.trim()
380            )));
381        }
382    };
383    p.log(&format!(
384        "{} {arch}",
385        uname.split_whitespace().next().unwrap_or("")
386    ));
387    if user != "root" {
388        ssh.run("check sudo", "sudo -n true", b"").map_err(|_| {
389            Error::invalid(format!(
390                "{}: the user needs passwordless sudo (or connect as root)",
391                o.ssh
392            ))
393        })?;
394    }
395    let sudo = if user == "root" { "" } else { "sudo -n " };
396    p.step("binary");
397    let binary = binary(o, arch, scratch, p)?;
398    let sha = sha256_hex(&binary);
399    p.step("upload");
400    p.log(&format!(
401        "uploading isb ({} MiB, sha256 {})",
402        binary.len() >> 20,
403        &sha[..16]
404    ));
405    let upload = ssh
406        .run(
407            "upload isb",
408            "f=$(mktemp /tmp/isb-agent.XXXXXX) && cat > \"$f\" && echo \"$f\"",
409            &binary,
410        )?
411        .trim()
412        .to_string();
413    if !upload.starts_with("/tmp/isb-agent.")
414        || !upload
415            .bytes()
416            .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'/' | b'.' | b'-' | b'_'))
417    {
418        return Err(Error::invalid(format!(
419            "upload isb: unexpected path {upload:?}"
420        )));
421    }
422    p.step("install");
423    p.log("installing incus, the agent and its unit (this takes a few minutes on a fresh box)");
424    let script = render_script(
425        &upload,
426        &sha,
427        ca_pem,
428        leaf,
429        o.agent_port,
430        &o.allow_from,
431        Some(o.ssh_port),
432        o.public_ingress,
433    );
434    let out = ssh.run(
435        "install the agent",
436        &format!("{sudo}bash -s"),
437        script.as_bytes(),
438    )?;
439    Ok(out.lines().last().unwrap_or("").to_string())
440}
441
442pub fn sha256_hex(b: &[u8]) -> String {
443    crate::machine::hex(ring::digest::digest(&ring::digest::SHA256, b).as_ref())
444}
445
446/// This process's own executable, when it is a Linux build for `arch`.
447pub fn own_binary(arch: &str) -> Result<Vec<u8>> {
448    let b = std::fs::read("/proc/self/exe")
449        .map_err(|e| Error::invalid(format!("this control plane's own binary: {e}")))?;
450    match elf_arch(&b) {
451        Some(a) if a == arch => Ok(b),
452        Some(a) => Err(Error::invalid(format!(
453            "this control plane's binary is for {a}; the box is {arch}"
454        ))),
455        None => Err(Error::invalid(
456            "this control plane's binary is not a Linux executable",
457        )),
458    }
459}
460
461/// The binary to install: this process's own, a file given, or a release
462/// (checked against its SHA256SUMS); checked to be for `arch`.
463fn binary(o: &AddOptions, arch: &str, scratch: &Path, p: &Provision) -> Result<Vec<u8>> {
464    if o.self_binary {
465        p.log(&format!(
466            "using this control plane's own binary (isb {})",
467            env!("CARGO_PKG_VERSION")
468        ));
469        return own_binary(arch);
470    }
471    let b = match &o.isb_binary {
472        Some(f) => {
473            p.log(&format!("using {}", f.display()));
474            std::fs::read(f).map_err(|e| Error::invalid(format!("{}: {e}", f.display())))?
475        }
476        None => {
477            let ver = o
478                .version
479                .clone()
480                .unwrap_or_else(|| env!("CARGO_PKG_VERSION").to_string());
481            p.log(&format!("downloading isb v{ver} for {arch}"));
482            let dst = scratch.join("isb");
483            crate::machine::download_release(&ver, arch, scratch, &dst)?;
484            std::fs::read(&dst)?
485        }
486    };
487    match elf_arch(&b) {
488        Some(a) if a == arch => Ok(b),
489        Some(a) => Err(Error::invalid(format!(
490            "the isb binary is for {a}; the box is {arch}"
491        ))),
492        None => Err(Error::invalid("the isb binary is not a Linux executable")),
493    }
494}
495
496#[cfg(test)]
497mod tests {
498    use super::*;
499
500    #[test]
501    fn names_and_targets_are_checked() {
502        validate_name("hel-1").unwrap();
503        validate_name(&format!("vm-{}", "a".repeat(31))).unwrap();
504        for bad in ["", "Local", "local", "1box", "a_b", &"a".repeat(41)] {
505            assert!(validate_name(bad).is_err(), "{bad}");
506        }
507        assert_eq!(
508            ssh_host("root@203.0.113.7").unwrap(),
509            ("root", "203.0.113.7")
510        );
511        for bad in ["host", "root@", "-oProxyCommand=x@h", "a@b;c", "a@-b"] {
512            assert!(ssh_host(bad).is_err(), "{bad}");
513        }
514        check_cidr("203.0.113.7").unwrap();
515        check_cidr("203.0.113.0/24").unwrap();
516        assert!(check_cidr("0.0.0.0/0; rm").is_err());
517    }
518
519    #[test]
520    fn the_script_checks_the_binary_and_firewalls_when_asked() {
521        let leaf = Leaf {
522            cert: "CERT\n".into(),
523            key: "KEY\n".into(),
524        };
525        let s = render_script(
526            "/tmp/isb-agent.x",
527            "abc",
528            "CA\n",
529            &leaf,
530            7443,
531            &["198.51.100.1".into()],
532            Some(22),
533            true,
534        );
535        assert!(s.contains("echo \"abc  /tmp/isb-agent.x\" | sha256sum -c"));
536        assert!(s.contains("ufw allow proto tcp from '198.51.100.1' to any port 7443"));
537        assert!(s.contains("ufw allow 22/tcp"));
538        assert!(s.contains("pkgs.zabbly.com/incus/stable"));
539        assert!(s.contains("--agent-listen 0.0.0.0:7443"));
540        assert!(s.contains("--ingress-https 0.0.0.0:443") && s.contains("--public-ingress"));
541        assert!(s.contains("systemctl enable --now isb-agent-upgrade.path"));
542        let open = render_script(
543            "/tmp/isb-agent.x",
544            "abc",
545            "CA\n",
546            &leaf,
547            7443,
548            &[],
549            Some(22),
550            false,
551        );
552        assert!(!open.contains("ufw") && !open.contains("ingress"));
553        let vm = render_script(
554            "/tmp/isb-agent.x",
555            "abc",
556            "CA\n",
557            &leaf,
558            7443,
559            &["10.0.3.1".into()],
560            None,
561            false,
562        );
563        assert!(
564            vm.contains("ufw allow proto tcp from '10.0.3.1' to any port 7443")
565                && !vm.contains("/tcp\nufw allow proto"),
566            "a dedicated VM opens the agent port to the host only, and no SSH"
567        );
568        assert!(!vm.contains("ufw allow 22"));
569        assert_eq!(
570            elf_arch(b"\x7fELF\x02\x01\x01\0\0\0\0\0\0\0\0\0\x02\0\x3e\0"),
571            Some("x86_64")
572        );
573        assert_eq!(elf_arch(b"#!/bin/sh"), None);
574    }
575}
isb_server/servers/bootstrap.rs

isb_server/servers/
bootstrap.rs
