runnerd 0.0.7

//! `runnerd doctor`: what the host is missing to run runners, each with the fix. It reads the host and changes
//! nothing; the user runs the fixes. The checks are `api::Check`s, so `GET /v1/doctor` serves the same ones.

use std::collections::BTreeMap;
use std::path::Path;
use std::process::Command;

use runnerd_core::api::{Advice, Check, CheckStatus as Status, Sizing};
use runnerd_core::config::{self, Admission, ByteSize, Config, Image, Isolation};
use runnerd_core::docker::{Docker, DockerApi as _};
use runnerd_core::record;
use runnerd_core::sizing::{Window, advise};
use runnerd_core::store;

/// What doctor reads from the host, gathered before any check so the checks can be tested without one.
pub(crate) struct Host {
    pub docker: Result<DockerFacts, String>,
    /// What Docker has of each profile's image, by reference; empty without a config or Docker.
    pub images: BTreeMap<String, Result<ImageFound, String>>,
    /// The disk the records of failed jobs take; `None` without a config or Docker.
    pub records: Option<Result<u64, String>>,
    /// Sizing advice from the job history; `None` without a config or a history. `read` leaves it out: the daemon
    /// reads its own store, `runnerd doctor` the database with `read_sizing`.
    pub sizing: Option<Result<Sizing, String>>,
    /// `/etc/os-release`, empty when missing.
    pub os_release: String,
    /// `None` without a config, which names the slice.
    pub slice: Option<Result<Slice, String>>,
    /// The scopes of `gh`'s token for github.com.
    pub gh_scopes: Result<Vec<String>, String>,
    /// Whether each of `SYSBOX_SERVICES` runs: Docker keeps the runtime registered without them.
    pub sysbox_services: [(&'static str, bool); 2],
    pub kvm: bool,
}

/// The services sysbox-runc needs; `sysbox.service` starts both.
pub(crate) const SYSBOX_SERVICES: [&str; 2] = ["sysbox-mgr", "sysbox-fs"];

pub(crate) struct DockerFacts {
    pub version: String,
    pub cgroup_version: String,
    pub cgroup_driver: String,
    pub runtimes: Vec<String>,
    /// The platform images must be built for, such as `linux/amd64`.
    pub platform: String,
}

/// A profile's image as Docker has it, or where it could come from.
pub(crate) enum ImageFound {
    /// Docker has it; `version` is its `runnerd.version` label, which published images carry.
    Present { version: Option<String> },
    /// Docker lacks it and its registry has it for `platforms`, such as `linux/amd64`; none when the registry does
    /// not say. runnerd pulls it before the first runner if one is Docker's.
    Pullable { platforms: Vec<String> },
    /// Docker lacks it and cannot get it: never built, or the registry refused, with why.
    Missing(String),
}

/// The label of a published image that names the runnerd release it was built for (`.github/workflows/images.yml`).
/// Not `org.opencontainers.image.version`: every image built on the runner base inherits Ubuntu's.
const VERSION_LABEL: &str = "runnerd.version";

pub(crate) struct Slice {
    /// The slice's unit file; without one, systemd creates the slice without limits.
    pub unit_file: Option<String>,
    pub memory_max: Option<ByteSize>,
    /// `CPUQuotaPerSecUSec` as `systemctl show` prints it, such as `16s` or `infinity`.
    pub cpu_quota: String,
}

impl Host {
    pub(crate) fn read(config: Option<&Config>) -> Host {
        let docker = read_docker();
        Host {
            images: match (config, &docker) {
                (Some(config), Ok(_)) => read_images(config),
                _ => BTreeMap::new(),
            },
            records: (config.is_some() && docker.is_ok()).then(read_records),
            sizing: None,
            docker,
            os_release: std::fs::read_to_string("/etc/os-release").unwrap_or_default(),
            slice: config.map(|config| read_slice(&config.host.slice)),
            gh_scopes: read_gh_scopes(),
            // One unit per call: `systemctl is-active` with several units succeeds when any one runs.
            sysbox_services: SYSBOX_SERVICES.map(|unit| {
                let active = Command::new("systemctl")
                    .args(["is-active", "--quiet", unit])
                    .status()
                    .is_ok_and(|status| status.success());
                (unit, active)
            }),
            kvm: Path::new("/dev/kvm").exists(),
        }
    }
}

/// The advice from the job history at `database`, read-only, while no daemon runs; `None` when it does not exist.
pub(crate) fn read_sizing(
    config: &Config,
    database: &Path,
    memory_max: Option<u64>,
    now: u64,
) -> Option<Result<Sizing, String>> {
    let window = Window::new(None, now);
    match store::sizing_at(database, window.since) {
        Ok(history) => history.map(|history| Ok(advise(config, memory_max, &history, window))),
        Err(error) => Some(Err(error.to_string())),
    }
}

fn read_docker() -> Result<DockerFacts, String> {
    let runtime = tokio::runtime::Builder::new_current_thread()
        .enable_all()
        .build()
        .map_err(|error| error.to_string())?;
    let info = runtime
        .block_on(async { Docker::connect()?.info().await })
        .map_err(|error| error.to_string())?;
    let mut runtimes: Vec<String> = info.runtimes.unwrap_or_default().into_keys().collect();
    runtimes.sort();
    Ok(DockerFacts {
        version: info.server_version.unwrap_or_default(),
        cgroup_version: info
            .cgroup_version
            .map(|v| v.to_string())
            .unwrap_or_default(),
        cgroup_driver: info
            .cgroup_driver
            .map(|d| d.to_string())
            .unwrap_or_default(),
        runtimes,
        platform: format!(
            "{}/{}",
            info.os_type.unwrap_or_default(),
            // Docker reports the kernel's name for it; images name it as Go does.
            match info.architecture.unwrap_or_default().as_str() {
                "x86_64" => "amd64".to_owned(),
                "aarch64" => "arm64".to_owned(),
                other => other.to_owned(),
            }
        ),
    })
}

fn read_images(config: &Config) -> BTreeMap<String, Result<ImageFound, String>> {
    let images = images(config);
    let read = async |docker: &Docker, reference: &str, image: &Image| {
        let Some(mut labels) = docker.image_labels(reference).await? else {
            return Ok(match image {
                Image::Stage(_) => ImageFound::Missing("not built".to_owned()),
                Image::Pulled(_) => match docker.registry_platforms(reference).await {
                    Ok(platforms) => ImageFound::Pullable { platforms },
                    Err(error) => ImageFound::Missing(error.to_string()),
                },
            });
        };
        Ok::<_, runnerd_core::docker::Error>(ImageFound::Present {
            version: labels.remove(VERSION_LABEL),
        })
    };
    let read_all = || -> Result<_, String> {
        let runtime = tokio::runtime::Builder::new_current_thread()
            .enable_all()
            .build()
            .map_err(|error| error.to_string())?;
        let docker = Docker::connect().map_err(|error| error.to_string())?;
        Ok(runtime.block_on(async {
            let mut found = BTreeMap::new();
            for (reference, image) in &images {
                let result = read(&docker, reference, image).await;
                found.insert(reference.clone(), result.map_err(|error| error.to_string()));
            }
            found
        }))
    };
    read_all().unwrap_or_else(|error| {
        images
            .keys()
            .map(|reference| (reference.clone(), Err(error.clone())))
            .collect()
    })
}

fn read_records() -> Result<u64, String> {
    let runtime = tokio::runtime::Builder::new_current_thread()
        .enable_all()
        .build()
        .map_err(|error| error.to_string())?;
    let images = runtime
        .block_on(async { Docker::connect()?.images(record::LABEL).await })
        .map_err(|error| error.to_string())?;
    Ok(images.iter().map(|image| image.bytes).sum())
}

/// The config's distinct images, by reference.
fn images(config: &Config) -> BTreeMap<String, &Image> {
    config
        .profiles
        .values()
        .map(|profile| (profile.image.reference(), &profile.image))
        .collect()
}

fn read_slice(slice: &str) -> Result<Slice, String> {
    let value = |property: &str| {
        let output = Command::new("systemctl")
            .args(["show", slice, &format!("--property={property}"), "--value"])
            .output()
            .map_err(|error| format!("systemctl: {error}"))?;
        if !output.status.success() {
            return Err(format!(
                "systemctl show {slice} failed: {}",
                String::from_utf8_lossy(&output.stderr).trim()
            ));
        }
        Ok(String::from_utf8_lossy(&output.stdout).trim().to_owned())
    };
    let unit_file = value("FragmentPath")?;
    let memory_max = value("MemoryMax")?;
    Ok(Slice {
        unit_file: (!unit_file.is_empty()).then_some(unit_file),
        memory_max: config::parse_memory_max(&memory_max).ok_or_else(|| {
            format!("systemctl show {slice}: unexpected MemoryMax {memory_max:?}")
        })?,
        cpu_quota: value("CPUQuotaPerSecUSec")?,
    })
}

fn read_gh_scopes() -> Result<Vec<String>, String> {
    let output = Command::new("gh")
        .args([
            "auth",
            "status",
            "--hostname",
            "github.com",
            "--json",
            "hosts",
            "--jq",
            r#".hosts["github.com"][] | select(.active) | .scopes"#,
        ])
        .output()
        .map_err(|error| format!("cannot run gh: {error}"))?;
    if !output.status.success() {
        return Err(format!(
            "gh is not logged in to github.com: {}",
            String::from_utf8_lossy(&output.stderr).trim()
        ));
    }
    Ok(String::from_utf8_lossy(&output.stdout)
        .split(',')
        .map(|scope| scope.trim().to_owned())
        .filter(|scope| !scope.is_empty())
        .collect())
}

/// Every check, in the order a new user fixes them.
#[allow(
    clippy::too_many_lines,
    reason = "one flat list of checks reads better than helpers per check"
)]
pub(crate) fn checks(host: &Host, config: Option<&Config>) -> Vec<Check> {
    let mut checks = Vec::new();
    let docker = match &host.docker {
        Ok(docker) => {
            checks.push(pass(
                "Docker",
                format!("reachable, Docker {}", docker.version),
            ));
            Some(docker)
        }
        Err(error) => {
            checks.push(fail(
                "Docker",
                format!("unreachable: {error}"),
                "start Docker and let your user reach its socket:\n\
                 sudo systemctl enable --now docker\n\
                 sudo usermod -aG docker \"$USER\"   (then log in again)\n\
                 runnerd reaches Docker only through a unix:// socket: unset DOCKER_HOST or point it at one",
            ));
            None
        }
    };
    checks.push(match docker {
        None => skip("cgroups", "needs Docker"),
        Some(docker) if docker.cgroup_version == "2" && docker.cgroup_driver == "systemd" => {
            pass("cgroups", "cgroup v2 with the systemd driver".to_owned())
        }
        Some(docker) => fail(
            "cgroups",
            format!(
                "cgroup v{} with the {} driver; runnerd places runners in a systemd slice, which needs \
                 cgroup v2 with the systemd driver",
                docker.cgroup_version, docker.cgroup_driver
            ),
            "boot with cgroup v2 (the default on current distributions), and remove any \
             \"native.cgroupdriver=cgroupfs\" from /etc/docker/daemon.json",
        ),
    });
    let isolation = config.map_or(Isolation::Sysbox, |config| config.host.isolation);
    checks.push(match isolation {
        Isolation::Sysbox => Check {
            name: "isolation".to_owned(),
            status: Status::Info,
            detail: format!(
                "sysbox: runners run unprivileged{}",
                if config.is_none() {
                    " (the default)"
                } else {
                    ""
                }
            ),
            fix: None,
        },
        Isolation::UnsafeDind => Check {
            name: "isolation".to_owned(),
            status: Status::Warn,
            detail: "unsafe-dind: runners run privileged, so a job can take over the host"
                .to_owned(),
            fix: None,
        },
    });
    checks.push(match docker {
        None => skip("sysbox", "needs Docker"),
        Some(_) if isolation == Isolation::UnsafeDind => skip("sysbox", "unsafe-dind does not use it"),
        Some(docker) if !docker.runtimes.iter().any(|r| r == "sysbox-runc") => fail(
            "sysbox",
            "Docker has no sysbox-runc runtime, which isolation = \"sysbox\" needs".to_owned(),
            &sysbox_fix(&host.os_release),
        ),
        Some(_) if host.sysbox_services.iter().any(|(_, active)| !active) => {
            let stopped: Vec<_> = host
                .sysbox_services
                .iter()
                .filter(|(_, active)| !active)
                .map(|(unit, _)| *unit)
                .collect();
            fail(
                "sysbox",
                format!(
                    "Docker has the sysbox-runc runtime, but not running: {}; runner containers fail to start",
                    stopped.join(", ")
                ),
                "sudo systemctl enable --now sysbox",
            )
        }
        Some(_) => pass(
            "sysbox",
            "Docker has the sysbox-runc runtime and sysbox's services run".to_owned(),
        ),
    });
    checks.extend(slice_checks(host, config));
    checks.extend(image_checks(host, config, docker));
    if let (Some(config), Some(records)) = (config, &host.records) {
        checks.push(records_check(records, config));
    }
    if let Some(sizing) = &host.sizing {
        checks.push(sizing_check(sizing));
    }
    checks.push(match &host.gh_scopes {
        Ok(scopes) if scopes.iter().any(|s| s == "repo") => pass(
            "gh",
            "logged in to github.com with the repo scope".to_owned(),
        ),
        Ok(scopes) => fail(
            "gh",
            format!(
                "the token has scopes {scopes:?} but not repo, which runner registration needs"
            ),
            "gh auth refresh --hostname github.com --scopes repo",
        ),
        Err(error) => fail(
            "gh",
            error.clone(),
            "install the GitHub CLI (https://cli.github.com), then:\n\
             gh auth login --hostname github.com --scopes repo",
        ),
    });
    checks.push(Check {
        name: "/dev/kvm".to_owned(),
        status: Status::Info,
        detail: if host.kvm {
            "present, for a future Kata Containers backend".to_owned()
        } else {
            "absent; only a future Kata Containers backend needs it".to_owned()
        },
        fix: None,
    });
    checks
}

fn slice_checks(host: &Host, config: Option<&Config>) -> Vec<Check> {
    let (Some(config), Some(slice)) = (config, &host.slice) else {
        return vec![
            skip("slice", "no config given, which names the slice"),
            skip("config", "no config given: runnerd doctor <runnerd.toml>"),
        ];
    };
    let name = &config.host.slice;
    let slice = match slice {
        Ok(slice) => slice,
        Err(error) => {
            return vec![
                fail("slice", error.clone(), "check that systemd runs"),
                skip("config", "needs the slice"),
            ];
        }
    };
    let slice_check = if let (Some(unit_file), Some(memory_max)) =
        (&slice.unit_file, slice.memory_max)
    {
        pass(
            "slice",
            format!(
                "{name} ({unit_file}): MemoryMax={memory_max}, CPU quota {}",
                slice.cpu_quota
            ),
        )
    } else {
        let caps = config.memory_caps().map_or_else(
            |_| "<total of your caps>".to_owned(),
            |caps| caps.to_string(),
        );
        fail(
            "slice",
            format!(
                "{name} has no unit file with a MemoryMax, so nothing holds runners to the budget"
            ),
            &format!(
                "create /etc/systemd/system/{name} with\n\
                     [Slice]\n\
                     MemoryMax={caps}\n\
                     then: sudo systemctl daemon-reload"
            ),
        )
    };
    let config_check = match config.validate(slice.memory_max) {
        Ok(warnings) if warnings.is_empty() => pass(
            "config",
            match (
                config.host.admission,
                config.memory_caps(),
                slice.memory_max,
            ) {
                (Admission::Measured, Ok(caps), Some(max)) if caps > max => format!(
                    "memory caps sum to {caps}, over {name}'s {max} MemoryMax: admission = \"measured\" starts \
                     what the predicted peaks fit"
                ),
                _ => format!("fits in {name}"),
            },
        ),
        Ok(warnings) => Check {
            name: "config".to_owned(),
            status: Status::Pass,
            detail: format!(
                "valid, with warnings: {}",
                warnings
                    .iter()
                    .map(ToString::to_string)
                    .collect::<Vec<_>>()
                    .join("; ")
            ),
            fix: None,
        },
        Err(error) => fail(
            "config",
            error.to_string(),
            "change runnerd.toml or the slice's limits",
        ),
    };
    vec![slice_check, config_check]
}

/// One check per distinct image of the config's profiles.
fn image_checks(host: &Host, config: Option<&Config>, docker: Option<&DockerFacts>) -> Vec<Check> {
    let Some(config) = config else {
        return vec![skip("image", "no config given, which names the images")];
    };
    let Some(docker) = docker else {
        return vec![skip("image", "needs Docker")];
    };
    // The release publishes linux/amd64 images only (.github/workflows/images.yml).
    let published = docker.platform == "linux/amd64";
    let version = env!("CARGO_PKG_VERSION");
    images(config)
        .into_iter()
        .map(|(reference, image)| match host.images.get(&reference) {
            None => skip("image", &format!("{reference}: not checked")),
            Some(Err(error)) => fail(
                "image",
                format!("cannot inspect {reference}: {error}"),
                "check that Docker runs",
            ),
            Some(Ok(ImageFound::Present { version: Some(built) })) if built != version => Check {
                name: "image".to_owned(),
                status: Status::Warn,
                detail: format!(
                    "{reference} is built for runnerd {built}, this is runnerd {version}; its scripts may not \
                     match the daemon's"
                ),
                fix: Some(format!(
                    "set image to this release's tag, such as {PUBLISHED}:<stage>-{version}"
                )),
            },
            Some(Ok(ImageFound::Present { .. })) => pass("image", format!("{reference}: present")),
            Some(Ok(ImageFound::Pullable { platforms }))
                if platforms.is_empty() || platforms.contains(&docker.platform) =>
            {
                pass(
                    "image",
                    format!("{reference}: not pulled yet; runnerd pulls it before the first runner"),
                )
            }
            Some(Ok(ImageFound::Pullable { platforms })) => fail(
                "image",
                format!(
                    "{reference} is built for {} only, and Docker here runs {}",
                    platforms.join(", "),
                    docker.platform
                ),
                "build the image on this host from a clone of the repository and use it with image_stage:\n\
                 docker build --target <stage> -t runnerd-runner:<stage> image",
            ),
            Some(Ok(ImageFound::Missing(why))) => match image {
                Image::Stage(stage) => {
                    let build = format!(
                        "build it from a clone of the repository:\n\
                         docker build --target {stage} -t {reference} image"
                    );
                    let fix = if published {
                        format!(
                            "use the published image instead of image_stage:\n\
                             image = \"{PUBLISHED}:{stage}-{version}\"\nor {build}"
                        )
                    } else {
                        build
                    };
                    fail("image", format!("{reference}: {why}"), &fix)
                }
                Image::Pulled(_) => fail(
                    "image",
                    format!("{reference}: not present, and cannot be pulled: {why}"),
                    &format!("check the reference, then try: docker pull {reference}"),
                ),
            },
        })
        .collect()
}

/// Where the release workflow publishes the runner images.
const PUBLISHED: &str = "ghcr.io/ayagmar/runnerd-runner";

/// Sysbox ships `.deb` packages only; Arch has it in the AUR; elsewhere it is built from source.
fn sysbox_fix(os_release: &str) -> String {
    let family: Vec<&str> = os_release
        .lines()
        .filter_map(|line| {
            line.strip_prefix("ID=")
                .or_else(|| line.strip_prefix("ID_LIKE="))
        })
        .flat_map(|value| value.trim_matches('"').split_whitespace())
        .collect();
    let restart = "sudo systemctl restart docker   (stops running containers)";
    if family.iter().any(|id| matches!(*id, "debian" | "ubuntu")) {
        "install the .deb from https://github.com/nestybox/sysbox/releases; it registers the runtime and \
         restarts Docker, which stops running containers:\n\
         sudo apt-get install ./sysbox-ce_<version>.linux_amd64.deb"
            .to_owned()
    } else if family.contains(&"arch") {
        format!(
            "install the AUR package sysbox-ce-bin, for example: paru -S sysbox-ce-bin\n\
             sudo systemctl enable --now sysbox\n\
             add to /etc/docker/daemon.json (Arch kernels are not on sysbox's allow-list):\n\
             {{\"runtimes\": {{\"sysbox-runc\": {{\"path\": \"/usr/bin/sysbox-runc\", \"runtimeArgs\": [\"--no-kernel-check\"]}}}}}}\n\
             {restart}"
        )
    } else {
        format!(
            "build sysbox from source: \
             https://github.com/nestybox/sysbox/blob/master/docs/developers-guide/build.md\n\
             register sysbox-runc as a runtime in /etc/docker/daemon.json\n\
             {restart}"
        )
    }
}

/// Warns when the records take 80% of `[records] max_total`, past which new ones evict the oldest.
fn records_check(records: &Result<u64, String>, config: &Config) -> Check {
    let max_total = config.records.max_total;
    match records {
        Err(error) => skip("records", &format!("cannot list them: {error}")),
        Ok(bytes) if u128::from(*bytes) * 5 >= u128::from(max_total.0) * 4 => Check {
            name: "records".to_owned(),
            status: Status::Warn,
            detail: format!(
                "the records of failed jobs take {} of [records] max_total = {max_total}; \
                 new ones remove the oldest to fit",
                ByteSize(*bytes)
            ),
            fix: Some(
                "runnerctl records rm --older-than 2d, or raise max_total under [records]"
                    .to_owned(),
            ),
        },
        Ok(bytes) => Check {
            name: "records".to_owned(),
            status: Status::Info,
            detail: format!(
                "the records of failed jobs take {} of [records] max_total = {max_total}",
                ByteSize(*bytes)
            ),
            fix: None,
        },
    }
}

/// Warns when a profile's jobs need more than its cap or than the slice holds; says which caps could be lowered.
fn sizing_check(sizing: &Result<Sizing, String>) -> Check {
    let sizing = match sizing {
        Ok(sizing) => sizing,
        Err(error) => return skip("sizing", &format!("cannot read the job history: {error}")),
    };
    let changes: Vec<_> = sizing
        .profiles
        .iter()
        .filter(|profile| {
            profile.advice.change().is_some()
                || matches!(profile.advice, Advice::SliceTooSmall { .. })
        })
        .collect();
    let raise = changes.iter().any(|profile| {
        matches!(
            profile.advice,
            Advice::Raise { .. } | Advice::SliceTooSmall { .. }
        )
    });
    let detail = if changes.is_empty() {
        let too_few = sizing
            .profiles
            .iter()
            .any(|profile| matches!(profile.advice, Advice::TooFew { .. }));
        if too_few {
            format!(
                "some profiles ran fewer than {} jobs; runnerctl sizing shows their peaks",
                sizing.min_jobs
            )
        } else {
            format!(
                "every profile's memory fits its jobs' largest peak + {}%",
                sizing.margin
            )
        }
    } else {
        changes
            .iter()
            .map(|profile| format!("{}: {}", profile.name, sizing.advice_line(profile)))
            .collect::<Vec<_>>()
            .join("; ")
    };
    Check {
        name: "sizing".to_owned(),
        status: if raise { Status::Warn } else { Status::Info },
        detail,
        fix: (!changes.is_empty()).then(|| {
            changes
                .iter()
                .map(|profile| match profile.advice {
                    Advice::SliceTooSmall { .. } => format!(
                        "raise MemoryMax in the slice's unit file, sudo systemctl daemon-reload, then: \
                         runnerctl sizing apply {}",
                        profile.name
                    ),
                    _ => format!("runnerctl sizing apply {}", profile.name),
                })
                .collect::<Vec<_>>()
                .join("\n")
        }),
    }
}

fn pass(name: &str, detail: String) -> Check {
    Check {
        name: name.to_owned(),
        status: Status::Pass,
        detail,
        fix: None,
    }
}

fn fail(name: &str, detail: String, fix: &str) -> Check {
    Check {
        name: name.to_owned(),
        status: Status::Fail,
        detail,
        fix: Some(fix.to_owned()),
    }
}

fn skip(name: &str, detail: &str) -> Check {
    Check {
        name: name.to_owned(),
        status: Status::Skip,
        detail: detail.to_owned(),
        fix: None,
    }
}

#[cfg(test)]
mod tests;