use std::collections::BTreeMap;
use std::path::Path;
use std::process::Command;
use runnerd_core::api::{Check, CheckStatus as Status};
use runnerd_core::config::{self, ByteSize, Config, Image, Isolation};
use runnerd_core::docker::Docker;
pub(crate) struct Host {
pub docker: Result<DockerFacts, String>,
pub images: BTreeMap<String, Result<ImageFound, String>>,
pub os_release: String,
pub slice: Option<Result<Slice, String>>,
pub gh_scopes: Result<Vec<String>, String>,
pub sysbox_services: [(&'static str, bool); 2],
pub kvm: bool,
}
pub(crate) const SYSBOX_SERVICES: [&str; 2] = ["sysbox-mgr", "sysbox-fs"];
pub(crate) struct DockerFacts {
pub version: String,
pub cgroup_version: String,
pub cgroup_driver: String,
pub runtimes: Vec<String>,
pub platform: String,
}
pub(crate) enum ImageFound {
Present { version: Option<String> },
Pullable { platforms: Vec<String> },
Missing(String),
}
const VERSION_LABEL: &str = "runnerd.version";
pub(crate) struct Slice {
pub unit_file: Option<String>,
pub memory_max: Option<ByteSize>,
pub cpu_quota: String,
}
impl Host {
pub(crate) fn read(config: Option<&Config>) -> Host {
let docker = read_docker();
Host {
images: match (config, &docker) {
(Some(config), Ok(_)) => read_images(config),
_ => BTreeMap::new(),
},
docker,
os_release: std::fs::read_to_string("/etc/os-release").unwrap_or_default(),
slice: config.map(|config| read_slice(&config.host.slice)),
gh_scopes: read_gh_scopes(),
sysbox_services: SYSBOX_SERVICES.map(|unit| {
let active = Command::new("systemctl")
.args(["is-active", "--quiet", unit])
.status()
.is_ok_and(|status| status.success());
(unit, active)
}),
kvm: Path::new("/dev/kvm").exists(),
}
}
}
fn read_docker() -> Result<DockerFacts, String> {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.map_err(|error| error.to_string())?;
let info = runtime
.block_on(async { Docker::connect()?.info().await })
.map_err(|error| error.to_string())?;
let mut runtimes: Vec<String> = info.runtimes.unwrap_or_default().into_keys().collect();
runtimes.sort();
Ok(DockerFacts {
version: info.server_version.unwrap_or_default(),
cgroup_version: info
.cgroup_version
.map(|v| v.to_string())
.unwrap_or_default(),
cgroup_driver: info
.cgroup_driver
.map(|d| d.to_string())
.unwrap_or_default(),
runtimes,
platform: format!(
"{}/{}",
info.os_type.unwrap_or_default(),
match info.architecture.unwrap_or_default().as_str() {
"x86_64" => "amd64".to_owned(),
"aarch64" => "arm64".to_owned(),
other => other.to_owned(),
}
),
})
}
fn read_images(config: &Config) -> BTreeMap<String, Result<ImageFound, String>> {
let images = images(config);
let read = async |docker: &Docker, reference: &str, image: &Image| {
let Some(mut labels) = docker.image_labels(reference).await? else {
return Ok(match image {
Image::Stage(_) => ImageFound::Missing("not built".to_owned()),
Image::Pulled(_) => match docker.registry_platforms(reference).await {
Ok(platforms) => ImageFound::Pullable { platforms },
Err(error) => ImageFound::Missing(error.to_string()),
},
});
};
Ok::<_, runnerd_core::docker::Error>(ImageFound::Present {
version: labels.remove(VERSION_LABEL),
})
};
let read_all = || -> Result<_, String> {
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.map_err(|error| error.to_string())?;
let docker = Docker::connect().map_err(|error| error.to_string())?;
Ok(runtime.block_on(async {
let mut found = BTreeMap::new();
for (reference, image) in &images {
let result = read(&docker, reference, image).await;
found.insert(reference.clone(), result.map_err(|error| error.to_string()));
}
found
}))
};
read_all().unwrap_or_else(|error| {
images
.keys()
.map(|reference| (reference.clone(), Err(error.clone())))
.collect()
})
}
fn images(config: &Config) -> BTreeMap<String, &Image> {
config
.profiles
.values()
.map(|profile| (profile.image.reference(), &profile.image))
.collect()
}
fn read_slice(slice: &str) -> Result<Slice, String> {
let value = |property: &str| {
let output = Command::new("systemctl")
.args(["show", slice, &format!("--property={property}"), "--value"])
.output()
.map_err(|error| format!("systemctl: {error}"))?;
if !output.status.success() {
return Err(format!(
"systemctl show {slice} failed: {}",
String::from_utf8_lossy(&output.stderr).trim()
));
}
Ok(String::from_utf8_lossy(&output.stdout).trim().to_owned())
};
let unit_file = value("FragmentPath")?;
let memory_max = value("MemoryMax")?;
Ok(Slice {
unit_file: (!unit_file.is_empty()).then_some(unit_file),
memory_max: config::parse_memory_max(&memory_max).ok_or_else(|| {
format!("systemctl show {slice}: unexpected MemoryMax {memory_max:?}")
})?,
cpu_quota: value("CPUQuotaPerSecUSec")?,
})
}
fn read_gh_scopes() -> Result<Vec<String>, String> {
let output = Command::new("gh")
.args([
"auth",
"status",
"--hostname",
"github.com",
"--json",
"hosts",
"--jq",
r#".hosts["github.com"][] | select(.active) | .scopes"#,
])
.output()
.map_err(|error| format!("cannot run gh: {error}"))?;
if !output.status.success() {
return Err(format!(
"gh is not logged in to github.com: {}",
String::from_utf8_lossy(&output.stderr).trim()
));
}
Ok(String::from_utf8_lossy(&output.stdout)
.split(',')
.map(|scope| scope.trim().to_owned())
.filter(|scope| !scope.is_empty())
.collect())
}
#[allow(
clippy::too_many_lines,
reason = "one flat list of checks reads better than helpers per check"
)]
pub(crate) fn checks(host: &Host, config: Option<&Config>) -> Vec<Check> {
let mut checks = Vec::new();
let docker = match &host.docker {
Ok(docker) => {
checks.push(pass(
"Docker",
format!("reachable, Docker {}", docker.version),
));
Some(docker)
}
Err(error) => {
checks.push(fail(
"Docker",
format!("unreachable: {error}"),
"start Docker and let your user reach its socket:\n\
sudo systemctl enable --now docker\n\
sudo usermod -aG docker \"$USER\" (then log in again)\n\
runnerd reaches Docker only through a unix:// socket: unset DOCKER_HOST or point it at one",
));
None
}
};
checks.push(match docker {
None => skip("cgroups", "needs Docker"),
Some(docker) if docker.cgroup_version == "2" && docker.cgroup_driver == "systemd" => {
pass("cgroups", "cgroup v2 with the systemd driver".to_owned())
}
Some(docker) => fail(
"cgroups",
format!(
"cgroup v{} with the {} driver; runnerd places runners in a systemd slice, which needs \
cgroup v2 with the systemd driver",
docker.cgroup_version, docker.cgroup_driver
),
"boot with cgroup v2 (the default on current distributions), and remove any \
\"native.cgroupdriver=cgroupfs\" from /etc/docker/daemon.json",
),
});
let isolation = config.map_or(Isolation::Sysbox, |config| config.host.isolation);
checks.push(match isolation {
Isolation::Sysbox => Check {
name: "isolation".to_owned(),
status: Status::Info,
detail: format!(
"sysbox: runners run unprivileged{}",
if config.is_none() {
" (the default)"
} else {
""
}
),
fix: None,
},
Isolation::UnsafeDind => Check {
name: "isolation".to_owned(),
status: Status::Warn,
detail: "unsafe-dind: runners run privileged, so a job can take over the host"
.to_owned(),
fix: None,
},
});
checks.push(match docker {
None => skip("sysbox", "needs Docker"),
Some(_) if isolation == Isolation::UnsafeDind => skip("sysbox", "unsafe-dind does not use it"),
Some(docker) if !docker.runtimes.iter().any(|r| r == "sysbox-runc") => fail(
"sysbox",
"Docker has no sysbox-runc runtime, which isolation = \"sysbox\" needs".to_owned(),
&sysbox_fix(&host.os_release),
),
Some(_) if host.sysbox_services.iter().any(|(_, active)| !active) => {
let stopped: Vec<_> = host
.sysbox_services
.iter()
.filter(|(_, active)| !active)
.map(|(unit, _)| *unit)
.collect();
fail(
"sysbox",
format!(
"Docker has the sysbox-runc runtime, but not running: {}; runner containers fail to start",
stopped.join(", ")
),
"sudo systemctl enable --now sysbox",
)
}
Some(_) => pass(
"sysbox",
"Docker has the sysbox-runc runtime and sysbox's services run".to_owned(),
),
});
checks.extend(slice_checks(host, config));
checks.extend(image_checks(host, config, docker));
checks.push(match &host.gh_scopes {
Ok(scopes) if scopes.iter().any(|s| s == "repo") => pass(
"gh",
"logged in to github.com with the repo scope".to_owned(),
),
Ok(scopes) => fail(
"gh",
format!(
"the token has scopes {scopes:?} but not repo, which runner registration needs"
),
"gh auth refresh --hostname github.com --scopes repo",
),
Err(error) => fail(
"gh",
error.clone(),
"install the GitHub CLI (https://cli.github.com), then:\n\
gh auth login --hostname github.com --scopes repo",
),
});
checks.push(Check {
name: "/dev/kvm".to_owned(),
status: Status::Info,
detail: if host.kvm {
"present, for a future Kata Containers backend".to_owned()
} else {
"absent; only a future Kata Containers backend needs it".to_owned()
},
fix: None,
});
checks
}
fn slice_checks(host: &Host, config: Option<&Config>) -> Vec<Check> {
let (Some(config), Some(slice)) = (config, &host.slice) else {
return vec![
skip("slice", "no config given, which names the slice"),
skip("config", "no config given: runnerd doctor <runnerd.toml>"),
];
};
let name = &config.host.slice;
let slice = match slice {
Ok(slice) => slice,
Err(error) => {
return vec![
fail("slice", error.clone(), "check that systemd runs"),
skip("config", "needs the slice"),
];
}
};
let slice_check = if let (Some(unit_file), Some(memory_max)) =
(&slice.unit_file, slice.memory_max)
{
pass(
"slice",
format!(
"{name} ({unit_file}): MemoryMax={memory_max}, CPU quota {}",
slice.cpu_quota
),
)
} else {
let caps = config.memory_caps().map_or_else(
|_| "<total of your caps>".to_owned(),
|caps| caps.to_string(),
);
fail(
"slice",
format!(
"{name} has no unit file with a MemoryMax, so nothing holds runners to the budget"
),
&format!(
"create /etc/systemd/system/{name} with\n\
[Slice]\n\
MemoryMax={caps}\n\
then: sudo systemctl daemon-reload"
),
)
};
let config_check = match config.validate(slice.memory_max) {
Ok(warnings) if warnings.is_empty() => pass("config", format!("fits in {name}")),
Ok(warnings) => Check {
name: "config".to_owned(),
status: Status::Pass,
detail: format!(
"valid, with warnings: {}",
warnings
.iter()
.map(ToString::to_string)
.collect::<Vec<_>>()
.join("; ")
),
fix: None,
},
Err(error) => fail(
"config",
error.to_string(),
"change runnerd.toml or the slice's limits",
),
};
vec![slice_check, config_check]
}
fn image_checks(host: &Host, config: Option<&Config>, docker: Option<&DockerFacts>) -> Vec<Check> {
let Some(config) = config else {
return vec![skip("image", "no config given, which names the images")];
};
let Some(docker) = docker else {
return vec![skip("image", "needs Docker")];
};
let published = docker.platform == "linux/amd64";
let version = env!("CARGO_PKG_VERSION");
images(config)
.into_iter()
.map(|(reference, image)| match host.images.get(&reference) {
None => skip("image", &format!("{reference}: not checked")),
Some(Err(error)) => fail(
"image",
format!("cannot inspect {reference}: {error}"),
"check that Docker runs",
),
Some(Ok(ImageFound::Present { version: Some(built) })) if built != version => Check {
name: "image".to_owned(),
status: Status::Warn,
detail: format!(
"{reference} is built for runnerd {built}, this is runnerd {version}; its scripts may not \
match the daemon's"
),
fix: Some(format!(
"set image to this release's tag, such as {PUBLISHED}:<stage>-{version}"
)),
},
Some(Ok(ImageFound::Present { .. })) => pass("image", format!("{reference}: present")),
Some(Ok(ImageFound::Pullable { platforms }))
if platforms.is_empty() || platforms.contains(&docker.platform) =>
{
pass(
"image",
format!("{reference}: not pulled yet; runnerd pulls it before the first runner"),
)
}
Some(Ok(ImageFound::Pullable { platforms })) => fail(
"image",
format!(
"{reference} is built for {} only, and Docker here runs {}",
platforms.join(", "),
docker.platform
),
"build the image on this host from a clone of the repository and use it with image_stage:\n\
docker build --target <stage> -t runnerd-runner:<stage> image",
),
Some(Ok(ImageFound::Missing(why))) => match image {
Image::Stage(stage) => {
let build = format!(
"build it from a clone of the repository:\n\
docker build --target {stage} -t {reference} image"
);
let fix = if published {
format!(
"use the published image instead of image_stage:\n\
image = \"{PUBLISHED}:{stage}-{version}\"\nor {build}"
)
} else {
build
};
fail("image", format!("{reference}: {why}"), &fix)
}
Image::Pulled(_) => fail(
"image",
format!("{reference}: not present, and cannot be pulled: {why}"),
&format!("check the reference, then try: docker pull {reference}"),
),
},
})
.collect()
}
const PUBLISHED: &str = "ghcr.io/ayagmar/runnerd-runner";
fn sysbox_fix(os_release: &str) -> String {
let family: Vec<&str> = os_release
.lines()
.filter_map(|line| {
line.strip_prefix("ID=")
.or_else(|| line.strip_prefix("ID_LIKE="))
})
.flat_map(|value| value.trim_matches('"').split_whitespace())
.collect();
let restart = "sudo systemctl restart docker (stops running containers)";
if family.iter().any(|id| matches!(*id, "debian" | "ubuntu")) {
"install the .deb from https://github.com/nestybox/sysbox/releases; it registers the runtime and \
restarts Docker, which stops running containers:\n\
sudo apt-get install ./sysbox-ce_<version>.linux_amd64.deb"
.to_owned()
} else if family.contains(&"arch") {
format!(
"install the AUR package sysbox-ce-bin, for example: paru -S sysbox-ce-bin\n\
sudo systemctl enable --now sysbox\n\
add to /etc/docker/daemon.json (Arch kernels are not on sysbox's allow-list):\n\
{{\"runtimes\": {{\"sysbox-runc\": {{\"path\": \"/usr/bin/sysbox-runc\", \"runtimeArgs\": [\"--no-kernel-check\"]}}}}}}\n\
{restart}"
)
} else {
format!(
"build sysbox from source: \
https://github.com/nestybox/sysbox/blob/master/docs/developers-guide/build.md\n\
register sysbox-runc as a runtime in /etc/docker/daemon.json\n\
{restart}"
)
}
}
fn pass(name: &str, detail: String) -> Check {
Check {
name: name.to_owned(),
status: Status::Pass,
detail,
fix: None,
}
}
fn fail(name: &str, detail: String, fix: &str) -> Check {
Check {
name: name.to_owned(),
status: Status::Fail,
detail,
fix: Some(fix.to_owned()),
}
}
fn skip(name: &str, detail: &str) -> Check {
Check {
name: name.to_owned(),
status: Status::Skip,
detail: detail.to_owned(),
fix: None,
}
}
#[cfg(test)]
mod tests;