use std::path::Path;
use anyhow::{Context, Result};
use clap::ValueEnum;
use crate::hostgpu::{self, HostGpu};
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
#[value(rename_all = "lower")]
pub enum ContainerManager {
Docker,
Podman,
}
impl ContainerManager {
pub fn binary(self) -> &'static str {
match self {
ContainerManager::Docker => "docker",
ContainerManager::Podman => "podman",
}
}
pub fn probe(self) -> Result<()> {
self.probe_with(true)
}
pub fn probe_cpu_only(self) -> Result<()> {
self.probe_with(false)
}
fn probe_with(self, require_gpu_toolkit: bool) -> Result<()> {
let cli = self.binary();
if crate::find_on_path(cli).is_none() {
anyhow::bail!("{cli} is not on PATH");
}
let mut cmd = std::process::Command::new(cli);
cmd.arg("info")
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null());
match run_with_timeout(cmd, PROBE_TIMEOUT).with_context(|| format!("run {cli} info"))? {
Some(status) if status.success() => {}
Some(_) => anyhow::bail!("{cli} info failed (is its daemon running?)"),
None => anyhow::bail!("{cli} info did not answer within {PROBE_TIMEOUT:?}"),
}
let wants_gpus = detect_backend().engine_args().iter().any(|a| a == "--gpus");
gpu_gate(require_gpu_toolkit, wants_gpus, cli, || {
nvidia_toolkit_present(self)
})
}
}
fn gpu_gate(
require_gpu_toolkit: bool,
wants_gpus: bool,
cli: &str,
toolkit_present: impl FnOnce() -> bool,
) -> Result<()> {
if require_gpu_toolkit && wants_gpus && !toolkit_present() {
anyhow::bail!(
"an NVIDIA GPU was detected but no NVIDIA Container Toolkit for {cli} \
(`{cli} run --gpus all` would fail)"
);
}
Ok(())
}
const PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
fn run_with_timeout(
mut cmd: std::process::Command,
timeout: std::time::Duration,
) -> Result<Option<std::process::ExitStatus>> {
let mut child = cmd.spawn()?;
let deadline = std::time::Instant::now() + timeout;
loop {
if let Some(status) = child.try_wait()? {
return Ok(Some(status));
}
if std::time::Instant::now() >= deadline {
let _ = child.kill();
let _ = child.wait();
return Ok(None);
}
std::thread::sleep(std::time::Duration::from_millis(50));
}
}
fn nvidia_toolkit_present(ociman: ContainerManager) -> bool {
if [
"nvidia-container-runtime-hook",
"nvidia-container-toolkit",
"nvidia-ctk",
]
.iter()
.any(|b| crate::find_on_path(b).is_some())
{
return true;
}
if ["/etc/cdi/nvidia.yaml", "/var/run/cdi/nvidia.yaml"]
.iter()
.any(|p| Path::new(p).exists())
{
return true;
}
if ociman == ContainerManager::Docker {
let runtimes = std::process::Command::new("docker")
.args([
"info",
"--format",
"{{range $k, $v := .Runtimes}}{{$k}} {{end}}",
])
.output()
.ok()
.filter(|o| o.status.success())
.map(|o| String::from_utf8_lossy(&o.stdout).into_owned())
.unwrap_or_default();
if runtimes.split_whitespace().any(|r| r == "nvidia") {
return true;
}
}
false
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum GpuBackend {
Cpu,
Cuda12,
Cuda13,
Rocm,
Vulkan,
}
impl GpuBackend {
fn image_tag(self) -> &'static str {
match self {
GpuBackend::Cpu => "server",
GpuBackend::Cuda12 => "server-cuda",
GpuBackend::Cuda13 => "server-cuda13",
GpuBackend::Rocm => "server-rocm",
GpuBackend::Vulkan => "server-vulkan",
}
}
fn image_ref(self, version: Option<&str>) -> String {
match version {
Some(v) => format!("ghcr.io/ggml-org/llama.cpp:{}-{v}", self.image_tag()),
None => format!("ghcr.io/ggml-org/llama.cpp:{}", self.image_tag()),
}
}
fn engine_args(self) -> Vec<String> {
match self {
GpuBackend::Cpu => vec![],
GpuBackend::Cuda12 | GpuBackend::Cuda13 => vec!["--gpus".into(), "all".into()],
GpuBackend::Rocm => vec![
"--device".into(),
"/dev/kfd".into(),
"--device".into(),
"/dev/dri".into(),
"--group-add".into(),
"video".into(),
],
GpuBackend::Vulkan => {
let mut args = Vec::new();
if Path::new("/dev/dri").exists() {
args.extend(["--device", "/dev/dri"].map(String::from));
}
if Path::new("/dev/nvidiactl").exists() {
args.extend(
["--gpus", "all", "-e", "NVIDIA_DRIVER_CAPABILITIES=all"].map(String::from),
);
}
args
}
}
}
}
pub(crate) fn detect_backend() -> GpuBackend {
backend_from_hostgpu(hostgpu::detect())
}
pub(crate) const SPEECH_BACKEND: GpuBackend = GpuBackend::Cpu;
fn backend_from_hostgpu(gpu: HostGpu) -> GpuBackend {
match gpu {
HostGpu::Cuda { major } if major >= 13 => GpuBackend::Cuda13,
HostGpu::Cuda { .. } => GpuBackend::Cuda12,
HostGpu::Rocm => GpuBackend::Rocm,
HostGpu::Vulkan => GpuBackend::Vulkan,
HostGpu::Metal | HostGpu::None | HostGpu::Opencl => GpuBackend::Cpu,
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum HostArch {
X86_64,
Aarch64,
}
impl HostArch {
fn parse(arch: &str) -> Option<HostArch> {
match arch {
"x86_64" => Some(HostArch::X86_64),
"aarch64" => Some(HostArch::Aarch64),
_ => None,
}
}
fn detect() -> Result<HostArch> {
HostArch::parse(std::env::consts::ARCH).with_context(|| {
format!(
"vLLM publishes no container image for {} hosts (only x86_64 and aarch64)",
std::env::consts::ARCH
)
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum VllmBackend {
Cuda12,
Cuda13,
Rocm,
Cpu,
}
impl VllmBackend {
fn from_gpu(backend: GpuBackend) -> VllmBackend {
match backend {
GpuBackend::Cuda12 => VllmBackend::Cuda12,
GpuBackend::Cuda13 => VllmBackend::Cuda13,
GpuBackend::Rocm => VllmBackend::Rocm,
GpuBackend::Cpu | GpuBackend::Vulkan => VllmBackend::Cpu,
}
}
fn detect() -> VllmBackend {
VllmBackend::from_gpu(detect_backend())
}
fn engine_image(
self,
engine: ContainerEngine,
version: Option<&str>,
vllm_image: Option<&str>,
) -> Result<String> {
if let (ContainerEngine::Vllm, Some(image)) = (engine, vllm_image) {
return Ok(image.to_string());
}
let arch = HostArch::detect()?;
match engine {
ContainerEngine::Vllm => self.image_ref(arch, version),
ContainerEngine::VllmOmni => self.omni_image_ref(arch, version),
ContainerEngine::Sglang => self.sglang_image_ref(arch, version),
ContainerEngine::LlamaServer | ContainerEngine::Speech => {
anyhow::bail!("llama-server and speech containers go through spawn")
}
}
}
fn image_ref(self, arch: HostArch, version: Option<&str>) -> Result<String> {
let v = version.unwrap_or("latest");
let (repo, suffix) = match (self, arch) {
(VllmBackend::Rocm, HostArch::X86_64) => return Ok(format!("docker.io/rocm/vllm:{v}")),
(VllmBackend::Rocm, HostArch::Aarch64) => anyhow::bail!(
"an AMD GPU was detected, but rocm/vllm publishes no aarch64 image \
(set LLMMAN_LLM_LIBRARY=cpu to run vllm/vllm-openai-cpu instead)"
),
(VllmBackend::Cuda12, HostArch::X86_64) => ("vllm-openai", "x86_64-cu129"),
(VllmBackend::Cuda12, HostArch::Aarch64) => ("vllm-openai", "aarch64-cu129"),
(VllmBackend::Cuda13, HostArch::X86_64) => ("vllm-openai", "x86_64"),
(VllmBackend::Cuda13, HostArch::Aarch64) => ("vllm-openai", "aarch64"),
(VllmBackend::Cpu, HostArch::X86_64) => ("vllm-openai-cpu", "x86_64"),
(VllmBackend::Cpu, HostArch::Aarch64) => ("vllm-openai-cpu", "arm64"),
};
Ok(format!("docker.io/vllm/{repo}:{v}-{suffix}"))
}
fn omni_image_ref(self, arch: HostArch, version: Option<&str>) -> Result<String> {
let v = version.unwrap_or("latest");
let suffix = match (self, arch) {
(VllmBackend::Cuda12 | VllmBackend::Cuda13, HostArch::X86_64) => "x86_64",
(VllmBackend::Cuda12 | VllmBackend::Cuda13, HostArch::Aarch64) => "aarch64",
(VllmBackend::Rocm, _) => anyhow::bail!(
"an AMD GPU was detected, but vllm/vllm-omni publishes only CUDA images \
(install vllm-omni locally and use --runtime path to serve this model)"
),
(VllmBackend::Cpu, _) => anyhow::bail!(
"no CUDA GPU was detected, and vllm/vllm-omni publishes only CUDA images"
),
};
Ok(format!("docker.io/vllm/vllm-omni:{v}-{suffix}"))
}
fn sglang_image_ref(self, arch: HostArch, version: Option<&str>) -> Result<String> {
let v = version.unwrap_or("latest");
let tag = match (self, arch) {
(VllmBackend::Cuda13, _) => v.to_string(),
(VllmBackend::Cuda12, _) => format!("{v}-cu129"),
(VllmBackend::Rocm, HostArch::X86_64) => version.map(str::to_string).context(
"an AMD GPU was detected, but lmsysorg/sglang publishes no floating ROCm \
tag: pass the whole tag for your GPU family as --sglang-version \
(e.g. v0.5.19-rocm700-mi30x; see hub.docker.com/r/lmsysorg/sglang/tags)",
)?,
(VllmBackend::Rocm, HostArch::Aarch64) => anyhow::bail!(
"an AMD GPU was detected, but lmsysorg/sglang publishes no aarch64 ROCm image"
),
(VllmBackend::Cpu, _) => anyhow::bail!(
"no CUDA or ROCm GPU was detected, and lmsysorg/sglang's CPU images (`-xeon`) \
need Intel AMX and launch flags llmman does not pass (install sglang \
locally and drop --ociman, or unset LLMMAN_SAFETENSORS_ENGINE to use vllm)"
),
};
Ok(format!("docker.io/lmsysorg/sglang:{tag}"))
}
fn engine_args(self) -> Vec<String> {
let mut args = match self {
VllmBackend::Cuda12 | VllmBackend::Cuda13 => GpuBackend::Cuda12.engine_args(),
VllmBackend::Rocm => GpuBackend::Rocm.engine_args(),
VllmBackend::Cpu => vec![
"--security-opt".into(),
"seccomp=unconfined".into(),
"--cap-add".into(),
"SYS_NICE".into(),
],
};
args.push("--ipc=host".into());
args
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ContainerEngine {
LlamaServer,
Vllm,
VllmOmni,
Sglang,
Speech,
}
fn image_for(
engine: ContainerEngine,
version: Option<&str>,
vllm_image: Option<&str>,
) -> Result<String> {
match engine {
ContainerEngine::LlamaServer => Ok(detect_backend().image_ref(version)),
ContainerEngine::Speech => Ok(SPEECH_BACKEND.image_ref(version)),
_ => VllmBackend::detect().engine_image(engine, version, vllm_image),
}
}
pub fn pull_image(
ociman: ContainerManager,
engine: ContainerEngine,
version: Option<&str>,
vllm_image: Option<&str>,
) -> Result<()> {
use std::io::Write;
let image = image_for(engine, version, vllm_image)?;
let cli = ociman.binary();
let header = format!("[llmman] {cli}: pulling {image}...");
eprintln!("{header}");
let (success, reason) = match pull_progress_sink() {
Some((mut sink, cols)) => {
let _ = write!(sink, "{header}\r\n");
let run = run_on_pty(cli, &["pull", &image], cols, &mut sink)?;
(run.success, run.last_line)
}
None => {
let status = std::process::Command::new(cli)
.args(["pull", &image])
.status()
.with_context(|| format!("run {cli} pull {image}"))?;
(status.success(), None)
}
};
if !success {
anyhow::bail!(
"{cli} pull {image} failed{}",
reason.map(|l| format!(": {l}")).unwrap_or_default()
);
}
Ok(())
}
pub const PULL_STATUS: &str = "pulling the llama.cpp container image";
pub const VERIFY_STATUS: &str = "checking that the llama.cpp container starts";
pub const PULL_PROGRESS_FILE_ENV: &str = "LLMMAN_PULL_PROGRESS_FILE";
pub const PULL_PROGRESS_COLS_ENV: &str = "LLMMAN_PULL_PROGRESS_COLS";
fn pull_progress_sink() -> Option<(std::fs::File, u16)> {
let path = std::env::var_os(PULL_PROGRESS_FILE_ENV)?;
let cols = std::env::var(PULL_PROGRESS_COLS_ENV)
.ok()?
.parse::<u16>()
.ok()
.filter(|c| *c > 0)?;
let file = std::fs::OpenOptions::new().append(true).open(path).ok()?;
Some((file, cols))
}
struct PtyRun {
success: bool,
last_line: Option<String>,
}
fn run_on_pty(
program: &str,
args: &[&str],
cols: u16,
sink: &mut dyn std::io::Write,
) -> Result<PtyRun> {
use portable_pty::{native_pty_system, CommandBuilder, PtySize};
use std::io::Read;
let pair = native_pty_system()
.openpty(PtySize {
rows: 24,
cols,
pixel_width: 0,
pixel_height: 0,
})
.context("open a pty")?;
let mut cmd = CommandBuilder::new(program);
cmd.args(args);
let mut child = pair
.slave
.spawn_command(cmd)
.with_context(|| format!("run {program}"))?;
drop(pair.slave);
let mut reader = pair
.master
.try_clone_reader()
.context("read from the pty")?;
const TAIL: usize = 4096;
let mut tail = Vec::new();
let mut buf = [0u8; 8192];
while let Ok(n @ 1..) = reader.read(&mut buf) {
let _ = sink.write_all(&buf[..n]);
tail.extend_from_slice(&buf[..n]);
if tail.len() > TAIL {
tail.drain(..tail.len() - TAIL);
}
}
let status = child.wait().with_context(|| format!("wait on {program}"))?;
drop(pair.master);
Ok(PtyRun {
success: status.success(),
last_line: last_output_line(&tail),
})
}
fn last_output_line(bytes: &[u8]) -> Option<String> {
let text = String::from_utf8_lossy(bytes);
let mut plain = String::new();
let mut chars = text.chars();
while let Some(c) = chars.next() {
match c {
'\x1b' => {
if chars.next() == Some('[') {
for n in chars.by_ref() {
if ('@'..='~').contains(&n) {
break;
}
}
}
}
'\r' | '\n' => plain.push('\n'),
c if c.is_control() => {}
c => plain.push(c),
}
}
plain
.lines()
.map(str::trim)
.rfind(|l| !l.is_empty())
.map(str::to_owned)
}
const VERIFY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60);
pub(crate) fn verify_llama_server_runs(
ociman: ContainerManager,
backend: GpuBackend,
version: Option<&str>,
) -> Result<()> {
let cli = ociman.binary();
let name = format!(
"llmman-verify-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| d.as_nanos())
);
let mut cmd = std::process::Command::new(cli);
cmd.args(verify_args(backend, version, &name))
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::null());
match run_with_timeout(cmd, VERIFY_TIMEOUT).with_context(|| format!("run {cli} run"))? {
Some(status) if status.success() => Ok(()),
Some(status) => anyhow::bail!("{cli} could not start a llama.cpp container ({status})"),
None => {
let mut rm = std::process::Command::new(cli);
rm.args(["rm", "-f", &name])
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null());
let _ = run_with_timeout(rm, PROBE_TIMEOUT);
anyhow::bail!("{cli} did not start a llama.cpp container within {VERIFY_TIMEOUT:?}")
}
}
}
fn verify_args(backend: GpuBackend, version: Option<&str>, name: &str) -> Vec<String> {
let mut args: Vec<String> = vec![
"run".into(),
"--rm".into(),
"--init".into(),
"--name".into(),
name.into(),
];
args.extend(backend.engine_args());
args.extend(forwarded_env_args(&[]));
args.push(backend.image_ref(version));
args.push("--version".into());
args
}
#[derive(Debug, Clone, Copy)]
pub struct LlamaOptions<'a> {
pub port: u16,
pub ctx_size: Option<u32>,
pub flash_attention: Option<&'a str>,
pub kv_cache_type: Option<&'a str>,
pub context_shift: bool,
pub split_mode: Option<&'a str>,
pub num_parallel: Option<u32>,
pub embeddings: bool,
pub batch_size: Option<u32>,
pub threads: Option<u32>,
pub cpus: Option<f64>,
pub extra_args: &'a [String],
}
pub fn spawn(
ociman: ContainerManager,
model_path: &Path,
mmproj_path: Option<&Path>,
llama_cpp_version: Option<&str>,
opts: LlamaOptions<'_>,
) -> Result<tokio::process::Child> {
let backend = detect_backend();
let image = backend.image_ref(llama_cpp_version);
eprintln!(
"[llmman] {}: detected {:?}, using image {:?}",
ociman.binary(),
backend,
image
);
let (model_dir, model_file) = mount_split(model_path, "model")?;
let mmproj = mmproj_path.map(|p| mount_split(p, "mmproj")).transpose()?;
let args = llama_run_args(
backend,
image,
&model_dir,
&model_file,
mmproj.as_ref().map(|(d, f)| (d.as_str(), f.as_str())),
crate::cmd::serve::LLAMA_CPP_ENV_PASSTHROUGH_VARS,
opts,
);
run(ociman, args)
}
fn mount_split(path: &Path, what: &str) -> Result<(String, String)> {
let dir = path
.parent()
.with_context(|| format!("{what} path has no parent directory"))?;
let file = path
.file_name()
.and_then(|n| n.to_str())
.with_context(|| format!("{what} path has no valid UTF-8 filename"))?;
Ok((mount_source(dir)?, file.to_owned()))
}
fn llama_run_args(
backend: GpuBackend,
image: String,
model_dir: &str,
model_file: &str,
mmproj: Option<(&str, &str)>,
passthrough_vars: &[&str],
opts: LlamaOptions<'_>,
) -> Vec<String> {
let LlamaOptions {
port,
ctx_size,
flash_attention,
kv_cache_type,
context_shift,
split_mode,
num_parallel,
embeddings,
batch_size,
threads,
cpus,
extra_args,
} = opts;
let mut args = run_args(backend.engine_args(), port, passthrough_vars, cpus);
args.push("-v".into());
args.push(format!("{model_dir}:/models:ro"));
if let Some((dir, _)) = mmproj {
args.push("-v".into());
args.push(format!("{dir}:/mmproj:ro"));
}
args.push(image);
args.extend([
"-m".into(),
format!("/models/{model_file}"),
"--port".into(),
port.to_string(),
"--host".into(),
"0.0.0.0".into(),
]);
if let Some((_, file)) = mmproj {
args.push("--mmproj".into());
args.push(format!("/mmproj/{file}"));
}
if let Some(n) = ctx_size {
args.push("--ctx-size".into());
args.push(n.to_string());
}
if let Some(mode) = flash_attention {
args.push("--flash-attn".into());
args.push(mode.to_string());
}
if let Some(t) = kv_cache_type {
args.push("--cache-type-k".into());
args.push(t.to_string());
args.push("--cache-type-v".into());
args.push(t.to_string());
}
args.push(
if context_shift {
"--context-shift"
} else {
"--no-context-shift"
}
.into(),
);
if let Some(mode) = split_mode {
args.push("--split-mode".into());
args.push(mode.to_string());
}
if let Some(n) = num_parallel {
args.push("--parallel".into());
args.push(n.to_string());
}
if embeddings {
args.push("--embeddings".into());
}
if let Some(n) = batch_size {
args.push("-b".into());
args.push(n.to_string());
args.push("-ub".into());
args.push(n.to_string());
}
if let Some(n) = threads {
args.push("--threads".into());
args.push(n.to_string());
}
args.extend(extra_args.iter().cloned());
args
}
fn run_args(
engine_args: Vec<String>,
port: u16,
passthrough_vars: &[&str],
cpus: Option<f64>,
) -> Vec<String> {
let mut args: Vec<String> = vec![
"run".into(),
"--rm".into(),
"--init".into(),
"-t".into(),
"-p".into(),
format!("127.0.0.1:{port}:{port}"),
];
if let Some(n) = cpus {
args.push("--cpus".into());
args.push(n.to_string());
}
args.extend(engine_args);
args.extend(forwarded_env_args(passthrough_vars));
args
}
fn forwarded_env_args(passthrough_vars: &[&str]) -> Vec<String> {
crate::cmd::serve::GPU_VISIBLE_DEVICE_VARS
.iter()
.chain(passthrough_vars)
.filter(|var| std::env::var_os(var).is_some())
.flat_map(|var| ["-e".to_string(), var.to_string()])
.collect()
}
impl ContainerEngine {
fn launcher(self) -> Option<(&'static [&'static str], &'static str)> {
match self {
ContainerEngine::LlamaServer | ContainerEngine::Speech => None,
ContainerEngine::Vllm | ContainerEngine::VllmOmni => Some((&["vllm"], "VLLM_")),
ContainerEngine::Sglang => {
Some((&["python3", "-m", "sglang.launch_server"], "SGLANG_"))
}
}
}
}
#[allow(clippy::too_many_arguments)]
pub fn spawn_engine(
ociman: ContainerManager,
engine: ContainerEngine,
model_dir: &Path,
version: Option<&str>,
vllm_image: Option<&str>,
port: u16,
cpus: Option<f64>,
serve_args: impl FnOnce(&str, &str) -> Vec<String>,
) -> Result<tokio::process::Child> {
let backend = VllmBackend::detect();
let image = backend.engine_image(engine, version, vllm_image)?;
let Some((launcher, env_prefix)) = engine.launcher() else {
unreachable!("LlamaServer and Speech bailed above");
};
eprintln!(
"[llmman] {}: detected {:?}, using image {:?}",
ociman.binary(),
backend,
image
);
let model_dir = mount_source(model_dir)?;
let vars: Vec<String> = std::env::vars_os()
.filter_map(|(k, _)| k.into_string().ok())
.filter(|k| {
k.starts_with(env_prefix) || matches!(k.as_str(), "HF_TOKEN" | "HUGGING_FACE_HUB_TOKEN")
})
.collect();
let vars: Vec<&str> = vars.iter().map(String::as_str).collect();
let prefix = run_args(backend.engine_args(), port, &vars, cpus);
run(
ociman,
engine_run_args(prefix, image, launcher, &model_dir, serve_args),
)
}
fn mount_source(dir: &Path) -> Result<String> {
let dir = dir
.canonicalize()
.with_context(|| format!("resolving {}", dir.display()))?;
dir.to_str()
.map(str::to_owned)
.with_context(|| format!("{} is not valid UTF-8", dir.display()))
}
fn engine_run_args(
mut args: Vec<String>,
image: String,
launcher: &[&str],
model_dir: &str,
serve_args: impl FnOnce(&str, &str) -> Vec<String>,
) -> Vec<String> {
args.extend([
"-v".into(),
format!("{model_dir}:/models:ro"),
"--entrypoint".into(),
launcher[0].into(),
image,
]);
args.extend(launcher[1..].iter().map(|a| a.to_string()));
args.extend(serve_args("/models", "0.0.0.0"));
args
}
fn run(ociman: ContainerManager, args: Vec<String>) -> Result<tokio::process::Child> {
crate::debug_log!("spawning {} {}", ociman.binary(), args.join(" "));
tokio::process::Command::new(ociman.binary())
.args(&args)
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.spawn()
.with_context(|| format!("spawn {} {}", ociman.binary(), args.join(" ")))
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn spawn_mediagen(
ociman: ContainerManager,
backend: GpuBackend,
model_ref: &str,
store_path: &Path,
cache_path: &Path,
llama_cpp_version: Option<&str>,
port: u16,
cpus: Option<f64>,
) -> Result<tokio::process::Child> {
let image = backend.image_ref(llama_cpp_version);
eprintln!(
"[llmman] {}: {:?} backend, using image {:?} for {model_ref}'s worker",
ociman.binary(),
backend,
image
);
let exe = std::env::current_exe().context("locating the llmman binary")?;
let utf8 = |p: &Path| -> Result<String> {
p.to_str()
.map(str::to_owned)
.with_context(|| format!("{} is not valid UTF-8", p.display()))
};
let (exe, store, cache) = (utf8(&exe)?, utf8(store_path)?, utf8(cache_path)?);
let passthrough = [
crate::cmd::serve::LLAMA_CPP_ENV_PASSTHROUGH_VARS,
crate::cmd::serve::MEDIAGEN_ENV_PASSTHROUGH_VARS,
]
.concat();
let mut args = run_args(backend.engine_args(), port, &passthrough, cpus);
args.extend([
"-v".into(),
format!("{exe}:/usr/local/bin/llmman:ro"),
"-v".into(),
format!("{store}:{store}:ro"),
"-v".into(),
format!("{cache}:{cache}:ro"),
"-e".into(),
format!("LLMMAN_MODELS={store}"),
"-e".into(),
"PATH=/app:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin".into(),
"--entrypoint".into(),
"/usr/local/bin/llmman".into(),
image,
"serve".into(),
model_ref.into(),
"--port".into(),
port.to_string(),
"--host".into(),
"0.0.0.0".into(),
"--runtime".into(),
"path".into(),
]);
run(ociman, args)
}
#[cfg(unix)]
pub fn stop(pid: u32) {
let result = unsafe { libc::kill(pid as libc::pid_t, libc::SIGTERM) };
if result != 0 {
let err = std::io::Error::last_os_error();
eprintln!("[llmman] warning: SIGTERM to container process {pid} failed: {err}");
}
}
#[cfg(not(unix))]
pub fn stop(_pid: u32) {
eprintln!("[llmman] warning: container::stop is a no-op on non-Unix platforms");
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn last_output_line_drops_control_sequences_and_redraws() {
let out = b"Pulling\r\n\x1b[1A\x1b[2K\rabc: Downloading 1MB/3MB\r\x1b[1B\x1b[1A\x1b[2K\rError response from daemon: denied \r\n\r\n";
assert_eq!(
last_output_line(out).as_deref(),
Some("Error response from daemon: denied")
);
assert_eq!(last_output_line(b"\x1b[2K\r\n"), None);
assert_eq!(last_output_line(b""), None);
}
#[cfg(unix)]
#[test]
fn run_on_pty_gives_the_program_a_terminal_and_relays_its_output() {
let mut sink = Vec::new();
let run = run_on_pty(
"sh",
&[
"-c",
"test -t 1 && stty size; printf 'half\\rdone\\n'; echo 'boom: nope' >&2; exit 3",
],
97,
&mut sink,
)
.unwrap();
let text = String::from_utf8_lossy(&sink);
assert!(text.contains("24 97"), "got: {text:?}");
assert!(text.contains("half\rdone"), "got: {text:?}");
assert!(!run.success);
assert_eq!(run.last_line.as_deref(), Some("boom: nope"));
}
#[cfg(unix)]
#[test]
fn run_on_pty_reports_success() {
let mut sink = Vec::new();
let run = run_on_pty("sh", &["-c", "echo ok"], 80, &mut sink).unwrap();
assert!(run.success);
assert_eq!(run.last_line.as_deref(), Some("ok"));
}
#[test]
fn run_on_pty_reports_a_missing_program() {
let err = run_on_pty("llmman-no-such-program", &[], 80, &mut Vec::new());
assert!(err.is_err());
}
#[test]
fn cuda_major_12_picks_cuda12_image() {
assert_eq!(
backend_from_hostgpu(HostGpu::Cuda { major: 12 }),
GpuBackend::Cuda12
);
assert_eq!(GpuBackend::Cuda12.image_tag(), "server-cuda");
}
#[test]
fn cuda_major_13_picks_cuda13_image() {
assert_eq!(
backend_from_hostgpu(HostGpu::Cuda { major: 13 }),
GpuBackend::Cuda13
);
assert_eq!(GpuBackend::Cuda13.image_tag(), "server-cuda13");
}
#[test]
fn cuda_major_above_13_still_picks_cuda13_image() {
assert_eq!(
backend_from_hostgpu(HostGpu::Cuda { major: 14 }),
GpuBackend::Cuda13
);
}
#[test]
fn non_cuda_hostgpu_variants_map_to_their_matching_backend() {
assert_eq!(backend_from_hostgpu(HostGpu::Rocm), GpuBackend::Rocm);
assert_eq!(backend_from_hostgpu(HostGpu::Opencl), GpuBackend::Cpu);
assert_eq!(backend_from_hostgpu(HostGpu::Vulkan), GpuBackend::Vulkan);
assert_eq!(backend_from_hostgpu(HostGpu::None), GpuBackend::Cpu);
assert_eq!(backend_from_hostgpu(HostGpu::Metal), GpuBackend::Cpu);
}
#[test]
fn image_tags_match_docs_docker_md() {
assert_eq!(GpuBackend::Cpu.image_tag(), "server");
assert_eq!(GpuBackend::Cuda12.image_tag(), "server-cuda");
assert_eq!(GpuBackend::Cuda13.image_tag(), "server-cuda13");
assert_eq!(GpuBackend::Rocm.image_tag(), "server-rocm");
assert_eq!(GpuBackend::Vulkan.image_tag(), "server-vulkan");
}
#[test]
fn the_cpu_only_probe_passes_where_the_toolkit_alone_refuses() {
let err = gpu_gate(true, true, "docker", || false)
.unwrap_err()
.to_string();
assert!(err.contains("NVIDIA Container Toolkit"), "{err}");
gpu_gate(false, true, "docker", || false).unwrap();
gpu_gate(true, true, "docker", || true).unwrap();
gpu_gate(true, false, "docker", || {
unreachable!("no GPU, no toolkit check")
})
.unwrap();
gpu_gate(false, false, "docker", || {
unreachable!("no GPU, no toolkit check")
})
.unwrap();
}
#[test]
fn speech_image_is_the_cpu_image_on_any_host() {
assert_eq!(
image_for(ContainerEngine::Speech, Some("b9994"), None).unwrap(),
GpuBackend::Cpu.image_ref(Some("b9994"))
);
assert_eq!(
image_for(ContainerEngine::Speech, None, None).unwrap(),
GpuBackend::Cpu.image_ref(None)
);
}
#[test]
fn image_ref_uses_floating_tag_when_no_version_given() {
assert_eq!(
GpuBackend::Cpu.image_ref(None),
"ghcr.io/ggml-org/llama.cpp:server"
);
assert_eq!(
GpuBackend::Cuda13.image_ref(None),
"ghcr.io/ggml-org/llama.cpp:server-cuda13"
);
}
#[test]
fn image_ref_pins_to_the_given_version() {
assert_eq!(
GpuBackend::Cpu.image_ref(Some("b9994")),
"ghcr.io/ggml-org/llama.cpp:server-b9994"
);
assert_eq!(
GpuBackend::Cuda13.image_ref(Some("b9994")),
"ghcr.io/ggml-org/llama.cpp:server-cuda13-b9994"
);
}
#[test]
fn cpu_backend_has_no_extra_engine_args() {
assert!(GpuBackend::Cpu.engine_args().is_empty());
}
#[test]
fn cuda_backend_requests_all_gpus() {
assert_eq!(GpuBackend::Cuda12.engine_args(), vec!["--gpus", "all"]);
assert_eq!(GpuBackend::Cuda13.engine_args(), vec!["--gpus", "all"]);
}
#[test]
fn rocm_backend_mounts_kfd_and_dri() {
let args = GpuBackend::Rocm.engine_args();
assert_eq!(
args,
vec![
"--device",
"/dev/kfd",
"--device",
"/dev/dri",
"--group-add",
"video"
]
);
}
#[test]
fn vulkan_backend_passes_the_devices_the_host_has() {
let args = GpuBackend::Vulkan.engine_args();
assert_eq!(
args.contains(&"/dev/dri".to_string()),
Path::new("/dev/dri").exists()
);
assert_eq!(
args.contains(&"--gpus".to_string()),
Path::new("/dev/nvidiactl").exists()
);
}
#[test]
fn container_manager_binary_names() {
assert_eq!(ContainerManager::Docker.binary(), "docker");
assert_eq!(ContainerManager::Podman.binary(), "podman");
}
#[test]
fn host_arch_parses_only_the_two_architectures_vllm_ships() {
assert_eq!(HostArch::parse("x86_64"), Some(HostArch::X86_64));
assert_eq!(HostArch::parse("aarch64"), Some(HostArch::Aarch64));
assert_eq!(HostArch::parse("s390x"), None);
assert_eq!(HostArch::parse("riscv64"), None);
assert_eq!(HostArch::parse(""), None);
}
#[test]
fn cuda_12_hosts_get_the_cu129_vllm_image() {
assert_eq!(
VllmBackend::from_gpu(GpuBackend::Cuda12),
VllmBackend::Cuda12
);
assert_eq!(
VllmBackend::from_gpu(GpuBackend::Cuda13),
VllmBackend::Cuda13
);
assert_eq!(
VllmBackend::Cuda12
.image_ref(HostArch::X86_64, None)
.unwrap(),
"docker.io/vllm/vllm-openai:latest-x86_64-cu129"
);
assert_eq!(
VllmBackend::Cuda12
.image_ref(HostArch::Aarch64, Some("v0.28.0"))
.unwrap(),
"docker.io/vllm/vllm-openai:v0.28.0-aarch64-cu129"
);
}
#[test]
fn vulkan_host_runs_the_vllm_cpu_image() {
assert_eq!(VllmBackend::from_gpu(GpuBackend::Vulkan), VllmBackend::Cpu);
assert_eq!(VllmBackend::from_gpu(GpuBackend::Cpu), VllmBackend::Cpu);
assert_eq!(VllmBackend::from_gpu(GpuBackend::Rocm), VllmBackend::Rocm);
}
#[test]
fn vllm_image_refs_match_docker_hub_tag_spellings() {
assert_eq!(
VllmBackend::Cuda13
.image_ref(HostArch::X86_64, None)
.unwrap(),
"docker.io/vllm/vllm-openai:latest-x86_64"
);
assert_eq!(
VllmBackend::Cuda13
.image_ref(HostArch::Aarch64, None)
.unwrap(),
"docker.io/vllm/vllm-openai:latest-aarch64"
);
assert_eq!(
VllmBackend::Rocm.image_ref(HostArch::X86_64, None).unwrap(),
"docker.io/rocm/vllm:latest"
);
assert_eq!(
VllmBackend::Cpu.image_ref(HostArch::X86_64, None).unwrap(),
"docker.io/vllm/vllm-openai-cpu:latest-x86_64"
);
assert_eq!(
VllmBackend::Cpu.image_ref(HostArch::Aarch64, None).unwrap(),
"docker.io/vllm/vllm-openai-cpu:latest-arm64"
);
}
#[test]
fn vllm_image_override_is_verbatim_and_for_plain_vllm_only() {
let img = Some("dhi.io/vllm-openai:0.32");
let backend = VllmBackend::Cuda13;
assert_eq!(
backend
.engine_image(ContainerEngine::Vllm, Some("v0.11.0"), img)
.unwrap(),
"dhi.io/vllm-openai:0.32"
);
for (engine, repo) in [
(ContainerEngine::VllmOmni, "vllm/vllm-omni:"),
(ContainerEngine::Sglang, "lmsysorg/sglang:"),
] {
let image = backend.engine_image(engine, None, img).unwrap();
assert!(image.contains(repo), "{image}");
}
}
#[test]
fn vllm_image_ref_pins_to_the_given_version() {
assert_eq!(
VllmBackend::Cuda13
.image_ref(HostArch::X86_64, Some("v0.11.0"))
.unwrap(),
"docker.io/vllm/vllm-openai:v0.11.0-x86_64"
);
assert_eq!(
VllmBackend::Cuda13
.image_ref(HostArch::Aarch64, Some("v0.11.0"))
.unwrap(),
"docker.io/vllm/vllm-openai:v0.11.0-aarch64"
);
assert_eq!(
VllmBackend::Cpu
.image_ref(HostArch::Aarch64, Some("v0.11.0"))
.unwrap(),
"docker.io/vllm/vllm-openai-cpu:v0.11.0-arm64"
);
assert_eq!(
VllmBackend::Rocm
.image_ref(
HostArch::X86_64,
Some("rocm7.14.1_cdna_ubuntu24.04_py3.14_pytorch_2.11_vllm_0.23.0")
)
.unwrap(),
"docker.io/rocm/vllm:rocm7.14.1_cdna_ubuntu24.04_py3.14_pytorch_2.11_vllm_0.23.0"
);
}
#[test]
fn rocm_vllm_has_no_aarch64_image() {
let err = VllmBackend::Rocm
.image_ref(HostArch::Aarch64, None)
.unwrap_err()
.to_string();
assert!(err.contains("rocm/vllm"), "{err}");
assert!(err.contains("LLMMAN_LLM_LIBRARY=cpu"), "{err}");
}
#[test]
fn vllm_omni_image_refs_match_docker_hub_tag_spellings() {
for cuda in [VllmBackend::Cuda12, VllmBackend::Cuda13] {
assert_eq!(
cuda.omni_image_ref(HostArch::X86_64, None).unwrap(),
"docker.io/vllm/vllm-omni:latest-x86_64"
);
assert_eq!(
cuda.omni_image_ref(HostArch::Aarch64, None).unwrap(),
"docker.io/vllm/vllm-omni:latest-aarch64"
);
assert_eq!(
cuda.omni_image_ref(HostArch::Aarch64, Some("v0.28.0"))
.unwrap(),
"docker.io/vllm/vllm-omni:v0.28.0-aarch64"
);
}
}
#[test]
fn vllm_omni_has_only_cuda_images() {
let err = VllmBackend::Rocm
.omni_image_ref(HostArch::X86_64, None)
.unwrap_err()
.to_string();
assert!(err.contains("vllm/vllm-omni"), "{err}");
assert!(err.contains("--runtime"), "{err}");
let err = VllmBackend::Cpu
.omni_image_ref(HostArch::X86_64, None)
.unwrap_err()
.to_string();
assert!(err.contains("CUDA"), "{err}");
}
#[test]
fn sglang_image_refs_match_docker_hub_tag_spellings() {
for arch in [HostArch::X86_64, HostArch::Aarch64] {
assert_eq!(
VllmBackend::Cuda13.sglang_image_ref(arch, None).unwrap(),
"docker.io/lmsysorg/sglang:latest"
);
assert_eq!(
VllmBackend::Cuda13
.sglang_image_ref(arch, Some("v0.5.19"))
.unwrap(),
"docker.io/lmsysorg/sglang:v0.5.19"
);
assert_eq!(
VllmBackend::Cuda12.sglang_image_ref(arch, None).unwrap(),
"docker.io/lmsysorg/sglang:latest-cu129"
);
assert_eq!(
VllmBackend::Cuda12
.sglang_image_ref(arch, Some("v0.5.19"))
.unwrap(),
"docker.io/lmsysorg/sglang:v0.5.19-cu129"
);
}
}
#[test]
fn sglang_rocm_image_is_the_whole_pinned_tag_or_nothing() {
assert_eq!(
VllmBackend::Rocm
.sglang_image_ref(HostArch::X86_64, Some("v0.5.19-rocm700-mi30x"))
.unwrap(),
"docker.io/lmsysorg/sglang:v0.5.19-rocm700-mi30x"
);
let err = VllmBackend::Rocm
.sglang_image_ref(HostArch::X86_64, None)
.unwrap_err()
.to_string();
assert!(err.contains("--sglang-version"), "{err}");
assert!(err.contains("rocm700-mi30x"), "{err}");
let err = VllmBackend::Rocm
.sglang_image_ref(HostArch::Aarch64, Some("v0.5.19-rocm700-mi30x"))
.unwrap_err()
.to_string();
assert!(err.contains("aarch64"), "{err}");
}
#[test]
fn sglang_has_no_cpu_image_here() {
for arch in [HostArch::X86_64, HostArch::Aarch64] {
let err = VllmBackend::Cpu
.sglang_image_ref(arch, Some("v0.5.19"))
.unwrap_err()
.to_string();
assert!(err.contains("xeon"), "{err}");
assert!(err.contains("LLMMAN_SAFETENSORS_ENGINE"), "{err}");
}
}
#[test]
fn sglang_run_args_launch_the_module_after_the_image_and_flags_after_that() {
let (launcher, _) = ContainerEngine::Sglang.launcher().unwrap();
let args = engine_run_args(
run_args(VllmBackend::Cuda13.engine_args(), 30000, &[], None),
"docker.io/lmsysorg/sglang:latest".into(),
launcher,
"/cache/abc/model",
|dir: &str, host: &str| {
vec![
"--model-path".into(),
dir.into(),
"--host".into(),
host.into(),
]
},
);
let image = args
.iter()
.position(|a| a == "docker.io/lmsysorg/sglang:latest")
.expect("image present");
assert_eq!(&args[image - 2..image], &["--entrypoint", "python3"]);
assert_eq!(
&args[image + 1..],
&[
"-m",
"sglang.launch_server",
"--model-path",
"/models",
"--host",
"0.0.0.0"
]
);
}
#[test]
fn vllm_engine_args_add_ipc_host_on_top_of_gpu_passthrough() {
assert_eq!(
VllmBackend::Cuda13.engine_args(),
vec!["--gpus", "all", "--ipc=host"]
);
let rocm = VllmBackend::Rocm.engine_args();
assert!(
rocm.starts_with(&GpuBackend::Rocm.engine_args()),
"{rocm:?}"
);
assert_eq!(rocm.last().map(String::as_str), Some("--ipc=host"));
}
#[test]
fn vllm_cpu_engine_args_grant_what_its_numa_thread_binding_needs() {
assert_eq!(
VllmBackend::Cpu.engine_args(),
vec![
"--security-opt",
"seccomp=unconfined",
"--cap-add",
"SYS_NICE",
"--ipc=host"
]
);
}
#[test]
fn vllm_run_args_put_the_mount_and_entrypoint_before_the_image_and_serve_after() {
let (launcher, _) = ContainerEngine::Vllm.launcher().unwrap();
let args = engine_run_args(
run_args(VllmBackend::Cuda13.engine_args(), 8000, &[], None),
"docker.io/vllm/vllm-openai:latest-x86_64".into(),
launcher,
"/cache/abc/model",
|dir: &str, host: &str| {
vec![
"serve".into(),
dir.into(),
"--host".into(),
host.into(),
"--served-model-name".into(),
"m".into(),
]
},
);
let image = args
.iter()
.position(|a| a == "docker.io/vllm/vllm-openai:latest-x86_64")
.expect("image present");
let before = &args[..image];
assert!(before.contains(&"--gpus".to_string()), "{before:?}");
assert!(before.contains(&"--ipc=host".to_string()), "{before:?}");
assert!(
before.contains(&"/cache/abc/model:/models:ro".to_string()),
"{before:?}"
);
assert_eq!(&before[before.len() - 2..], &["--entrypoint", "vllm"]);
assert_eq!(
&args[image + 1..],
&[
"serve",
"/models",
"--host",
"0.0.0.0",
"--served-model-name",
"m"
]
);
}
#[test]
fn run_args_forward_only_the_requested_passthrough_vars() {
const VAR: &str = "LLMMAN_TEST_RUN_ARGS_PASSTHROUGH";
std::env::set_var(VAR, "1");
let args = run_args(vec![], 8080, &[], None);
assert_eq!(
&args[..6],
&["run", "--rm", "--init", "-t", "-p", "127.0.0.1:8080:8080"]
);
assert!(!args.contains(&VAR.to_string()), "{args:?}");
let args = run_args(vec![], 8080, &[VAR], None);
assert!(args.windows(2).any(|w| w == ["-e", VAR]), "{args:?}");
assert!(!args.iter().any(|a| a.starts_with(&format!("{VAR}="))));
}
#[test]
fn verify_runs_the_served_image_with_its_gpu_passthrough() {
for (backend, image) in [
(GpuBackend::Cpu, "ghcr.io/ggml-org/llama.cpp:server-b9994"),
(
GpuBackend::Cuda13,
"ghcr.io/ggml-org/llama.cpp:server-cuda13-b9994",
),
] {
let args = verify_args(backend, Some("b9994"), "llmman-verify-1");
assert_eq!(
&args[..5],
["run", "--rm", "--init", "--name", "llmman-verify-1"]
);
let at = args.iter().position(|a| a == image).expect("image present");
let mut between = backend.engine_args();
between.extend(forwarded_env_args(&[]));
assert_eq!(&args[5..at], between.as_slice(), "{backend:?}");
assert_eq!(&args[at + 1..], ["--version"]);
}
}
#[test]
fn cpus_is_forwarded_as_a_run_flag_only_when_a_limit_binds() {
let args = run_args(vec![], 8080, &[], None);
assert!(!args.contains(&"--cpus".to_string()), "{args:?}");
let args = run_args(vec!["--gpus".into(), "all".into()], 8080, &[], Some(2.0));
let cpus = args.iter().position(|a| a == "--cpus").unwrap();
assert_eq!(args[cpus + 1], "2");
assert!(cpus < args.iter().position(|a| a == "--gpus").unwrap());
let args = run_args(vec![], 8080, &[], Some(0.5));
assert!(args.windows(2).any(|w| w == ["--cpus", "0.5"]), "{args:?}");
}
#[test]
fn llama_container_gets_cpus_before_the_image_and_threads_after() {
let opts = |threads, cpus| LlamaOptions {
port: 8080,
ctx_size: None,
flash_attention: None,
kv_cache_type: None,
context_shift: true,
split_mode: None,
num_parallel: None,
embeddings: false,
batch_size: None,
threads,
cpus,
extra_args: &[],
};
let image = "ghcr.io/ggml-org/llama.cpp:server";
let argv = |o| {
llama_run_args(
GpuBackend::Cpu,
image.into(),
"/cache/blobs",
"sha256-abc",
None,
&[],
o,
)
};
let args = argv(opts(Some(2), Some(2.0)));
let at = |flag: &str| args.iter().position(|a| a == flag).unwrap();
assert!(at("--cpus") < at(image), "{args:?}");
assert!(at(image) < at("--threads"), "{args:?}");
assert_eq!(args[at("--cpus") + 1], "2");
assert_eq!(args[at("--threads") + 1], "2");
let args = argv(opts(None, None));
assert!(
!args.iter().any(|a| a == "--cpus" || a == "--threads"),
"{args:?}"
);
}
#[test]
fn llama_container_ends_with_the_extra_args() {
let opts = LlamaOptions {
port: 8080,
ctx_size: None,
flash_attention: None,
kv_cache_type: None,
context_shift: true,
split_mode: None,
num_parallel: None,
embeddings: false,
batch_size: None,
threads: Some(2),
cpus: None,
extra_args: &["--slot-prompt-similarity".into(), "0.5".into()],
};
let args = llama_run_args(
GpuBackend::Cpu,
"ghcr.io/ggml-org/llama.cpp:server".into(),
"/cache/blobs",
"sha256-abc",
None,
&[],
opts,
);
assert_eq!(
args[args.len() - 4..],
["--threads", "2", "--slot-prompt-similarity", "0.5"]
);
}
#[test]
fn vllm_containers_get_cpus_too() {
let (launcher, _) = ContainerEngine::Vllm.launcher().unwrap();
let args = engine_run_args(
run_args(VllmBackend::Cpu.engine_args(), 8000, &[], Some(3.0)),
"docker.io/vllm/vllm-openai-cpu:latest-x86_64".into(),
launcher,
"/cache/abc/model",
|dir: &str, host: &str| vec!["serve".into(), dir.into(), "--host".into(), host.into()],
);
assert!(args.windows(2).any(|w| w == ["--cpus", "3"]), "{args:?}");
}
#[test]
#[ignore = "result depends on this host's actual GPU/driver setup"]
fn detect_backend_reports_this_hosts_real_hardware() {
let backend = detect_backend();
println!(
"container::detect_backend() -> {backend:?} ({})",
backend.image_ref(None)
);
}
}