use std::path::Path;
use anyhow::{Context, Result};
use clap::ValueEnum;
use crate::hostgpu::{self, HostGpu};
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
#[value(rename_all = "lower")]
pub enum ContainerManager {
Docker,
Podman,
}
impl ContainerManager {
pub fn binary(self) -> &'static str {
match self {
ContainerManager::Docker => "docker",
ContainerManager::Podman => "podman",
}
}
pub fn probe(self) -> Result<()> {
let cli = self.binary();
if crate::find_on_path(cli).is_none() {
anyhow::bail!("{cli} is not on PATH");
}
let mut cmd = std::process::Command::new(cli);
cmd.arg("info")
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null());
match run_with_timeout(cmd, PROBE_TIMEOUT).with_context(|| format!("run {cli} info"))? {
Some(status) if status.success() => {}
Some(_) => anyhow::bail!("{cli} info failed (is its daemon running?)"),
None => anyhow::bail!("{cli} info did not answer within {PROBE_TIMEOUT:?}"),
}
let wants_gpus = detect_backend().engine_args().iter().any(|a| a == "--gpus");
if wants_gpus && !nvidia_toolkit_present(self) {
anyhow::bail!(
"an NVIDIA GPU was detected but no NVIDIA Container Toolkit for {cli} \
(`{cli} run --gpus all` would fail)"
);
}
Ok(())
}
}
const PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
fn run_with_timeout(
mut cmd: std::process::Command,
timeout: std::time::Duration,
) -> Result<Option<std::process::ExitStatus>> {
let mut child = cmd.spawn()?;
let deadline = std::time::Instant::now() + timeout;
loop {
if let Some(status) = child.try_wait()? {
return Ok(Some(status));
}
if std::time::Instant::now() >= deadline {
let _ = child.kill();
let _ = child.wait();
return Ok(None);
}
std::thread::sleep(std::time::Duration::from_millis(50));
}
}
fn nvidia_toolkit_present(ociman: ContainerManager) -> bool {
if [
"nvidia-container-runtime-hook",
"nvidia-container-toolkit",
"nvidia-ctk",
]
.iter()
.any(|b| crate::find_on_path(b).is_some())
{
return true;
}
if ["/etc/cdi/nvidia.yaml", "/var/run/cdi/nvidia.yaml"]
.iter()
.any(|p| Path::new(p).exists())
{
return true;
}
if ociman == ContainerManager::Docker {
let runtimes = std::process::Command::new("docker")
.args([
"info",
"--format",
"{{range $k, $v := .Runtimes}}{{$k}} {{end}}",
])
.output()
.ok()
.filter(|o| o.status.success())
.map(|o| String::from_utf8_lossy(&o.stdout).into_owned())
.unwrap_or_default();
if runtimes.split_whitespace().any(|r| r == "nvidia") {
return true;
}
}
false
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum GpuBackend {
Cpu,
Cuda12,
Cuda13,
Rocm,
Vulkan,
}
impl GpuBackend {
fn image_tag(self) -> &'static str {
match self {
GpuBackend::Cpu => "server",
GpuBackend::Cuda12 => "server-cuda",
GpuBackend::Cuda13 => "server-cuda13",
GpuBackend::Rocm => "server-rocm",
GpuBackend::Vulkan => "server-vulkan",
}
}
fn image_ref(self, version: Option<&str>) -> String {
match version {
Some(v) => format!("ghcr.io/ggml-org/llama.cpp:{}-{v}", self.image_tag()),
None => format!("ghcr.io/ggml-org/llama.cpp:{}", self.image_tag()),
}
}
fn engine_args(self) -> Vec<String> {
match self {
GpuBackend::Cpu => vec![],
GpuBackend::Cuda12 | GpuBackend::Cuda13 => vec!["--gpus".into(), "all".into()],
GpuBackend::Rocm => vec![
"--device".into(),
"/dev/kfd".into(),
"--device".into(),
"/dev/dri".into(),
"--group-add".into(),
"video".into(),
],
GpuBackend::Vulkan => {
let mut args = Vec::new();
if Path::new("/dev/dri").exists() {
args.extend(["--device", "/dev/dri"].map(String::from));
}
if Path::new("/dev/nvidiactl").exists() {
args.extend(
["--gpus", "all", "-e", "NVIDIA_DRIVER_CAPABILITIES=all"].map(String::from),
);
}
args
}
}
}
}
fn detect_backend() -> GpuBackend {
backend_from_hostgpu(hostgpu::detect())
}
fn backend_from_hostgpu(gpu: HostGpu) -> GpuBackend {
match gpu {
HostGpu::Cuda { major } if major >= 13 => GpuBackend::Cuda13,
HostGpu::Cuda { .. } => GpuBackend::Cuda12,
HostGpu::Rocm => GpuBackend::Rocm,
HostGpu::Vulkan => GpuBackend::Vulkan,
HostGpu::Metal | HostGpu::None => GpuBackend::Cpu,
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum HostArch {
X86_64,
Aarch64,
}
impl HostArch {
fn parse(arch: &str) -> Option<HostArch> {
match arch {
"x86_64" => Some(HostArch::X86_64),
"aarch64" => Some(HostArch::Aarch64),
_ => None,
}
}
fn detect() -> Result<HostArch> {
HostArch::parse(std::env::consts::ARCH).with_context(|| {
format!(
"vLLM publishes no container image for {} hosts (only x86_64 and aarch64)",
std::env::consts::ARCH
)
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum VllmBackend {
Cuda12,
Cuda13,
Rocm,
Cpu,
}
impl VllmBackend {
fn from_gpu(backend: GpuBackend) -> VllmBackend {
match backend {
GpuBackend::Cuda12 => VllmBackend::Cuda12,
GpuBackend::Cuda13 => VllmBackend::Cuda13,
GpuBackend::Rocm => VllmBackend::Rocm,
GpuBackend::Cpu | GpuBackend::Vulkan => VllmBackend::Cpu,
}
}
fn detect() -> Result<(VllmBackend, HostArch)> {
Ok((VllmBackend::from_gpu(detect_backend()), HostArch::detect()?))
}
fn image_ref(self, arch: HostArch, version: Option<&str>) -> Result<String> {
let v = version.unwrap_or("latest");
let (repo, suffix) = match (self, arch) {
(VllmBackend::Rocm, HostArch::X86_64) => return Ok(format!("docker.io/rocm/vllm:{v}")),
(VllmBackend::Rocm, HostArch::Aarch64) => anyhow::bail!(
"an AMD GPU was detected, but rocm/vllm publishes no aarch64 image \
(set LLMMAN_LLM_LIBRARY=cpu to run vllm/vllm-openai-cpu instead)"
),
(VllmBackend::Cuda12, HostArch::X86_64) => ("vllm-openai", "x86_64-cu129"),
(VllmBackend::Cuda12, HostArch::Aarch64) => ("vllm-openai", "aarch64-cu129"),
(VllmBackend::Cuda13, HostArch::X86_64) => ("vllm-openai", "x86_64"),
(VllmBackend::Cuda13, HostArch::Aarch64) => ("vllm-openai", "aarch64"),
(VllmBackend::Cpu, HostArch::X86_64) => ("vllm-openai-cpu", "x86_64"),
(VllmBackend::Cpu, HostArch::Aarch64) => ("vllm-openai-cpu", "arm64"),
};
Ok(format!("docker.io/vllm/{repo}:{v}-{suffix}"))
}
fn omni_image_ref(self, arch: HostArch, version: Option<&str>) -> Result<String> {
let v = version.unwrap_or("latest");
let suffix = match (self, arch) {
(VllmBackend::Cuda12 | VllmBackend::Cuda13, HostArch::X86_64) => "x86_64",
(VllmBackend::Cuda12 | VllmBackend::Cuda13, HostArch::Aarch64) => "aarch64",
(VllmBackend::Rocm, _) => anyhow::bail!(
"an AMD GPU was detected, but vllm/vllm-omni publishes only CUDA images \
(install vllm-omni locally and use --runtime path to serve this model)"
),
(VllmBackend::Cpu, _) => anyhow::bail!(
"no CUDA GPU was detected, and vllm/vllm-omni publishes only CUDA images"
),
};
Ok(format!("docker.io/vllm/vllm-omni:{v}-{suffix}"))
}
fn sglang_image_ref(self, arch: HostArch, version: Option<&str>) -> Result<String> {
let v = version.unwrap_or("latest");
let tag = match (self, arch) {
(VllmBackend::Cuda13, _) => v.to_string(),
(VllmBackend::Cuda12, _) => format!("{v}-cu129"),
(VllmBackend::Rocm, HostArch::X86_64) => version.map(str::to_string).context(
"an AMD GPU was detected, but lmsysorg/sglang publishes no floating ROCm \
tag: pass the whole tag for your GPU family as --sglang-version \
(e.g. v0.5.19-rocm700-mi30x; see hub.docker.com/r/lmsysorg/sglang/tags)",
)?,
(VllmBackend::Rocm, HostArch::Aarch64) => anyhow::bail!(
"an AMD GPU was detected, but lmsysorg/sglang publishes no aarch64 ROCm image"
),
(VllmBackend::Cpu, _) => anyhow::bail!(
"no CUDA or ROCm GPU was detected, and lmsysorg/sglang's CPU images (`-xeon`) \
need Intel AMX and launch flags llmman does not pass (install sglang \
locally and drop --ociman, or unset LLMMAN_SAFETENSORS_ENGINE to use vllm)"
),
};
Ok(format!("docker.io/lmsysorg/sglang:{tag}"))
}
fn engine_args(self) -> Vec<String> {
let mut args = match self {
VllmBackend::Cuda12 | VllmBackend::Cuda13 => GpuBackend::Cuda12.engine_args(),
VllmBackend::Rocm => GpuBackend::Rocm.engine_args(),
VllmBackend::Cpu => vec![
"--security-opt".into(),
"seccomp=unconfined".into(),
"--cap-add".into(),
"SYS_NICE".into(),
],
};
args.push("--ipc=host".into());
args
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ContainerEngine {
LlamaServer,
Vllm,
VllmOmni,
Sglang,
}
fn image_for(engine: ContainerEngine, version: Option<&str>) -> Result<String> {
match engine {
ContainerEngine::LlamaServer => Ok(detect_backend().image_ref(version)),
ContainerEngine::Vllm => {
let (backend, arch) = VllmBackend::detect()?;
backend.image_ref(arch, version)
}
ContainerEngine::VllmOmni => {
let (backend, arch) = VllmBackend::detect()?;
backend.omni_image_ref(arch, version)
}
ContainerEngine::Sglang => {
let (backend, arch) = VllmBackend::detect()?;
backend.sglang_image_ref(arch, version)
}
}
}
pub fn pull_image(
ociman: ContainerManager,
engine: ContainerEngine,
version: Option<&str>,
) -> Result<()> {
let image = image_for(engine, version)?;
eprintln!("[llmman] {}: pulling {image}...", ociman.binary());
let status = std::process::Command::new(ociman.binary())
.args(["pull", &image])
.status()
.with_context(|| format!("run {} pull {image}", ociman.binary()))?;
if !status.success() {
anyhow::bail!("{} pull {image} failed", ociman.binary());
}
Ok(())
}
#[derive(Debug, Clone, Copy)]
pub struct LlamaOptions<'a> {
pub port: u16,
pub ctx_size: Option<u32>,
pub flash_attention: Option<&'a str>,
pub kv_cache_type: Option<&'a str>,
pub context_shift: bool,
pub split_mode: Option<&'a str>,
pub num_parallel: Option<u32>,
pub embeddings: bool,
pub batch_size: Option<u32>,
pub threads: Option<u32>,
pub cpus: Option<f64>,
}
pub fn spawn(
ociman: ContainerManager,
model_path: &Path,
mmproj_path: Option<&Path>,
llama_cpp_version: Option<&str>,
opts: LlamaOptions<'_>,
) -> Result<tokio::process::Child> {
let backend = detect_backend();
let image = backend.image_ref(llama_cpp_version);
eprintln!(
"[llmman] {}: detected {:?}, using image {:?}",
ociman.binary(),
backend,
image
);
let (model_dir, model_file) = mount_split(model_path, "model")?;
let mmproj = mmproj_path.map(|p| mount_split(p, "mmproj")).transpose()?;
let args = llama_run_args(
backend,
image,
&model_dir,
&model_file,
mmproj.as_ref().map(|(d, f)| (d.as_str(), f.as_str())),
crate::cmd::serve::LLAMA_CPP_ENV_PASSTHROUGH_VARS,
opts,
);
run(ociman, args)
}
fn mount_split(path: &Path, what: &str) -> Result<(String, String)> {
let dir = path
.parent()
.with_context(|| format!("{what} path has no parent directory"))?;
let file = path
.file_name()
.and_then(|n| n.to_str())
.with_context(|| format!("{what} path has no valid UTF-8 filename"))?;
Ok((mount_source(dir)?, file.to_owned()))
}
fn llama_run_args(
backend: GpuBackend,
image: String,
model_dir: &str,
model_file: &str,
mmproj: Option<(&str, &str)>,
passthrough_vars: &[&str],
opts: LlamaOptions<'_>,
) -> Vec<String> {
let LlamaOptions {
port,
ctx_size,
flash_attention,
kv_cache_type,
context_shift,
split_mode,
num_parallel,
embeddings,
batch_size,
threads,
cpus,
} = opts;
let mut args = run_args(backend.engine_args(), port, passthrough_vars, cpus);
args.push("-v".into());
args.push(format!("{model_dir}:/models:ro"));
if let Some((dir, _)) = mmproj {
args.push("-v".into());
args.push(format!("{dir}:/mmproj:ro"));
}
args.push(image);
args.extend([
"-m".into(),
format!("/models/{model_file}"),
"--port".into(),
port.to_string(),
"--host".into(),
"0.0.0.0".into(),
]);
if let Some((_, file)) = mmproj {
args.push("--mmproj".into());
args.push(format!("/mmproj/{file}"));
}
if let Some(n) = ctx_size {
args.push("--ctx-size".into());
args.push(n.to_string());
}
if let Some(mode) = flash_attention {
args.push("--flash-attn".into());
args.push(mode.to_string());
}
if let Some(t) = kv_cache_type {
args.push("--cache-type-k".into());
args.push(t.to_string());
args.push("--cache-type-v".into());
args.push(t.to_string());
}
args.push(
if context_shift {
"--context-shift"
} else {
"--no-context-shift"
}
.into(),
);
if let Some(mode) = split_mode {
args.push("--split-mode".into());
args.push(mode.to_string());
}
if let Some(n) = num_parallel {
args.push("--parallel".into());
args.push(n.to_string());
}
if embeddings {
args.push("--embeddings".into());
}
if let Some(n) = batch_size {
args.push("-b".into());
args.push(n.to_string());
args.push("-ub".into());
args.push(n.to_string());
}
if let Some(n) = threads {
args.push("--threads".into());
args.push(n.to_string());
}
args
}
fn run_args(
engine_args: Vec<String>,
port: u16,
passthrough_vars: &[&str],
cpus: Option<f64>,
) -> Vec<String> {
let mut args: Vec<String> = vec![
"run".into(),
"--rm".into(),
"--init".into(),
"-t".into(),
"-p".into(),
format!("127.0.0.1:{port}:{port}"),
];
if let Some(n) = cpus {
args.push("--cpus".into());
args.push(n.to_string());
}
args.extend(engine_args);
for var in crate::cmd::serve::GPU_VISIBLE_DEVICE_VARS
.iter()
.chain(passthrough_vars)
{
if std::env::var_os(var).is_some() {
args.push("-e".into());
args.push(var.to_string());
}
}
args
}
impl ContainerEngine {
fn launcher(self) -> Option<(&'static [&'static str], &'static str)> {
match self {
ContainerEngine::LlamaServer => None,
ContainerEngine::Vllm | ContainerEngine::VllmOmni => Some((&["vllm"], "VLLM_")),
ContainerEngine::Sglang => {
Some((&["python3", "-m", "sglang.launch_server"], "SGLANG_"))
}
}
}
}
pub fn spawn_engine(
ociman: ContainerManager,
engine: ContainerEngine,
model_dir: &Path,
version: Option<&str>,
port: u16,
cpus: Option<f64>,
serve_args: impl FnOnce(&str, &str) -> Vec<String>,
) -> Result<tokio::process::Child> {
let (backend, arch) = VllmBackend::detect()?;
let image = match engine {
ContainerEngine::VllmOmni => backend.omni_image_ref(arch, version)?,
ContainerEngine::Sglang => backend.sglang_image_ref(arch, version)?,
ContainerEngine::Vllm => backend.image_ref(arch, version)?,
ContainerEngine::LlamaServer => anyhow::bail!("llama-server containers go through spawn"),
};
let Some((launcher, env_prefix)) = engine.launcher() else {
unreachable!("LlamaServer bailed above");
};
eprintln!(
"[llmman] {}: detected {:?} on {:?}, using image {:?}",
ociman.binary(),
backend,
arch,
image
);
let model_dir = mount_source(model_dir)?;
let vars: Vec<String> = std::env::vars_os()
.filter_map(|(k, _)| k.into_string().ok())
.filter(|k| {
k.starts_with(env_prefix) || matches!(k.as_str(), "HF_TOKEN" | "HUGGING_FACE_HUB_TOKEN")
})
.collect();
let vars: Vec<&str> = vars.iter().map(String::as_str).collect();
let prefix = run_args(backend.engine_args(), port, &vars, cpus);
run(
ociman,
engine_run_args(prefix, image, launcher, &model_dir, serve_args),
)
}
fn mount_source(dir: &Path) -> Result<String> {
let dir = dir
.canonicalize()
.with_context(|| format!("resolving {}", dir.display()))?;
dir.to_str()
.map(str::to_owned)
.with_context(|| format!("{} is not valid UTF-8", dir.display()))
}
fn engine_run_args(
mut args: Vec<String>,
image: String,
launcher: &[&str],
model_dir: &str,
serve_args: impl FnOnce(&str, &str) -> Vec<String>,
) -> Vec<String> {
args.extend([
"-v".into(),
format!("{model_dir}:/models:ro"),
"--entrypoint".into(),
launcher[0].into(),
image,
]);
args.extend(launcher[1..].iter().map(|a| a.to_string()));
args.extend(serve_args("/models", "0.0.0.0"));
args
}
fn run(ociman: ContainerManager, args: Vec<String>) -> Result<tokio::process::Child> {
crate::debug_log!("spawning {} {}", ociman.binary(), args.join(" "));
tokio::process::Command::new(ociman.binary())
.args(&args)
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.spawn()
.with_context(|| format!("spawn {} {}", ociman.binary(), args.join(" ")))
}
pub fn spawn_mediagen(
ociman: ContainerManager,
model_ref: &str,
store_path: &Path,
cache_path: &Path,
llama_cpp_version: Option<&str>,
port: u16,
cpus: Option<f64>,
) -> Result<tokio::process::Child> {
let backend = detect_backend();
let image = backend.image_ref(llama_cpp_version);
eprintln!(
"[llmman] {}: detected {:?}, using image {:?} for media generation",
ociman.binary(),
backend,
image
);
let exe = std::env::current_exe().context("locating the llmman binary")?;
let utf8 = |p: &Path| -> Result<String> {
p.to_str()
.map(str::to_owned)
.with_context(|| format!("{} is not valid UTF-8", p.display()))
};
let (exe, store, cache) = (utf8(&exe)?, utf8(store_path)?, utf8(cache_path)?);
let passthrough = [
crate::cmd::serve::LLAMA_CPP_ENV_PASSTHROUGH_VARS,
crate::cmd::serve::MEDIAGEN_ENV_PASSTHROUGH_VARS,
]
.concat();
let mut args = run_args(backend.engine_args(), port, &passthrough, cpus);
args.extend([
"-v".into(),
format!("{exe}:/usr/local/bin/llmman:ro"),
"-v".into(),
format!("{store}:{store}:ro"),
"-v".into(),
format!("{cache}:{cache}:ro"),
"-e".into(),
format!("LLMMAN_MODELS={store}"),
"-e".into(),
"PATH=/app:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin".into(),
"--entrypoint".into(),
"/usr/local/bin/llmman".into(),
image,
"serve".into(),
model_ref.into(),
"--port".into(),
port.to_string(),
"--host".into(),
"0.0.0.0".into(),
"--runtime".into(),
"path".into(),
]);
run(ociman, args)
}
#[cfg(unix)]
pub fn stop(pid: u32) {
let result = unsafe { libc::kill(pid as libc::pid_t, libc::SIGTERM) };
if result != 0 {
let err = std::io::Error::last_os_error();
eprintln!("[llmman] warning: SIGTERM to container process {pid} failed: {err}");
}
}
#[cfg(not(unix))]
pub fn stop(_pid: u32) {
eprintln!("[llmman] warning: container::stop is a no-op on non-Unix platforms");
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn cuda_major_12_picks_cuda12_image() {
assert_eq!(
backend_from_hostgpu(HostGpu::Cuda { major: 12 }),
GpuBackend::Cuda12
);
assert_eq!(GpuBackend::Cuda12.image_tag(), "server-cuda");
}
#[test]
fn cuda_major_13_picks_cuda13_image() {
assert_eq!(
backend_from_hostgpu(HostGpu::Cuda { major: 13 }),
GpuBackend::Cuda13
);
assert_eq!(GpuBackend::Cuda13.image_tag(), "server-cuda13");
}
#[test]
fn cuda_major_above_13_still_picks_cuda13_image() {
assert_eq!(
backend_from_hostgpu(HostGpu::Cuda { major: 14 }),
GpuBackend::Cuda13
);
}
#[test]
fn non_cuda_hostgpu_variants_map_to_their_matching_backend() {
assert_eq!(backend_from_hostgpu(HostGpu::Rocm), GpuBackend::Rocm);
assert_eq!(backend_from_hostgpu(HostGpu::Vulkan), GpuBackend::Vulkan);
assert_eq!(backend_from_hostgpu(HostGpu::None), GpuBackend::Cpu);
assert_eq!(backend_from_hostgpu(HostGpu::Metal), GpuBackend::Cpu);
}
#[test]
fn image_tags_match_docs_docker_md() {
assert_eq!(GpuBackend::Cpu.image_tag(), "server");
assert_eq!(GpuBackend::Cuda12.image_tag(), "server-cuda");
assert_eq!(GpuBackend::Cuda13.image_tag(), "server-cuda13");
assert_eq!(GpuBackend::Rocm.image_tag(), "server-rocm");
assert_eq!(GpuBackend::Vulkan.image_tag(), "server-vulkan");
}
#[test]
fn image_ref_uses_floating_tag_when_no_version_given() {
assert_eq!(
GpuBackend::Cpu.image_ref(None),
"ghcr.io/ggml-org/llama.cpp:server"
);
assert_eq!(
GpuBackend::Cuda13.image_ref(None),
"ghcr.io/ggml-org/llama.cpp:server-cuda13"
);
}
#[test]
fn image_ref_pins_to_the_given_version() {
assert_eq!(
GpuBackend::Cpu.image_ref(Some("b9994")),
"ghcr.io/ggml-org/llama.cpp:server-b9994"
);
assert_eq!(
GpuBackend::Cuda13.image_ref(Some("b9994")),
"ghcr.io/ggml-org/llama.cpp:server-cuda13-b9994"
);
}
#[test]
fn cpu_backend_has_no_extra_engine_args() {
assert!(GpuBackend::Cpu.engine_args().is_empty());
}
#[test]
fn cuda_backend_requests_all_gpus() {
assert_eq!(GpuBackend::Cuda12.engine_args(), vec!["--gpus", "all"]);
assert_eq!(GpuBackend::Cuda13.engine_args(), vec!["--gpus", "all"]);
}
#[test]
fn rocm_backend_mounts_kfd_and_dri() {
let args = GpuBackend::Rocm.engine_args();
assert_eq!(
args,
vec![
"--device",
"/dev/kfd",
"--device",
"/dev/dri",
"--group-add",
"video"
]
);
}
#[test]
fn vulkan_backend_passes_the_devices_the_host_has() {
let args = GpuBackend::Vulkan.engine_args();
assert_eq!(
args.contains(&"/dev/dri".to_string()),
Path::new("/dev/dri").exists()
);
assert_eq!(
args.contains(&"--gpus".to_string()),
Path::new("/dev/nvidiactl").exists()
);
}
#[test]
fn container_manager_binary_names() {
assert_eq!(ContainerManager::Docker.binary(), "docker");
assert_eq!(ContainerManager::Podman.binary(), "podman");
}
#[test]
fn host_arch_parses_only_the_two_architectures_vllm_ships() {
assert_eq!(HostArch::parse("x86_64"), Some(HostArch::X86_64));
assert_eq!(HostArch::parse("aarch64"), Some(HostArch::Aarch64));
assert_eq!(HostArch::parse("s390x"), None);
assert_eq!(HostArch::parse("riscv64"), None);
assert_eq!(HostArch::parse(""), None);
}
#[test]
fn cuda_12_hosts_get_the_cu129_vllm_image() {
assert_eq!(
VllmBackend::from_gpu(GpuBackend::Cuda12),
VllmBackend::Cuda12
);
assert_eq!(
VllmBackend::from_gpu(GpuBackend::Cuda13),
VllmBackend::Cuda13
);
assert_eq!(
VllmBackend::Cuda12
.image_ref(HostArch::X86_64, None)
.unwrap(),
"docker.io/vllm/vllm-openai:latest-x86_64-cu129"
);
assert_eq!(
VllmBackend::Cuda12
.image_ref(HostArch::Aarch64, Some("v0.28.0"))
.unwrap(),
"docker.io/vllm/vllm-openai:v0.28.0-aarch64-cu129"
);
}
#[test]
fn vulkan_host_runs_the_vllm_cpu_image() {
assert_eq!(VllmBackend::from_gpu(GpuBackend::Vulkan), VllmBackend::Cpu);
assert_eq!(VllmBackend::from_gpu(GpuBackend::Cpu), VllmBackend::Cpu);
assert_eq!(VllmBackend::from_gpu(GpuBackend::Rocm), VllmBackend::Rocm);
}
#[test]
fn vllm_image_refs_match_docker_hub_tag_spellings() {
assert_eq!(
VllmBackend::Cuda13
.image_ref(HostArch::X86_64, None)
.unwrap(),
"docker.io/vllm/vllm-openai:latest-x86_64"
);
assert_eq!(
VllmBackend::Cuda13
.image_ref(HostArch::Aarch64, None)
.unwrap(),
"docker.io/vllm/vllm-openai:latest-aarch64"
);
assert_eq!(
VllmBackend::Rocm.image_ref(HostArch::X86_64, None).unwrap(),
"docker.io/rocm/vllm:latest"
);
assert_eq!(
VllmBackend::Cpu.image_ref(HostArch::X86_64, None).unwrap(),
"docker.io/vllm/vllm-openai-cpu:latest-x86_64"
);
assert_eq!(
VllmBackend::Cpu.image_ref(HostArch::Aarch64, None).unwrap(),
"docker.io/vllm/vllm-openai-cpu:latest-arm64"
);
}
#[test]
fn vllm_image_ref_pins_to_the_given_version() {
assert_eq!(
VllmBackend::Cuda13
.image_ref(HostArch::X86_64, Some("v0.11.0"))
.unwrap(),
"docker.io/vllm/vllm-openai:v0.11.0-x86_64"
);
assert_eq!(
VllmBackend::Cuda13
.image_ref(HostArch::Aarch64, Some("v0.11.0"))
.unwrap(),
"docker.io/vllm/vllm-openai:v0.11.0-aarch64"
);
assert_eq!(
VllmBackend::Cpu
.image_ref(HostArch::Aarch64, Some("v0.11.0"))
.unwrap(),
"docker.io/vllm/vllm-openai-cpu:v0.11.0-arm64"
);
assert_eq!(
VllmBackend::Rocm
.image_ref(
HostArch::X86_64,
Some("rocm7.14.1_cdna_ubuntu24.04_py3.14_pytorch_2.11_vllm_0.23.0")
)
.unwrap(),
"docker.io/rocm/vllm:rocm7.14.1_cdna_ubuntu24.04_py3.14_pytorch_2.11_vllm_0.23.0"
);
}
#[test]
fn rocm_vllm_has_no_aarch64_image() {
let err = VllmBackend::Rocm
.image_ref(HostArch::Aarch64, None)
.unwrap_err()
.to_string();
assert!(err.contains("rocm/vllm"), "{err}");
assert!(err.contains("LLMMAN_LLM_LIBRARY=cpu"), "{err}");
}
#[test]
fn vllm_omni_image_refs_match_docker_hub_tag_spellings() {
for cuda in [VllmBackend::Cuda12, VllmBackend::Cuda13] {
assert_eq!(
cuda.omni_image_ref(HostArch::X86_64, None).unwrap(),
"docker.io/vllm/vllm-omni:latest-x86_64"
);
assert_eq!(
cuda.omni_image_ref(HostArch::Aarch64, None).unwrap(),
"docker.io/vllm/vllm-omni:latest-aarch64"
);
assert_eq!(
cuda.omni_image_ref(HostArch::Aarch64, Some("v0.28.0"))
.unwrap(),
"docker.io/vllm/vllm-omni:v0.28.0-aarch64"
);
}
}
#[test]
fn vllm_omni_has_only_cuda_images() {
let err = VllmBackend::Rocm
.omni_image_ref(HostArch::X86_64, None)
.unwrap_err()
.to_string();
assert!(err.contains("vllm/vllm-omni"), "{err}");
assert!(err.contains("--runtime"), "{err}");
let err = VllmBackend::Cpu
.omni_image_ref(HostArch::X86_64, None)
.unwrap_err()
.to_string();
assert!(err.contains("CUDA"), "{err}");
}
#[test]
fn sglang_image_refs_match_docker_hub_tag_spellings() {
for arch in [HostArch::X86_64, HostArch::Aarch64] {
assert_eq!(
VllmBackend::Cuda13.sglang_image_ref(arch, None).unwrap(),
"docker.io/lmsysorg/sglang:latest"
);
assert_eq!(
VllmBackend::Cuda13
.sglang_image_ref(arch, Some("v0.5.19"))
.unwrap(),
"docker.io/lmsysorg/sglang:v0.5.19"
);
assert_eq!(
VllmBackend::Cuda12.sglang_image_ref(arch, None).unwrap(),
"docker.io/lmsysorg/sglang:latest-cu129"
);
assert_eq!(
VllmBackend::Cuda12
.sglang_image_ref(arch, Some("v0.5.19"))
.unwrap(),
"docker.io/lmsysorg/sglang:v0.5.19-cu129"
);
}
}
#[test]
fn sglang_rocm_image_is_the_whole_pinned_tag_or_nothing() {
assert_eq!(
VllmBackend::Rocm
.sglang_image_ref(HostArch::X86_64, Some("v0.5.19-rocm700-mi30x"))
.unwrap(),
"docker.io/lmsysorg/sglang:v0.5.19-rocm700-mi30x"
);
let err = VllmBackend::Rocm
.sglang_image_ref(HostArch::X86_64, None)
.unwrap_err()
.to_string();
assert!(err.contains("--sglang-version"), "{err}");
assert!(err.contains("rocm700-mi30x"), "{err}");
let err = VllmBackend::Rocm
.sglang_image_ref(HostArch::Aarch64, Some("v0.5.19-rocm700-mi30x"))
.unwrap_err()
.to_string();
assert!(err.contains("aarch64"), "{err}");
}
#[test]
fn sglang_has_no_cpu_image_here() {
for arch in [HostArch::X86_64, HostArch::Aarch64] {
let err = VllmBackend::Cpu
.sglang_image_ref(arch, Some("v0.5.19"))
.unwrap_err()
.to_string();
assert!(err.contains("xeon"), "{err}");
assert!(err.contains("LLMMAN_SAFETENSORS_ENGINE"), "{err}");
}
}
#[test]
fn sglang_run_args_launch_the_module_after_the_image_and_flags_after_that() {
let (launcher, _) = ContainerEngine::Sglang.launcher().unwrap();
let args = engine_run_args(
run_args(VllmBackend::Cuda13.engine_args(), 30000, &[], None),
"docker.io/lmsysorg/sglang:latest".into(),
launcher,
"/cache/abc/model",
|dir: &str, host: &str| {
vec![
"--model-path".into(),
dir.into(),
"--host".into(),
host.into(),
]
},
);
let image = args
.iter()
.position(|a| a == "docker.io/lmsysorg/sglang:latest")
.expect("image present");
assert_eq!(&args[image - 2..image], &["--entrypoint", "python3"]);
assert_eq!(
&args[image + 1..],
&[
"-m",
"sglang.launch_server",
"--model-path",
"/models",
"--host",
"0.0.0.0"
]
);
}
#[test]
fn vllm_engine_args_add_ipc_host_on_top_of_gpu_passthrough() {
assert_eq!(
VllmBackend::Cuda13.engine_args(),
vec!["--gpus", "all", "--ipc=host"]
);
let rocm = VllmBackend::Rocm.engine_args();
assert!(
rocm.starts_with(&GpuBackend::Rocm.engine_args()),
"{rocm:?}"
);
assert_eq!(rocm.last().map(String::as_str), Some("--ipc=host"));
}
#[test]
fn vllm_cpu_engine_args_grant_what_its_numa_thread_binding_needs() {
assert_eq!(
VllmBackend::Cpu.engine_args(),
vec![
"--security-opt",
"seccomp=unconfined",
"--cap-add",
"SYS_NICE",
"--ipc=host"
]
);
}
#[test]
fn vllm_run_args_put_the_mount_and_entrypoint_before_the_image_and_serve_after() {
let (launcher, _) = ContainerEngine::Vllm.launcher().unwrap();
let args = engine_run_args(
run_args(VllmBackend::Cuda13.engine_args(), 8000, &[], None),
"docker.io/vllm/vllm-openai:latest-x86_64".into(),
launcher,
"/cache/abc/model",
|dir: &str, host: &str| {
vec![
"serve".into(),
dir.into(),
"--host".into(),
host.into(),
"--served-model-name".into(),
"m".into(),
]
},
);
let image = args
.iter()
.position(|a| a == "docker.io/vllm/vllm-openai:latest-x86_64")
.expect("image present");
let before = &args[..image];
assert!(before.contains(&"--gpus".to_string()), "{before:?}");
assert!(before.contains(&"--ipc=host".to_string()), "{before:?}");
assert!(
before.contains(&"/cache/abc/model:/models:ro".to_string()),
"{before:?}"
);
assert_eq!(&before[before.len() - 2..], &["--entrypoint", "vllm"]);
assert_eq!(
&args[image + 1..],
&[
"serve",
"/models",
"--host",
"0.0.0.0",
"--served-model-name",
"m"
]
);
}
#[test]
fn run_args_forward_only_the_requested_passthrough_vars() {
const VAR: &str = "LLMMAN_TEST_RUN_ARGS_PASSTHROUGH";
std::env::set_var(VAR, "1");
let args = run_args(vec![], 8080, &[], None);
assert_eq!(
&args[..6],
&["run", "--rm", "--init", "-t", "-p", "127.0.0.1:8080:8080"]
);
assert!(!args.contains(&VAR.to_string()), "{args:?}");
let args = run_args(vec![], 8080, &[VAR], None);
assert!(args.windows(2).any(|w| w == ["-e", VAR]), "{args:?}");
assert!(!args.iter().any(|a| a.starts_with(&format!("{VAR}="))));
}
#[test]
fn cpus_is_forwarded_as_a_run_flag_only_when_a_limit_binds() {
let args = run_args(vec![], 8080, &[], None);
assert!(!args.contains(&"--cpus".to_string()), "{args:?}");
let args = run_args(vec!["--gpus".into(), "all".into()], 8080, &[], Some(2.0));
let cpus = args.iter().position(|a| a == "--cpus").unwrap();
assert_eq!(args[cpus + 1], "2");
assert!(cpus < args.iter().position(|a| a == "--gpus").unwrap());
let args = run_args(vec![], 8080, &[], Some(0.5));
assert!(args.windows(2).any(|w| w == ["--cpus", "0.5"]), "{args:?}");
}
#[test]
fn llama_container_gets_cpus_before_the_image_and_threads_after() {
let opts = |threads, cpus| LlamaOptions {
port: 8080,
ctx_size: None,
flash_attention: None,
kv_cache_type: None,
context_shift: true,
split_mode: None,
num_parallel: None,
embeddings: false,
batch_size: None,
threads,
cpus,
};
let image = "ghcr.io/ggml-org/llama.cpp:server";
let argv = |o| {
llama_run_args(
GpuBackend::Cpu,
image.into(),
"/cache/blobs",
"sha256-abc",
None,
&[],
o,
)
};
let args = argv(opts(Some(2), Some(2.0)));
let at = |flag: &str| args.iter().position(|a| a == flag).unwrap();
assert!(at("--cpus") < at(image), "{args:?}");
assert!(at(image) < at("--threads"), "{args:?}");
assert_eq!(args[at("--cpus") + 1], "2");
assert_eq!(args[at("--threads") + 1], "2");
let args = argv(opts(None, None));
assert!(
!args.iter().any(|a| a == "--cpus" || a == "--threads"),
"{args:?}"
);
}
#[test]
fn vllm_containers_get_cpus_too() {
let (launcher, _) = ContainerEngine::Vllm.launcher().unwrap();
let args = engine_run_args(
run_args(VllmBackend::Cpu.engine_args(), 8000, &[], Some(3.0)),
"docker.io/vllm/vllm-openai-cpu:latest-x86_64".into(),
launcher,
"/cache/abc/model",
|dir: &str, host: &str| vec!["serve".into(), dir.into(), "--host".into(), host.into()],
);
assert!(args.windows(2).any(|w| w == ["--cpus", "3"]), "{args:?}");
}
#[test]
#[ignore = "result depends on this host's actual GPU/driver setup"]
fn detect_backend_reports_this_hosts_real_hardware() {
let backend = detect_backend();
println!(
"container::detect_backend() -> {backend:?} ({})",
backend.image_ref(None)
);
}
}