use std::path::Path;
use anyhow::{Context, Result};
use clap::ValueEnum;
use crate::hostgpu::{self, HostGpu};
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
#[value(rename_all = "lower")]
pub enum ContainerManager {
Docker,
Podman,
}
impl ContainerManager {
pub fn binary(self) -> &'static str {
match self {
ContainerManager::Docker => "docker",
ContainerManager::Podman => "podman",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum GpuBackend {
Cpu,
Cuda12,
Cuda13,
Rocm,
Vulkan,
}
impl GpuBackend {
fn image_tag(self) -> &'static str {
match self {
GpuBackend::Cpu => "server",
GpuBackend::Cuda12 => "server-cuda",
GpuBackend::Cuda13 => "server-cuda13",
GpuBackend::Rocm => "server-rocm",
GpuBackend::Vulkan => "server-vulkan",
}
}
fn image_ref(self, version: Option<&str>) -> String {
match version {
Some(v) => format!("ghcr.io/ggml-org/llama.cpp:{}-{v}", self.image_tag()),
None => format!("ghcr.io/ggml-org/llama.cpp:{}", self.image_tag()),
}
}
fn engine_args(self) -> Vec<String> {
match self {
GpuBackend::Cpu => vec![],
GpuBackend::Cuda12 | GpuBackend::Cuda13 => vec!["--gpus".into(), "all".into()],
GpuBackend::Rocm => vec![
"--device".into(),
"/dev/kfd".into(),
"--device".into(),
"/dev/dri".into(),
"--group-add".into(),
"video".into(),
],
GpuBackend::Vulkan => {
let mut args = Vec::new();
if Path::new("/dev/dri").exists() {
args.extend(["--device", "/dev/dri"].map(String::from));
}
if Path::new("/dev/nvidiactl").exists() {
args.extend(
["--gpus", "all", "-e", "NVIDIA_DRIVER_CAPABILITIES=all"].map(String::from),
);
}
args
}
}
}
}
fn detect_backend() -> GpuBackend {
backend_from_hostgpu(hostgpu::detect())
}
fn backend_from_hostgpu(gpu: HostGpu) -> GpuBackend {
match gpu {
HostGpu::Cuda { major } if major >= 13 => GpuBackend::Cuda13,
HostGpu::Cuda { .. } => GpuBackend::Cuda12,
HostGpu::Rocm => GpuBackend::Rocm,
HostGpu::Vulkan => GpuBackend::Vulkan,
HostGpu::Metal | HostGpu::None => GpuBackend::Cpu,
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum HostArch {
X86_64,
Aarch64,
}
impl HostArch {
fn parse(arch: &str) -> Option<HostArch> {
match arch {
"x86_64" => Some(HostArch::X86_64),
"aarch64" => Some(HostArch::Aarch64),
_ => None,
}
}
fn detect() -> Result<HostArch> {
HostArch::parse(std::env::consts::ARCH).with_context(|| {
format!(
"vLLM publishes no container image for {} hosts (only x86_64 and aarch64)",
std::env::consts::ARCH
)
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum VllmBackend {
Cuda12,
Cuda13,
Rocm,
Cpu,
}
impl VllmBackend {
fn from_gpu(backend: GpuBackend) -> VllmBackend {
match backend {
GpuBackend::Cuda12 => VllmBackend::Cuda12,
GpuBackend::Cuda13 => VllmBackend::Cuda13,
GpuBackend::Rocm => VllmBackend::Rocm,
GpuBackend::Cpu | GpuBackend::Vulkan => VllmBackend::Cpu,
}
}
fn detect() -> Result<(VllmBackend, HostArch)> {
Ok((VllmBackend::from_gpu(detect_backend()), HostArch::detect()?))
}
fn image_ref(self, arch: HostArch, version: Option<&str>) -> Result<String> {
let v = version.unwrap_or("latest");
let (repo, suffix) = match (self, arch) {
(VllmBackend::Rocm, HostArch::X86_64) => return Ok(format!("docker.io/rocm/vllm:{v}")),
(VllmBackend::Rocm, HostArch::Aarch64) => anyhow::bail!(
"an AMD GPU was detected, but rocm/vllm publishes no aarch64 image \
(set LLMMAN_LLM_LIBRARY=cpu to run vllm/vllm-openai-cpu instead)"
),
(VllmBackend::Cuda12, HostArch::X86_64) => ("vllm-openai", "x86_64-cu129"),
(VllmBackend::Cuda12, HostArch::Aarch64) => ("vllm-openai", "aarch64-cu129"),
(VllmBackend::Cuda13, HostArch::X86_64) => ("vllm-openai", "x86_64"),
(VllmBackend::Cuda13, HostArch::Aarch64) => ("vllm-openai", "aarch64"),
(VllmBackend::Cpu, HostArch::X86_64) => ("vllm-openai-cpu", "x86_64"),
(VllmBackend::Cpu, HostArch::Aarch64) => ("vllm-openai-cpu", "arm64"),
};
Ok(format!("docker.io/vllm/{repo}:{v}-{suffix}"))
}
fn omni_image_ref(self, arch: HostArch, version: Option<&str>) -> Result<String> {
let v = version.unwrap_or("latest");
let suffix = match (self, arch) {
(VllmBackend::Cuda12 | VllmBackend::Cuda13, HostArch::X86_64) => "x86_64",
(VllmBackend::Cuda12 | VllmBackend::Cuda13, HostArch::Aarch64) => "aarch64",
(VllmBackend::Rocm, _) => anyhow::bail!(
"an AMD GPU was detected, but vllm/vllm-omni publishes only CUDA images \
(install vllm-omni locally and drop --ociman to serve this model)"
),
(VllmBackend::Cpu, _) => anyhow::bail!(
"no CUDA GPU was detected, and vllm/vllm-omni publishes only CUDA images"
),
};
Ok(format!("docker.io/vllm/vllm-omni:{v}-{suffix}"))
}
fn engine_args(self) -> Vec<String> {
let mut args = match self {
VllmBackend::Cuda12 | VllmBackend::Cuda13 => GpuBackend::Cuda12.engine_args(),
VllmBackend::Rocm => GpuBackend::Rocm.engine_args(),
VllmBackend::Cpu => vec![
"--security-opt".into(),
"seccomp=unconfined".into(),
"--cap-add".into(),
"SYS_NICE".into(),
],
};
args.push("--ipc=host".into());
args
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ContainerEngine {
LlamaServer,
Vllm,
VllmOmni,
}
fn image_for(engine: ContainerEngine, version: Option<&str>) -> Result<String> {
match engine {
ContainerEngine::LlamaServer => Ok(detect_backend().image_ref(version)),
ContainerEngine::Vllm => {
let (backend, arch) = VllmBackend::detect()?;
backend.image_ref(arch, version)
}
ContainerEngine::VllmOmni => {
let (backend, arch) = VllmBackend::detect()?;
backend.omni_image_ref(arch, version)
}
}
}
pub fn pull_image(
ociman: ContainerManager,
engine: ContainerEngine,
version: Option<&str>,
) -> Result<()> {
let image = image_for(engine, version)?;
eprintln!("[llmman] {}: pulling {image}...", ociman.binary());
let status = std::process::Command::new(ociman.binary())
.args(["pull", &image])
.status()
.with_context(|| format!("run {} pull {image}", ociman.binary()))?;
if !status.success() {
anyhow::bail!("{} pull {image} failed", ociman.binary());
}
Ok(())
}
#[derive(Debug, Clone, Copy)]
pub struct LlamaOptions<'a> {
pub port: u16,
pub ctx_size: Option<u32>,
pub flash_attention: Option<&'a str>,
pub kv_cache_type: Option<&'a str>,
pub context_shift: bool,
pub split_mode: Option<&'a str>,
pub num_parallel: Option<u32>,
pub embeddings: bool,
pub batch_size: Option<u32>,
pub threads: Option<u32>,
}
pub fn spawn(
ociman: ContainerManager,
model_path: &Path,
mmproj_path: Option<&Path>,
llama_cpp_version: Option<&str>,
opts: LlamaOptions<'_>,
) -> Result<tokio::process::Child> {
let LlamaOptions {
port,
ctx_size,
flash_attention,
kv_cache_type,
context_shift,
split_mode,
num_parallel,
embeddings,
batch_size,
threads: _,
} = opts;
let backend = detect_backend();
let image = backend.image_ref(llama_cpp_version);
eprintln!(
"[llmman] {}: detected {:?}, using image {:?}",
ociman.binary(),
backend,
image
);
let model_dir = model_path
.parent()
.context("model path has no parent directory")?;
let model_file = model_path
.file_name()
.and_then(|n| n.to_str())
.context("model path has no valid UTF-8 filename")?;
let model_dir = mount_source(model_dir)?;
let mut args = run_args(
backend.engine_args(),
port,
crate::cmd::serve::LLAMA_CPP_ENV_PASSTHROUGH_VARS,
);
args.push("-v".into());
args.push(format!("{model_dir}:/models:ro"));
let mmproj_file = mmproj_path
.map(|p| -> Result<&str> {
let dir = p.parent().context("mmproj path has no parent directory")?;
args.push("-v".into());
args.push(format!("{}:/mmproj:ro", mount_source(dir)?));
p.file_name()
.and_then(|n| n.to_str())
.context("mmproj path has no valid UTF-8 filename")
})
.transpose()?;
args.push(image);
args.extend([
"-m".into(),
format!("/models/{model_file}"),
"--port".into(),
port.to_string(),
"--host".into(),
"0.0.0.0".into(),
]);
if let Some(mmproj_file) = mmproj_file {
args.push("--mmproj".into());
args.push(format!("/mmproj/{mmproj_file}"));
}
if let Some(n) = ctx_size {
args.push("--ctx-size".into());
args.push(n.to_string());
}
if let Some(mode) = flash_attention {
args.push("--flash-attn".into());
args.push(mode.to_string());
}
if let Some(t) = kv_cache_type {
args.push("--cache-type-k".into());
args.push(t.to_string());
args.push("--cache-type-v".into());
args.push(t.to_string());
}
args.push(
if context_shift {
"--context-shift"
} else {
"--no-context-shift"
}
.into(),
);
if let Some(mode) = split_mode {
args.push("--split-mode".into());
args.push(mode.to_string());
}
if let Some(n) = num_parallel {
args.push("--parallel".into());
args.push(n.to_string());
}
if embeddings {
args.push("--embeddings".into());
}
if let Some(n) = batch_size {
args.push("-b".into());
args.push(n.to_string());
args.push("-ub".into());
args.push(n.to_string());
}
run(ociman, args)
}
fn run_args(engine_args: Vec<String>, port: u16, passthrough_vars: &[&str]) -> Vec<String> {
let mut args: Vec<String> = vec![
"run".into(),
"--rm".into(),
"--init".into(),
"-t".into(),
"-p".into(),
format!("127.0.0.1:{port}:{port}"),
];
args.extend(engine_args);
for var in crate::cmd::serve::GPU_VISIBLE_DEVICE_VARS
.iter()
.chain(passthrough_vars)
{
if std::env::var_os(var).is_some() {
args.push("-e".into());
args.push(var.to_string());
}
}
args
}
pub fn spawn_vllm(
ociman: ContainerManager,
engine: ContainerEngine,
model_dir: &Path,
vllm_version: Option<&str>,
port: u16,
serve_args: impl FnOnce(&str, &str) -> Vec<String>,
) -> Result<tokio::process::Child> {
let (backend, arch) = VllmBackend::detect()?;
let image = match engine {
ContainerEngine::VllmOmni => backend.omni_image_ref(arch, vllm_version)?,
ContainerEngine::Vllm | ContainerEngine::LlamaServer => {
backend.image_ref(arch, vllm_version)?
}
};
eprintln!(
"[llmman] {}: detected {:?} on {:?}, using image {:?}",
ociman.binary(),
backend,
arch,
image
);
let model_dir = mount_source(model_dir)?;
let vllm_vars: Vec<String> = std::env::vars_os()
.filter_map(|(k, _)| k.into_string().ok())
.filter(|k| {
k.starts_with("VLLM_") || matches!(k.as_str(), "HF_TOKEN" | "HUGGING_FACE_HUB_TOKEN")
})
.collect();
let vllm_vars: Vec<&str> = vllm_vars.iter().map(String::as_str).collect();
let args = vllm_run_args(backend, image, &model_dir, port, &vllm_vars, serve_args);
run(ociman, args)
}
fn mount_source(dir: &Path) -> Result<String> {
let dir = dir
.canonicalize()
.with_context(|| format!("resolving {}", dir.display()))?;
dir.to_str()
.map(str::to_owned)
.with_context(|| format!("{} is not valid UTF-8", dir.display()))
}
fn vllm_run_args(
backend: VllmBackend,
image: String,
model_dir: &str,
port: u16,
passthrough_vars: &[&str],
serve_args: impl FnOnce(&str, &str) -> Vec<String>,
) -> Vec<String> {
let mut args = run_args(backend.engine_args(), port, passthrough_vars);
args.extend([
"-v".into(),
format!("{model_dir}:/models:ro"),
"--entrypoint".into(),
"vllm".into(),
image,
]);
args.extend(serve_args("/models", "0.0.0.0"));
args
}
fn run(ociman: ContainerManager, args: Vec<String>) -> Result<tokio::process::Child> {
crate::debug_log!("spawning {} {}", ociman.binary(), args.join(" "));
tokio::process::Command::new(ociman.binary())
.args(&args)
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.spawn()
.with_context(|| format!("spawn {} {}", ociman.binary(), args.join(" ")))
}
pub fn spawn_mediagen(
ociman: ContainerManager,
model_ref: &str,
store_path: &Path,
cache_path: &Path,
llama_cpp_version: Option<&str>,
port: u16,
) -> Result<tokio::process::Child> {
let backend = detect_backend();
let image = backend.image_ref(llama_cpp_version);
eprintln!(
"[llmman] {}: detected {:?}, using image {:?} for media generation",
ociman.binary(),
backend,
image
);
let exe = std::env::current_exe().context("locating the llmman binary")?;
let utf8 = |p: &Path| -> Result<String> {
p.to_str()
.map(str::to_owned)
.with_context(|| format!("{} is not valid UTF-8", p.display()))
};
let (exe, store, cache) = (utf8(&exe)?, utf8(store_path)?, utf8(cache_path)?);
let passthrough = [
crate::cmd::serve::LLAMA_CPP_ENV_PASSTHROUGH_VARS,
crate::cmd::serve::MEDIAGEN_ENV_PASSTHROUGH_VARS,
]
.concat();
let mut args = run_args(backend.engine_args(), port, &passthrough);
args.extend([
"-v".into(),
format!("{exe}:/usr/local/bin/llmman:ro"),
"-v".into(),
format!("{store}:{store}:ro"),
"-v".into(),
format!("{cache}:{cache}:ro"),
"-e".into(),
format!("LLMMAN_MODELS={store}"),
"-e".into(),
"PATH=/app:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin".into(),
"--entrypoint".into(),
"/usr/local/bin/llmman".into(),
image,
"serve".into(),
model_ref.into(),
"--port".into(),
port.to_string(),
"--host".into(),
"0.0.0.0".into(),
]);
run(ociman, args)
}
#[cfg(unix)]
pub fn stop(pid: u32) {
let result = unsafe { libc::kill(pid as libc::pid_t, libc::SIGTERM) };
if result != 0 {
let err = std::io::Error::last_os_error();
eprintln!("[llmman] warning: SIGTERM to container process {pid} failed: {err}");
}
}
#[cfg(not(unix))]
pub fn stop(_pid: u32) {
eprintln!("[llmman] warning: container::stop is a no-op on non-Unix platforms");
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn cuda_major_12_picks_cuda12_image() {
assert_eq!(
backend_from_hostgpu(HostGpu::Cuda { major: 12 }),
GpuBackend::Cuda12
);
assert_eq!(GpuBackend::Cuda12.image_tag(), "server-cuda");
}
#[test]
fn cuda_major_13_picks_cuda13_image() {
assert_eq!(
backend_from_hostgpu(HostGpu::Cuda { major: 13 }),
GpuBackend::Cuda13
);
assert_eq!(GpuBackend::Cuda13.image_tag(), "server-cuda13");
}
#[test]
fn cuda_major_above_13_still_picks_cuda13_image() {
assert_eq!(
backend_from_hostgpu(HostGpu::Cuda { major: 14 }),
GpuBackend::Cuda13
);
}
#[test]
fn non_cuda_hostgpu_variants_map_to_their_matching_backend() {
assert_eq!(backend_from_hostgpu(HostGpu::Rocm), GpuBackend::Rocm);
assert_eq!(backend_from_hostgpu(HostGpu::Vulkan), GpuBackend::Vulkan);
assert_eq!(backend_from_hostgpu(HostGpu::None), GpuBackend::Cpu);
assert_eq!(backend_from_hostgpu(HostGpu::Metal), GpuBackend::Cpu);
}
#[test]
fn image_tags_match_docs_docker_md() {
assert_eq!(GpuBackend::Cpu.image_tag(), "server");
assert_eq!(GpuBackend::Cuda12.image_tag(), "server-cuda");
assert_eq!(GpuBackend::Cuda13.image_tag(), "server-cuda13");
assert_eq!(GpuBackend::Rocm.image_tag(), "server-rocm");
assert_eq!(GpuBackend::Vulkan.image_tag(), "server-vulkan");
}
#[test]
fn image_ref_uses_floating_tag_when_no_version_given() {
assert_eq!(
GpuBackend::Cpu.image_ref(None),
"ghcr.io/ggml-org/llama.cpp:server"
);
assert_eq!(
GpuBackend::Cuda13.image_ref(None),
"ghcr.io/ggml-org/llama.cpp:server-cuda13"
);
}
#[test]
fn image_ref_pins_to_the_given_version() {
assert_eq!(
GpuBackend::Cpu.image_ref(Some("b9994")),
"ghcr.io/ggml-org/llama.cpp:server-b9994"
);
assert_eq!(
GpuBackend::Cuda13.image_ref(Some("b9994")),
"ghcr.io/ggml-org/llama.cpp:server-cuda13-b9994"
);
}
#[test]
fn cpu_backend_has_no_extra_engine_args() {
assert!(GpuBackend::Cpu.engine_args().is_empty());
}
#[test]
fn cuda_backend_requests_all_gpus() {
assert_eq!(GpuBackend::Cuda12.engine_args(), vec!["--gpus", "all"]);
assert_eq!(GpuBackend::Cuda13.engine_args(), vec!["--gpus", "all"]);
}
#[test]
fn rocm_backend_mounts_kfd_and_dri() {
let args = GpuBackend::Rocm.engine_args();
assert_eq!(
args,
vec![
"--device",
"/dev/kfd",
"--device",
"/dev/dri",
"--group-add",
"video"
]
);
}
#[test]
fn vulkan_backend_passes_the_devices_the_host_has() {
let args = GpuBackend::Vulkan.engine_args();
assert_eq!(
args.contains(&"/dev/dri".to_string()),
Path::new("/dev/dri").exists()
);
assert_eq!(
args.contains(&"--gpus".to_string()),
Path::new("/dev/nvidiactl").exists()
);
}
#[test]
fn container_manager_binary_names() {
assert_eq!(ContainerManager::Docker.binary(), "docker");
assert_eq!(ContainerManager::Podman.binary(), "podman");
}
#[test]
fn host_arch_parses_only_the_two_architectures_vllm_ships() {
assert_eq!(HostArch::parse("x86_64"), Some(HostArch::X86_64));
assert_eq!(HostArch::parse("aarch64"), Some(HostArch::Aarch64));
assert_eq!(HostArch::parse("s390x"), None);
assert_eq!(HostArch::parse("riscv64"), None);
assert_eq!(HostArch::parse(""), None);
}
#[test]
fn cuda_12_hosts_get_the_cu129_vllm_image() {
assert_eq!(
VllmBackend::from_gpu(GpuBackend::Cuda12),
VllmBackend::Cuda12
);
assert_eq!(
VllmBackend::from_gpu(GpuBackend::Cuda13),
VllmBackend::Cuda13
);
assert_eq!(
VllmBackend::Cuda12
.image_ref(HostArch::X86_64, None)
.unwrap(),
"docker.io/vllm/vllm-openai:latest-x86_64-cu129"
);
assert_eq!(
VllmBackend::Cuda12
.image_ref(HostArch::Aarch64, Some("v0.28.0"))
.unwrap(),
"docker.io/vllm/vllm-openai:v0.28.0-aarch64-cu129"
);
}
#[test]
fn vulkan_host_runs_the_vllm_cpu_image() {
assert_eq!(VllmBackend::from_gpu(GpuBackend::Vulkan), VllmBackend::Cpu);
assert_eq!(VllmBackend::from_gpu(GpuBackend::Cpu), VllmBackend::Cpu);
assert_eq!(VllmBackend::from_gpu(GpuBackend::Rocm), VllmBackend::Rocm);
}
#[test]
fn vllm_image_refs_match_docker_hub_tag_spellings() {
assert_eq!(
VllmBackend::Cuda13
.image_ref(HostArch::X86_64, None)
.unwrap(),
"docker.io/vllm/vllm-openai:latest-x86_64"
);
assert_eq!(
VllmBackend::Cuda13
.image_ref(HostArch::Aarch64, None)
.unwrap(),
"docker.io/vllm/vllm-openai:latest-aarch64"
);
assert_eq!(
VllmBackend::Rocm.image_ref(HostArch::X86_64, None).unwrap(),
"docker.io/rocm/vllm:latest"
);
assert_eq!(
VllmBackend::Cpu.image_ref(HostArch::X86_64, None).unwrap(),
"docker.io/vllm/vllm-openai-cpu:latest-x86_64"
);
assert_eq!(
VllmBackend::Cpu.image_ref(HostArch::Aarch64, None).unwrap(),
"docker.io/vllm/vllm-openai-cpu:latest-arm64"
);
}
#[test]
fn vllm_image_ref_pins_to_the_given_version() {
assert_eq!(
VllmBackend::Cuda13
.image_ref(HostArch::X86_64, Some("v0.11.0"))
.unwrap(),
"docker.io/vllm/vllm-openai:v0.11.0-x86_64"
);
assert_eq!(
VllmBackend::Cuda13
.image_ref(HostArch::Aarch64, Some("v0.11.0"))
.unwrap(),
"docker.io/vllm/vllm-openai:v0.11.0-aarch64"
);
assert_eq!(
VllmBackend::Cpu
.image_ref(HostArch::Aarch64, Some("v0.11.0"))
.unwrap(),
"docker.io/vllm/vllm-openai-cpu:v0.11.0-arm64"
);
assert_eq!(
VllmBackend::Rocm
.image_ref(
HostArch::X86_64,
Some("rocm7.14.1_cdna_ubuntu24.04_py3.14_pytorch_2.11_vllm_0.23.0")
)
.unwrap(),
"docker.io/rocm/vllm:rocm7.14.1_cdna_ubuntu24.04_py3.14_pytorch_2.11_vllm_0.23.0"
);
}
#[test]
fn rocm_vllm_has_no_aarch64_image() {
let err = VllmBackend::Rocm
.image_ref(HostArch::Aarch64, None)
.unwrap_err()
.to_string();
assert!(err.contains("rocm/vllm"), "{err}");
assert!(err.contains("LLMMAN_LLM_LIBRARY=cpu"), "{err}");
}
#[test]
fn vllm_omni_image_refs_match_docker_hub_tag_spellings() {
for cuda in [VllmBackend::Cuda12, VllmBackend::Cuda13] {
assert_eq!(
cuda.omni_image_ref(HostArch::X86_64, None).unwrap(),
"docker.io/vllm/vllm-omni:latest-x86_64"
);
assert_eq!(
cuda.omni_image_ref(HostArch::Aarch64, None).unwrap(),
"docker.io/vllm/vllm-omni:latest-aarch64"
);
assert_eq!(
cuda.omni_image_ref(HostArch::Aarch64, Some("v0.28.0"))
.unwrap(),
"docker.io/vllm/vllm-omni:v0.28.0-aarch64"
);
}
}
#[test]
fn vllm_omni_has_only_cuda_images() {
let err = VllmBackend::Rocm
.omni_image_ref(HostArch::X86_64, None)
.unwrap_err()
.to_string();
assert!(err.contains("vllm/vllm-omni"), "{err}");
assert!(err.contains("--ociman"), "{err}");
let err = VllmBackend::Cpu
.omni_image_ref(HostArch::X86_64, None)
.unwrap_err()
.to_string();
assert!(err.contains("CUDA"), "{err}");
}
#[test]
fn vllm_engine_args_add_ipc_host_on_top_of_gpu_passthrough() {
assert_eq!(
VllmBackend::Cuda13.engine_args(),
vec!["--gpus", "all", "--ipc=host"]
);
let rocm = VllmBackend::Rocm.engine_args();
assert!(
rocm.starts_with(&GpuBackend::Rocm.engine_args()),
"{rocm:?}"
);
assert_eq!(rocm.last().map(String::as_str), Some("--ipc=host"));
}
#[test]
fn vllm_cpu_engine_args_grant_what_its_numa_thread_binding_needs() {
assert_eq!(
VllmBackend::Cpu.engine_args(),
vec![
"--security-opt",
"seccomp=unconfined",
"--cap-add",
"SYS_NICE",
"--ipc=host"
]
);
}
#[test]
fn vllm_run_args_put_the_mount_and_entrypoint_before_the_image_and_serve_after() {
let args = vllm_run_args(
VllmBackend::Cuda13,
"docker.io/vllm/vllm-openai:latest-x86_64".into(),
"/cache/abc/model",
8000,
&[],
|dir, host| {
vec![
"serve".into(),
dir.into(),
"--host".into(),
host.into(),
"--served-model-name".into(),
"m".into(),
]
},
);
let image = args
.iter()
.position(|a| a == "docker.io/vllm/vllm-openai:latest-x86_64")
.expect("image present");
let before = &args[..image];
assert!(before.contains(&"--gpus".to_string()), "{before:?}");
assert!(before.contains(&"--ipc=host".to_string()), "{before:?}");
assert!(
before.contains(&"/cache/abc/model:/models:ro".to_string()),
"{before:?}"
);
assert_eq!(&before[before.len() - 2..], &["--entrypoint", "vllm"]);
assert_eq!(
&args[image + 1..],
&[
"serve",
"/models",
"--host",
"0.0.0.0",
"--served-model-name",
"m"
]
);
}
#[test]
fn run_args_forward_only_the_requested_passthrough_vars() {
const VAR: &str = "LLMMAN_TEST_RUN_ARGS_PASSTHROUGH";
std::env::set_var(VAR, "1");
let args = run_args(vec![], 8080, &[]);
assert_eq!(
&args[..6],
&["run", "--rm", "--init", "-t", "-p", "127.0.0.1:8080:8080"]
);
assert!(!args.contains(&VAR.to_string()), "{args:?}");
let args = run_args(vec![], 8080, &[VAR]);
assert!(args.windows(2).any(|w| w == ["-e", VAR]), "{args:?}");
assert!(!args.iter().any(|a| a.starts_with(&format!("{VAR}="))));
}
#[test]
#[ignore = "result depends on this host's actual GPU/driver setup"]
fn detect_backend_reports_this_hosts_real_hardware() {
let backend = detect_backend();
println!(
"container::detect_backend() -> {backend:?} ({})",
backend.image_ref(None)
);
}
}