use std::path::Path;
use anyhow::{Context, Result};
use clap::ValueEnum;
use crate::hostgpu::{self, HostGpu};
pub(crate) mod identity;
#[derive(Debug, Clone, Copy, PartialEq, Eq, ValueEnum)]
#[value(rename_all = "lower")]
pub enum ContainerManager {
Docker,
Podman,
}
impl ContainerManager {
pub fn binary(self) -> &'static str {
match self {
ContainerManager::Docker => "docker",
ContainerManager::Podman => "podman",
}
}
pub fn probe(self) -> Result<()> {
self.probe_with(true)
}
pub fn probe_cpu_only(self) -> Result<()> {
self.probe_with(false)
}
fn probe_with(self, require_gpu_toolkit: bool) -> Result<()> {
let cli = self.binary();
if crate::find_on_path(cli).is_none() {
anyhow::bail!("{cli} is not on PATH");
}
let mut cmd = std::process::Command::new(cli);
cmd.arg("info")
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null());
match run_with_timeout(cmd, PROBE_TIMEOUT).with_context(|| format!("run {cli} info"))? {
Some(status) if status.success() => {}
Some(_) => anyhow::bail!("{cli} info failed (is its daemon running?)"),
None => anyhow::bail!("{cli} info did not answer within {PROBE_TIMEOUT:?}"),
}
if self == ContainerManager::Docker && cfg!(target_os = "linux") {
let _ = identity::docker_is_rootless();
}
let wants_gpus = detect_backend().engine_args().iter().any(|a| a == "--gpus");
gpu_gate(require_gpu_toolkit, wants_gpus, cli, || {
nvidia_toolkit_present(self)
})
}
}
fn gpu_gate(
require_gpu_toolkit: bool,
wants_gpus: bool,
cli: &str,
toolkit_present: impl FnOnce() -> bool,
) -> Result<()> {
if require_gpu_toolkit && wants_gpus && !toolkit_present() {
anyhow::bail!(
"an NVIDIA GPU was detected but no NVIDIA Container Toolkit for {cli} \
(`{cli} run --gpus all` would fail)"
);
}
Ok(())
}
const PROBE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10);
fn run_with_timeout(
mut cmd: std::process::Command,
timeout: std::time::Duration,
) -> Result<Option<std::process::ExitStatus>> {
wait_with_timeout(&mut cmd.spawn()?, timeout)
}
fn wait_with_timeout(
child: &mut std::process::Child,
timeout: std::time::Duration,
) -> Result<Option<std::process::ExitStatus>> {
let deadline = std::time::Instant::now() + timeout;
loop {
if let Some(status) = child.try_wait()? {
return Ok(Some(status));
}
if std::time::Instant::now() >= deadline {
let _ = child.kill();
let _ = child.wait();
return Ok(None);
}
std::thread::sleep(std::time::Duration::from_millis(50));
}
}
fn nvidia_toolkit_present(ociman: ContainerManager) -> bool {
if [
"nvidia-container-runtime-hook",
"nvidia-container-toolkit",
"nvidia-ctk",
]
.iter()
.any(|b| crate::find_on_path(b).is_some())
{
return true;
}
if ["/etc/cdi/nvidia.yaml", "/var/run/cdi/nvidia.yaml"]
.iter()
.any(|p| Path::new(p).exists())
{
return true;
}
if ociman == ContainerManager::Docker {
let runtimes = std::process::Command::new("docker")
.args([
"info",
"--format",
"{{range $k, $v := .Runtimes}}{{$k}} {{end}}",
])
.output()
.ok()
.filter(|o| o.status.success())
.map(|o| String::from_utf8_lossy(&o.stdout).into_owned())
.unwrap_or_default();
if runtimes.split_whitespace().any(|r| r == "nvidia") {
return true;
}
}
false
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum GpuBackend {
Cpu,
Cuda12,
Cuda13,
Rocm,
Vulkan,
}
impl GpuBackend {
fn image_tag(self) -> &'static str {
match self {
GpuBackend::Cpu => "server",
GpuBackend::Cuda12 => "server-cuda",
GpuBackend::Cuda13 => "server-cuda13",
GpuBackend::Rocm => "server-rocm",
GpuBackend::Vulkan => "server-vulkan",
}
}
fn image_ref(self, version: Option<&str>) -> String {
match version {
Some(v) => format!("ghcr.io/ggml-org/llama.cpp:{}-{v}", self.image_tag()),
None => format!("ghcr.io/ggml-org/llama.cpp:{}", self.image_tag()),
}
}
fn engine_args(self) -> Vec<String> {
match self {
GpuBackend::Cpu => vec![],
GpuBackend::Cuda12 | GpuBackend::Cuda13 => vec!["--gpus".into(), "all".into()],
GpuBackend::Rocm => vec![
"--device".into(),
"/dev/kfd".into(),
"--device".into(),
"/dev/dri".into(),
"--group-add".into(),
"video".into(),
],
GpuBackend::Vulkan => {
let mut args = Vec::new();
if Path::new("/dev/dri").exists() {
args.extend(["--device", "/dev/dri"].map(String::from));
}
if Path::new("/dev/nvidiactl").exists() {
args.extend(
["--gpus", "all", "-e", "NVIDIA_DRIVER_CAPABILITIES=all"].map(String::from),
);
}
args
}
}
}
}
pub(crate) fn detect_backend() -> GpuBackend {
backend_from_hostgpu(hostgpu::detect())
}
pub(crate) const SPEECH_BACKEND: GpuBackend = GpuBackend::Cpu;
fn backend_from_hostgpu(gpu: HostGpu) -> GpuBackend {
match gpu {
HostGpu::Cuda { major } if major >= 13 => GpuBackend::Cuda13,
HostGpu::Cuda { .. } => GpuBackend::Cuda12,
HostGpu::Rocm => GpuBackend::Rocm,
HostGpu::Vulkan => GpuBackend::Vulkan,
HostGpu::Metal | HostGpu::None | HostGpu::Opencl => GpuBackend::Cpu,
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum HostArch {
X86_64,
Aarch64,
}
impl HostArch {
fn parse(arch: &str) -> Option<HostArch> {
match arch {
"x86_64" => Some(HostArch::X86_64),
"aarch64" => Some(HostArch::Aarch64),
_ => None,
}
}
fn detect() -> Result<HostArch> {
HostArch::parse(std::env::consts::ARCH).with_context(|| {
format!(
"vLLM publishes no container image for {} hosts (only x86_64 and aarch64)",
std::env::consts::ARCH
)
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum VllmBackend {
Cuda12,
Cuda13,
Rocm,
Cpu,
}
impl VllmBackend {
fn from_gpu(backend: GpuBackend) -> VllmBackend {
match backend {
GpuBackend::Cuda12 => VllmBackend::Cuda12,
GpuBackend::Cuda13 => VllmBackend::Cuda13,
GpuBackend::Rocm => VllmBackend::Rocm,
GpuBackend::Cpu | GpuBackend::Vulkan => VllmBackend::Cpu,
}
}
fn detect() -> VllmBackend {
VllmBackend::from_gpu(detect_backend())
}
fn engine_image(
self,
engine: ContainerEngine,
version: Option<&str>,
vllm_image: Option<&str>,
) -> Result<String> {
if let (ContainerEngine::Vllm, Some(image)) = (engine, vllm_image) {
return Ok(image.to_string());
}
let arch = HostArch::detect()?;
match engine {
ContainerEngine::Vllm => self.image_ref(arch, version),
ContainerEngine::VllmOmni => self.omni_image_ref(arch, version),
ContainerEngine::Sglang => self.sglang_image_ref(arch, version),
ContainerEngine::LlamaServer | ContainerEngine::Speech => {
anyhow::bail!("llama-server and speech containers go through spawn")
}
}
}
fn image_ref(self, arch: HostArch, version: Option<&str>) -> Result<String> {
let v = version.unwrap_or("latest");
let (repo, suffix) = match (self, arch) {
(VllmBackend::Rocm, HostArch::X86_64) => return Ok(format!("docker.io/rocm/vllm:{v}")),
(VllmBackend::Rocm, HostArch::Aarch64) => anyhow::bail!(
"an AMD GPU was detected, but rocm/vllm publishes no aarch64 image \
(set LLMMAN_LLM_LIBRARY=cpu to run vllm/vllm-openai-cpu instead)"
),
(VllmBackend::Cuda12, HostArch::X86_64) => ("vllm-openai", "x86_64-cu129"),
(VllmBackend::Cuda12, HostArch::Aarch64) => ("vllm-openai", "aarch64-cu129"),
(VllmBackend::Cuda13, HostArch::X86_64) => ("vllm-openai", "x86_64"),
(VllmBackend::Cuda13, HostArch::Aarch64) => ("vllm-openai", "aarch64"),
(VllmBackend::Cpu, HostArch::X86_64) => ("vllm-openai-cpu", "x86_64"),
(VllmBackend::Cpu, HostArch::Aarch64) => ("vllm-openai-cpu", "arm64"),
};
Ok(format!("docker.io/vllm/{repo}:{v}-{suffix}"))
}
fn omni_image_ref(self, arch: HostArch, version: Option<&str>) -> Result<String> {
let v = version.unwrap_or("latest");
let suffix = match (self, arch) {
(VllmBackend::Cuda12 | VllmBackend::Cuda13, HostArch::X86_64) => "x86_64",
(VllmBackend::Cuda12 | VllmBackend::Cuda13, HostArch::Aarch64) => "aarch64",
(VllmBackend::Rocm, _) => anyhow::bail!(
"an AMD GPU was detected, but vllm/vllm-omni publishes only CUDA images \
(install vllm-omni locally and use --runtime path to serve this model)"
),
(VllmBackend::Cpu, _) => anyhow::bail!(
"no CUDA GPU was detected, and vllm/vllm-omni publishes only CUDA images"
),
};
Ok(format!("docker.io/vllm/vllm-omni:{v}-{suffix}"))
}
fn sglang_image_ref(self, arch: HostArch, version: Option<&str>) -> Result<String> {
let v = version.unwrap_or("latest");
let tag = match (self, arch) {
(VllmBackend::Cuda13, _) => v.to_string(),
(VllmBackend::Cuda12, _) => format!("{v}-cu129"),
(VllmBackend::Rocm, HostArch::X86_64) => version.map(str::to_string).context(
"an AMD GPU was detected, but lmsysorg/sglang publishes no floating ROCm \
tag: pass the whole tag for your GPU family as --sglang-version \
(e.g. v0.5.19-rocm700-mi30x; see hub.docker.com/r/lmsysorg/sglang/tags)",
)?,
(VllmBackend::Rocm, HostArch::Aarch64) => anyhow::bail!(
"an AMD GPU was detected, but lmsysorg/sglang publishes no aarch64 ROCm image"
),
(VllmBackend::Cpu, _) => anyhow::bail!(
"no CUDA or ROCm GPU was detected, and lmsysorg/sglang's CPU images (`-xeon`) \
need Intel AMX and launch flags llmman does not pass (install sglang \
locally and drop --ociman, or unset LLMMAN_SAFETENSORS_ENGINE to use vllm)"
),
};
Ok(format!("docker.io/lmsysorg/sglang:{tag}"))
}
fn engine_args(self) -> Vec<String> {
let mut args = match self {
VllmBackend::Cuda12 | VllmBackend::Cuda13 => GpuBackend::Cuda12.engine_args(),
VllmBackend::Rocm => GpuBackend::Rocm.engine_args(),
VllmBackend::Cpu => vec![
"--security-opt".into(),
"seccomp=unconfined".into(),
"--cap-add".into(),
"SYS_NICE".into(),
],
};
args.push("--ipc=host".into());
args
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ContainerEngine {
LlamaServer,
Vllm,
VllmOmni,
Sglang,
Speech,
}
fn image_for(
engine: ContainerEngine,
version: Option<&str>,
vllm_image: Option<&str>,
) -> Result<String> {
match engine {
ContainerEngine::LlamaServer => Ok(detect_backend().image_ref(version)),
ContainerEngine::Speech => Ok(SPEECH_BACKEND.image_ref(version)),
_ => VllmBackend::detect().engine_image(engine, version, vllm_image),
}
}
pub fn pull_image(
ociman: ContainerManager,
engine: ContainerEngine,
version: Option<&str>,
vllm_image: Option<&str>,
) -> Result<()> {
use std::io::Write;
let image = image_for(engine, version, vllm_image)?;
let cli = ociman.binary();
let header = format!("[llmman] {cli}: pulling {image}...");
eprintln!("{header}");
let (success, reason) = match pull_progress_sink() {
Some((mut sink, cols)) => {
let _ = write!(sink, "{header}\r\n");
let run = run_on_pty(cli, &["pull", &image], cols, &mut sink)?;
(run.success, run.last_line)
}
None => {
let status = std::process::Command::new(cli)
.args(["pull", &image])
.status()
.with_context(|| format!("run {cli} pull {image}"))?;
(status.success(), None)
}
};
if !success {
anyhow::bail!(
"{cli} pull {image} failed{}",
reason.map(|l| format!(": {l}")).unwrap_or_default()
);
}
Ok(())
}
pub const PULL_STATUS: &str = "pulling the llama.cpp container image";
pub const VERIFY_STATUS: &str = "checking that the llama.cpp container starts";
pub const PULL_PROGRESS_FILE_ENV: &str = "LLMMAN_PULL_PROGRESS_FILE";
pub const PULL_PROGRESS_COLS_ENV: &str = "LLMMAN_PULL_PROGRESS_COLS";
fn pull_progress_sink() -> Option<(std::fs::File, u16)> {
let path = std::env::var_os(PULL_PROGRESS_FILE_ENV)?;
let cols = std::env::var(PULL_PROGRESS_COLS_ENV)
.ok()?
.parse::<u16>()
.ok()
.filter(|c| *c > 0)?;
let file = std::fs::OpenOptions::new().append(true).open(path).ok()?;
Some((file, cols))
}
struct PtyRun {
success: bool,
last_line: Option<String>,
}
fn run_on_pty(
program: &str,
args: &[&str],
cols: u16,
sink: &mut dyn std::io::Write,
) -> Result<PtyRun> {
use portable_pty::{native_pty_system, CommandBuilder, PtySize};
use std::io::Read;
let pair = native_pty_system()
.openpty(PtySize {
rows: 24,
cols,
pixel_width: 0,
pixel_height: 0,
})
.context("open a pty")?;
let mut cmd = CommandBuilder::new(program);
cmd.args(args);
let mut child = pair
.slave
.spawn_command(cmd)
.with_context(|| format!("run {program}"))?;
drop(pair.slave);
let mut reader = pair
.master
.try_clone_reader()
.context("read from the pty")?;
const TAIL: usize = 4096;
let mut tail = Vec::new();
let mut buf = [0u8; 8192];
while let Ok(n @ 1..) = reader.read(&mut buf) {
let _ = sink.write_all(&buf[..n]);
tail.extend_from_slice(&buf[..n]);
if tail.len() > TAIL {
tail.drain(..tail.len() - TAIL);
}
}
let status = child.wait().with_context(|| format!("wait on {program}"))?;
drop(pair.master);
Ok(PtyRun {
success: status.success(),
last_line: last_output_line(&tail),
})
}
fn last_output_line(bytes: &[u8]) -> Option<String> {
let text = String::from_utf8_lossy(bytes);
let mut plain = String::new();
let mut chars = text.chars();
while let Some(c) = chars.next() {
match c {
'\x1b' => {
if chars.next() == Some('[') {
for n in chars.by_ref() {
if ('@'..='~').contains(&n) {
break;
}
}
}
}
'\r' | '\n' => plain.push('\n'),
c if c.is_control() => {}
c => plain.push(c),
}
}
plain
.lines()
.map(str::trim)
.rfind(|l| !l.is_empty())
.map(str::to_owned)
}
const VERIFY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60);
pub(crate) fn verify_llama_server_runs(
ociman: ContainerManager,
backend: GpuBackend,
version: Option<&str>,
hardened: bool,
) -> Result<()> {
let cli = ociman.binary();
let name = format!(
"llmman-verify-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |d| d.as_nanos())
);
let user = hardened
.then(|| HardenedUser::resolve(ociman, None))
.flatten();
let mut cmd = std::process::Command::new(cli);
cmd.args(verify_args(
backend,
version,
&name,
hardened,
user.as_ref(),
))
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::null());
let engine_args = backend.engine_args();
let suffix = if hardened {
format!(
"; {}",
hardened_note(
&hardened_flags(&engine_args, &llama_tmpfs(&engine_args), user.as_ref()),
None,
)
)
} else {
String::new()
};
match run_with_timeout(cmd, VERIFY_TIMEOUT).with_context(|| format!("run {cli} run"))? {
Some(status) if status.success() => Ok(()),
Some(status) => {
anyhow::bail!("{cli} could not start a llama.cpp container ({status}){suffix}")
}
None => {
let mut rm = std::process::Command::new(cli);
rm.args(["rm", "-f", &name])
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null());
let _ = run_with_timeout(rm, PROBE_TIMEOUT);
anyhow::bail!(
"{cli} did not start a llama.cpp container within {VERIFY_TIMEOUT:?}{suffix}"
)
}
}
}
fn verify_args(
backend: GpuBackend,
version: Option<&str>,
name: &str,
hardened: bool,
user: Option<&HardenedUser>,
) -> Vec<String> {
let mut args: Vec<String> = vec![
"run".into(),
"--rm".into(),
"--init".into(),
"--name".into(),
name.into(),
];
let engine_args = backend.engine_args();
if hardened {
args.extend(hardened_flags(
&engine_args,
&llama_tmpfs(&engine_args),
user,
));
}
args.extend(engine_args);
args.extend(forwarded_env_args(&[]));
args.push(backend.image_ref(version));
args.push("--version".into());
args
}
#[derive(Debug, Clone, Copy)]
pub struct LlamaOptions<'a> {
pub port: u16,
pub ctx_size: Option<u32>,
pub flash_attention: Option<&'a str>,
pub kv_cache_type: Option<&'a str>,
pub context_shift: bool,
pub split_mode: Option<&'a str>,
pub num_parallel: Option<u32>,
pub embeddings: bool,
pub batch_size: Option<u32>,
pub threads: Option<u32>,
pub cpus: Option<f64>,
pub hardened: bool,
pub extra_args: &'a [String],
}
pub fn spawn(
ociman: ContainerManager,
model_path: &Path,
mmproj_path: Option<&Path>,
llama_cpp_version: Option<&str>,
opts: LlamaOptions<'_>,
) -> Result<tokio::process::Child> {
let backend = detect_backend();
let image = backend.image_ref(llama_cpp_version);
eprintln!(
"[llmman] {}: detected {:?}, using image {:?}",
ociman.binary(),
backend,
image
);
let (model_dir, model_file) = mount_split(model_path, "model")?;
let mmproj = mmproj_path.map(|p| mount_split(p, "mmproj")).transpose()?;
let user = opts
.hardened
.then(|| HardenedUser::resolve(ociman, Some(Path::new(&model_dir))))
.flatten();
let args = llama_run_args(
backend,
image,
&model_dir,
&model_file,
mmproj.as_ref().map(|(d, f)| (d.as_str(), f.as_str())),
crate::cmd::serve::LLAMA_CPP_ENV_PASSTHROUGH_VARS,
opts,
user.as_ref(),
);
run(ociman, args)
}
fn mount_split(path: &Path, what: &str) -> Result<(String, String)> {
let dir = path
.parent()
.with_context(|| format!("{what} path has no parent directory"))?;
let file = path
.file_name()
.and_then(|n| n.to_str())
.with_context(|| format!("{what} path has no valid UTF-8 filename"))?;
Ok((mount_source(dir)?, file.to_owned()))
}
#[allow(clippy::too_many_arguments)]
fn llama_run_args(
backend: GpuBackend,
image: String,
model_dir: &str,
model_file: &str,
mmproj: Option<(&str, &str)>,
passthrough_vars: &[&str],
opts: LlamaOptions<'_>,
user: Option<&HardenedUser>,
) -> Vec<String> {
let LlamaOptions {
port,
ctx_size,
flash_attention,
kv_cache_type,
context_shift,
split_mode,
num_parallel,
embeddings,
batch_size,
threads,
cpus,
hardened,
extra_args,
} = opts;
let engine_args = backend.engine_args();
let tmpfs = llama_tmpfs(&engine_args);
let mut args = run_args(
engine_args,
port,
passthrough_vars,
cpus,
hardened,
user,
&tmpfs,
);
args.push("-v".into());
args.push(format!("{model_dir}:/models:ro"));
if let Some((dir, _)) = mmproj {
args.push("-v".into());
args.push(format!("{dir}:/mmproj:ro"));
}
args.push(image);
args.extend([
"-m".into(),
format!("/models/{model_file}"),
"--port".into(),
port.to_string(),
"--host".into(),
"0.0.0.0".into(),
]);
if let Some((_, file)) = mmproj {
args.push("--mmproj".into());
args.push(format!("/mmproj/{file}"));
}
if let Some(n) = ctx_size {
args.push("--ctx-size".into());
args.push(n.to_string());
}
if let Some(mode) = flash_attention {
args.push("--flash-attn".into());
args.push(mode.to_string());
}
if let Some(t) = kv_cache_type {
args.push("--cache-type-k".into());
args.push(t.to_string());
args.push("--cache-type-v".into());
args.push(t.to_string());
}
args.push(
if context_shift {
"--context-shift"
} else {
"--no-context-shift"
}
.into(),
);
if let Some(mode) = split_mode {
args.push("--split-mode".into());
args.push(mode.to_string());
}
if let Some(n) = num_parallel {
args.push("--parallel".into());
args.push(n.to_string());
}
if embeddings {
args.push("--embeddings".into());
}
if let Some(n) = batch_size {
args.push("-b".into());
args.push(n.to_string());
args.push("-ub".into());
args.push(n.to_string());
}
if let Some(n) = threads {
args.push("--threads".into());
args.push(n.to_string());
}
args.extend(extra_args.iter().cloned());
args
}
fn run_args(
engine_args: Vec<String>,
port: u16,
passthrough_vars: &[&str],
cpus: Option<f64>,
hardened: bool,
user: Option<&HardenedUser>,
tmpfs: &[&str],
) -> Vec<String> {
let mut args: Vec<String> = vec![
"run".into(),
"--rm".into(),
"--init".into(),
"-t".into(),
"-p".into(),
format!("127.0.0.1:{port}:{port}"),
];
if let Some(n) = cpus {
args.push("--cpus".into());
args.push(n.to_string());
}
if hardened {
args.extend(hardened_flags(&engine_args, tmpfs, user));
}
args.extend(engine_args);
args.extend(forwarded_env_args(passthrough_vars));
args
}
const LLAMA_TMPFS: &[&str] = &["/tmp"];
const GPU_CACHE_TMPFS: &str = "/root/.cache";
fn llama_tmpfs(engine_args: &[String]) -> Vec<&'static str> {
let mut tmpfs = LLAMA_TMPFS.to_vec();
if passes_devices(engine_args) {
tmpfs.push(GPU_CACHE_TMPFS);
}
tmpfs
}
pub(crate) fn vllm_rpc_tmpfs(
value: Option<&str>,
tmpfs_roots: &[&str],
) -> Result<Option<String>, String> {
let Some(value) = value else {
return Ok(None);
};
if !value.starts_with('/') {
return Err(format!(
"VLLM_RPC_BASE_PATH must be an absolute path under hardening, got {value:?}; \
unset it or pass --no-hardened"
));
}
if value.contains([':', ',']) || value.split('/').any(|s| s == "..") {
return Err(format!(
"VLLM_RPC_BASE_PATH must not contain ':', ',' or a '..' segment under hardening, \
got {value:?}; unset it or pass --no-hardened"
));
}
let mounted = tmpfs_roots.iter().any(|root| {
value == *root
|| value
.strip_prefix(root)
.is_some_and(|rest| rest.starts_with('/'))
});
Ok((!mounted).then(|| format!("{value}:exec")))
}
fn engine_rpc_tmpfs(
env_prefix: &str,
hardened: bool,
rpc_base: impl FnOnce() -> Option<String>,
) -> Result<Option<String>, String> {
if !hardened || env_prefix != "VLLM_" {
return Ok(None);
}
vllm_rpc_tmpfs(rpc_base().as_deref(), ENGINE_TMPFS)
}
const ENGINE_TMPFS: &[&str] = &["/tmp", "/root/.cache", "/root/.triton", "/root/.config"];
fn passes_gpus(engine_args: &[String]) -> bool {
engine_args.iter().any(|a| a == "--gpus")
}
fn passes_devices(engine_args: &[String]) -> bool {
engine_args.iter().any(|a| a == "--gpus" || a == "--device")
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct HardenedUser {
pub(crate) uid: u32,
pub(crate) gid: u32,
pub(crate) device_gids: Vec<u32>,
}
impl HardenedUser {
pub(crate) fn resolve(ociman: ContainerManager, mounted: Option<&Path>) -> Option<Self> {
let linux = cfg!(target_os = "linux");
let rootless =
ociman == ContainerManager::Docker && linux && identity::docker_is_rootless();
let identity =
identity::Identity::from_parts(Some(ociman), linux, rootless, identity::user_ids());
Self::from_parts(
&identity,
rootless,
mounted.and_then(path_owner),
device_gids(),
)
}
pub(crate) fn from_parts(
identity: &identity::Identity,
rootless: bool,
owner: Option<(u32, u32)>,
device_gids: Vec<u32>,
) -> Option<Self> {
match *identity {
identity::Identity::User { uid, gid } => {
let (uid, gid, device_gids) = if rootless {
(uid, gid, Vec::new())
} else {
let (uid, gid) = owner.unwrap_or((uid, gid));
(uid, gid, device_gids)
};
Some(Self {
uid,
gid,
device_gids,
})
}
identity::Identity::Image | identity::Identity::KeepId => None,
}
}
fn args(&self, devices: bool) -> Vec<String> {
let mut args = vec![
"--user".to_string(),
identity::Identity::user_arg(self.uid, self.gid),
];
if devices {
args.extend(
self.device_gids
.iter()
.flat_map(|gid| ["--group-add".to_string(), gid.to_string()]),
);
}
args.extend(
HARDENED_USER_ENV
.iter()
.flat_map(|pair| ["-e".to_string(), pair.to_string()]),
);
args
}
}
const HARDENED_USER_ENV: &[&str] = &[
"HOME=/tmp",
"XDG_CACHE_HOME=/tmp/.cache",
"XDG_CONFIG_HOME=/tmp/.config",
"TRITON_CACHE_DIR=/tmp/.triton",
];
#[cfg(unix)]
fn path_owner(path: &Path) -> Option<(u32, u32)> {
use std::os::unix::fs::MetadataExt;
let meta = std::fs::metadata(path).ok()?;
Some((meta.uid(), meta.gid()))
}
#[cfg(not(unix))]
fn path_owner(_path: &Path) -> Option<(u32, u32)> {
None
}
#[cfg(unix)]
fn device_gids() -> Vec<u32> {
use std::os::unix::fs::MetadataExt;
let mut nodes = vec![std::path::PathBuf::from("/dev/kfd")];
if let Ok(dir) = std::fs::read_dir("/dev/dri") {
nodes.extend(dir.flatten().map(|e| e.path()).filter(|p| {
p.file_name()
.and_then(|n| n.to_str())
.is_some_and(|n| n.starts_with("renderD") || n.starts_with("card"))
}));
}
let mut gids: Vec<u32> = nodes
.iter()
.filter_map(|p| std::fs::metadata(p).ok())
.map(|m| m.gid())
.collect();
gids.sort_unstable();
gids.dedup();
gids
}
#[cfg(not(unix))]
fn device_gids() -> Vec<u32> {
Vec::new()
}
fn hardened_flags(
engine_args: &[String],
tmpfs: &[&str],
user: Option<&HardenedUser>,
) -> Vec<String> {
let mut args = hardened_args(!passes_gpus(engine_args), tmpfs);
if let Some(user) = user {
args.extend(user.args(passes_devices(engine_args)));
}
args
}
pub(crate) fn hardened_note(flags: &[String], mounted: Option<&str>) -> String {
let fix = match mounted {
Some(path) => format!("chmod -R go+rX {path} lets the container read it as any user, and "),
None => String::new(),
};
format!(
"started hardened ({}); {fix}--no-hardened turns these flags off",
flags.join(" ")
)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum HardenedSpawn {
LlamaServer,
Speech,
Vllm,
Sglang,
}
impl HardenedSpawn {
fn note(
self,
host: GpuBackend,
user: Option<&HardenedUser>,
rpc_tmpfs: Option<&str>,
mounted: &str,
) -> String {
let mut flags = match self {
Self::Vllm | Self::Sglang => hardened_flags(
&VllmBackend::from_gpu(host).engine_args(),
ENGINE_TMPFS,
user,
),
Self::LlamaServer => {
let engine_args = host.engine_args();
hardened_flags(&engine_args, &llama_tmpfs(&engine_args), user)
}
Self::Speech => {
let engine_args = SPEECH_BACKEND.engine_args();
hardened_flags(&engine_args, &llama_tmpfs(&engine_args), user)
}
};
if let (Self::Vllm, Some(tmpfs)) = (self, rpc_tmpfs) {
flags.extend(["--tmpfs".to_string(), tmpfs.to_string()]);
}
hardened_note(&flags, Some(mounted))
}
}
pub(crate) fn spawn_hardened_note(
spawn: HardenedSpawn,
ociman: ContainerManager,
mounted: &Path,
) -> String {
let user = HardenedUser::resolve(ociman, Some(mounted));
let rpc_tmpfs = vllm_rpc_tmpfs(
std::env::var("VLLM_RPC_BASE_PATH").ok().as_deref(),
ENGINE_TMPFS,
)
.ok()
.flatten();
spawn.note(
detect_backend(),
user.as_ref(),
rpc_tmpfs.as_deref(),
&mounted.display().to_string(),
)
}
pub(crate) fn hardened_args(read_only: bool, tmpfs: &[&str]) -> Vec<String> {
let mut args: Vec<String> = vec![
"--cap-drop".into(),
"all".into(),
"--security-opt".into(),
"no-new-privileges".into(),
];
if read_only {
args.push("--read-only".into());
}
args.extend(
tmpfs
.iter()
.flat_map(|path| ["--tmpfs".to_string(), format!("{path}:exec")]),
);
args
}
fn forwarded_env_args(passthrough_vars: &[&str]) -> Vec<String> {
crate::cmd::serve::GPU_VISIBLE_DEVICE_VARS
.iter()
.chain(passthrough_vars)
.filter(|var| std::env::var_os(var).is_some())
.flat_map(|var| ["-e".to_string(), var.to_string()])
.collect()
}
impl ContainerEngine {
fn launcher(self) -> Option<(&'static [&'static str], &'static str)> {
match self {
ContainerEngine::LlamaServer | ContainerEngine::Speech => None,
ContainerEngine::Vllm | ContainerEngine::VllmOmni => Some((&["vllm"], "VLLM_")),
ContainerEngine::Sglang => {
Some((&["python3", "-m", "sglang.launch_server"], "SGLANG_"))
}
}
}
}
#[allow(clippy::too_many_arguments)]
pub fn spawn_engine(
ociman: ContainerManager,
engine: ContainerEngine,
model_dir: &Path,
version: Option<&str>,
vllm_image: Option<&str>,
port: u16,
cpus: Option<f64>,
hardened: bool,
serve_args: impl FnOnce(&str, &str) -> Vec<String>,
) -> Result<tokio::process::Child> {
let backend = VllmBackend::detect();
let image = backend.engine_image(engine, version, vllm_image)?;
let Some((launcher, env_prefix)) = engine.launcher() else {
unreachable!("LlamaServer and Speech bailed above");
};
eprintln!(
"[llmman] {}: detected {:?}, using image {:?}",
ociman.binary(),
backend,
image
);
let model_dir = mount_source(model_dir)?;
let vars: Vec<String> = std::env::vars_os()
.filter_map(|(k, _)| k.into_string().ok())
.filter(|k| {
k.starts_with(env_prefix) || matches!(k.as_str(), "HF_TOKEN" | "HUGGING_FACE_HUB_TOKEN")
})
.collect();
let vars: Vec<&str> = vars.iter().map(String::as_str).collect();
let rpc_tmpfs = engine_rpc_tmpfs(env_prefix, hardened, || {
std::env::var("VLLM_RPC_BASE_PATH").ok()
})
.map_err(anyhow::Error::msg)?;
let user = hardened
.then(|| HardenedUser::resolve(ociman, Some(Path::new(&model_dir))))
.flatten();
let prefix = run_args(
backend.engine_args(),
port,
&vars,
cpus,
hardened,
user.as_ref(),
ENGINE_TMPFS,
);
run(
ociman,
engine_run_args(
prefix,
rpc_tmpfs.as_deref(),
image,
launcher,
&model_dir,
serve_args,
),
)
}
fn mount_source(dir: &Path) -> Result<String> {
let dir = dir
.canonicalize()
.with_context(|| format!("resolving {}", dir.display()))?;
dir.to_str()
.map(str::to_owned)
.with_context(|| format!("{} is not valid UTF-8", dir.display()))
}
fn engine_run_args(
mut args: Vec<String>,
rpc_tmpfs: Option<&str>,
image: String,
launcher: &[&str],
model_dir: &str,
serve_args: impl FnOnce(&str, &str) -> Vec<String>,
) -> Vec<String> {
if let Some(tmpfs) = rpc_tmpfs {
args.extend(["--tmpfs".to_string(), tmpfs.to_string()]);
}
args.extend([
"-v".into(),
format!("{model_dir}:/models:ro"),
"--entrypoint".into(),
launcher[0].into(),
image,
]);
args.extend(launcher[1..].iter().map(|a| a.to_string()));
args.extend(serve_args("/models", "0.0.0.0"));
args
}
fn run(ociman: ContainerManager, args: Vec<String>) -> Result<tokio::process::Child> {
crate::debug_log!("spawning {} {}", ociman.binary(), args.join(" "));
tokio::process::Command::new(ociman.binary())
.args(&args)
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.spawn()
.with_context(|| format!("spawn {} {}", ociman.binary(), args.join(" ")))
}
#[allow(clippy::too_many_arguments)]
pub(crate) fn spawn_mediagen(
ociman: ContainerManager,
backend: GpuBackend,
model_ref: &str,
store_path: &Path,
cache_path: &Path,
llama_cpp_version: Option<&str>,
port: u16,
cpus: Option<f64>,
hardened: bool,
) -> Result<tokio::process::Child> {
let image = backend.image_ref(llama_cpp_version);
eprintln!(
"[llmman] {}: {:?} backend, using image {:?} for {model_ref}'s worker",
ociman.binary(),
backend,
image
);
let exe = std::env::current_exe().context("locating the llmman binary")?;
let utf8 = |p: &Path| -> Result<String> {
p.to_str()
.map(str::to_owned)
.with_context(|| format!("{} is not valid UTF-8", p.display()))
};
let (exe, store, cache) = (utf8(&exe)?, utf8(store_path)?, utf8(cache_path)?);
let user = hardened
.then(|| HardenedUser::resolve(ociman, Some(store_path)))
.flatten();
let args = mediagen_run_args(
backend,
image,
&exe,
&store,
&cache,
model_ref,
port,
cpus,
hardened,
user.as_ref(),
);
run(ociman, args)
}
#[allow(clippy::too_many_arguments)]
fn mediagen_run_args(
backend: GpuBackend,
image: String,
exe: &str,
store: &str,
cache: &str,
model_ref: &str,
port: u16,
cpus: Option<f64>,
hardened: bool,
user: Option<&HardenedUser>,
) -> Vec<String> {
let passthrough = [
crate::cmd::serve::LLAMA_CPP_ENV_PASSTHROUGH_VARS,
crate::cmd::serve::MEDIAGEN_ENV_PASSTHROUGH_VARS,
]
.concat();
let engine_args = backend.engine_args();
let tmpfs = llama_tmpfs(&engine_args);
let mut args = run_args(
engine_args,
port,
&passthrough,
cpus,
hardened,
user,
&tmpfs,
);
args.extend([
"-v".into(),
format!("{exe}:/usr/local/bin/llmman:ro"),
"-v".into(),
format!("{store}:{store}:ro"),
"-v".into(),
format!("{cache}:{cache}:ro"),
"-e".into(),
format!("LLMMAN_MODELS={store}"),
"-e".into(),
"PATH=/app:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin".into(),
"--entrypoint".into(),
"/usr/local/bin/llmman".into(),
image,
"serve".into(),
model_ref.into(),
"--port".into(),
port.to_string(),
"--host".into(),
"0.0.0.0".into(),
"--runtime".into(),
"path".into(),
]);
args
}
#[cfg(unix)]
pub fn stop(pid: u32) {
let result = unsafe { libc::kill(pid as libc::pid_t, libc::SIGTERM) };
if result != 0 {
let err = std::io::Error::last_os_error();
eprintln!("[llmman] warning: SIGTERM to container process {pid} failed: {err}");
}
}
#[cfg(not(unix))]
pub fn stop(_pid: u32) {
eprintln!("[llmman] warning: container::stop is a no-op on non-Unix platforms");
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn last_output_line_drops_control_sequences_and_redraws() {
let out = b"Pulling\r\n\x1b[1A\x1b[2K\rabc: Downloading 1MB/3MB\r\x1b[1B\x1b[1A\x1b[2K\rError response from daemon: denied \r\n\r\n";
assert_eq!(
last_output_line(out).as_deref(),
Some("Error response from daemon: denied")
);
assert_eq!(last_output_line(b"\x1b[2K\r\n"), None);
assert_eq!(last_output_line(b""), None);
}
#[cfg(unix)]
#[test]
fn run_on_pty_gives_the_program_a_terminal_and_relays_its_output() {
let mut sink = Vec::new();
let run = run_on_pty(
"sh",
&[
"-c",
"test -t 1 && stty size; printf 'half\\rdone\\n'; echo 'boom: nope' >&2; exit 3",
],
97,
&mut sink,
)
.unwrap();
let text = String::from_utf8_lossy(&sink);
assert!(text.contains("24 97"), "got: {text:?}");
assert!(text.contains("half\rdone"), "got: {text:?}");
assert!(!run.success);
assert_eq!(run.last_line.as_deref(), Some("boom: nope"));
}
#[cfg(unix)]
#[test]
fn run_on_pty_reports_success() {
let mut sink = Vec::new();
let run = run_on_pty("sh", &["-c", "echo ok"], 80, &mut sink).unwrap();
assert!(run.success);
assert_eq!(run.last_line.as_deref(), Some("ok"));
}
#[test]
fn run_on_pty_reports_a_missing_program() {
let err = run_on_pty("llmman-no-such-program", &[], 80, &mut Vec::new());
assert!(err.is_err());
}
#[test]
fn cuda_major_12_picks_cuda12_image() {
assert_eq!(
backend_from_hostgpu(HostGpu::Cuda { major: 12 }),
GpuBackend::Cuda12
);
assert_eq!(GpuBackend::Cuda12.image_tag(), "server-cuda");
}
#[test]
fn cuda_major_13_picks_cuda13_image() {
assert_eq!(
backend_from_hostgpu(HostGpu::Cuda { major: 13 }),
GpuBackend::Cuda13
);
assert_eq!(GpuBackend::Cuda13.image_tag(), "server-cuda13");
}
#[test]
fn cuda_major_above_13_still_picks_cuda13_image() {
assert_eq!(
backend_from_hostgpu(HostGpu::Cuda { major: 14 }),
GpuBackend::Cuda13
);
}
#[test]
fn non_cuda_hostgpu_variants_map_to_their_matching_backend() {
assert_eq!(backend_from_hostgpu(HostGpu::Rocm), GpuBackend::Rocm);
assert_eq!(backend_from_hostgpu(HostGpu::Opencl), GpuBackend::Cpu);
assert_eq!(backend_from_hostgpu(HostGpu::Vulkan), GpuBackend::Vulkan);
assert_eq!(backend_from_hostgpu(HostGpu::None), GpuBackend::Cpu);
assert_eq!(backend_from_hostgpu(HostGpu::Metal), GpuBackend::Cpu);
}
#[test]
fn image_tags_match_docs_docker_md() {
assert_eq!(GpuBackend::Cpu.image_tag(), "server");
assert_eq!(GpuBackend::Cuda12.image_tag(), "server-cuda");
assert_eq!(GpuBackend::Cuda13.image_tag(), "server-cuda13");
assert_eq!(GpuBackend::Rocm.image_tag(), "server-rocm");
assert_eq!(GpuBackend::Vulkan.image_tag(), "server-vulkan");
}
#[test]
fn the_cpu_only_probe_passes_where_the_toolkit_alone_refuses() {
let err = gpu_gate(true, true, "docker", || false)
.unwrap_err()
.to_string();
assert!(err.contains("NVIDIA Container Toolkit"), "{err}");
gpu_gate(false, true, "docker", || false).unwrap();
gpu_gate(true, true, "docker", || true).unwrap();
gpu_gate(true, false, "docker", || {
unreachable!("no GPU, no toolkit check")
})
.unwrap();
gpu_gate(false, false, "docker", || {
unreachable!("no GPU, no toolkit check")
})
.unwrap();
}
#[test]
fn speech_image_is_the_cpu_image_on_any_host() {
assert_eq!(
image_for(ContainerEngine::Speech, Some("b9994"), None).unwrap(),
GpuBackend::Cpu.image_ref(Some("b9994"))
);
assert_eq!(
image_for(ContainerEngine::Speech, None, None).unwrap(),
GpuBackend::Cpu.image_ref(None)
);
}
#[test]
fn image_ref_uses_floating_tag_when_no_version_given() {
assert_eq!(
GpuBackend::Cpu.image_ref(None),
"ghcr.io/ggml-org/llama.cpp:server"
);
assert_eq!(
GpuBackend::Cuda13.image_ref(None),
"ghcr.io/ggml-org/llama.cpp:server-cuda13"
);
}
#[test]
fn image_ref_pins_to_the_given_version() {
assert_eq!(
GpuBackend::Cpu.image_ref(Some("b9994")),
"ghcr.io/ggml-org/llama.cpp:server-b9994"
);
assert_eq!(
GpuBackend::Cuda13.image_ref(Some("b9994")),
"ghcr.io/ggml-org/llama.cpp:server-cuda13-b9994"
);
}
#[test]
fn cpu_backend_has_no_extra_engine_args() {
assert!(GpuBackend::Cpu.engine_args().is_empty());
}
#[test]
fn cuda_backend_requests_all_gpus() {
assert_eq!(GpuBackend::Cuda12.engine_args(), vec!["--gpus", "all"]);
assert_eq!(GpuBackend::Cuda13.engine_args(), vec!["--gpus", "all"]);
}
#[test]
fn rocm_backend_mounts_kfd_and_dri() {
let args = GpuBackend::Rocm.engine_args();
assert_eq!(
args,
vec![
"--device",
"/dev/kfd",
"--device",
"/dev/dri",
"--group-add",
"video"
]
);
}
#[test]
fn vulkan_backend_passes_the_devices_the_host_has() {
let args = GpuBackend::Vulkan.engine_args();
assert_eq!(
args.contains(&"/dev/dri".to_string()),
Path::new("/dev/dri").exists()
);
assert_eq!(
args.contains(&"--gpus".to_string()),
Path::new("/dev/nvidiactl").exists()
);
}
#[test]
fn container_manager_binary_names() {
assert_eq!(ContainerManager::Docker.binary(), "docker");
assert_eq!(ContainerManager::Podman.binary(), "podman");
}
#[test]
fn host_arch_parses_only_the_two_architectures_vllm_ships() {
assert_eq!(HostArch::parse("x86_64"), Some(HostArch::X86_64));
assert_eq!(HostArch::parse("aarch64"), Some(HostArch::Aarch64));
assert_eq!(HostArch::parse("s390x"), None);
assert_eq!(HostArch::parse("riscv64"), None);
assert_eq!(HostArch::parse(""), None);
}
#[test]
fn cuda_12_hosts_get_the_cu129_vllm_image() {
assert_eq!(
VllmBackend::from_gpu(GpuBackend::Cuda12),
VllmBackend::Cuda12
);
assert_eq!(
VllmBackend::from_gpu(GpuBackend::Cuda13),
VllmBackend::Cuda13
);
assert_eq!(
VllmBackend::Cuda12
.image_ref(HostArch::X86_64, None)
.unwrap(),
"docker.io/vllm/vllm-openai:latest-x86_64-cu129"
);
assert_eq!(
VllmBackend::Cuda12
.image_ref(HostArch::Aarch64, Some("v0.28.0"))
.unwrap(),
"docker.io/vllm/vllm-openai:v0.28.0-aarch64-cu129"
);
}
#[test]
fn vulkan_host_runs_the_vllm_cpu_image() {
assert_eq!(VllmBackend::from_gpu(GpuBackend::Vulkan), VllmBackend::Cpu);
assert_eq!(VllmBackend::from_gpu(GpuBackend::Cpu), VllmBackend::Cpu);
assert_eq!(VllmBackend::from_gpu(GpuBackend::Rocm), VllmBackend::Rocm);
}
#[test]
fn vllm_image_refs_match_docker_hub_tag_spellings() {
assert_eq!(
VllmBackend::Cuda13
.image_ref(HostArch::X86_64, None)
.unwrap(),
"docker.io/vllm/vllm-openai:latest-x86_64"
);
assert_eq!(
VllmBackend::Cuda13
.image_ref(HostArch::Aarch64, None)
.unwrap(),
"docker.io/vllm/vllm-openai:latest-aarch64"
);
assert_eq!(
VllmBackend::Rocm.image_ref(HostArch::X86_64, None).unwrap(),
"docker.io/rocm/vllm:latest"
);
assert_eq!(
VllmBackend::Cpu.image_ref(HostArch::X86_64, None).unwrap(),
"docker.io/vllm/vllm-openai-cpu:latest-x86_64"
);
assert_eq!(
VllmBackend::Cpu.image_ref(HostArch::Aarch64, None).unwrap(),
"docker.io/vllm/vllm-openai-cpu:latest-arm64"
);
}
#[test]
fn vllm_image_override_is_verbatim_and_for_plain_vllm_only() {
let img = Some("dhi.io/vllm-openai:0.32");
let backend = VllmBackend::Cuda13;
assert_eq!(
backend
.engine_image(ContainerEngine::Vllm, Some("v0.11.0"), img)
.unwrap(),
"dhi.io/vllm-openai:0.32"
);
for (engine, repo) in [
(ContainerEngine::VllmOmni, "vllm/vllm-omni:"),
(ContainerEngine::Sglang, "lmsysorg/sglang:"),
] {
let image = backend.engine_image(engine, None, img).unwrap();
assert!(image.contains(repo), "{image}");
}
}
#[test]
fn vllm_image_ref_pins_to_the_given_version() {
assert_eq!(
VllmBackend::Cuda13
.image_ref(HostArch::X86_64, Some("v0.11.0"))
.unwrap(),
"docker.io/vllm/vllm-openai:v0.11.0-x86_64"
);
assert_eq!(
VllmBackend::Cuda13
.image_ref(HostArch::Aarch64, Some("v0.11.0"))
.unwrap(),
"docker.io/vllm/vllm-openai:v0.11.0-aarch64"
);
assert_eq!(
VllmBackend::Cpu
.image_ref(HostArch::Aarch64, Some("v0.11.0"))
.unwrap(),
"docker.io/vllm/vllm-openai-cpu:v0.11.0-arm64"
);
assert_eq!(
VllmBackend::Rocm
.image_ref(
HostArch::X86_64,
Some("rocm7.14.1_cdna_ubuntu24.04_py3.14_pytorch_2.11_vllm_0.23.0")
)
.unwrap(),
"docker.io/rocm/vllm:rocm7.14.1_cdna_ubuntu24.04_py3.14_pytorch_2.11_vllm_0.23.0"
);
}
#[test]
fn rocm_vllm_has_no_aarch64_image() {
let err = VllmBackend::Rocm
.image_ref(HostArch::Aarch64, None)
.unwrap_err()
.to_string();
assert!(err.contains("rocm/vllm"), "{err}");
assert!(err.contains("LLMMAN_LLM_LIBRARY=cpu"), "{err}");
}
#[test]
fn vllm_omni_image_refs_match_docker_hub_tag_spellings() {
for cuda in [VllmBackend::Cuda12, VllmBackend::Cuda13] {
assert_eq!(
cuda.omni_image_ref(HostArch::X86_64, None).unwrap(),
"docker.io/vllm/vllm-omni:latest-x86_64"
);
assert_eq!(
cuda.omni_image_ref(HostArch::Aarch64, None).unwrap(),
"docker.io/vllm/vllm-omni:latest-aarch64"
);
assert_eq!(
cuda.omni_image_ref(HostArch::Aarch64, Some("v0.28.0"))
.unwrap(),
"docker.io/vllm/vllm-omni:v0.28.0-aarch64"
);
}
}
#[test]
fn vllm_omni_has_only_cuda_images() {
let err = VllmBackend::Rocm
.omni_image_ref(HostArch::X86_64, None)
.unwrap_err()
.to_string();
assert!(err.contains("vllm/vllm-omni"), "{err}");
assert!(err.contains("--runtime"), "{err}");
let err = VllmBackend::Cpu
.omni_image_ref(HostArch::X86_64, None)
.unwrap_err()
.to_string();
assert!(err.contains("CUDA"), "{err}");
}
#[test]
fn sglang_image_refs_match_docker_hub_tag_spellings() {
for arch in [HostArch::X86_64, HostArch::Aarch64] {
assert_eq!(
VllmBackend::Cuda13.sglang_image_ref(arch, None).unwrap(),
"docker.io/lmsysorg/sglang:latest"
);
assert_eq!(
VllmBackend::Cuda13
.sglang_image_ref(arch, Some("v0.5.19"))
.unwrap(),
"docker.io/lmsysorg/sglang:v0.5.19"
);
assert_eq!(
VllmBackend::Cuda12.sglang_image_ref(arch, None).unwrap(),
"docker.io/lmsysorg/sglang:latest-cu129"
);
assert_eq!(
VllmBackend::Cuda12
.sglang_image_ref(arch, Some("v0.5.19"))
.unwrap(),
"docker.io/lmsysorg/sglang:v0.5.19-cu129"
);
}
}
#[test]
fn sglang_rocm_image_is_the_whole_pinned_tag_or_nothing() {
assert_eq!(
VllmBackend::Rocm
.sglang_image_ref(HostArch::X86_64, Some("v0.5.19-rocm700-mi30x"))
.unwrap(),
"docker.io/lmsysorg/sglang:v0.5.19-rocm700-mi30x"
);
let err = VllmBackend::Rocm
.sglang_image_ref(HostArch::X86_64, None)
.unwrap_err()
.to_string();
assert!(err.contains("--sglang-version"), "{err}");
assert!(err.contains("rocm700-mi30x"), "{err}");
let err = VllmBackend::Rocm
.sglang_image_ref(HostArch::Aarch64, Some("v0.5.19-rocm700-mi30x"))
.unwrap_err()
.to_string();
assert!(err.contains("aarch64"), "{err}");
}
#[test]
fn sglang_has_no_cpu_image_here() {
for arch in [HostArch::X86_64, HostArch::Aarch64] {
let err = VllmBackend::Cpu
.sglang_image_ref(arch, Some("v0.5.19"))
.unwrap_err()
.to_string();
assert!(err.contains("xeon"), "{err}");
assert!(err.contains("LLMMAN_SAFETENSORS_ENGINE"), "{err}");
}
}
#[test]
fn sglang_run_args_launch_the_module_after_the_image_and_flags_after_that() {
let (launcher, _) = ContainerEngine::Sglang.launcher().unwrap();
let args = engine_run_args(
run_args(
VllmBackend::Cuda13.engine_args(),
30000,
&[],
None,
false,
None,
&[],
),
None,
"docker.io/lmsysorg/sglang:latest".into(),
launcher,
"/cache/abc/model",
|dir: &str, host: &str| {
vec![
"--model-path".into(),
dir.into(),
"--host".into(),
host.into(),
]
},
);
let image = args
.iter()
.position(|a| a == "docker.io/lmsysorg/sglang:latest")
.expect("image present");
assert_eq!(&args[image - 2..image], &["--entrypoint", "python3"]);
assert_eq!(
&args[image + 1..],
&[
"-m",
"sglang.launch_server",
"--model-path",
"/models",
"--host",
"0.0.0.0"
]
);
}
#[test]
fn vllm_engine_args_add_ipc_host_on_top_of_gpu_passthrough() {
assert_eq!(
VllmBackend::Cuda13.engine_args(),
vec!["--gpus", "all", "--ipc=host"]
);
let rocm = VllmBackend::Rocm.engine_args();
assert!(
rocm.starts_with(&GpuBackend::Rocm.engine_args()),
"{rocm:?}"
);
assert_eq!(rocm.last().map(String::as_str), Some("--ipc=host"));
}
#[test]
fn vllm_cpu_engine_args_grant_what_its_numa_thread_binding_needs() {
assert_eq!(
VllmBackend::Cpu.engine_args(),
vec![
"--security-opt",
"seccomp=unconfined",
"--cap-add",
"SYS_NICE",
"--ipc=host"
]
);
}
#[test]
fn vllm_run_args_put_the_mount_and_entrypoint_before_the_image_and_serve_after() {
let (launcher, _) = ContainerEngine::Vllm.launcher().unwrap();
let args = engine_run_args(
run_args(
VllmBackend::Cuda13.engine_args(),
8000,
&[],
None,
false,
None,
&[],
),
None,
"docker.io/vllm/vllm-openai:latest-x86_64".into(),
launcher,
"/cache/abc/model",
|dir: &str, host: &str| {
vec![
"serve".into(),
dir.into(),
"--host".into(),
host.into(),
"--served-model-name".into(),
"m".into(),
]
},
);
let image = args
.iter()
.position(|a| a == "docker.io/vllm/vllm-openai:latest-x86_64")
.expect("image present");
let before = &args[..image];
assert!(before.contains(&"--gpus".to_string()), "{before:?}");
assert!(before.contains(&"--ipc=host".to_string()), "{before:?}");
assert!(
before.contains(&"/cache/abc/model:/models:ro".to_string()),
"{before:?}"
);
assert_eq!(&before[before.len() - 2..], &["--entrypoint", "vllm"]);
assert_eq!(
&args[image + 1..],
&[
"serve",
"/models",
"--host",
"0.0.0.0",
"--served-model-name",
"m"
]
);
}
#[test]
fn run_args_forward_only_the_requested_passthrough_vars() {
const VAR: &str = "LLMMAN_TEST_RUN_ARGS_PASSTHROUGH";
std::env::set_var(VAR, "1");
let args = run_args(vec![], 8080, &[], None, false, None, &[]);
assert_eq!(
&args[..6],
&["run", "--rm", "--init", "-t", "-p", "127.0.0.1:8080:8080"]
);
assert!(!args.contains(&VAR.to_string()), "{args:?}");
let args = run_args(vec![], 8080, &[VAR], None, false, None, &[]);
assert!(args.windows(2).any(|w| w == ["-e", VAR]), "{args:?}");
assert!(!args.iter().any(|a| a.starts_with(&format!("{VAR}="))));
}
#[test]
fn verify_runs_the_served_image_with_its_gpu_passthrough() {
for (backend, image) in [
(GpuBackend::Cpu, "ghcr.io/ggml-org/llama.cpp:server-b9994"),
(
GpuBackend::Cuda13,
"ghcr.io/ggml-org/llama.cpp:server-cuda13-b9994",
),
] {
let args = verify_args(backend, Some("b9994"), "llmman-verify-1", false, None);
assert_eq!(
&args[..5],
["run", "--rm", "--init", "--name", "llmman-verify-1"]
);
let at = args.iter().position(|a| a == image).expect("image present");
let mut between = backend.engine_args();
between.extend(forwarded_env_args(&[]));
assert_eq!(&args[5..at], between.as_slice(), "{backend:?}");
assert_eq!(&args[at + 1..], ["--version"]);
}
}
fn assert_hardened_between(
args: &[String],
prefix_len: usize,
read_only: bool,
tmpfs: &[&str],
engine_args: &[String],
) {
let mut expected = hardened_args(read_only, tmpfs);
expected.extend(engine_args.iter().cloned());
assert_eq!(
&args[prefix_len..prefix_len + expected.len()],
expected.as_slice(),
"{args:?}"
);
}
fn user(device_gids: &[u32]) -> HardenedUser {
HardenedUser {
uid: 1000,
gid: 1000,
device_gids: device_gids.to_vec(),
}
}
#[test]
fn hardened_user_follows_the_engine_the_owner_and_rootlessness() {
use identity::Identity;
let me = Identity::User {
uid: 1000,
gid: 1000,
};
assert_eq!(
HardenedUser::from_parts(&me, false, Some((1001, 1002)), vec![44]),
Some(HardenedUser {
uid: 1001,
gid: 1002,
device_gids: vec![44]
})
);
assert_eq!(
HardenedUser::from_parts(&me, false, None, vec![]),
Some(user(&[]))
);
let root = Identity::User { uid: 0, gid: 0 };
assert_eq!(
HardenedUser::from_parts(&root, true, Some((1000, 1000)), vec![44]),
Some(HardenedUser {
uid: 0,
gid: 0,
device_gids: vec![]
})
);
assert_eq!(
HardenedUser::from_parts(&Identity::KeepId, false, Some((1000, 1000)), vec![44]),
None
);
assert_eq!(
HardenedUser::from_parts(&Identity::Image, false, Some((1000, 1000)), vec![44]),
None
);
}
#[test]
fn hardened_user_args_add_the_user_device_groups_and_tmp_caches() {
assert_eq!(
user(&[44, 993]).args(true),
[
"--user",
"1000:1000",
"--group-add",
"44",
"--group-add",
"993",
"-e",
"HOME=/tmp",
"-e",
"XDG_CACHE_HOME=/tmp/.cache",
"-e",
"XDG_CONFIG_HOME=/tmp/.config",
"-e",
"TRITON_CACHE_DIR=/tmp/.triton",
]
);
let cpu = user(&[44]).args(false);
assert!(!cpu.contains(&"--group-add".to_string()), "{cpu:?}");
assert_eq!(&cpu[..2], ["--user", "1000:1000"]);
assert_eq!(cpu.len(), 2 + 2 * HARDENED_USER_ENV.len());
let root = HardenedUser {
uid: 0,
gid: 0,
device_gids: vec![],
};
assert_eq!(&root.args(true)[..2], ["--user", "0:0"]);
}
#[test]
fn passes_devices_covers_device_nodes_while_the_read_only_skip_is_gpus_only() {
for (backend, gpus, devices) in [
(GpuBackend::Cpu, false, false),
(GpuBackend::Cuda13, true, true),
(GpuBackend::Rocm, false, true),
] {
let engine_args = backend.engine_args();
assert_eq!(passes_gpus(&engine_args), gpus, "{backend:?}");
assert_eq!(passes_devices(&engine_args), devices, "{backend:?}");
let args = run_args(engine_args, 8080, &[], None, true, None, LLAMA_TMPFS);
assert_eq!(
args.contains(&"--read-only".to_string()),
!gpus,
"{backend:?}: {args:?}"
);
}
}
#[test]
fn hardened_user_lands_after_the_block_and_before_the_engine_args_in_every_builder() {
let gpu_user = user(&[44, 993]);
let engine_args = GpuBackend::Rocm.engine_args();
let args = run_args(
engine_args.clone(),
8080,
&[],
None,
true,
Some(&gpu_user),
LLAMA_TMPFS,
);
let mut expected = hardened_args(true, LLAMA_TMPFS);
expected.extend(gpu_user.args(true));
expected.extend(engine_args);
assert_eq!(
&args[6..6 + expected.len()],
expected.as_slice(),
"{args:?}"
);
let at = |flag: &str| args.iter().position(|a| a == flag).unwrap();
assert!(at("--user") < at("--device"), "{args:?}");
assert!(at("--group-add") < at("--device"), "{args:?}");
let args = run_args(
GpuBackend::Cpu.engine_args(),
8080,
&[],
None,
true,
Some(&gpu_user),
LLAMA_TMPFS,
);
assert!(
args.windows(2).any(|w| w == ["--user", "1000:1000"]),
"{args:?}"
);
assert!(!args.contains(&"--group-add".to_string()), "{args:?}");
assert!(
args.windows(2).any(|w| w == ["-e", "HOME=/tmp"]),
"{args:?}"
);
let args = run_args(
GpuBackend::Cpu.engine_args(),
8080,
&[],
None,
false,
Some(&gpu_user),
LLAMA_TMPFS,
);
assert!(!args.contains(&"--user".to_string()), "{args:?}");
assert!(!args.iter().any(|a| a == "HOME=/tmp"), "{args:?}");
let args = verify_args(GpuBackend::Cuda13, None, "v", true, Some(&gpu_user));
let at = |flag: &str| args.iter().position(|a| a == flag).unwrap();
assert!(
at("--cap-drop") < at("--user") && at("--user") < at("--gpus"),
"{args:?}"
);
assert!(
args.windows(2).any(|w| w == ["--group-add", "44"]),
"{args:?}"
);
let args = verify_args(GpuBackend::Cuda13, None, "v", true, None);
assert!(!args.contains(&"--user".to_string()), "{args:?}");
let args = mediagen_run_args(
GpuBackend::Cpu,
"image".into(),
"/usr/bin/llmman",
"/store",
"/cache",
"a-model",
8090,
None,
true,
Some(&user(&[])),
);
let at = |flag: &str| args.iter().position(|a| a == flag).unwrap();
assert!(at("--user") < at("-v"), "{args:?}");
assert_eq!(args[at("--user") + 1], "1000:1000");
}
#[test]
fn hardened_note_names_the_flags_and_the_opt_out() {
assert_eq!(
hardened_note(&hardened_args(true, LLAMA_TMPFS), None),
"started hardened (--cap-drop all --security-opt no-new-privileges --read-only --tmpfs /tmp:exec); --no-hardened turns these flags off"
);
assert_eq!(
hardened_note(&hardened_args(false, LLAMA_TMPFS), Some("/srv/store")),
"started hardened (--cap-drop all --security-opt no-new-privileges --tmpfs /tmp:exec); chmod -R go+rX /srv/store lets the container read it as any user, and --no-hardened turns these flags off"
);
}
#[test]
fn speech_worker_note_is_read_only_on_a_gpu_host() {
for host in [GpuBackend::Cuda13, GpuBackend::Vulkan] {
let speech = HardenedSpawn::Speech.note(host, None, None, "/srv/store");
assert!(speech.contains("--read-only"), "{host:?}: {speech}");
assert_eq!(
speech,
hardened_note(
&hardened_args(true, &llama_tmpfs(&SPEECH_BACKEND.engine_args())),
Some("/srv/store")
)
);
assert!(speech.contains("chmod -R go+rX /srv/store"), "{speech}");
let llama = HardenedSpawn::LlamaServer.note(host, None, None, "/srv/store");
assert_eq!(
llama.contains("--read-only"),
!passes_gpus(&host.engine_args()),
"{host:?}: {llama}"
);
let vllm = HardenedSpawn::Vllm.note(host, None, None, "/srv/store");
assert!(vllm.contains("/root/.triton:exec"), "{host:?}: {vllm}");
}
}
#[test]
fn spawn_note_carries_the_user_block_and_the_rpc_tmpfs_the_argv_had() {
let gpu_user = user(&[44]);
let vllm = HardenedSpawn::Vllm.note(
GpuBackend::Rocm,
Some(&gpu_user),
Some("/run/vllm:exec"),
"/srv/store",
);
let expected = format!(
"started hardened ({} --user 1000:1000 --group-add 44 -e HOME=/tmp -e XDG_CACHE_HOME=/tmp/.cache -e XDG_CONFIG_HOME=/tmp/.config -e TRITON_CACHE_DIR=/tmp/.triton --tmpfs /run/vllm:exec); chmod -R go+rX /srv/store lets the container read it as any user, and --no-hardened turns these flags off",
hardened_args(true, ENGINE_TMPFS).join(" ")
);
assert_eq!(vllm, expected);
let sglang = HardenedSpawn::Sglang.note(
GpuBackend::Cpu,
Some(&gpu_user),
Some("/run/vllm:exec"),
"/srv/store",
);
assert!(!sglang.contains("/run/vllm"), "{sglang}");
assert!(!sglang.contains("--group-add"), "{sglang}");
assert!(sglang.contains("--user 1000:1000 -e HOME=/tmp"), "{sglang}");
let engine_args = GpuBackend::Rocm.engine_args();
let flags = hardened_flags(&engine_args, &llama_tmpfs(&engine_args), Some(&gpu_user));
let verify = verify_args(GpuBackend::Rocm, None, "x", true, Some(&gpu_user));
assert_eq!(&verify[5..5 + flags.len()], flags.as_slice(), "{verify:?}");
}
#[test]
fn hardened_args_drop_caps_and_mount_each_tmpfs() {
assert_eq!(
hardened_args(true, &["/tmp", "/root/.cache"]),
[
"--cap-drop",
"all",
"--security-opt",
"no-new-privileges",
"--read-only",
"--tmpfs",
"/tmp:exec",
"--tmpfs",
"/root/.cache:exec",
]
);
let gpu = hardened_args(false, &["/tmp"]);
assert!(!gpu.contains(&"--read-only".to_string()), "{gpu:?}");
assert_eq!(
&gpu[..4],
["--cap-drop", "all", "--security-opt", "no-new-privileges"]
);
assert_eq!(&gpu[4..], ["--tmpfs", "/tmp:exec"]);
}
#[test]
fn hardened_llama_container_is_read_only_on_cpu_but_not_behind_gpus() {
let opts = |hardened| LlamaOptions {
port: 8080,
ctx_size: None,
flash_attention: None,
kv_cache_type: None,
context_shift: true,
split_mode: None,
num_parallel: None,
embeddings: false,
batch_size: None,
threads: None,
cpus: Some(2.0),
hardened,
extra_args: &[],
};
let argv = |backend: GpuBackend, hardened| {
llama_run_args(
backend,
backend.image_ref(None),
"/cache/blobs",
"sha256-abc",
None,
&[],
opts(hardened),
None,
)
};
let args = argv(GpuBackend::Cpu, true);
assert_hardened_between(&args, 8, true, LLAMA_TMPFS, &GpuBackend::Cpu.engine_args());
let image = args.iter().position(|a| a.starts_with("ghcr.io/")).unwrap();
assert!(
args[..image].contains(&"--read-only".to_string()),
"{args:?}"
);
let args = argv(GpuBackend::Cuda13, true);
assert_hardened_between(
&args,
8,
false,
&llama_tmpfs(&GpuBackend::Cuda13.engine_args()),
&GpuBackend::Cuda13.engine_args(),
);
assert!(!args.contains(&"--read-only".to_string()), "{args:?}");
assert!(
args.windows(2).any(|w| w == ["--cap-drop", "all"]),
"{args:?}"
);
assert!(
args.windows(2).any(|w| w == ["--tmpfs", "/tmp:exec"]),
"{args:?}"
);
assert!(args.windows(2).any(|w| w == ["--gpus", "all"]), "{args:?}");
let args = argv(GpuBackend::Cpu, false);
assert!(
!args
.iter()
.any(|a| a == "--cap-drop" || a == "--read-only" || a == "--tmpfs"),
"{args:?}"
);
}
#[test]
fn hardened_engine_containers_keep_their_own_grants_after_the_drop() {
let serve =
|dir: &str, host: &str| vec!["serve".into(), dir.into(), "--host".into(), host.into()];
let (vllm, _) = ContainerEngine::Vllm.launcher().unwrap();
let (sglang, _) = ContainerEngine::Sglang.launcher().unwrap();
for (backend, launcher, read_only) in [
(VllmBackend::Cpu, vllm, true),
(VllmBackend::Cuda13, vllm, false),
(VllmBackend::Cuda13, sglang, false),
] {
let engine_args = backend.engine_args();
let args = engine_run_args(
run_args(
engine_args.clone(),
8000,
&[],
None,
true,
None,
ENGINE_TMPFS,
),
None,
"image:tag".into(),
launcher,
"/cache/abc/model",
serve,
);
assert_hardened_between(&args, 6, read_only, ENGINE_TMPFS, &engine_args);
assert_eq!(
args.contains(&"--read-only".to_string()),
read_only,
"{args:?}"
);
assert_eq!(
args.iter().filter(|a| *a == "--tmpfs").count(),
ENGINE_TMPFS.len()
);
assert!(args.contains(&"--ipc=host".to_string()), "{args:?}");
let image = args.iter().position(|a| a == "image:tag").unwrap();
assert_eq!(&args[image - 2..image - 1], &["--entrypoint"]);
}
for launcher in [vllm, sglang] {
let args = engine_run_args(
run_args(
VllmBackend::Cuda13.engine_args(),
8000,
&[],
None,
false,
None,
&[],
),
None,
"image:tag".into(),
launcher,
"/cache/abc/model",
serve,
);
for flag in ["--cap-drop", "--security-opt", "--read-only", "--tmpfs"] {
assert!(!args.contains(&flag.to_string()), "{args:?}");
}
}
let args = run_args(
VllmBackend::Cpu.engine_args(),
8000,
&[],
None,
true,
None,
ENGINE_TMPFS,
);
let at = |flag: &str| args.iter().position(|a| a == flag).unwrap();
assert!(at("--cap-drop") < at("--cap-add"), "{args:?}");
assert_eq!(args[at("--cap-add") + 1], "SYS_NICE");
assert!(args.contains(&"seccomp=unconfined".to_string()), "{args:?}");
}
#[test]
fn hardened_verify_runs_with_the_same_block_as_spawn() {
for (backend, read_only) in [(GpuBackend::Cpu, true), (GpuBackend::Cuda13, false)] {
let args = verify_args(backend, Some("b9994"), "llmman-verify-1", true, None);
assert_eq!(
&args[..5],
["run", "--rm", "--init", "--name", "llmman-verify-1"]
);
assert_hardened_between(
&args,
5,
read_only,
&llama_tmpfs(&backend.engine_args()),
&backend.engine_args(),
);
let at = args
.iter()
.position(|a| a == &backend.image_ref(Some("b9994")))
.unwrap();
assert_eq!(&args[at + 1..], ["--version"]);
}
}
fn worker_args(backend: GpuBackend, hardened: bool) -> Vec<String> {
mediagen_run_args(
backend,
backend.image_ref(Some("b9994")),
"/usr/bin/llmman",
"/store",
"/cache",
"a-model",
8090,
None,
hardened,
None,
)
}
#[test]
fn worker_run_args_mount_the_binary_store_and_cache_read_only_then_serve() {
for backend in [GpuBackend::Cpu, GpuBackend::Cuda12, SPEECH_BACKEND] {
let args = worker_args(backend, false);
let engine_args = backend.engine_args();
assert_eq!(
&args[..6],
["run", "--rm", "--init", "-t", "-p", "127.0.0.1:8090:8090"]
);
assert_eq!(&args[6..6 + engine_args.len()], engine_args.as_slice());
for flag in ["--cap-drop", "--read-only", "--tmpfs"] {
assert!(!args.contains(&flag.to_string()), "{backend:?}: {args:?}");
}
let image = backend.image_ref(Some("b9994"));
let at = args.iter().position(|a| a == &image).unwrap();
assert_eq!(
&args[at - 12..at],
[
"-v",
"/usr/bin/llmman:/usr/local/bin/llmman:ro",
"-v",
"/store:/store:ro",
"-v",
"/cache:/cache:ro",
"-e",
"LLMMAN_MODELS=/store",
"-e",
"PATH=/app:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
"--entrypoint",
"/usr/local/bin/llmman",
]
);
assert_eq!(
&args[at + 1..],
[
"serve",
"a-model",
"--port",
"8090",
"--host",
"0.0.0.0",
"--runtime",
"path"
]
);
}
}
#[test]
fn hardened_workers_drop_caps_and_otherwise_match_the_opt_out() {
for (backend, read_only) in [
(GpuBackend::Cpu, true),
(GpuBackend::Cuda12, false),
(SPEECH_BACKEND, true),
] {
let args = worker_args(backend, true);
let tmpfs = llama_tmpfs(&backend.engine_args());
assert_hardened_between(&args, 6, read_only, &tmpfs, &backend.engine_args());
let mut off = worker_args(backend, false);
off.splice(6..6, hardened_args(read_only, &tmpfs));
assert_eq!(args, off, "{backend:?}");
}
}
#[test]
fn cpus_is_forwarded_as_a_run_flag_only_when_a_limit_binds() {
let args = run_args(vec![], 8080, &[], None, false, None, &[]);
assert!(!args.contains(&"--cpus".to_string()), "{args:?}");
let args = run_args(
vec!["--gpus".into(), "all".into()],
8080,
&[],
Some(2.0),
false,
None,
&[],
);
let cpus = args.iter().position(|a| a == "--cpus").unwrap();
assert_eq!(args[cpus + 1], "2");
assert!(cpus < args.iter().position(|a| a == "--gpus").unwrap());
let args = run_args(vec![], 8080, &[], Some(0.5), false, None, &[]);
assert!(args.windows(2).any(|w| w == ["--cpus", "0.5"]), "{args:?}");
}
#[test]
fn llama_container_gets_cpus_before_the_image_and_threads_after() {
let opts = |threads, cpus| LlamaOptions {
port: 8080,
ctx_size: None,
flash_attention: None,
kv_cache_type: None,
context_shift: true,
split_mode: None,
num_parallel: None,
embeddings: false,
batch_size: None,
threads,
cpus,
hardened: false,
extra_args: &[],
};
let image = "ghcr.io/ggml-org/llama.cpp:server";
let argv = |o| {
llama_run_args(
GpuBackend::Cpu,
image.into(),
"/cache/blobs",
"sha256-abc",
None,
&[],
o,
None,
)
};
let args = argv(opts(Some(2), Some(2.0)));
let at = |flag: &str| args.iter().position(|a| a == flag).unwrap();
assert!(at("--cpus") < at(image), "{args:?}");
assert!(at(image) < at("--threads"), "{args:?}");
assert_eq!(args[at("--cpus") + 1], "2");
assert_eq!(args[at("--threads") + 1], "2");
let args = argv(opts(None, None));
assert!(
!args.iter().any(|a| a == "--cpus" || a == "--threads"),
"{args:?}"
);
}
#[test]
fn llama_container_ends_with_the_extra_args() {
let opts = LlamaOptions {
port: 8080,
ctx_size: None,
flash_attention: None,
kv_cache_type: None,
context_shift: true,
split_mode: None,
num_parallel: None,
embeddings: false,
batch_size: None,
threads: Some(2),
cpus: None,
hardened: false,
extra_args: &["--slot-prompt-similarity".into(), "0.5".into()],
};
let args = llama_run_args(
GpuBackend::Cpu,
"ghcr.io/ggml-org/llama.cpp:server".into(),
"/cache/blobs",
"sha256-abc",
None,
&[],
opts,
None,
);
assert_eq!(
args[args.len() - 4..],
["--threads", "2", "--slot-prompt-similarity", "0.5"]
);
}
#[test]
fn vllm_containers_get_cpus_too() {
let (launcher, _) = ContainerEngine::Vllm.launcher().unwrap();
let args = engine_run_args(
run_args(
VllmBackend::Cpu.engine_args(),
8000,
&[],
Some(3.0),
false,
None,
&[],
),
None,
"docker.io/vllm/vllm-openai-cpu:latest-x86_64".into(),
launcher,
"/cache/abc/model",
|dir: &str, host: &str| vec!["serve".into(), dir.into(), "--host".into(), host.into()],
);
assert!(args.windows(2).any(|w| w == ["--cpus", "3"]), "{args:?}");
}
#[test]
#[ignore = "result depends on this host's actual GPU/driver setup"]
fn detect_backend_reports_this_hosts_real_hardware() {
let backend = detect_backend();
println!(
"container::detect_backend() -> {backend:?} ({})",
backend.image_ref(None)
);
}
#[test]
fn gpu_llama_containers_get_the_shader_cache_tmpfs() {
let opts = LlamaOptions {
port: 8080,
ctx_size: None,
flash_attention: None,
kv_cache_type: None,
context_shift: true,
split_mode: None,
num_parallel: None,
embeddings: false,
batch_size: None,
threads: None,
cpus: None,
hardened: true,
extra_args: &[],
};
let argv = |backend: GpuBackend| {
llama_run_args(
backend,
backend.image_ref(None),
"/cache/blobs",
"sha256-abc",
None,
&[],
opts,
None,
)
};
let cache = |args: &[String]| {
args.windows(2)
.filter(|w| w == &["--tmpfs", "/root/.cache:exec"])
.count()
};
let args = argv(GpuBackend::Rocm);
assert!(args.contains(&"/dev/kfd".to_string()), "{args:?}");
let image = args.iter().position(|a| a.starts_with("ghcr.io/")).unwrap();
assert_eq!(cache(&args[..image]), 1, "{args:?}");
assert_eq!(cache(&args[image..]), 0, "{args:?}");
let vulkan = ["--device", "/dev/dri"].map(String::from);
assert_eq!(llama_tmpfs(&vulkan), ["/tmp", "/root/.cache"]);
let args = run_args(
vulkan.to_vec(),
8080,
&[],
None,
true,
None,
&llama_tmpfs(&vulkan),
);
assert_eq!(cache(&args), 1, "{args:?}");
assert!(args.contains(&"--read-only".to_string()), "{args:?}");
let args = argv(GpuBackend::Cpu);
assert_eq!(cache(&args), 0, "{args:?}");
assert_eq!(
args.iter().filter(|a| *a == "--tmpfs").count(),
1,
"{args:?}"
);
assert_eq!(llama_tmpfs(&GpuBackend::Cpu.engine_args()), ["/tmp"]);
let args = verify_args(GpuBackend::Rocm, None, "v", true, None);
assert_eq!(cache(&args), 1, "{args:?}");
let args = verify_args(GpuBackend::Cpu, None, "v", true, None);
assert_eq!(cache(&args), 0, "{args:?}");
}
#[test]
fn vllm_rpc_tmpfs_mounts_an_absolute_path_outside_the_engine_tmpfs() {
for (value, expected) in [
(None, Ok(None)),
(Some("/run/vllm"), Ok(Some("/run/vllm:exec".to_string()))),
(Some("/tmp/vllm"), Ok(None)),
(Some("/tmp"), Ok(None)),
(Some("/root/.cache/rpc"), Ok(None)),
(Some("/tmpfs"), Ok(Some("/tmpfs:exec".to_string()))),
(Some("relative/path"), Err(())),
(Some(""), Err(())),
(Some("/run/rpc:rw"), Err(())),
(Some("/run/rpc,exec"), Err(())),
(Some("/run/../tmp/rpc"), Err(())),
(Some("/run/.."), Err(())),
(Some("/run/..rpc"), Ok(Some("/run/..rpc:exec".to_string()))),
] {
let got = vllm_rpc_tmpfs(value, ENGINE_TMPFS);
match expected {
Ok(expected) => assert_eq!(got, Ok(expected), "{value:?}"),
Err(()) => {
let err = got.expect_err(&format!("{value:?}"));
assert!(err.contains("VLLM_RPC_BASE_PATH"), "{err}");
assert!(err.contains(&format!("{:?}", value.unwrap())), "{err}");
}
}
}
}
#[test]
fn engine_rpc_tmpfs_only_applies_to_a_hardened_vllm_container() {
let relative = || Some("relative/path".to_string());
let absolute = || Some("/run/vllm".to_string());
for engine in [ContainerEngine::Vllm, ContainerEngine::VllmOmni] {
let (_, prefix) = engine.launcher().unwrap();
assert_eq!(
engine_rpc_tmpfs(prefix, true, absolute),
Ok(Some("/run/vllm:exec".to_string()))
);
assert!(engine_rpc_tmpfs(prefix, true, relative).is_err());
assert_eq!(engine_rpc_tmpfs(prefix, false, relative), Ok(None));
}
let (_, prefix) = ContainerEngine::Sglang.launcher().unwrap();
assert_eq!(engine_rpc_tmpfs(prefix, true, absolute), Ok(None));
assert_eq!(engine_rpc_tmpfs(prefix, true, relative), Ok(None));
}
#[test]
fn engine_run_args_mount_the_rpc_tmpfs_before_the_model() {
let (launcher, _) = ContainerEngine::Vllm.launcher().unwrap();
let engine_args = VllmBackend::Cpu.engine_args();
let rpc = vllm_rpc_tmpfs(Some("/run/vllm"), ENGINE_TMPFS).unwrap();
let args = engine_run_args(
run_args(engine_args, 8000, &[], None, true, None, ENGINE_TMPFS),
rpc.as_deref(),
"image:tag".into(),
launcher,
"/cache/abc/model",
|dir, host| vec!["serve".into(), dir.into(), "--host".into(), host.into()],
);
let at = |flag: &str| args.iter().position(|a| a == flag).unwrap();
assert!(
args.windows(2).any(|w| w == ["--tmpfs", "/run/vllm:exec"]),
"{args:?}"
);
assert!(at("/run/vllm:exec") < at("-v"), "{args:?}");
assert_eq!(
args.iter().filter(|a| *a == "--tmpfs").count(),
ENGINE_TMPFS.len() + 1
);
let (launcher, prefix) = ContainerEngine::Sglang.launcher().unwrap();
let rpc = engine_rpc_tmpfs(prefix, true, || Some("/run/vllm".to_string())).unwrap();
let args = engine_run_args(
run_args(
VllmBackend::Cpu.engine_args(),
8000,
&[],
None,
true,
None,
ENGINE_TMPFS,
),
rpc.as_deref(),
"image:tag".into(),
launcher,
"/cache/abc/model",
|dir, host| vec!["serve".into(), dir.into(), "--host".into(), host.into()],
);
assert!(!args.iter().any(|a| a == "/run/vllm:exec"), "{args:?}");
assert_eq!(
args.iter().filter(|a| *a == "--tmpfs").count(),
ENGINE_TMPFS.len()
);
}
}