use std::collections::VecDeque;
use std::net::TcpListener;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex as StdMutex};
use anyhow::{anyhow, Context};
use reqwest::Client;
use tokio::io::{AsyncBufReadExt, AsyncRead, BufReader};
use tokio::time::{sleep, Duration, Instant};
use super::sched::parse_keep_alive_str;
use super::{canonical_ref, AppState, Engine, ModelProcess};
use crate::modelpack::{resolve_model, ModelPath};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(super) enum SafetensorsEngine {
Auto,
Vllm,
Sglang,
}
pub(super) const SAFETENSORS_ENGINE_VAR: &str = "LLMMAN_SAFETENSORS_ENGINE";
pub(super) fn parse_safetensors_engine(value: Option<&str>) -> Option<SafetensorsEngine> {
let value = value.map(str::trim).unwrap_or_default();
if value.is_empty() {
return Some(SafetensorsEngine::Auto);
}
match value.to_ascii_lowercase().as_str() {
"vllm" => Some(SafetensorsEngine::Vllm),
"sglang" => Some(SafetensorsEngine::Sglang),
_ => None,
}
}
pub(super) fn safetensors_engine_from_env() -> SafetensorsEngine {
static WARNED: std::sync::Once = std::sync::Once::new();
let raw = std::env::var(SAFETENSORS_ENGINE_VAR).ok();
parse_safetensors_engine(raw.as_deref()).unwrap_or_else(|| {
WARNED.call_once(|| {
eprintln!(
"[llmman] warning: {SAFETENSORS_ENGINE_VAR}={:?} is not one of vllm/sglang; ignoring it",
raw.unwrap_or_default()
);
});
SafetensorsEngine::Auto
})
}
pub(super) fn use_mlx_for_safetensors() -> bool {
cfg!(target_os = "macos") && safetensors_engine_from_env() == SafetensorsEngine::Auto
}
pub(super) fn find_free_port() -> anyhow::Result<u16> {
let l = TcpListener::bind("127.0.0.1:0")?;
Ok(l.local_addr()?.port())
}
pub(super) type OutputTail = Arc<StdMutex<VecDeque<String>>>;
const TAIL_LINES: usize = 20;
fn spawn_tail_relay(
reader: impl AsyncRead + Unpin + Send + 'static,
tail: OutputTail,
to_stderr: bool,
) {
tokio::spawn(async move {
let mut lines = BufReader::new(reader).lines();
while let Ok(Some(line)) = lines.next_line().await {
if to_stderr {
eprintln!("{line}");
} else {
println!("{line}");
}
if let Ok(mut buf) = tail.lock() {
if buf.len() >= TAIL_LINES {
buf.pop_front();
}
buf.push_back(line);
}
}
});
}
pub(super) async fn spawn_llama_server(
bin: &Path,
model: &Path,
mmproj: Option<&Path>,
opts: crate::container::LlamaOptions<'_>,
) -> anyhow::Result<(tokio::process::Child, OutputTail)> {
let crate::container::LlamaOptions {
port,
ctx_size,
flash_attention,
kv_cache_type,
context_shift,
split_mode,
num_parallel,
embeddings,
batch_size,
threads,
cpus: _,
} = opts;
let mut cmd = tokio::process::Command::new(bin);
cmd.args([
"--model",
model.to_str().context("non-UTF-8 model path")?,
"--port",
&port.to_string(),
"--host",
"127.0.0.1",
]);
if let Some(mmproj) = mmproj {
cmd.args([
"--mmproj",
mmproj.to_str().context("non-UTF-8 mmproj path")?,
]);
}
if let Some(n) = ctx_size {
cmd.args(["--ctx-size", &n.to_string()]);
}
if let Some(mode) = flash_attention {
cmd.args(["--flash-attn", mode]);
}
if let Some(t) = kv_cache_type {
cmd.args(["--cache-type-k", t, "--cache-type-v", t]);
}
cmd.arg(if context_shift {
"--context-shift"
} else {
"--no-context-shift"
});
if let Some(mode) = split_mode {
cmd.args(["--split-mode", mode]);
}
if let Some(n) = num_parallel {
cmd.args(["--parallel", &n.to_string()]);
}
if let Some(n) = threads {
cmd.args(["--threads", &n.to_string()]);
}
if embeddings {
cmd.arg("--embeddings");
}
if let Some(n) = batch_size {
let n = n.to_string();
cmd.args(["-b", &n, "-ub", &n]);
}
for var in GPU_VISIBLE_DEVICE_VARS {
if let Ok(val) = std::env::var(var) {
cmd.env(var, val);
}
}
for var in LLAMA_CPP_ENV_PASSTHROUGH_VARS {
if let Ok(val) = std::env::var(var) {
cmd.env(var, val);
}
}
crate::debug_log!("spawning {}: {:?}", bin.display(), cmd);
cmd.stdout(std::process::Stdio::piped());
cmd.stderr(std::process::Stdio::piped());
let mut child = cmd
.kill_on_drop(true)
.spawn()
.with_context(|| format!("spawn llama-server from {}", bin.display()))?;
let tail = tail_child_output(&mut child);
Ok((child, tail))
}
pub(super) fn tail_child_output(child: &mut tokio::process::Child) -> OutputTail {
let tail: OutputTail = Arc::new(StdMutex::new(VecDeque::with_capacity(TAIL_LINES)));
if let Some(stdout) = child.stdout.take() {
spawn_tail_relay(stdout, tail.clone(), false);
}
if let Some(stderr) = child.stderr.take() {
spawn_tail_relay(stderr, tail.clone(), true);
}
tail
}
pub(super) fn vllm_max_model_len(ctx_size: Option<u32>, ctx_size_explicit: bool) -> Option<u32> {
ctx_size.filter(|n| ctx_size_explicit && *n > 0)
}
pub(super) const VLLM_ARGS_VAR: &str = "LLMMAN_VLLM_ARGS";
pub(super) const SGLANG_ARGS_VAR: &str = "LLMMAN_SGLANG_ARGS";
pub(super) fn split_extra_args(value: Option<&str>) -> Vec<String> {
value
.unwrap_or_default()
.split_whitespace()
.map(String::from)
.collect()
}
fn extra_args_from_env(var: &str) -> Vec<String> {
split_extra_args(std::env::var(var).ok().as_deref())
}
pub(super) fn vllm_serve_args(
model_dir: &str,
host: &str,
port: u16,
model_name: &str,
max_model_len: Option<u32>,
) -> Vec<String> {
let mut args = vec![
"serve".into(),
model_dir.into(),
"--port".into(),
port.to_string(),
"--host".into(),
host.into(),
"--served-model-name".into(),
model_name.into(),
];
if let Some(n) = max_model_len {
args.push("--max-model-len".into());
args.push(n.to_string());
}
args
}
pub(super) fn vllm_serve_args_from_env(
model_dir: &str,
host: &str,
port: u16,
model_name: &str,
max_model_len: Option<u32>,
) -> Vec<String> {
let mut args = vllm_serve_args(model_dir, host, port, model_name, max_model_len);
args.extend(extra_args_from_env(VLLM_ARGS_VAR));
args
}
fn grouped_command(bin: &Path, args: Vec<String>) -> tokio::process::Command {
let mut cmd = tokio::process::Command::new(bin);
cmd.args(args).kill_on_drop(true);
#[cfg(unix)]
cmd.process_group(0);
cmd
}
pub(super) async fn spawn_vllm_server(
model_dir: &Path,
port: u16,
model_name: &str,
max_model_len: Option<u32>,
) -> anyhow::Result<tokio::process::Child> {
let vllm = which_binary("vllm")?;
let args = vllm_serve_args_from_env(
model_dir.to_str().context("non-UTF-8 model path")?,
"127.0.0.1",
port,
model_name,
max_model_len,
);
let mut cmd = grouped_command(&vllm, args);
crate::debug_log!("spawning {}: {:?}", vllm.display(), cmd);
cmd.spawn()
.with_context(|| format!("spawn vllm from {}", vllm.display()))
}
const VLLM_OMNI_GUARDRAILS_VAR: &str = "LLMMAN_VLLM_OMNI_GUARDRAILS";
fn vllm_omni_serve_args(
model_dir: &str,
host: &str,
port: u16,
model_name: &str,
guardrails: bool,
init_timeout: Duration,
) -> Vec<String> {
let mut args = vllm_serve_args(model_dir, host, port, model_name, None);
args.push("--omni".into());
if !guardrails {
args.push("--no-guardrails".into());
}
args.push("--init-timeout".into());
args.push(init_timeout.as_secs().max(1).to_string());
args
}
pub(super) fn vllm_omni_serve_args_from_env(
model_dir: &str,
host: &str,
port: u16,
model_name: &str,
) -> Vec<String> {
let guardrails = crate::env_flag_set(VLLM_OMNI_GUARDRAILS_VAR);
let init_timeout = load_timeout_from_env().unwrap_or(Duration::from_secs(24 * 3600));
let mut args =
vllm_omni_serve_args(model_dir, host, port, model_name, guardrails, init_timeout);
args.extend(extra_args_from_env(VLLM_ARGS_VAR));
args
}
pub(super) async fn spawn_vllm_omni_server(
model_dir: &Path,
port: u16,
model_name: &str,
) -> anyhow::Result<(tokio::process::Child, OutputTail)> {
let vllm = which_binary("vllm")?;
if !vllm_has_omni_plugin(&vllm).await {
anyhow::bail!(
"{} has no vllm-omni plugin, which a Diffusers-layout model needs \
(`uv pip install vllm-omni` into the same environment, or serve it \
with --runtime docker to use the vllm/vllm-omni image)",
vllm.display()
);
}
let args = vllm_omni_serve_args_from_env(
model_dir.to_str().context("non-UTF-8 model path")?,
"127.0.0.1",
port,
model_name,
);
spawn_piped(grouped_command(&vllm, args), &vllm, "vllm --omni")
}
fn spawn_piped(
mut cmd: tokio::process::Command,
bin: &Path,
what: &str,
) -> anyhow::Result<(tokio::process::Child, OutputTail)> {
cmd.stdout(std::process::Stdio::piped());
cmd.stderr(std::process::Stdio::piped());
crate::debug_log!("spawning {}: {:?}", bin.display(), cmd);
let mut child = cmd
.spawn()
.with_context(|| format!("spawn {what} from {}", bin.display()))?;
let tail = tail_child_output(&mut child);
Ok((child, tail))
}
pub(super) fn sglang_served_model_name(model_ref: &str) -> String {
model_ref.replace(':', "-")
}
pub(super) fn sglang_serve_args(
model_dir: &str,
host: &str,
port: u16,
model_name: &str,
context_length: Option<u32>,
) -> Vec<String> {
let mut args = vec![
"--model-path".into(),
model_dir.into(),
"--port".into(),
port.to_string(),
"--host".into(),
host.into(),
"--served-model-name".into(),
sglang_served_model_name(model_name),
];
if let Some(n) = context_length {
args.push("--context-length".into());
args.push(n.to_string());
}
args
}
pub(super) fn sglang_serve_args_from_env(
model_dir: &str,
host: &str,
port: u16,
model_name: &str,
context_length: Option<u32>,
) -> Vec<String> {
let mut args = sglang_serve_args(model_dir, host, port, model_name, context_length);
args.extend(extra_args_from_env(SGLANG_ARGS_VAR));
args
}
pub(super) async fn spawn_sglang_server(
model_dir: &Path,
port: u16,
model_name: &str,
context_length: Option<u32>,
) -> anyhow::Result<(tokio::process::Child, OutputTail)> {
let sglang = which_binary("sglang").map_err(|e| {
anyhow!(
"{e} (`uv pip install sglang` puts it there, or drop {SAFETENSORS_ENGINE_VAR}=sglang \
to serve this model with vllm)"
)
})?;
let mut args = vec!["serve".to_string()];
args.extend(sglang_serve_args_from_env(
model_dir.to_str().context("non-UTF-8 model path")?,
"127.0.0.1",
port,
model_name,
context_length,
));
spawn_piped(grouped_command(&sglang, args), &sglang, "sglang")
}
const OMNI_PLUGIN_PROBE_TIMEOUT: Duration = Duration::from_secs(60);
async fn vllm_has_omni_plugin(vllm: &Path) -> bool {
let Some(python) = console_script_interpreter(vllm) else {
return true;
};
let probe = tokio::process::Command::new(&python)
.args(["-c", "import vllm_omni"])
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.kill_on_drop(true)
.status();
match tokio::time::timeout(OMNI_PLUGIN_PROBE_TIMEOUT, probe).await {
Ok(Ok(status)) => status.success(),
Ok(Err(_)) | Err(_) => true,
}
}
fn console_script_interpreter(script: &Path) -> Option<PathBuf> {
let mut head = [0u8; 512];
let n = std::fs::File::open(script)
.and_then(|mut f| std::io::Read::read(&mut f, &mut head))
.ok()?;
let first = std::str::from_utf8(&head[..n]).ok()?.lines().next()?;
let interp = first.strip_prefix("#!")?.split_whitespace().next()?;
(!interp.ends_with("/env") && interp.contains("python")).then(|| PathBuf::from(interp))
}
pub(super) async fn spawn_mlx_server(
state: &AppState,
port: u16,
) -> anyhow::Result<tokio::process::Child> {
let mlx =
match mlx_source(state.0.runtime.requested()) {
None => which_binary("mlx_lm.server").context("--runtime path")?,
Some(fallback) => tokio::task::spawn_blocking(move || {
crate::mlx_release::ensure_mlx_server(fallback)
})
.await
.context("ensure mlx_lm.server task panicked")?
.map_err(|e| {
anyhow!("{e:#} (set {SAFETENSORS_ENGINE_VAR}=vllm to serve this model with vllm)")
})?,
};
let mut cmd = tokio::process::Command::new(&mlx);
cmd.args(["--port", &port.to_string(), "--host", "127.0.0.1"]);
cmd.kill_on_drop(true)
.spawn()
.with_context(|| format!("spawn mlx_lm.server from {}", mlx.display()))
}
pub(super) fn mlx_source(runtime: super::runtime::Runtime) -> Option<crate::mlx_release::Fallback> {
use super::runtime::Runtime;
use crate::mlx_release::Fallback;
match runtime {
Runtime::Path => None,
Runtime::Auto => Some(Fallback::Path),
Runtime::Bin | Runtime::Docker | Runtime::Podman => Some(Fallback::None),
}
}
fn which_binary(name: &str) -> anyhow::Result<PathBuf> {
crate::find_on_path(name).ok_or_else(|| anyhow::anyhow!("{name} not found on PATH"))
}
const DEFAULT_LOAD_TIMEOUT: Duration = Duration::from_secs(600);
fn load_timeout_from_env() -> Option<Duration> {
match std::env::var("LLMMAN_LOAD_TIMEOUT") {
Ok(v) => parse_load_timeout(&v).unwrap_or(Some(DEFAULT_LOAD_TIMEOUT)),
Err(_) => Some(DEFAULT_LOAD_TIMEOUT),
}
}
fn parse_load_timeout(value: &str) -> Option<Option<Duration>> {
let trimmed = value.trim();
if let Some(magnitude) = trimmed.strip_prefix('-') {
return parse_keep_alive_str(magnitude).map(|_| None);
}
Some(match parse_keep_alive_str(trimmed)? {
Some(d) if d.is_zero() => None,
other => other,
})
}
pub(super) const POLL_INTERVAL: Duration = Duration::from_millis(500);
const HEALTH_TIMEOUT: Duration = Duration::from_secs(30);
pub(super) async fn wait_for_ready(
client: &Client,
port: u16,
process: &mut ModelProcess,
stderr_tail: Option<&OutputTail>,
) -> anyhow::Result<()> {
let url = format!("http://127.0.0.1:{port}/health");
let load_timeout = load_timeout_from_env();
let deadline = load_timeout.and_then(|d| Instant::now().checked_add(d));
loop {
let remaining = deadline.map(|d| d.saturating_duration_since(Instant::now()));
if remaining.is_some_and(|r| r.is_zero()) {
return Err(anyhow!(
"inference server on port {port} did not become ready within {:?}",
load_timeout.unwrap_or_default()
));
}
if !process.is_alive() {
let detail = stderr_tail.and_then(|t| {
let lines = t.lock().ok()?;
(!lines.is_empty()).then(|| lines.iter().cloned().collect::<Vec<_>>().join(" | "))
});
return Err(match detail {
Some(detail) => anyhow!(
"inference server on port {port} exited before becoming ready: {detail}"
),
None => anyhow!("inference server on port {port} exited before becoming ready"),
});
}
let bound = remaining.map_or(HEALTH_TIMEOUT, |r| r.min(HEALTH_TIMEOUT));
let attempt_start = Instant::now();
if let Ok(resp) = client.get(&url).timeout(bound).send().await {
if resp.status().is_success() {
return Ok(());
}
}
let sleep_for = POLL_INTERVAL.saturating_sub(attempt_start.elapsed());
let remaining = deadline.map(|d| d.saturating_duration_since(Instant::now()));
sleep(remaining.map_or(sleep_for, |r| sleep_for.min(r))).await;
}
}
pub const GPU_VISIBLE_DEVICE_VARS: &[&str] = &[
"CUDA_VISIBLE_DEVICES",
"HIP_VISIBLE_DEVICES",
"ROCR_VISIBLE_DEVICES",
"GGML_VK_VISIBLE_DEVICES",
"GPU_DEVICE_ORDINAL",
"HSA_OVERRIDE_GFX_VERSION",
];
pub const LLAMA_CPP_ENV_PASSTHROUGH_VARS: &[&str] = &[
"LLAMA_ARG_FIT",
"LLAMA_ARG_FIT_TARGET",
"LLAMA_ARG_THREADS",
"LLAMA_ARG_N_GPU_LAYERS",
];
pub(super) async fn would_use_mlx(state: &AppState, model_ref: &str) -> Option<String> {
let model_ref = crate::shortnames::resolve_ollama_api(model_ref).ok()?;
let model_ref = crate::storage::default_tag(&model_ref);
let model_ref = canonical_ref(&state.0.store_path, &model_ref);
{
let mgr = state.0.manager.lock().await;
if let Some(running) = mgr.running.get(&model_ref) {
return matches!(running.process, ModelProcess::Local(Engine::Mlx, _, _))
.then(|| model_ref.clone());
}
}
if !use_mlx_for_safetensors() {
return None;
}
let store_path = state.0.store_path.clone();
let cache_path = state.0.cache_path.clone();
let lookup_ref = model_ref.clone();
let is_safetensors = tokio::task::spawn_blocking(move || {
resolve_model(&store_path, &cache_path, &lookup_ref)
.map(|p| matches!(p, ModelPath::SafeTensors(_)))
.unwrap_or(false)
})
.await
.unwrap_or(false);
is_safetensors.then_some(model_ref)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parse_load_timeout_accepts_the_same_duration_syntax_as_keep_alive() {
assert_eq!(
parse_load_timeout("300"),
Some(Some(Duration::from_secs(300)))
);
assert_eq!(
parse_load_timeout("10m"),
Some(Some(Duration::from_secs(600)))
);
assert_eq!(parse_load_timeout("garbage"), None);
}
#[test]
fn parse_load_timeout_treats_zero_or_negative_as_infinite() {
assert_eq!(parse_load_timeout("0"), Some(None));
assert_eq!(parse_load_timeout("-1"), Some(None));
assert_eq!(parse_load_timeout("-10m"), Some(None));
}
#[test]
fn parse_load_timeout_rejects_a_dash_prefixed_non_duration_instead_of_disabling_forever() {
assert_eq!(parse_load_timeout("-garbage"), None);
assert_eq!(parse_load_timeout("-"), None);
}
#[test]
fn vllm_max_model_len_uses_only_explicit_ctx_size() {
assert_eq!(vllm_max_model_len(Some(4096), true), Some(4096));
assert_eq!(vllm_max_model_len(Some(0), true), None);
assert_eq!(vllm_max_model_len(Some(65536), false), None);
assert_eq!(vllm_max_model_len(None, true), None);
assert_eq!(vllm_max_model_len(None, false), None);
}
#[test]
fn vllm_serve_args_handles_max_model_len() {
for (max_model_len, expected) in [
(Some(256), Some("256")),
(Some(1024), Some("1024")),
(Some(4096), Some("4096")),
(None, None),
] {
let args = vllm_serve_args(
"/models/qwen",
"127.0.0.1",
8000,
"qwen3.5:0.8b",
max_model_len,
);
let pos = args.iter().position(|arg| arg == "--max-model-len");
match expected {
Some(value) => {
let i = pos.expect("--max-model-len should be present");
assert_eq!(args.get(i + 1).map(String::as_str), Some(value));
}
None => assert!(pos.is_none()),
}
}
}
#[test]
fn vllm_serve_args_differ_between_local_and_container_only_in_dir_and_host() {
let local = vllm_serve_args("/cache/abc/model", "127.0.0.1", 8000, "m", Some(4096));
let container = vllm_serve_args("/models", "0.0.0.0", 8000, "m", Some(4096));
assert_eq!(local[0], "serve");
assert_eq!(local[1], "/cache/abc/model");
assert_eq!(container[1], "/models");
let host = |args: &[String]| {
let i = args.iter().position(|a| a == "--host").unwrap();
args[i + 1].clone()
};
assert_eq!(host(&local), "127.0.0.1");
assert_eq!(host(&container), "0.0.0.0");
let rest = |args: &[String]| {
let i = args.iter().position(|a| a == "--host").unwrap();
[&args[2..i], &args[i + 2..]].concat()
};
assert_eq!(rest(&local), rest(&container));
}
#[test]
fn vllm_omni_serve_args_add_omni_and_disable_guardrails_by_default() {
let args = vllm_omni_serve_args(
"/models",
"0.0.0.0",
8000,
"nvidia/Cosmos3-Edge",
false,
Duration::from_secs(600),
);
assert_eq!(args[0], "serve");
assert_eq!(args[1], "/models");
assert!(args.contains(&"--omni".to_string()));
assert!(args.contains(&"--no-guardrails".to_string()));
let i = args.iter().position(|a| a == "--init-timeout").unwrap();
assert_eq!(args[i + 1], "600");
let i = args
.iter()
.position(|a| a == "--served-model-name")
.unwrap();
assert_eq!(args[i + 1], "nvidia/Cosmos3-Edge");
assert!(!args.contains(&"--max-model-len".to_string()));
}
#[test]
fn vllm_omni_serve_args_keep_guardrails_when_asked() {
let args = vllm_omni_serve_args("/m", "127.0.0.1", 8000, "m", true, Duration::from_secs(1));
assert!(args.contains(&"--omni".to_string()));
assert!(!args.contains(&"--no-guardrails".to_string()));
}
#[test]
fn parse_safetensors_engine_accepts_vllm_sglang_or_nothing() {
assert_eq!(
parse_safetensors_engine(None),
Some(SafetensorsEngine::Auto)
);
assert_eq!(
parse_safetensors_engine(Some("")),
Some(SafetensorsEngine::Auto)
);
assert_eq!(
parse_safetensors_engine(Some(" ")),
Some(SafetensorsEngine::Auto)
);
assert_eq!(
parse_safetensors_engine(Some("vllm")),
Some(SafetensorsEngine::Vllm)
);
assert_eq!(
parse_safetensors_engine(Some("SGLang")),
Some(SafetensorsEngine::Sglang)
);
assert_eq!(
parse_safetensors_engine(Some(" sglang ")),
Some(SafetensorsEngine::Sglang)
);
assert_eq!(parse_safetensors_engine(Some("mlx")), None);
assert_eq!(parse_safetensors_engine(Some("tgi")), None);
}
#[test]
fn split_extra_args_splits_on_whitespace_only() {
assert_eq!(split_extra_args(None), Vec::<String>::new());
assert_eq!(split_extra_args(Some("")), Vec::<String>::new());
assert_eq!(split_extra_args(Some(" \t ")), Vec::<String>::new());
assert_eq!(
split_extra_args(Some(" --dtype bfloat16\t--enforce-eager\n--tp 2 ")),
["--dtype", "bfloat16", "--enforce-eager", "--tp", "2"]
);
assert_eq!(
split_extra_args(Some("--chat-template 'a b'")),
["--chat-template", "'a", "b'"]
);
}
#[test]
fn sglang_serve_args_speak_sglangs_flag_names() {
let args = sglang_serve_args("/models", "0.0.0.0", 30000, "qwen3.5:0.8b", Some(4096));
assert_eq!(&args[..2], &["--model-path", "/models"]);
assert!(!args.contains(&"serve".to_string()));
let value = |flag: &str| {
let i = args.iter().position(|a| a == flag).unwrap();
args[i + 1].clone()
};
assert_eq!(value("--host"), "0.0.0.0");
assert_eq!(value("--port"), "30000");
assert_eq!(value("--served-model-name"), "qwen3.5-0.8b");
assert_eq!(value("--context-length"), "4096");
assert!(!args.contains(&"--max-model-len".to_string()));
}
#[test]
fn sglang_served_model_name_drops_the_colon_sglang_reserves() {
assert_eq!(
sglang_served_model_name("docker.io/ai/qwen3.5:0.8b-safetensors"),
"docker.io/ai/qwen3.5-0.8b-safetensors"
);
assert_eq!(
sglang_served_model_name("hf.co/HuggingFaceTB/SmolLM2-135M-Instruct:latest"),
"hf.co/HuggingFaceTB/SmolLM2-135M-Instruct-latest"
);
}
#[test]
fn sglang_serve_args_leave_context_length_to_the_model_when_unset() {
let args = sglang_serve_args("/models", "127.0.0.1", 30000, "m", None);
assert!(!args.contains(&"--context-length".to_string()));
}
#[test]
fn sglang_serve_args_differ_between_local_and_container_only_in_dir_and_host() {
let local = sglang_serve_args("/cache/abc/model", "127.0.0.1", 30000, "m", Some(4096));
let container = sglang_serve_args("/models", "0.0.0.0", 30000, "m", Some(4096));
assert_eq!(local[1], "/cache/abc/model");
assert_eq!(container[1], "/models");
let host = |args: &[String]| {
let i = args.iter().position(|a| a == "--host").unwrap();
args[i + 1].clone()
};
assert_eq!(host(&local), "127.0.0.1");
assert_eq!(host(&container), "0.0.0.0");
let rest = |args: &[String]| {
let i = args.iter().position(|a| a == "--host").unwrap();
[&args[2..i], &args[i + 2..]].concat()
};
assert_eq!(rest(&local), rest(&container));
}
#[test]
fn console_script_interpreter_reads_a_python_shebang_only() {
let dir = std::env::temp_dir().join(format!("llmman-shebang-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let script = dir.join("vllm");
std::fs::write(
&script,
"#!/opt/venv/bin/python3\n# -*- coding: utf-8 -*-\n",
)
.unwrap();
assert_eq!(
console_script_interpreter(&script),
Some(PathBuf::from("/opt/venv/bin/python3"))
);
std::fs::write(&script, "#!/usr/bin/env python3\nimport sys\n").unwrap();
assert_eq!(console_script_interpreter(&script), None);
std::fs::write(&script, "#!/bin/sh\nexec vllm \"$@\"\n").unwrap();
assert_eq!(console_script_interpreter(&script), None);
std::fs::write(&script, b"\x7fELF\x02\x01\x01").unwrap();
assert_eq!(console_script_interpreter(&script), None);
let _ = std::fs::remove_dir_all(&dir);
}
}