use std::collections::VecDeque;
use std::net::TcpListener;
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex as StdMutex};
use anyhow::{anyhow, Context};
use reqwest::Client;
use tokio::io::{AsyncBufReadExt, AsyncRead, BufReader};
use tokio::time::{sleep, Duration, Instant};
use super::sched::parse_keep_alive_str;
use super::{canonical_ref, AppState, Engine, ModelProcess};
use crate::modelpack::{resolve_model, ModelPath};
pub(super) fn use_mlx_for_safetensors() -> bool {
crate::hostgpu::detect() == crate::hostgpu::HostGpu::Metal
&& which_binary("mlx_lm.server").is_ok()
}
pub(super) fn find_free_port() -> anyhow::Result<u16> {
let l = TcpListener::bind("127.0.0.1:0")?;
Ok(l.local_addr()?.port())
}
pub(super) type OutputTail = Arc<StdMutex<VecDeque<String>>>;
const TAIL_LINES: usize = 20;
fn spawn_tail_relay(
reader: impl AsyncRead + Unpin + Send + 'static,
tail: OutputTail,
to_stderr: bool,
) {
tokio::spawn(async move {
let mut lines = BufReader::new(reader).lines();
while let Ok(Some(line)) = lines.next_line().await {
if to_stderr {
eprintln!("{line}");
} else {
println!("{line}");
}
if let Ok(mut buf) = tail.lock() {
if buf.len() >= TAIL_LINES {
buf.pop_front();
}
buf.push_back(line);
}
}
});
}
pub(super) async fn spawn_llama_server(
bin: &Path,
model: &Path,
mmproj: Option<&Path>,
opts: crate::container::LlamaOptions<'_>,
) -> anyhow::Result<(tokio::process::Child, OutputTail)> {
let crate::container::LlamaOptions {
port,
ctx_size,
flash_attention,
kv_cache_type,
context_shift,
split_mode,
num_parallel,
embeddings,
batch_size,
threads,
} = opts;
let mut cmd = tokio::process::Command::new(bin);
cmd.args([
"--model",
model.to_str().context("non-UTF-8 model path")?,
"--port",
&port.to_string(),
"--host",
"127.0.0.1",
]);
if let Some(mmproj) = mmproj {
cmd.args([
"--mmproj",
mmproj.to_str().context("non-UTF-8 mmproj path")?,
]);
}
if let Some(n) = ctx_size {
cmd.args(["--ctx-size", &n.to_string()]);
}
if let Some(mode) = flash_attention {
cmd.args(["--flash-attn", mode]);
}
if let Some(t) = kv_cache_type {
cmd.args(["--cache-type-k", t, "--cache-type-v", t]);
}
cmd.arg(if context_shift {
"--context-shift"
} else {
"--no-context-shift"
});
if let Some(mode) = split_mode {
cmd.args(["--split-mode", mode]);
}
if let Some(n) = num_parallel {
cmd.args(["--parallel", &n.to_string()]);
}
if let Some(n) = threads {
cmd.args(["--threads", &n.to_string()]);
}
if embeddings {
cmd.arg("--embeddings");
}
if let Some(n) = batch_size {
let n = n.to_string();
cmd.args(["-b", &n, "-ub", &n]);
}
for var in GPU_VISIBLE_DEVICE_VARS {
if let Ok(val) = std::env::var(var) {
cmd.env(var, val);
}
}
for var in LLAMA_CPP_ENV_PASSTHROUGH_VARS {
if let Ok(val) = std::env::var(var) {
cmd.env(var, val);
}
}
crate::debug_log!("spawning {}: {:?}", bin.display(), cmd);
cmd.stdout(std::process::Stdio::piped());
cmd.stderr(std::process::Stdio::piped());
let mut child = cmd
.kill_on_drop(true)
.spawn()
.with_context(|| format!("spawn llama-server from {}", bin.display()))?;
let tail = tail_child_output(&mut child);
Ok((child, tail))
}
pub(super) fn tail_child_output(child: &mut tokio::process::Child) -> OutputTail {
let tail: OutputTail = Arc::new(StdMutex::new(VecDeque::with_capacity(TAIL_LINES)));
if let Some(stdout) = child.stdout.take() {
spawn_tail_relay(stdout, tail.clone(), false);
}
if let Some(stderr) = child.stderr.take() {
spawn_tail_relay(stderr, tail.clone(), true);
}
tail
}
pub(super) fn vllm_max_model_len(ctx_size: Option<u32>, ctx_size_explicit: bool) -> Option<u32> {
ctx_size.filter(|n| ctx_size_explicit && *n > 0)
}
pub(super) fn vllm_serve_args(
model_dir: &str,
host: &str,
port: u16,
model_name: &str,
max_model_len: Option<u32>,
) -> Vec<String> {
let mut args = vec![
"serve".into(),
model_dir.into(),
"--port".into(),
port.to_string(),
"--host".into(),
host.into(),
"--served-model-name".into(),
model_name.into(),
];
if let Some(n) = max_model_len {
args.push("--max-model-len".into());
args.push(n.to_string());
}
args
}
fn vllm_command(vllm: &Path, args: Vec<String>) -> tokio::process::Command {
let mut cmd = tokio::process::Command::new(vllm);
cmd.args(args).kill_on_drop(true);
#[cfg(unix)]
cmd.process_group(0);
cmd
}
pub(super) async fn spawn_vllm_server(
model_dir: &Path,
port: u16,
model_name: &str,
max_model_len: Option<u32>,
) -> anyhow::Result<tokio::process::Child> {
let vllm = which_binary("vllm")?;
let args = vllm_serve_args(
model_dir.to_str().context("non-UTF-8 model path")?,
"127.0.0.1",
port,
model_name,
max_model_len,
);
vllm_command(&vllm, args)
.spawn()
.with_context(|| format!("spawn vllm from {}", vllm.display()))
}
const VLLM_OMNI_GUARDRAILS_VAR: &str = "LLMMAN_VLLM_OMNI_GUARDRAILS";
fn vllm_omni_serve_args(
model_dir: &str,
host: &str,
port: u16,
model_name: &str,
guardrails: bool,
init_timeout: Duration,
) -> Vec<String> {
let mut args = vllm_serve_args(model_dir, host, port, model_name, None);
args.push("--omni".into());
if !guardrails {
args.push("--no-guardrails".into());
}
args.push("--init-timeout".into());
args.push(init_timeout.as_secs().max(1).to_string());
args
}
pub(super) fn vllm_omni_serve_args_from_env(
model_dir: &str,
host: &str,
port: u16,
model_name: &str,
) -> Vec<String> {
let guardrails = crate::env_flag_set(VLLM_OMNI_GUARDRAILS_VAR);
let init_timeout = load_timeout_from_env().unwrap_or(Duration::from_secs(24 * 3600));
vllm_omni_serve_args(model_dir, host, port, model_name, guardrails, init_timeout)
}
pub(super) async fn spawn_vllm_omni_server(
model_dir: &Path,
port: u16,
model_name: &str,
) -> anyhow::Result<(tokio::process::Child, OutputTail)> {
let vllm = which_binary("vllm")?;
if !vllm_has_omni_plugin(&vllm).await {
anyhow::bail!(
"{} has no vllm-omni plugin, which a Diffusers-layout model needs \
(`uv pip install vllm-omni` into the same environment, or serve it \
with --ociman docker to use the vllm/vllm-omni image)",
vllm.display()
);
}
let args = vllm_omni_serve_args_from_env(
model_dir.to_str().context("non-UTF-8 model path")?,
"127.0.0.1",
port,
model_name,
);
let mut cmd = vllm_command(&vllm, args);
cmd.stdout(std::process::Stdio::piped());
cmd.stderr(std::process::Stdio::piped());
crate::debug_log!("spawning {}: {:?}", vllm.display(), cmd);
let mut child = cmd
.spawn()
.with_context(|| format!("spawn vllm --omni from {}", vllm.display()))?;
let tail = tail_child_output(&mut child);
Ok((child, tail))
}
const OMNI_PLUGIN_PROBE_TIMEOUT: Duration = Duration::from_secs(60);
async fn vllm_has_omni_plugin(vllm: &Path) -> bool {
let Some(python) = console_script_interpreter(vllm) else {
return true;
};
let probe = tokio::process::Command::new(&python)
.args(["-c", "import vllm_omni"])
.stdin(std::process::Stdio::null())
.stdout(std::process::Stdio::null())
.stderr(std::process::Stdio::null())
.kill_on_drop(true)
.status();
match tokio::time::timeout(OMNI_PLUGIN_PROBE_TIMEOUT, probe).await {
Ok(Ok(status)) => status.success(),
Ok(Err(_)) | Err(_) => true,
}
}
fn console_script_interpreter(script: &Path) -> Option<PathBuf> {
let mut head = [0u8; 512];
let n = std::fs::File::open(script)
.and_then(|mut f| std::io::Read::read(&mut f, &mut head))
.ok()?;
let first = std::str::from_utf8(&head[..n]).ok()?.lines().next()?;
let interp = first.strip_prefix("#!")?.split_whitespace().next()?;
(!interp.ends_with("/env") && interp.contains("python")).then(|| PathBuf::from(interp))
}
pub(super) async fn spawn_mlx_server(port: u16) -> anyhow::Result<tokio::process::Child> {
let mlx = which_binary("mlx_lm.server")?;
let mut cmd = tokio::process::Command::new(&mlx);
cmd.args(["--port", &port.to_string(), "--host", "127.0.0.1"]);
cmd.kill_on_drop(true)
.spawn()
.with_context(|| format!("spawn mlx_lm.server from {}", mlx.display()))
}
fn find_on_path(name: &str) -> Option<PathBuf> {
let path_var = std::env::var_os("PATH")?;
for dir in std::env::split_paths(&path_var) {
#[cfg(windows)]
let candidate = dir.join(format!("{name}.exe"));
#[cfg(not(windows))]
let candidate = dir.join(name);
if candidate.is_file() {
return Some(candidate);
}
}
None
}
fn which_binary(name: &str) -> anyhow::Result<PathBuf> {
find_on_path(name).ok_or_else(|| anyhow::anyhow!("{name} not found on PATH"))
}
const DEFAULT_LOAD_TIMEOUT: Duration = Duration::from_secs(600);
fn load_timeout_from_env() -> Option<Duration> {
match std::env::var("LLMMAN_LOAD_TIMEOUT") {
Ok(v) => parse_load_timeout(&v).unwrap_or(Some(DEFAULT_LOAD_TIMEOUT)),
Err(_) => Some(DEFAULT_LOAD_TIMEOUT),
}
}
fn parse_load_timeout(value: &str) -> Option<Option<Duration>> {
let trimmed = value.trim();
if let Some(magnitude) = trimmed.strip_prefix('-') {
return parse_keep_alive_str(magnitude).map(|_| None);
}
Some(match parse_keep_alive_str(trimmed)? {
Some(d) if d.is_zero() => None,
other => other,
})
}
pub(super) const POLL_INTERVAL: Duration = Duration::from_millis(500);
pub(super) async fn wait_for_ready(
client: &Client,
port: u16,
process: &mut ModelProcess,
stderr_tail: Option<&OutputTail>,
) -> anyhow::Result<()> {
let url = format!("http://127.0.0.1:{port}/health");
let load_timeout = load_timeout_from_env();
let deadline = load_timeout.and_then(|d| Instant::now().checked_add(d));
loop {
let remaining = deadline.map(|d| d.saturating_duration_since(Instant::now()));
if remaining.is_some_and(|r| r.is_zero()) {
return Err(anyhow!(
"inference server on port {port} did not become ready within {:?}",
load_timeout.unwrap_or_default()
));
}
if !process.is_alive() {
let detail = stderr_tail.and_then(|t| {
let lines = t.lock().ok()?;
(!lines.is_empty()).then(|| lines.iter().cloned().collect::<Vec<_>>().join(" | "))
});
return Err(match detail {
Some(detail) => anyhow!(
"inference server on port {port} exited before becoming ready: {detail}"
),
None => anyhow!("inference server on port {port} exited before becoming ready"),
});
}
let bound = remaining.map_or(POLL_INTERVAL, |r| r.min(POLL_INTERVAL));
let attempt_start = Instant::now();
if let Ok(resp) = client.get(&url).timeout(bound).send().await {
if resp.status().is_success() {
return Ok(());
}
}
let sleep_for = POLL_INTERVAL.saturating_sub(attempt_start.elapsed());
let remaining = deadline.map(|d| d.saturating_duration_since(Instant::now()));
sleep(remaining.map_or(sleep_for, |r| sleep_for.min(r))).await;
}
}
pub const GPU_VISIBLE_DEVICE_VARS: &[&str] = &[
"CUDA_VISIBLE_DEVICES",
"HIP_VISIBLE_DEVICES",
"ROCR_VISIBLE_DEVICES",
"GGML_VK_VISIBLE_DEVICES",
"GPU_DEVICE_ORDINAL",
"HSA_OVERRIDE_GFX_VERSION",
];
pub const LLAMA_CPP_ENV_PASSTHROUGH_VARS: &[&str] = &[
"LLAMA_ARG_FIT",
"LLAMA_ARG_FIT_TARGET",
"LLAMA_ARG_THREADS",
"LLAMA_ARG_N_GPU_LAYERS",
];
pub(super) fn resolve_llama_server(pinned_version: Option<&str>) -> anyhow::Result<PathBuf> {
if pinned_version.is_none() {
if let Some(p) = find_on_path("llama-server") {
return Ok(p);
}
}
let resolved = crate::llama_release::ensure_llama_server(pinned_version)
.context("no llama-server on PATH and automatic download failed")?;
eprintln!(
"[llmman] using downloaded llama-server ({}): {}",
resolved.backend_label,
resolved.bin.display()
);
Ok(resolved.bin)
}
pub(super) async fn local_llama_server_bin(state: &AppState) -> anyhow::Result<PathBuf> {
let current = state
.0
.llama_server_bin
.lock()
.unwrap_or_else(|e| e.into_inner())
.clone();
let Some(bin) = current else {
anyhow::bail!("no local llama-server binary resolved and --ociman was not set")
};
if bin.exists() {
return Ok(bin);
}
eprintln!(
"[llmman] llama-server at {} no longer exists; re-resolving",
bin.display()
);
let pinned = state.0.llama_cpp_version.clone();
let resolved = tokio::task::spawn_blocking(move || resolve_llama_server(pinned.as_deref()))
.await
.context("resolve llama-server task panicked")??;
*state
.0
.llama_server_bin
.lock()
.unwrap_or_else(|e| e.into_inner()) = Some(resolved.clone());
Ok(resolved)
}
pub(super) async fn would_use_mlx(state: &AppState, model_ref: &str) -> Option<String> {
let model_ref = crate::shortnames::resolve_ollama_api(model_ref).ok()?;
let model_ref = crate::storage::default_tag(&model_ref);
let model_ref = canonical_ref(&state.0.store_path, &model_ref);
{
let mgr = state.0.manager.lock().await;
if let Some(running) = mgr.running.get(&model_ref) {
return matches!(running.process, ModelProcess::Local(Engine::Mlx, _, _))
.then(|| model_ref.clone());
}
}
if !use_mlx_for_safetensors() {
return None;
}
let store_path = state.0.store_path.clone();
let cache_path = state.0.cache_path.clone();
let lookup_ref = model_ref.clone();
let is_safetensors = tokio::task::spawn_blocking(move || {
resolve_model(&store_path, &cache_path, &lookup_ref)
.map(|p| matches!(p, ModelPath::SafeTensors(_)))
.unwrap_or(false)
})
.await
.unwrap_or(false);
is_safetensors.then_some(model_ref)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parse_load_timeout_accepts_the_same_duration_syntax_as_keep_alive() {
assert_eq!(
parse_load_timeout("300"),
Some(Some(Duration::from_secs(300)))
);
assert_eq!(
parse_load_timeout("10m"),
Some(Some(Duration::from_secs(600)))
);
assert_eq!(parse_load_timeout("garbage"), None);
}
#[test]
fn parse_load_timeout_treats_zero_or_negative_as_infinite() {
assert_eq!(parse_load_timeout("0"), Some(None));
assert_eq!(parse_load_timeout("-1"), Some(None));
assert_eq!(parse_load_timeout("-10m"), Some(None));
}
#[test]
fn parse_load_timeout_rejects_a_dash_prefixed_non_duration_instead_of_disabling_forever() {
assert_eq!(parse_load_timeout("-garbage"), None);
assert_eq!(parse_load_timeout("-"), None);
}
#[test]
fn vllm_max_model_len_uses_only_explicit_ctx_size() {
assert_eq!(vllm_max_model_len(Some(4096), true), Some(4096));
assert_eq!(vllm_max_model_len(Some(0), true), None);
assert_eq!(vllm_max_model_len(Some(65536), false), None);
assert_eq!(vllm_max_model_len(None, true), None);
assert_eq!(vllm_max_model_len(None, false), None);
}
#[test]
fn vllm_serve_args_handles_max_model_len() {
for (max_model_len, expected) in [
(Some(256), Some("256")),
(Some(1024), Some("1024")),
(Some(4096), Some("4096")),
(None, None),
] {
let args = vllm_serve_args(
"/models/qwen",
"127.0.0.1",
8000,
"qwen3.5:0.8b",
max_model_len,
);
let pos = args.iter().position(|arg| arg == "--max-model-len");
match expected {
Some(value) => {
let i = pos.expect("--max-model-len should be present");
assert_eq!(args.get(i + 1).map(String::as_str), Some(value));
}
None => assert!(pos.is_none()),
}
}
}
#[test]
fn vllm_serve_args_differ_between_local_and_container_only_in_dir_and_host() {
let local = vllm_serve_args("/cache/abc/model", "127.0.0.1", 8000, "m", Some(4096));
let container = vllm_serve_args("/models", "0.0.0.0", 8000, "m", Some(4096));
assert_eq!(local[0], "serve");
assert_eq!(local[1], "/cache/abc/model");
assert_eq!(container[1], "/models");
let host = |args: &[String]| {
let i = args.iter().position(|a| a == "--host").unwrap();
args[i + 1].clone()
};
assert_eq!(host(&local), "127.0.0.1");
assert_eq!(host(&container), "0.0.0.0");
let rest = |args: &[String]| {
let i = args.iter().position(|a| a == "--host").unwrap();
[&args[2..i], &args[i + 2..]].concat()
};
assert_eq!(rest(&local), rest(&container));
}
#[test]
fn vllm_omni_serve_args_add_omni_and_disable_guardrails_by_default() {
let args = vllm_omni_serve_args(
"/models",
"0.0.0.0",
8000,
"nvidia/Cosmos3-Edge",
false,
Duration::from_secs(600),
);
assert_eq!(args[0], "serve");
assert_eq!(args[1], "/models");
assert!(args.contains(&"--omni".to_string()));
assert!(args.contains(&"--no-guardrails".to_string()));
let i = args.iter().position(|a| a == "--init-timeout").unwrap();
assert_eq!(args[i + 1], "600");
let i = args
.iter()
.position(|a| a == "--served-model-name")
.unwrap();
assert_eq!(args[i + 1], "nvidia/Cosmos3-Edge");
assert!(!args.contains(&"--max-model-len".to_string()));
}
#[test]
fn vllm_omni_serve_args_keep_guardrails_when_asked() {
let args = vllm_omni_serve_args("/m", "127.0.0.1", 8000, "m", true, Duration::from_secs(1));
assert!(args.contains(&"--omni".to_string()));
assert!(!args.contains(&"--no-guardrails".to_string()));
}
#[test]
fn console_script_interpreter_reads_a_python_shebang_only() {
let dir = std::env::temp_dir().join(format!("llmman-shebang-{}", std::process::id()));
std::fs::create_dir_all(&dir).unwrap();
let script = dir.join("vllm");
std::fs::write(
&script,
"#!/opt/venv/bin/python3\n# -*- coding: utf-8 -*-\n",
)
.unwrap();
assert_eq!(
console_script_interpreter(&script),
Some(PathBuf::from("/opt/venv/bin/python3"))
);
std::fs::write(&script, "#!/usr/bin/env python3\nimport sys\n").unwrap();
assert_eq!(console_script_interpreter(&script), None);
std::fs::write(&script, "#!/bin/sh\nexec vllm \"$@\"\n").unwrap();
assert_eq!(console_script_interpreter(&script), None);
std::fs::write(&script, b"\x7fELF\x02\x01\x01").unwrap();
assert_eq!(console_script_interpreter(&script), None);
let _ = std::fs::remove_dir_all(&dir);
}
}