use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::{Arc, LazyLock, Mutex as StdMutex};
use anyhow::{anyhow, Context};
use base64::Engine as _;
use axum::body::{Body, Bytes, HttpBody as _};
use axum::extract::{
DefaultBodyLimit, FromRequest, MatchedPath, Path as UrlPath, Query, Request, State,
};
use axum::http::{HeaderMap, StatusCode};
use axum::middleware::{self, Next};
use axum::response::{IntoResponse, Response};
use axum::routing::{delete, get, post, put};
use axum::{Json, Router};
use clap::Args;
use futures::StreamExt;
use reqwest::Client;
use serde::{Deserialize, Serialize};
use tokio::sync::Mutex;
use tokio::time::{sleep, Duration, Instant};
use crate::default_store;
use crate::metrics::{self, UnloadReason};
use crate::modelpack::{resolve_model, ModelPath};
use crate::providers::{Wire, PLACEHOLDER_API_KEY};
use crate::storage::OciStore;
mod aggregation;
mod anthropic;
mod auth;
mod backend;
mod config;
mod gemini;
mod hybrid;
mod managed;
mod messages;
mod ollama;
mod openai;
mod realtime;
mod refusal;
mod relay;
mod responses;
pub mod runtime;
mod sched;
pub(crate) use sched::parse_keep_alive_str;
mod search;
mod shell;
mod stream;
mod types;
mod usage;
mod webui;
use backend::{
backend_json, ds4_context_length, find_free_port, gguf_arm, gguf_engine_from_env,
gguf_goes_to_ds4, safetensors_engine_from_env, sglang_serve_args_from_env,
sglang_served_model_name, spawn_ds4_server, spawn_llama_server, spawn_mlx_server,
spawn_sglang_server, spawn_vllm_omni_server, spawn_vllm_server, tail_child_output,
use_mlx_for_safetensors, vllm_family_args, vllm_max_model_len, vllm_omni_serve_args_from_env,
vllm_serve_args_from_env, wait_for_ready, GgufArm, GgufEngine, OutputTail, SafetensorsEngine,
POLL_INTERVAL,
};
pub use backend::{GPU_VISIBLE_DEVICE_VARS, LLAMA_CPP_ENV_PASSTHROUGH_VARS};
use config::{
backend_ctx_size, container_cpu_limit, effective_num_parallel,
embedding_context_length_from_env, embedding_model_ctx, flash_attention_from_env,
gguf_trained_ctx, initial_ctx_size, kv_cache_type_from_env, looks_like_oom,
max_loaded_models_from_env, max_queue_from_env, metrics_enabled_from_env,
next_ctx_size_after_oom, num_parallel_from_env, sched_spread_from_env, supports_context_shift,
threads_from_env_or_host, tls_from_env, whole_prompt_batch, MAX_CTX_SHRINK_ATTEMPTS,
};
pub use config::{context_length_from_env, DEFAULT_CTX_SIZE};
use gemini::{gemini_method, handle_pinned_gemini};
use hybrid::resolve_hybrid_side;
use ollama::{
handle_blob_head, handle_blob_upload, handle_copy, handle_create, handle_delete, handle_embed,
handle_embeddings, handle_ollama_chat, handle_ollama_generate, handle_ps, handle_pull,
handle_push, handle_show, handle_tags, handle_version,
};
use openai::{
handle_openai_chat, handle_openai_completions, handle_openai_embeddings, handle_openai_images,
handle_openai_models, handle_openai_speech, handle_openai_systemone,
handle_openai_transcriptions, handle_openai_video_get, handle_openai_videos, handle_tts,
local_engine, TRANSCRIPTION_BODY_LIMIT_BYTES,
};
use relay::proxy;
pub use runtime::Runtime;
use runtime::Workload;
use sched::{
begin_activity, default_keep_alive, reap_idle_models, refresh_activity, ActivityGuard,
};
use stream::bytes_to_lines;
use types::*;
const SERVE_ENV_HELP: &str = "\
Environment Variables:
LLMMAN_DEBUG Show additional debug information (e.g. LLMMAN_DEBUG=1)
LLMMAN_HOST [scheme://][host][:port] to bind (default \"127.0.0.1:17434\"; https:// tells clients to expect TLS)
LLMMAN_API_KEYS Comma-separated API keys every request must present (overrides [auth] in llmman.conf; required off loopback)
LLMMAN_AUTH off: serve without keys even on a bind the network can reach
LLMMAN_PEER_API_KEY The key presented to aggregation peers (default: the first of LLMMAN_API_KEYS)
LLMMAN_TLS_CERT PEM certificate chain to terminate TLS with (with LLMMAN_TLS_KEY)
LLMMAN_TLS_KEY PEM private key for LLMMAN_TLS_CERT
LLMMAN_TLS_CA PEM bundle of extra roots to trust when reaching peers (and, for the CLI, the daemon)
LLMMAN_CONTEXT_LENGTH Context size for llama-server/vLLM/SGLang when set (default 262144 for llama-server)
LLMMAN_EMBEDDING_CONTEXT_LENGTH Context size for embedding models, over LLMMAN_CONTEXT_LENGTH (0: trained context)
LLMMAN_HYBRID_LOCAL_BYTES Largest request a hybrid pair serves locally, in bytes (0 disables; default: from the context length)
LLMMAN_KEEP_ALIVE The duration that models stay loaded in memory (default \"5m\")
LLMMAN_MAX_LOADED_MODELS Maximum number of loaded models (default: unbounded)
LLMMAN_MAX_TRANSFER_STREAMS Maximum parallel transfer streams for safetensors model pulls (default 4)
LLMMAN_MAX_QUEUE Maximum number of queued requests (default 512)
LLMMAN_METRICS Serve a Prometheus scrape endpoint at /metrics (default: off)
LLMMAN_MODELS The path to the models directory
LLMMAN_NUM_PARALLEL Maximum number of parallel requests per model (GGUF only; default: llama-server's auto, one for embedding models)
LLMMAN_MLX_LM_VERSION Exact mlx-lm release to install on macOS (default: PyPI's current)
LLMMAN_SAFETENSORS_ENGINE Engine for safetensors models: vllm or sglang (default: mlx_lm.server on macOS, installed at startup; else vllm)
LLMMAN_VLLM_IMAGE Container image to run vLLM from for safetensors models, e.g. dhi.io/vllm-openai:0.32 — same as --vllm-image
LLMMAN_VLLM_ARGS Extra whitespace-separated arguments ('quote' one to keep its spaces) appended to every `vllm serve`, before llmman's --port/--host (e.g. \"--dtype bfloat16 --tp 2\")
LLMMAN_SGLANG_ARGS Extra whitespace-separated arguments ('quote' one to keep its spaces) appended to every sglang launch, before llmman's --port/--host (e.g. \"--disable-cuda-graph\")
LLMMAN_GGUF_ENGINE Engine for GGUF generation models: ds4 or llama-server (default: ds4-server on PATH for DeepSeek V4.1, else llama-server)
LLMMAN_DS4_ARGS Extra whitespace-separated arguments ('quote' one to keep its spaces) appended to every ds4-server launch, before llmman's --port/--host (e.g. \"--ssd-streaming\")
LLMMAN_LLAMA_ARGS Extra whitespace-separated arguments ('quote' one to keep its spaces) appended to every llama-server launch; --port/--host are refused (e.g. \"--slot-prompt-similarity 0.5\")
LLMMAN_NOHISTORY Do not record prompts for `llmman log`
LLMMAN_NOUSAGE Do not record token usage and cost for `llmman usage`
LLMMAN_NOPRUNE Do not prune model blobs on startup
LLMMAN_ORIGINS A comma separated list of allowed CORS origins
LLMMAN_PEERS A comma separated list of peer daemons ([scheme://]host[:port]) to pool hardware with (overrides [aggregation] in llmman.conf)
LLMMAN_REGISTRY_MIRRORS A comma separated list of Docker Hub mirrors ([scheme://]host[:port]) to try before docker.io (overrides [registries.\"docker.io\"] in llmman.conf)
LLMMAN_SCHED_SPREAD Always schedule model across all GPUs
LLMMAN_FLASH_ATTENTION Enable flash attention
LLMMAN_KV_CACHE_TYPE Quantization type for the K/V cache (default: f16)
LLMMAN_RUNTIME Where the inference engine comes from: auto (default), docker, podman, bin or path — same as --runtime
LLMMAN_NO_HARDENED Run the engine containers without the hardened flags (read-only rootfs, every capability dropped), same as --no-hardened
LLMMAN_LLM_LIBRARY Set backend (cpu/cuda/cuda13/rocm/opencl/vulkan/metal) to bypass GPU autodetection
LLMMAN_IGPU_ENABLE Enable integrated GPUs
LLMMAN_LOAD_TIMEOUT How long to allow model loads to stall before giving up (default \"20m\")
LLMMAN_VLLM_OMNI_GUARDRAILS Keep a Diffusers-layout model's vLLM-Omni safety guardrails on (default: off)
LLMMAN_TMPDIR Staging directory for llama-server release downloads
LLAMA_ARG_FIT Enable llama.cpp automatic fit of unset memory options (default \"on\")
LLAMA_ARG_FIT_TARGET Target free VRAM margin per device for llama.cpp fit (MiB)
LLAMA_ARG_THREADS Thread count for llama-server (default: half the available CPUs above 8, else all of them)
LLMMAN_SHELL The web UI's Shell: off to remove it, or a command to run ('quote' an argument to keep its spaces) (default: the login shell)
";
#[derive(Args, Debug)]
#[command(after_help = SERVE_ENV_HELP)]
pub struct ServeArgs {
#[arg(value_name = "MODEL")]
pub model: Option<String>,
#[arg(long, conflicts_with_all = ["pull_only", "port"])]
pub detach: bool,
#[arg(long, value_enum, default_value = "auto", env = "LLMMAN_RUNTIME")]
pub runtime: Runtime,
#[arg(long)]
pub no_hardened: bool,
#[arg(long, value_name = "TAG|latest")]
pub llama_cpp_version: Option<String>,
#[arg(long, value_name = "TAG")]
pub vllm_version: Option<String>,
#[arg(
long,
value_name = "IMAGE",
env = "LLMMAN_VLLM_IMAGE",
conflicts_with = "vllm_version"
)]
pub vllm_image: Option<String>,
#[arg(long, value_name = "TAG")]
pub sglang_version: Option<String>,
#[arg(long)]
pub pull_only: bool,
#[arg(long, hide = true, requires = "model")]
pub port: Option<u16>,
#[arg(long, hide = true, requires = "port", default_value = "127.0.0.1")]
pub host: std::net::IpAddr,
}
#[derive(Clone)]
struct AppState(Arc<Inner>);
pub(crate) fn hardened_from(no_hardened: bool, env: Option<&str>) -> bool {
!(no_hardened || crate::parse_env_flag(env))
}
struct Inner {
manager: Mutex<ModelManager>,
exe: Option<PathBuf>,
runtime: runtime::Lazy,
llama_cpp_version: Option<String>,
vllm_version: Option<String>,
vllm_image: Option<String>,
sglang_version: Option<String>,
ctx_size: Option<u32>,
ctx_size_explicit: bool,
embedding_ctx_size: Option<u32>,
hybrid_local_bytes: Option<u64>,
hybrid_local_bytes_explicit: bool,
flash_attention: Option<String>,
kv_cache_type: Option<String>,
split_mode: Option<&'static str>,
num_parallel: Option<u32>,
threads: Option<u32>,
cpu_limit: Option<f64>,
hardened: bool,
max_queue: usize,
max_loaded_models: usize,
peers: Vec<String>,
memory: u64,
store_path: PathBuf,
cache_path: PathBuf,
prompt_log: Option<PathBuf>,
usage_log: Option<PathBuf>,
shell: shell::Policy,
auth: auth::Policy,
peer_key: Option<String>,
client: Client,
}
impl Inner {
fn ctx_setting(&self, embedding: bool) -> (Option<u32>, bool) {
match self.embedding_ctx_size.filter(|_| embedding) {
Some(n) => (Some(n), true),
None => (self.ctx_size, self.ctx_size_explicit),
}
}
}
struct ModelManager {
running: HashMap<String, RunningModel>,
pending_loads: usize,
}
struct RunningModel {
process: ModelProcess,
port: u16,
digest: String,
size: u64,
started_at: String,
last_active: Instant,
last_active_wall: chrono::DateTime<chrono::Utc>,
keep_alive: Option<Duration>,
in_flight: u32,
parallel_slots: Option<u32>,
backend_model_path: Option<String>,
context_window: Option<u32>,
decision: bool,
effort_budget: bool,
effort_budget_message: bool,
thinking: Option<crate::chat_template::Thinking>,
trained_ctx: Option<u32>,
retiring: Arc<std::sync::atomic::AtomicBool>,
}
impl RunningModel {
fn needs_context(&self, want: Option<u32>) -> bool {
match want {
Some(want) if self.process.engine() == Engine::LlamaServer => {
self.context_window != Some(effective_ctx(want, self.trained_ctx))
}
_ => false,
}
}
fn processor(&self) -> String {
match &self.process {
ModelProcess::Local(engine, _, _) => format!("{} (local)", engine.label()),
ModelProcess::Container(ociman, engine, _) => {
format!("{} (container/{})", engine.label(), ociman.binary())
}
}
}
fn pid(&self) -> Option<u32> {
match &self.process {
ModelProcess::Local(_, child, _) => child.id(),
ModelProcess::Container(_, _, child) => child.id(),
}
}
fn engine_label(&self) -> &'static str {
match &self.process {
ModelProcess::Local(engine, _, _) | ModelProcess::Container(_, engine, _) => {
engine.label()
}
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Engine {
LlamaServer,
Vllm,
VllmOmni,
Sglang,
Mlx,
Ds4,
Speech,
}
impl Engine {
fn label(self) -> &'static str {
match self {
Engine::LlamaServer => "llama-server",
Engine::Vllm => "vllm",
Engine::VllmOmni => "vllm-omni",
Engine::Sglang => "sglang",
Engine::Mlx => "mlx",
Engine::Ds4 => "ds4",
Engine::Speech => "speech",
}
}
fn ready_path(self) -> &'static str {
match self {
Engine::Ds4 => "/v1/models",
_ => "/health",
}
}
}
enum ModelProcess {
Local(
Engine,
tokio::process::Child,
#[cfg_attr(not(unix), allow(dead_code))] Option<u32>,
),
Container(
crate::container::ContainerManager,
Engine,
tokio::process::Child,
),
}
impl Drop for ModelProcess {
fn drop(&mut self) {
match self {
ModelProcess::Container(_, _, child) => {
if let Some(pid) = child.id() {
crate::container::stop(pid);
}
}
#[cfg(unix)]
ModelProcess::Local(
engine @ (Engine::Vllm | Engine::VllmOmni | Engine::Sglang),
_,
pid,
) => {
if let Some(pid) = pid {
let result = unsafe { libc::kill(-(*pid as libc::pid_t), libc::SIGKILL) };
if result != 0 {
let err = std::io::Error::last_os_error();
eprintln!(
"[llmman] warning: SIGKILL to {} process group {pid} failed: {err}",
engine.label()
);
}
}
}
#[cfg(not(unix))]
ModelProcess::Local(Engine::Vllm | Engine::VllmOmni | Engine::Sglang, _, _) => {}
ModelProcess::Local(Engine::Mlx, _, _) => {}
ModelProcess::Local(Engine::LlamaServer | Engine::Ds4 | Engine::Speech, _, _) => {}
}
}
}
impl ModelProcess {
fn engine(&self) -> Engine {
match self {
ModelProcess::Local(engine, _, _) | ModelProcess::Container(_, engine, _) => *engine,
}
}
fn is_alive(&mut self) -> bool {
let child = match self {
ModelProcess::Local(_, child, _) => child,
ModelProcess::Container(_, _, child) => child,
};
matches!(child.try_wait(), Ok(None))
}
async fn stop_and_wait(&mut self) {
match self {
ModelProcess::Container(_, _, child) => {
if let Some(pid) = child.id() {
crate::container::stop(pid);
}
}
#[cfg(unix)]
ModelProcess::Local(Engine::Vllm | Engine::VllmOmni | Engine::Sglang, _, pid) => {
if let Some(pid) = pid {
unsafe { libc::kill(-(*pid as libc::pid_t), libc::SIGKILL) };
}
}
ModelProcess::Local(_, _, _) => {}
}
let child = match self {
ModelProcess::Local(_, child, _) => child,
ModelProcess::Container(_, _, child) => child,
};
let _ = child.kill().await;
}
}
#[derive(Debug)]
struct AppError(anyhow::Error, StatusCode);
impl<E: Into<anyhow::Error>> From<E> for AppError {
fn from(e: E) -> Self {
Self(e.into(), StatusCode::INTERNAL_SERVER_ERROR)
}
}
impl AppError {
fn status(status: StatusCode, message: impl Into<String>) -> Self {
Self(anyhow!(message.into()), status)
}
fn bad_request(e: crate::shortnames::InvalidReference) -> Self {
Self(e.into(), StatusCode::BAD_REQUEST)
}
}
impl IntoResponse for AppError {
fn into_response(self) -> Response {
let body = serde_json::json!({ "error": format!("{:#}", self.0) });
(self.1, Json(body)).into_response()
}
}
fn now_rfc3339() -> String {
chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true)
}
static MODEL_LOCKS: LazyLock<StdMutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>> =
LazyLock::new(|| StdMutex::new(HashMap::new()));
static LOAD_LOCKS: LazyLock<StdMutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>> =
LazyLock::new(|| StdMutex::new(HashMap::new()));
fn keyed_lock(
registry: &StdMutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>,
key: &str,
) -> Arc<tokio::sync::Mutex<()>> {
let mut locks = registry.lock().unwrap();
locks
.entry(key.to_owned())
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
.clone()
}
fn release_keyed_lock(
registry: &StdMutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>,
key: &str,
) {
let mut locks = registry.lock().unwrap();
if let Some(arc) = locks.get(key) {
if Arc::strong_count(arc) <= 1 {
locks.remove(key);
}
}
}
fn model_lock(model: &str) -> Arc<tokio::sync::Mutex<()>> {
keyed_lock(&MODEL_LOCKS, model)
}
fn release_model_lock(model: &str) {
release_keyed_lock(&MODEL_LOCKS, model)
}
fn load_lock(model: &str) -> Arc<tokio::sync::Mutex<()>> {
keyed_lock(&LOAD_LOCKS, model)
}
fn release_load_lock(model: &str) {
release_keyed_lock(&LOAD_LOCKS, model)
}
struct LoadLockGuard {
model: String,
guard: Option<tokio::sync::OwnedMutexGuard<()>>,
}
impl Drop for LoadLockGuard {
fn drop(&mut self) {
self.guard.take(); release_load_lock(&self.model);
}
}
async fn acquire_load_lock(model: &str) -> LoadLockGuard {
let guard = load_lock(model).lock_owned().await;
LoadLockGuard {
model: model.to_owned(),
guard: Some(guard),
}
}
fn ffi_pull_ref<'a>(progress_key: &'a str, classified: &str) -> &'a str {
debug_assert!(
classified == progress_key || classified.strip_prefix(progress_key) == Some(":latest"),
"classify should differ from the progress key only by a defaulted tag",
);
progress_key
}
fn pull_serialized(store_path: &std::path::Path, model: &str) -> anyhow::Result<Vec<String>> {
let lock = model_lock(model);
let result = {
let _guard = lock.blocking_lock();
tokio::runtime::Handle::current().block_on(pull_checked(store_path, model))
};
drop(lock);
release_model_lock(model);
result
}
pub(crate) async fn pull_checked(
store_path: &std::path::Path,
model: &str,
) -> anyhow::Result<Vec<String>> {
let existing = OciStore::open(store_path).and_then(|s| s.find(model)).ok();
if let Some(desc) = existing {
if !crate::verify::is_registry_reference(model) {
return Ok(Vec::new());
}
if !crate::verify::is_enabled_for(model)? {
return Ok(Vec::new());
}
return verify_stored(store_path, model, &desc.digest);
}
let layout_dir = store_path
.to_str()
.ok_or_else(|| anyhow!("store path is not valid UTF-8"))?;
match crate::hf::classify(model).await {
crate::hf::ClassifiedRef::Hf(reference) => {
if crate::verify::is_registry_reference(model) && crate::verify::is_enabled_for(model)?
{
return Err(anyhow!(concat!(
"is covered by a signature policy but could not be confirmed ",
"to be an OCI registry (the /v2/ probe failed); refusing to ",
"pull it as a HuggingFace repository unchecked",
))
.context(model.to_string()));
}
crate::hf::pull::pull(&reference, store_path, model)
.await
.map(|()| Vec::new())
}
crate::hf::ClassifiedRef::Source(reference) => {
crate::sources::pull(&reference, store_path, model)
.await
.map(|()| Vec::new())
}
crate::hf::ClassifiedRef::Other(normalized) => {
let guard = crate::verify::PullGuard::check(&normalized)?;
crate::ffi::pull(ffi_pull_ref(model, &normalized), layout_dir)?;
let stored = OciStore::open(store_path)?.find(&normalized)?;
match guard.confirm(&stored.digest) {
Ok(notices) => Ok(notices),
Err(e) => Err(reject_stored(store_path, &normalized, e)),
}
}
}
}
fn pull_failure(requested: &str, e: anyhow::Error) -> AppError {
if crate::storage::oci::is_source_not_found(&e) {
return AppError(
anyhow!("model '{requested}' not found: {e:#}"),
StatusCode::NOT_FOUND,
);
}
AppError::from(e.context("pull failed"))
}
fn verify_stored(
store_path: &std::path::Path,
model: &str,
digest: &str,
) -> anyhow::Result<Vec<String>> {
debug_assert!(crate::verify::is_registry_reference(model));
match crate::verify::check(model, Some(digest)) {
Ok(verdict) => Ok(verdict.notices),
Err(e) if crate::verify::is_indeterminate(&e) => Err(e),
Err(e) => Err(reject_stored(store_path, model, e)),
}
}
pub(crate) fn verify_layers_found(
store_path: &std::path::Path,
reference: &str,
desc: &crate::storage::oci::Descriptor,
) -> anyhow::Result<(String, Vec<String>)> {
let mode = crate::verify::layers_mode()?;
let notices = if mode == crate::verify::Mode::Off {
Vec::new()
} else {
let store = OciStore::open(store_path)?;
crate::verify::verify_layers(mode, &store, reference, desc)?
};
Ok((crate::verify::pinned_reference(reference, desc), notices))
}
fn reject_stored(
store_path: &std::path::Path,
reference: &str,
cause: anyhow::Error,
) -> anyhow::Error {
match OciStore::open(store_path).and_then(|s| s.remove(reference)) {
Ok(_) => cause,
Err(e) => cause.context(format!(
"could not remove the rejected model {reference} from the store ({e:#}); \
it will keep being refused, but `llmman rm {reference}` is needed to clear it"
)),
}
}
fn canonical_ref(store_path: &std::path::Path, model_ref: &str) -> String {
let Ok(store) = crate::storage::OciStore::open(store_path) else {
return model_ref.to_owned();
};
let Ok(desc) = store.find(model_ref) else {
return model_ref.to_owned();
};
desc.annotations
.as_ref()
.and_then(|a| a.get("org.opencontainers.image.ref.name"))
.cloned()
.unwrap_or_else(|| model_ref.to_owned())
}
fn running_key(state: &AppState, model_ref: &str) -> String {
let resolved =
crate::shortnames::resolve_ollama_api(model_ref).unwrap_or_else(|_| model_ref.to_string());
canonical_ref(&state.0.store_path, &crate::storage::default_tag(&resolved))
}
fn load_identity(
store_path: &std::path::Path,
model: &str,
) -> Result<String, crate::shortnames::InvalidReference> {
if crate::providers::is_remote_ref(model) {
return Ok(model.to_string());
}
let resolved = crate::shortnames::resolve_ollama_api(model)?;
let (without_digest, digest) = crate::storage::split_ref_digest(&resolved);
if digest.is_some() {
if let Some(tag) = stored_tag_for_digest(store_path, &resolved) {
return Ok(tag);
}
}
Ok(crate::storage::default_tag(without_digest))
}
fn stored_tag_for_digest(store_path: &std::path::Path, reference: &str) -> Option<String> {
crate::storage::split_ref_digest(reference).1?;
let desc = OciStore::open(store_path).ok()?.find(reference).ok()?;
let stored = desc
.annotations
.as_ref()?
.get("org.opencontainers.image.ref.name")?;
crate::storage::split_ref_digest(stored)
.1
.is_none()
.then(|| crate::storage::default_tag(stored))
}
async fn unload_model(state: &AppState, model: &str) -> Result<(), AppError> {
let lock_key = load_identity(&state.0.store_path, model).map_err(AppError::bad_request)?;
let _guard = acquire_load_lock(&lock_key).await;
let canonical = if crate::providers::is_remote_ref(model) {
lock_key
} else {
let resolved =
crate::shortnames::resolve_ollama_api(model).map_err(AppError::bad_request)?;
canonical_ref(&state.0.store_path, &crate::storage::default_tag(&resolved))
};
if state
.0
.manager
.lock()
.await
.running
.remove(&canonical)
.is_some()
{
metrics::record_model_unload(&canonical, UnloadReason::Requested);
return Ok(());
}
if crate::providers::is_remote_ref(model) {
return Ok(());
}
let stored = OciStore::open(&state.0.store_path)?.find(&canonical).ok();
let (base, digest) = crate::storage::split_ref_digest(&canonical);
let digest = digest
.map(str::to_owned)
.or_else(|| stored.as_ref().map(|d| d.digest.clone()));
if let Some(digest) = digest {
let spelled = crate::storage::default_tag(base);
let mut mgr = state.0.manager.lock().await;
let key = mgr
.running
.iter()
.filter(|(key, m)| {
m.digest.eq_ignore_ascii_case(&digest)
&& crate::storage::repo_name(key) == crate::storage::repo_name(base)
})
.map(|(key, _)| key.clone())
.min_by_key(|key| *key != spelled);
if let Some(key) = key {
mgr.running.remove(&key);
return Ok(());
}
}
if stored.is_none() {
return Err(AppError(
anyhow!("model '{model}' not found"),
StatusCode::NOT_FOUND,
));
}
Ok(())
}
async fn forward_ollama<T: Serialize>(
state: &AppState,
target: &Target,
route: &str,
headers: &HeaderMap,
req: &T,
model: &str,
guard: ActivityGuard,
) -> Result<Response, AppError> {
let mut req = serde_json::to_value(req).context("re-serialize Ollama request")?;
if req["model"]
.as_str()
.is_some_and(crate::hybrid::is_hybrid_ref)
{
req["model"] = serde_json::Value::String(model.to_string());
}
let body = Bytes::from(serde_json::to_vec(&req).context("re-serialize Ollama request")?);
let activity = begin_activity(guard, None).await;
proxy(&state.0.client, target, route, headers, body, activity).await
}
async fn unload_everywhere(
state: &AppState,
model: &str,
headers: &HeaderMap,
) -> Result<(), AppError> {
let local = unload_model(state, model).await;
if aggregation::unload(state, model, headers).await {
return Ok(());
}
local
}
async fn check_running(state: &AppState, model_ref: &str) -> Option<(u16, ActivityGuard)> {
check_running_for(state, model_ref, None).await
}
async fn check_running_for(
state: &AppState,
model_ref: &str,
ctx: Option<u32>,
) -> Option<(u16, ActivityGuard)> {
let mut mgr = state.0.manager.lock().await;
if let Some(m) = mgr.running.get_mut(model_ref) {
if m.process.is_alive() {
if m.needs_context(ctx) || m.retiring.load(std::sync::atomic::Ordering::SeqCst) {
return None;
}
m.in_flight += 1;
return Some((m.port, ActivityGuard::new(state, model_ref)));
}
eprintln!(
"[llmman] {model_ref} was marked running on port {} but its process has exited — reloading",
m.port
);
mgr.running.remove(model_ref);
metrics::record_model_unload(model_ref, UnloadReason::Crashed);
}
None
}
fn running_by_digest<'a>(
running: &'a HashMap<String, RunningModel>,
model_ref: &str,
digest: &str,
) -> Option<(&'a String, &'a RunningModel)> {
let repo = crate::storage::repo_name(model_ref);
running.iter().find(|(key, m)| {
!digest.is_empty()
&& crate::storage::repo_name(key) == repo
&& m.digest.eq_ignore_ascii_case(digest)
})
}
async fn check_running_by_digest(
state: &AppState,
model_ref: &str,
digest: &str,
ctx: Option<u32>,
) -> Option<(String, u16, ActivityGuard)> {
let mut mgr = state.0.manager.lock().await;
let key = running_by_digest(&mgr.running, model_ref, digest)?
.0
.clone();
let m = mgr.running.get_mut(&key)?;
if !m.process.is_alive() {
eprintln!(
"[llmman] {key} was marked running on port {} but its process has exited — reloading",
m.port
);
mgr.running.remove(&key);
return None;
}
if m.needs_context(ctx) {
return None;
}
m.in_flight += 1;
let port = m.port;
Some((key.clone(), port, ActivityGuard::new(state, &key)))
}
async fn evict_other_models(state: &AppState, model_ref: &str) -> bool {
let mut mgr = state.0.manager.lock().await;
let other_keys: Vec<String> = mgr
.running
.iter()
.filter(|(k, m)| k.as_str() != model_ref && m.in_flight == 0)
.map(|(k, _)| k.clone())
.collect();
let mut evicted: Vec<(String, RunningModel)> = Vec::with_capacity(other_keys.len());
for key in other_keys {
if let Some(running) = mgr.running.remove(&key) {
metrics::record_model_unload(&key, UnloadReason::Oom);
evicted.push((key, running));
}
}
drop(mgr); let any = !evicted.is_empty();
for (name, mut running) in evicted {
eprintln!("[llmman] evicting {name} to free memory before retrying {model_ref}");
running.process.stop_and_wait().await;
}
any
}
struct RetireMark(Arc<std::sync::atomic::AtomicBool>);
impl RetireMark {
fn new(flag: Arc<std::sync::atomic::AtomicBool>) -> Self {
flag.store(true, std::sync::atomic::Ordering::SeqCst);
Self(flag)
}
}
impl Drop for RetireMark {
fn drop(&mut self) {
self.0.store(false, std::sync::atomic::Ordering::SeqCst);
}
}
async fn retire_for_context(
state: &AppState,
model_ref: &str,
ctx: Option<u32>,
) -> Option<PendingLoadGuard> {
let want = ctx?;
let mut mark = None;
loop {
let mut mgr = state.0.manager.lock().await;
let model = mgr.running.get(model_ref)?;
if !model.needs_context(ctx) {
return None;
}
if model.in_flight > 0 {
mark.get_or_insert_with(|| RetireMark::new(model.retiring.clone()));
drop(mgr);
sleep(Duration::from_millis(100)).await;
continue;
}
let mut retired = mgr.running.remove(model_ref)?;
mgr.pending_loads += 1;
let reserved = PendingLoadGuard {
state: state.clone(),
armed: true,
};
metrics::record_model_unload(model_ref, UnloadReason::Reloaded);
drop(mgr);
eprintln!(
"[llmman] {model_ref} is loaded with a context of {}; unloading it to load it with {}",
retired
.context_window
.map_or_else(|| "an unknown size".to_string(), |n| n.to_string()),
effective_ctx(want, retired.trained_ctx),
);
retired.process.stop_and_wait().await;
return Some(reserved);
}
}
struct PendingLoadGuard {
state: AppState,
armed: bool,
}
impl PendingLoadGuard {
fn release_into(&mut self, mgr: &mut ModelManager) {
if !std::mem::take(&mut self.armed) {
return;
}
mgr.pending_loads = mgr.pending_loads.saturating_sub(1);
}
}
impl std::fmt::Debug for PendingLoadGuard {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("PendingLoadGuard")
.field("armed", &self.armed)
.finish()
}
}
impl Drop for PendingLoadGuard {
fn drop(&mut self) {
if !self.armed {
return;
}
let state = self.state.clone();
let Ok(handle) = tokio::runtime::Handle::try_current() else {
return;
};
handle.spawn(async move {
let mut mgr = state.0.manager.lock().await;
mgr.pending_loads = mgr.pending_loads.saturating_sub(1);
});
}
}
async fn enforce_max_loaded_models(
state: &AppState,
max_loaded: usize,
) -> Result<PendingLoadGuard, AppError> {
if max_loaded == 0 {
state.0.manager.lock().await.pending_loads += 1;
return Ok(PendingLoadGuard {
state: state.clone(),
armed: true,
});
}
loop {
let mut mgr = state.0.manager.lock().await;
if mgr.running.len() + mgr.pending_loads < max_loaded {
mgr.pending_loads += 1;
return Ok(PendingLoadGuard {
state: state.clone(),
armed: true,
});
}
if mgr.running.len() < max_loaded {
drop(mgr);
sleep(POLL_INTERVAL).await;
continue;
}
let victim = mgr
.running
.iter()
.filter(|(_, m)| m.in_flight == 0)
.min_by_key(|(_, m)| m.last_active)
.map(|(k, _)| k.clone());
let Some(victim) = victim else {
return Err(AppError::status(
StatusCode::SERVICE_UNAVAILABLE,
format!(
"max loaded models ({max_loaded}) reached, and every loaded model is busy — try again"
),
));
};
mgr.pending_loads += 1;
let mut running = mgr
.running
.remove(&victim)
.expect("victim key was just looked up under this same lock");
metrics::record_model_unload(&victim, UnloadReason::Evicted);
drop(mgr); eprintln!(
"[llmman] evicting {victim} to free a loaded-model slot (LLMMAN_MAX_LOADED_MODELS={max_loaded})"
);
running.process.stop_and_wait().await;
return Ok(PendingLoadGuard {
state: state.clone(),
armed: true,
});
}
}
const MAX_QUEUE_ERROR: &str = "server busy, please try again. maximum pending requests exceeded";
static PENDING_REQUESTS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
struct QueueGuard(&'static std::sync::atomic::AtomicUsize);
impl Drop for QueueGuard {
fn drop(&mut self) {
self.0.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
}
}
fn try_admit_against(
counter: &'static std::sync::atomic::AtomicUsize,
max_queue: usize,
) -> Result<QueueGuard, AppError> {
let cap = max_queue.max(1);
let admitted = counter
.try_update(
std::sync::atomic::Ordering::SeqCst,
std::sync::atomic::Ordering::SeqCst,
|n| (n < cap).then_some(n + 1),
)
.is_ok();
if admitted {
Ok(QueueGuard(counter))
} else {
Err(AppError::status(
StatusCode::SERVICE_UNAVAILABLE,
MAX_QUEUE_ERROR,
))
}
}
fn try_admit(max_queue: usize) -> Result<QueueGuard, AppError> {
let admitted = try_admit_against(&PENDING_REQUESTS, max_queue);
if admitted.is_err() {
metrics::record_scheduling_rejection();
}
admitted
}
#[derive(Clone, Debug)]
enum Target {
Local(u16),
Peer(Arc<PeerTarget>),
Remote(Arc<RemoteTarget>),
}
struct PeerTarget {
origin: String,
api_key: Option<String>,
}
impl std::fmt::Debug for PeerTarget {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("PeerTarget")
.field("origin", &self.origin)
.field("api_key", &self.api_key.as_ref().map(|_| "<redacted>"))
.finish()
}
}
struct RemoteTarget {
provider: String,
base_url: String,
wire: Wire,
model: String,
max_output: Option<u32>,
cost: Option<crate::providers::Cost>,
api_key: Option<String>,
session: Option<String>,
}
impl RemoteTarget {
fn refuses_stream_options(&self) -> bool {
self.provider == "cohere"
}
fn server_errors_on_responses(&self) -> bool {
self.provider == "opencode"
}
fn sends_opencode_session(&self) -> bool {
self.provider == "opencode-go"
}
fn adopt_session(&mut self, headers: Option<&HeaderMap>) {
if self.sends_opencode_session() {
self.session = client_opencode_session(headers);
}
}
}
impl std::fmt::Debug for RemoteTarget {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RemoteTarget")
.field("provider", &self.provider)
.field("base_url", &self.base_url)
.field("wire", &self.wire)
.field("model", &self.model)
.field("api_key", &self.api_key.as_ref().map(|_| "<redacted>"))
.field("session", &self.session.as_ref().map(|_| "<redacted>"))
.finish()
}
}
impl Target {
fn url(&self, route: &str) -> String {
match self {
Self::Local(port) => format!("http://127.0.0.1:{port}{route}"),
Self::Peer(peer) => format!("{}{route}", peer.origin),
Self::Remote(remote) => crate::providers::rebase_url(&remote.base_url, route),
}
}
fn authorize(&self, req: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
match self {
Self::Local(_) => req,
Self::Peer(peer) => aggregation::hop(req, peer.api_key.as_deref()),
Self::Remote(remote) => {
let req = match (remote.wire, remote.api_key.as_deref()) {
(Wire::OpenAi, Some(key)) => req.bearer_auth(key),
(Wire::OpenAi, None) => req,
(Wire::Anthropic, key) => {
let req = req.header("anthropic-version", anthropic::VERSION);
match key {
Some(key) => req.header("x-api-key", key),
None => req,
}
}
};
if remote.sends_opencode_session() {
req.header(
OPENCODE_SESSION_HEADER,
remote.session.as_deref().unwrap_or(DAEMON_SESSION.as_str()),
)
.header(reqwest::header::USER_AGENT, UPSTREAM_USER_AGENT)
} else {
req
}
}
}
}
fn is_remote(&self) -> bool {
matches!(self, Self::Remote(_))
}
fn wire(&self) -> Option<Wire> {
match self {
Self::Remote(remote) => Some(remote.wire),
_ => None,
}
}
fn is_anthropic(&self) -> bool {
self.wire() == Some(Wire::Anthropic)
}
fn describe(&self) -> String {
match self {
Self::Local(_) => "inference backend".to_string(),
Self::Peer(peer) => format!("peer {}", peer.origin),
Self::Remote(remote) => format!("provider {}", remote.provider),
}
}
}
const CHAT_COMPLETIONS_ROUTE: &str = "/v1/chat/completions";
const OPENCODE_SESSION_HEADER: &str = "x-opencode-session";
fn client_opencode_session(headers: Option<&HeaderMap>) -> Option<String> {
headers
.and_then(|h| h.get(OPENCODE_SESSION_HEADER))
.and_then(|v| v.to_str().ok())
.map(str::trim)
.filter(|s| !s.is_empty())
.map(str::to_owned)
}
static DAEMON_SESSION: LazyLock<String> = LazyLock::new(|| format!("llmman-{}", types::gen_id()));
const UPSTREAM_USER_AGENT: &str = concat!("llmman/", env!("CARGO_PKG_VERSION"));
fn client_api_key(headers: Option<&HeaderMap>) -> Option<String> {
let headers = headers?;
let usable = |k: &str| {
let k = k.trim();
(!k.is_empty() && k != PLACEHOLDER_API_KEY).then(|| k.to_string())
};
let bearer = auth::bearer(headers).and_then(usable);
bearer
.or_else(|| {
headers
.get("x-api-key")
.and_then(|v| v.to_str().ok())
.and_then(usable)
})
.or_else(|| {
headers
.get("x-goog-api-key")
.and_then(|v| v.to_str().ok())
.and_then(usable)
})
}
fn is_cross_site(headers: Option<&HeaderMap>) -> bool {
headers
.and_then(|h| h.get("sec-fetch-site"))
.and_then(|v| v.to_str().ok())
.is_some_and(|site| {
let site = site.trim();
site.eq_ignore_ascii_case("cross-site") || site.eq_ignore_ascii_case("same-site")
})
}
fn daemon_key_spendable(state: &AppState, headers: Option<&HeaderMap>) -> bool {
state.0.auth.enforced() || (crate::daemon::reachable_only_locally() && !is_cross_site(headers))
}
async fn provider_catalog() -> Result<Arc<crate::providers::Catalog>, AppError> {
catalog_blocking(crate::providers::catalog).await
}
async fn provider_catalog_listing(
provider: &str,
model: &str,
) -> Result<Arc<crate::providers::Catalog>, AppError> {
let (provider, model) = (provider.to_string(), model.to_string());
catalog_blocking(move || crate::providers::catalog_listing(&provider, &model)).await
}
async fn catalog_blocking(
load: impl FnOnce() -> anyhow::Result<Arc<crate::providers::Catalog>> + Send + 'static,
) -> Result<Arc<crate::providers::Catalog>, AppError> {
tokio::task::spawn_blocking(load)
.await
.context("provider catalog task panicked")?
.map_err(|e| AppError(e, StatusCode::BAD_GATEWAY))
}
async fn resolve_remote_target(
state: &AppState,
model_ref: &str,
headers: Option<&HeaderMap>,
) -> Result<Option<Target>, AppError> {
let Some((provider_id, model)) = crate::providers::split_remote_ref(model_ref) else {
return Ok(None);
};
let (provider_id, model) = (provider_id.to_string(), model.to_string());
let catalog = provider_catalog().await?;
let provider = catalog.get(&provider_id).ok_or_else(|| {
AppError(
crate::providers::unknown_provider_error(&provider_id, &catalog),
StatusCode::BAD_REQUEST,
)
})?;
let trusted = daemon_key_spendable(state, headers);
let own_key = || trusted.then(|| provider.api_key()).flatten();
let api_key = match client_api_key(headers).or_else(own_key) {
Some(key) => Some(key),
None if provider.key_optional => None,
None => {
return Err(AppError(
anyhow!(
"no API key for provider {provider_id:?} — send it as an Authorization \
header{}",
if is_cross_site(headers) && !state.0.auth.enforced() {
". This request came from another site, so llmman serve's own \
environment is deliberately not used"
.to_string()
} else if trusted {
format!(
", or give llmman serve a key of its own: {}",
crate::providers::key_hint(&provider.id, provider.key_env.as_deref())
)
} else {
". llmman serve is not bound to loopback, so its own environment is \
deliberately not used"
.to_string()
}
),
StatusCode::UNAUTHORIZED,
));
}
};
let listed = provider.models.iter().find(|m| m.id == model);
let mut target = RemoteTarget {
provider: provider_id,
base_url: provider.base_url.clone(),
wire: provider.wire,
max_output: listed.and_then(|m| m.max_output),
cost: listed.and_then(|m| m.cost.clone()),
model,
api_key,
session: None,
};
target.adopt_session(headers);
eprintln!(
"[llmman] routing {} to provider {} ({}, {} wire)",
target.model,
target.provider,
target.base_url,
target.wire.as_str()
);
Ok(Some(Target::Remote(Arc::new(target))))
}
async fn ensure_model(
state: &AppState,
model_ref: &str,
headers: Option<&HeaderMap>,
request_threads: Option<u32>,
) -> Result<(String, Target, ActivityGuard), AppError> {
let opts = LoadOptions {
threads: request_threads,
..LoadOptions::default()
};
ensure_model_for(state, model_ref, headers, opts, Task::Other).await
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
struct LoadOptions {
threads: Option<u32>,
ctx: Option<u32>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Task {
Generate,
Embed,
Forward,
Other,
}
impl Task {
fn refuse_decision(self, model: &str) -> Option<AppError> {
let message = match self {
Self::Generate => format!(
"{model} is a decision model: it answers questions through /v1/systemone \
(llmman run {model} --ask ...) and does not generate text"
),
Self::Embed => {
format!("{model} is a decision model, which does not support embeddings")
}
Self::Forward | Self::Other => return None,
};
Some(AppError::status(StatusCode::BAD_REQUEST, message))
}
fn refuse_speech(self, model: &str) -> Option<AppError> {
let what = match self {
Self::Generate => "does not generate text",
Self::Embed => "does not support embeddings",
Self::Forward => "serves no other route",
Self::Other => return None,
};
Some(AppError::status(
StatusCode::BAD_REQUEST,
format!(
"{model} is a speech model: it transcribes audio through \
/v1/audio/transcriptions and {what}"
),
))
}
}
fn is_speech_model(info: &crate::gguf::Info) -> bool {
info.is_speech()
}
async fn ensure_model_for(
state: &AppState,
model_ref: &str,
headers: Option<&HeaderMap>,
opts: LoadOptions,
task: Task,
) -> Result<(String, Target, ActivityGuard), AppError> {
let (model, target, guard) = resolve_target(state, model_ref, headers, opts, task).await?;
if task != Task::Other && matches!(target, Target::Local(_)) {
let (decision, speech) = {
let mgr = state.0.manager.lock().await;
let running = mgr.running.get(&model);
(
running.is_some_and(|m| m.decision),
running.is_some_and(|m| m.process.engine() == Engine::Speech),
)
};
let refusal = decision
.then(|| task.refuse_decision(&model))
.flatten()
.or_else(|| speech.then(|| task.refuse_speech(&model)).flatten());
if let Some(refusal) = refusal {
guard.release_idle();
return Err(refusal);
}
}
usage::note_target(&model, &target);
Ok((model, target, guard))
}
async fn resolve_target(
state: &AppState,
model_ref: &str,
headers: Option<&HeaderMap>,
opts: LoadOptions,
task: Task,
) -> Result<(String, Target, ActivityGuard), AppError> {
let hybrid_side = match crate::hybrid::split_ref(model_ref) {
Some(pair) => Some(resolve_hybrid_side(state, &pair, headers).await?),
None => None,
};
let model_ref = hybrid_side.as_deref().unwrap_or(model_ref);
if let Some(target) = resolve_remote_target(state, model_ref, headers).await? {
return Ok((
model_ref.to_string(),
target,
ActivityGuard::new(state, model_ref),
));
}
let load_id = load_identity(&state.0.store_path, model_ref).map_err(AppError::bad_request)?;
let requested = model_ref;
let model_ref =
crate::shortnames::resolve_ollama_api(model_ref).map_err(AppError::bad_request)?;
let model_ref = crate::storage::default_tag(&model_ref);
let model_ref = canonical_ref(&state.0.store_path, &model_ref);
let model_ref = model_ref.as_str();
if let Some((port, guard)) = check_running_for(state, model_ref, opts.ctx).await {
return Ok((model_ref.to_string(), Target::Local(port), guard));
}
if let Some(peer) = aggregation::route(state, model_ref, headers).await {
return Ok((
model_ref.to_string(),
aggregation::target(state, peer),
ActivityGuard::new(state, model_ref),
));
}
let load_started = Instant::now();
let _queue_guard = try_admit(state.0.max_queue)?;
let _guard = acquire_load_lock(&load_id).await;
if let Some((port, guard)) = check_running_for(state, model_ref, opts.ctx).await {
return Ok((model_ref.to_string(), Target::Local(port), guard));
}
{
let present = crate::storage::OciStore::open(&state.0.store_path)
.and_then(|s| s.find(model_ref))
.is_ok();
if !present {
eprintln!("[llmman] {model_ref} not in store — pulling");
}
let store_path = state.0.store_path.clone();
let model_ref_owned = model_ref.to_owned();
let notices =
tokio::task::spawn_blocking(move || pull_serialized(&store_path, &model_ref_owned))
.await
.context("pull task panicked")?
.map_err(|e| pull_failure(requested, e))?;
for notice in notices {
eprintln!("[llmman] {notice}");
}
}
let model_ref = canonical_ref(&state.0.store_path, model_ref);
let model_ref = model_ref.as_str();
if let Some((port, guard)) = check_running_for(state, model_ref, opts.ctx).await {
return Ok((model_ref.to_string(), Target::Local(port), guard));
}
let desc = OciStore::open(&state.0.store_path)?
.find(model_ref)
.with_context(|| format!("model not found in store: {model_ref}"))?;
if let Some((key, port, guard)) =
check_running_by_digest(state, model_ref, &desc.digest, opts.ctx).await
{
return Ok((key, Target::Local(port), guard));
}
let load_ref = {
let store_path = state.0.store_path.clone();
let model_ref_owned = model_ref.to_owned();
let desc = desc.clone();
let (load_ref, notices) = tokio::task::spawn_blocking(move || {
verify_layers_found(&store_path, &model_ref_owned, &desc)
})
.await
.context("layer verification task panicked")?
.with_context(|| format!("verify layers of {model_ref}"))?;
for notice in notices {
eprintln!("[llmman] {notice}");
}
load_ref
};
let size = OciStore::open(&state.0.store_path)?.total_size(&desc);
let digest = desc.digest;
let model_path = resolve_model(&state.0.store_path, &state.0.cache_path, &load_ref)
.with_context(|| format!("resolve model {model_ref}"))?;
let context_shift = supports_context_shift(model_ref);
let gguf_info = match &model_path {
ModelPath::Gguf(path, _) => crate::gguf::read_info(path).ok(),
_ => None,
};
let trained_ctx = gguf_info.as_ref().and_then(gguf_trained_ctx);
let embedding_ctx = gguf_info.as_ref().and_then(embedding_model_ctx);
let decision = gguf_info.as_ref().is_some_and(|info| info.is_decision());
let speech = gguf_info.as_ref().is_some_and(is_speech_model);
let thinking_controls = gguf_info
.as_ref()
.and_then(|info| info.str("tokenizer.chat_template"))
.map(crate::chat_template::thinking_controls);
if decision {
if let Some(refusal) = task.refuse_decision(model_ref) {
return Err(refusal);
}
}
if speech {
if let Some(refusal) = task.refuse_speech(model_ref) {
return Err(refusal);
}
}
let ds4 = !speech
&& gguf_goes_to_ds4(
gguf_engine_from_env(),
&model_path,
gguf_info.as_ref().and_then(|info| info.architecture()),
embedding_ctx.is_some(),
decision,
backend::ds4_server_on_path,
);
let load_ctx = ctx_for_load(opts.ctx, &model_path, ds4);
let mut pending_load_guard = match retire_for_context(state, model_ref, load_ctx).await {
Some(reserved) => reserved,
None => enforce_max_loaded_models(state, state.0.max_loaded_models).await?,
};
let (ctx_configured, ctx_explicit) = match load_ctx {
Some(num_ctx) => (Some(effective_ctx(num_ctx, trained_ctx)), true),
None => state.0.ctx_setting(embedding_ctx.is_some()),
};
let mut ctx_size = initial_ctx_size(
ctx_configured,
ctx_explicit,
trained_ctx,
embedding_ctx.is_some(),
);
let mut split_mode = state.0.split_mode;
let num_parallel =
effective_num_parallel(ctx_size, state.0.num_parallel, embedding_ctx.is_some());
if state.0.num_parallel.is_some() && num_parallel != state.0.num_parallel {
eprintln!(
"[llmman] {model_ref}: no explicit ctx-size to scale, ignoring LLMMAN_NUM_PARALLEL for this load"
);
}
let mut shrink_attempts = 0u32;
let mut evicted_others = false;
let mut split_mode_relaxed = false;
let mut process;
let mut port = find_free_port()?;
let llama_extra_args = backend::extra_args_from_env(backend::LLAMA_ARGS_VAR);
loop {
eprintln!("[llmman] loading {model_ref} on port {port}");
let scaled_ctx_size = backend_ctx_size(ctx_size, num_parallel);
let llama_opts = crate::container::LlamaOptions {
port,
ctx_size: scaled_ctx_size,
flash_attention: state.0.flash_attention.as_deref(),
kv_cache_type: state.0.kv_cache_type.as_deref(),
context_shift,
split_mode,
num_parallel,
embeddings: embedding_ctx.is_some(),
batch_size: whole_prompt_batch(
embedding_ctx.is_some(),
decision,
ctx_size,
ctx_explicit,
trained_ctx,
),
threads: opts.threads.or(state.0.threads),
cpus: state.0.cpu_limit,
hardened: state.0.hardened,
extra_args: &llama_extra_args,
};
let mut stderr_tail: Option<OutputTail> = None;
let mut oom_retryable = false;
let max_model_len = vllm_max_model_len(ctx_size, ctx_explicit);
let safetensors_engine = safetensors_engine_from_env();
let ociman = if ds4 {
None
} else if speech {
state.0.runtime.ociman_for(Workload::CpuOnly).await?
} else {
state.0.runtime.ociman().await?
};
process = match (&model_path, ociman) {
(ModelPath::Gguf(path, mmproj), ociman) => {
match (gguf_arm(ds4, speech, ociman.is_some()), ociman) {
(GgufArm::Ds4, _) => {
let (child, tail) =
spawn_ds4_server(path, mmproj.as_deref(), port, max_model_len).await?;
stderr_tail = Some(tail);
ModelProcess::Local(Engine::Ds4, child, None)
}
(GgufArm::Speech { container: true }, Some(ociman)) => {
let mut child = crate::container::spawn_mediagen(
ociman,
crate::container::SPEECH_BACKEND,
&load_ref,
&state.0.store_path,
&state.0.cache_path,
state.0.llama_cpp_version.as_deref(),
port,
state.0.cpu_limit,
state.0.hardened,
)?;
stderr_tail = Some(tail_child_output(&mut child));
ModelProcess::Container(ociman, Engine::Speech, child)
}
(GgufArm::Speech { .. }, _) => {
let (child, tail) =
spawn_mediagen_backend(&load_ref, port, state, Workload::CpuOnly)
.await?;
stderr_tail = Some(tail);
ModelProcess::Local(Engine::Speech, child, None)
}
(GgufArm::LlamaServer { container: true }, Some(ociman)) => {
let mut child = crate::container::spawn(
ociman,
path,
mmproj.as_deref(),
state.0.llama_cpp_version.as_deref(),
llama_opts,
)?;
stderr_tail = Some(tail_child_output(&mut child));
oom_retryable = true;
ModelProcess::Container(ociman, Engine::LlamaServer, child)
}
(GgufArm::LlamaServer { .. }, _) => {
let bin = state.0.runtime.local_llama_server_bin().await?;
let (child, tail) =
spawn_llama_server(&bin, path, mmproj.as_deref(), llama_opts).await?;
stderr_tail = Some(tail);
oom_retryable = true;
ModelProcess::Local(Engine::LlamaServer, child, None)
}
}
}
(ModelPath::Diffusion(_), Some(ociman)) => {
let mut child = crate::container::spawn_mediagen(
ociman,
crate::container::detect_backend(),
&load_ref,
&state.0.store_path,
&state.0.cache_path,
state.0.llama_cpp_version.as_deref(),
port,
state.0.cpu_limit,
state.0.hardened,
)?;
stderr_tail = Some(tail_child_output(&mut child));
oom_retryable = true;
ModelProcess::Container(ociman, Engine::LlamaServer, child)
}
(ModelPath::Diffusion(_), None) => {
let (child, tail) =
spawn_mediagen_backend(&load_ref, port, state, Workload::General).await?;
stderr_tail = Some(tail);
oom_retryable = true;
ModelProcess::Local(Engine::LlamaServer, child, None)
}
(ModelPath::SafeTensors(dir), Some(ociman)) => {
let sglang = safetensors_engine == SafetensorsEngine::Sglang;
let (engine, container_engine, version) = if sglang {
(
Engine::Sglang,
crate::container::ContainerEngine::Sglang,
state.0.sglang_version.as_deref(),
)
} else {
(
Engine::Vllm,
crate::container::ContainerEngine::Vllm,
state.0.vllm_version.as_deref(),
)
};
let mut child = crate::container::spawn_engine(
ociman,
container_engine,
dir,
version,
state.0.vllm_image.as_deref(),
port,
state.0.cpu_limit,
state.0.hardened,
|model_dir, host| {
if sglang {
sglang_serve_args_from_env(
model_dir,
host,
port,
model_ref,
max_model_len,
)
} else {
vllm_serve_args_from_env(
model_dir,
host,
port,
model_ref,
max_model_len,
&vllm_family_args(dir),
)
}
},
)?;
stderr_tail = Some(tail_child_output(&mut child));
ModelProcess::Container(ociman, engine, child)
}
(ModelPath::Omni(dir), Some(ociman)) => {
let mut child = crate::container::spawn_engine(
ociman,
crate::container::ContainerEngine::VllmOmni,
dir,
state.0.vllm_version.as_deref(),
None,
port,
state.0.cpu_limit,
state.0.hardened,
|model_dir, host| {
vllm_omni_serve_args_from_env(model_dir, host, port, model_ref)
},
)?;
stderr_tail = Some(tail_child_output(&mut child));
ModelProcess::Container(ociman, Engine::VllmOmni, child)
}
(ModelPath::Omni(dir), None) => {
let (child, tail) = spawn_vllm_omni_server(dir, port, model_ref).await?;
stderr_tail = Some(tail);
let pid = child.id();
ModelProcess::Local(Engine::VllmOmni, child, pid)
}
(ModelPath::SafeTensors(dir), None)
if safetensors_engine == SafetensorsEngine::Sglang =>
{
let (child, tail) =
spawn_sglang_server(dir, port, model_ref, max_model_len).await?;
stderr_tail = Some(tail);
let pid = child.id();
ModelProcess::Local(Engine::Sglang, child, pid)
}
(ModelPath::SafeTensors(_dir), None) if use_mlx_for_safetensors() => {
let child = spawn_mlx_server(state, port).await?;
let pid = child.id();
ModelProcess::Local(Engine::Mlx, child, pid)
}
(ModelPath::SafeTensors(dir), None) => {
let child = spawn_vllm_server(dir, port, model_ref, max_model_len).await?;
let pid = child.id();
ModelProcess::Local(Engine::Vllm, child, pid)
}
};
match wait_for_ready(&state.0.client, port, &mut process, stderr_tail.as_ref()).await {
Ok(()) => break,
Err(e) => {
let looks_oom = oom_retryable && looks_like_oom(&e.to_string());
if !looks_oom {
if let (true, ModelProcess::Container(ociman, engine, _)) =
(state.0.hardened, &process)
{
use crate::container::HardenedSpawn;
let spawn = match engine {
Engine::Vllm | Engine::VllmOmni => HardenedSpawn::Vllm,
Engine::Sglang => HardenedSpawn::Sglang,
Engine::Speech => HardenedSpawn::Speech,
_ => HardenedSpawn::LlamaServer,
};
let mounted: &std::path::Path = match &model_path {
ModelPath::Gguf(path, _) if spawn != HardenedSpawn::Speech => {
path.parent().unwrap_or(&state.0.store_path)
}
ModelPath::SafeTensors(dir) | ModelPath::Omni(dir) => dir,
ModelPath::Gguf(..) | ModelPath::Diffusion(_) => &state.0.store_path,
};
let note = crate::container::spawn_hardened_note(spawn, *ociman, mounted);
return Err(anyhow!("{e:#}; {note}").into());
}
return Err(e.into());
}
process.stop_and_wait().await;
if !evicted_others {
evicted_others = true;
if evict_other_models(state, model_ref).await {
metrics::record_oom_retry(model_ref, metrics::OomRetry::EvictOthers);
eprintln!(
"[llmman] {model_ref} failed to load on port {port}, which looks like an out-of-memory error — evicted other loaded models and retrying: {:#}",
e
);
port = find_free_port()?;
continue;
}
}
if !split_mode_relaxed && split_mode == Some("none") {
split_mode_relaxed = true;
split_mode = Some("layer");
metrics::record_oom_retry(model_ref, metrics::OomRetry::SplitMode);
eprintln!(
"[llmman] {model_ref} failed to load on port {port} with --split-mode none, which looks like an out-of-memory error — retrying with --split-mode layer (spread across every GPU) instead of failing outright: {:#}",
e
);
port = find_free_port()?;
continue;
}
let can_shrink = !ctx_explicit && shrink_attempts < MAX_CTX_SHRINK_ATTEMPTS;
let Some(next) = can_shrink
.then(|| ctx_size.and_then(next_ctx_size_after_oom))
.flatten()
else {
return Err(e.into());
};
eprintln!(
"[llmman] {model_ref} failed to load on port {port}, which looks like an out-of-memory error — retrying with --ctx-size {next} (was {}): {:#}",
ctx_size
.map(|n| n.to_string())
.unwrap_or_else(|| "model default".to_string()),
e
);
ctx_size = Some(next);
shrink_attempts += 1;
metrics::record_oom_retry(model_ref, metrics::OomRetry::CtxShrink);
port = find_free_port()?;
}
}
}
eprintln!("[llmman] {model_ref} ready on port {port}");
let backend_model_path = match process.engine() {
Engine::Mlx => model_path.path().to_str().map(|s| s.to_string()),
Engine::Sglang => Some(sglang_served_model_name(model_ref)),
Engine::LlamaServer | Engine::Vllm | Engine::VllmOmni | Engine::Ds4 | Engine::Speech => {
None
}
};
let configured_window = loaded_context_window(
process.engine(),
ctx_size,
trained_ctx,
num_parallel,
ctx_explicit,
);
let context_window = match process.engine() {
Engine::Ds4 => ds4_context_length(&state.0.client, port)
.await
.or(configured_window),
_ => configured_window,
};
let inherits_env = matches!(process, ModelProcess::Local(..));
let (thinking, budgeted) = match (&thinking_controls, process.engine()) {
(Some(controls), Engine::LlamaServer) => {
backend::llama_thinking(controls, &llama_extra_args, inherits_env)
}
_ => (None, false),
};
let effort_budget = budgeted
&& thinking_controls
.as_ref()
.is_some_and(|c| c.thinks_without_levels());
let effort_budget_message = !backend::llama_sets(
&llama_extra_args,
"--reasoning-budget-message",
inherits_env.then_some("LLAMA_ARG_THINK_BUDGET_MESSAGE"),
);
let parallel_slots = num_parallel.filter(|_| {
process.engine() == Engine::LlamaServer
&& !matches!(model_path, ModelPath::Diffusion(_))
&& !backend::llama_sets_parallel(&llama_extra_args, inherits_env)
});
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
model_ref.to_string(),
RunningModel {
process,
port,
digest,
size,
started_at: now_rfc3339(),
last_active: Instant::now(),
last_active_wall: chrono::Utc::now(),
backend_model_path,
context_window,
keep_alive: default_keep_alive(),
in_flight: 1,
parallel_slots,
decision,
effort_budget,
effort_budget_message,
thinking,
trained_ctx,
retiring: Default::default(),
},
);
pending_load_guard.release_into(&mut mgr);
drop(mgr);
metrics::record_model_load(model_ref, load_started.elapsed());
Ok((
model_ref.to_string(),
Target::Local(port),
ActivityGuard::new(state, model_ref),
))
}
fn ctx_for_load(request: Option<u32>, model: &ModelPath, ds4: bool) -> Option<u32> {
request.filter(|_| match model {
ModelPath::Gguf(..) => !ds4,
ModelPath::Diffusion(_) => true,
ModelPath::SafeTensors(_) | ModelPath::Omni(_) => false,
})
}
fn effective_ctx(num_ctx: u32, trained: Option<u32>) -> u32 {
let num_ctx = num_ctx.max(4);
trained
.filter(|t| *t > 0)
.map_or(num_ctx, |t| num_ctx.min(t))
}
fn loaded_context_window(
engine: Engine,
ctx_size: Option<u32>,
trained_ctx: Option<u32>,
num_parallel: Option<u32>,
ctx_size_explicit: bool,
) -> Option<u32> {
match engine {
Engine::LlamaServer if ctx_size == Some(0) => {
trained_ctx.map(|trained| trained / num_parallel.unwrap_or(1))
}
Engine::LlamaServer => ctx_size,
Engine::Vllm | Engine::Sglang | Engine::Ds4 => {
vllm_max_model_len(ctx_size, ctx_size_explicit)
}
Engine::VllmOmni | Engine::Mlx | Engine::Speech => None,
}
.filter(|n| *n > 0)
}
async fn backend_wire_model(state: &AppState, target: &Target, canonical_model: &str) -> String {
if let Target::Remote(remote) = target {
return remote.model.clone();
}
state
.0
.manager
.lock()
.await
.running
.get(canonical_model)
.and_then(|r| r.backend_model_path.clone())
.unwrap_or_else(|| canonical_model.to_string())
}
fn apply_default_repeat_penalty_typed(oai_req: &mut OAIChatRequest) {
if oai_req.repeat_penalty.is_none() {
oai_req.repeat_penalty = Some(DEFAULT_REPEAT_PENALTY);
}
}
fn repeat_penalty_applies(target: &Target) -> bool {
!target.is_remote()
}
const LLAMA_FIELDS: &[&str] = &[
"repeat_penalty",
"chat_template_kwargs",
"top_k",
"min_p",
"typical_p",
"n_probs",
"cache_prompt",
"samplers",
"id_slot",
"n_keep",
"n_indent",
"dynatemp_range",
"dynatemp_exponent",
"mirostat",
"mirostat_tau",
"mirostat_eta",
"dry_multiplier",
"dry_base",
"dry_allowed_length",
"dry_penalty_last_n",
"dry_sequence_breakers",
"xtc_probability",
"xtc_threshold",
"top_n_sigma",
"penalty_last_n",
"repeat_last_n",
"n_predict",
"grammar",
"grammar_lazy",
"grammar_triggers",
"preserved_tokens",
"json_schema",
"reasoning_format",
"reasoning_budget",
"thinking_forced_open",
"return_progress",
"return_tokens",
"timings_per_token",
"post_sampling_probs",
"response_fields",
"min_keep",
"t_max_predict_ms",
"t_max_prompt_ms",
"ignore_eos",
"lora",
];
fn strip_llama_fields(req: &mut serde_json::Value) {
if let Some(o) = req.as_object_mut() {
for field in LLAMA_FIELDS {
o.remove(*field);
}
}
}
fn provider_compat(remote: &RemoteTarget, req: &mut serde_json::Value) {
let Some(o) = req.as_object_mut() else {
return;
};
if let Some(kwargs) = o.remove("chat_template_kwargs") {
let level = kwargs.get("reasoning_effort").and_then(|v| v.as_str());
let enabled = kwargs.get("enable_thinking").and_then(|v| v.as_bool());
let effort = match (level, enabled, remote.wire) {
(Some(level), _, _) => Some(level.to_string()),
(None, Some(true), Wire::Anthropic) => Some("medium".to_string()),
(None, Some(false), Wire::Anthropic) => Some("none".to_string()),
_ => None,
};
if let Some(effort) = effort {
o.entry("reasoning_effort")
.or_insert(serde_json::Value::String(effort));
}
}
for field in LLAMA_FIELDS {
o.remove(*field);
}
if remote.refuses_stream_options() {
o.remove("stream_options");
}
if remote.wire == Wire::Anthropic {
anthropic::sampling_compat(&remote.model, o);
}
if remote.provider != "openai" {
return;
}
if !openai_reasoning_model(&remote.model) {
o.remove("reasoning_effort");
return;
}
if let Some(max) = o.remove("max_tokens") {
o.entry("max_completion_tokens").or_insert(max);
}
if o.get("temperature").and_then(|t| t.as_f64()) != Some(1.0) {
o.remove("temperature");
}
for field in [
"top_p",
"presence_penalty",
"frequency_penalty",
"logprobs",
"top_logprobs",
] {
o.remove(field);
}
if remote.model.contains("gpt-5") || remote.model.contains("gpt-6") {
for field in ["stop", "logit_bias", "modalities", "prediction", "audio"] {
o.remove(field);
}
}
}
fn openai_reasoning_model(model: &str) -> bool {
let name = model.rsplit('/').next().unwrap_or(model);
let o_series = name
.strip_prefix('o')
.and_then(|rest| rest.chars().next())
.is_some_and(|c| c.is_ascii_digit());
let gpt = (name.contains("gpt-5") || name.contains("gpt-6")) && !name.starts_with("gpt-5-chat");
o_series || gpt
}
type ChatBody = futures::stream::BoxStream<'static, reqwest::Result<Bytes>>;
struct ChatUpstream {
status: StatusCode,
headers: HeaderMap,
body: ChatBody,
}
impl ChatUpstream {
fn relay(resp: reqwest::Response) -> Self {
Self {
status: resp.status(),
headers: resp.headers().clone(),
body: resp.bytes_stream().boxed(),
}
}
fn translated(resp: &reqwest::Response, content_type: &'static str) -> HeaderMap {
let mut headers = resp.headers().clone();
headers.remove(reqwest::header::CONTENT_LENGTH);
headers.remove(reqwest::header::TRANSFER_ENCODING);
headers.insert(
reqwest::header::CONTENT_TYPE,
reqwest::header::HeaderValue::from_static(content_type),
);
headers
}
async fn text(self) -> String {
String::from_utf8_lossy(&collect_body(self.body).await).into_owned()
}
}
async fn collect_body(mut body: ChatBody) -> Vec<u8> {
let mut out = Vec::new();
while let Some(Ok(chunk)) = body.next().await {
out.extend_from_slice(&chunk);
}
out
}
async fn send_chat_completion<T: Serialize + ?Sized>(
client: &Client,
target: &Target,
req: &T,
response_model: &str,
) -> Result<ChatUpstream, AppError> {
let Target::Remote(remote) = target else {
let resp = target
.authorize(client.post(target.url(CHAT_COMPLETIONS_ROUTE)).json(req))
.send()
.await
.with_context(|| format!("send to {}", target.describe()))?;
let upstream = ChatUpstream::relay(resp);
if matches!(target, Target::Local(_)) && upstream.status == StatusCode::BAD_REQUEST {
return retry_merging_roles(client, target, req, upstream).await;
}
return Ok(upstream);
};
let mut req = serde_json::to_value(req).context("serialize chat request")?;
provider_compat(remote, &mut req);
if remote.wire == Wire::OpenAi {
let resp = target
.authorize(client.post(target.url(CHAT_COMPLETIONS_ROUTE)).json(&req))
.send()
.await
.with_context(|| format!("send to {}", target.describe()))?;
return Ok(ChatUpstream::relay(resp));
}
let streaming = req
.get("stream")
.and_then(serde_json::Value::as_bool)
.unwrap_or(false);
if let Some(ceiling) = remote.max_output {
for key in ["max_tokens", "max_completion_tokens"] {
if req
.get(key)
.and_then(serde_json::Value::as_u64)
.is_some_and(|n| n > u64::from(ceiling))
{
req[key] = serde_json::json!(ceiling);
}
}
}
let default_max_tokens = remote.max_output.unwrap_or(anthropic::DEFAULT_MAX_TOKENS);
let messages_req = anthropic::from_chat_request(&req, default_max_tokens)
.map_err(|e| AppError(e, StatusCode::BAD_REQUEST))?;
let mut upstream = target.authorize(client.post(target.url(anthropic::MESSAGES_ROUTE)));
if anthropic::manual_thinking(&messages_req) {
upstream = upstream.header("anthropic-beta", anthropic::INTERLEAVED_THINKING_BETA);
}
let resp = upstream
.json(&messages_req)
.send()
.await
.with_context(|| format!("send to {}", target.describe()))?;
if !resp.status().is_success() {
return Ok(ChatUpstream::relay(resp));
}
let mut converter =
anthropic::StreamConverter::new(response_model).json_tool(anthropic::json_tool_name(&req));
if !streaming {
let headers = ChatUpstream::translated(&resp, "application/json");
let lines: Vec<String> = bytes_to_lines(resp.bytes_stream()).collect().await;
for line in &lines {
converter.line(line);
}
converter.finish();
let (status, body) = match converter.error() {
Some(error) => (
StatusCode::BAD_GATEWAY,
serde_json::json!({ "error": error }),
),
None if !converter.finished() => (
StatusCode::BAD_GATEWAY,
serde_json::json!({ "error": {
"message": format!("{} closed the stream before finishing", target.describe()),
"type": "server_error",
} }),
),
None => (StatusCode::OK, converter.completion()),
};
let body = Bytes::from(serde_json::to_vec(&body).unwrap_or_default());
return Ok(ChatUpstream {
status,
headers,
body: futures::stream::once(futures::future::ready(Ok(body))).boxed(),
});
}
let headers = ChatUpstream::translated(&resp, "text/event-stream");
let body = bytes_to_lines(resp.bytes_stream())
.map(Some)
.chain(futures::stream::once(futures::future::ready(None)))
.map(move |line| {
let out = match line {
Some(line) => converter.line(&line),
None => converter.finish(),
};
Ok(Bytes::from(out))
})
.boxed();
Ok(ChatUpstream {
status: StatusCode::OK,
headers,
body,
})
}
async fn retry_merging_roles<T: Serialize + ?Sized>(
client: &Client,
target: &Target,
req: &T,
refused: ChatUpstream,
) -> Result<ChatUpstream, AppError> {
let ChatUpstream {
status,
headers,
body,
} = refused;
let reply = Bytes::from(collect_body(body).await);
let sent = serde_json::to_vec(req).context("serialize chat request")?;
if let Some(merged) = openai::merged_after_refusal(target, status, &reply, &sent) {
let resp = target
.authorize(client.post(target.url(CHAT_COMPLETIONS_ROUTE)))
.header(reqwest::header::CONTENT_TYPE, "application/json")
.body(merged)
.send()
.await
.with_context(|| format!("send to {}", target.describe()))?;
return Ok(ChatUpstream::relay(resp));
}
Ok(ChatUpstream {
status,
headers,
body: futures::stream::once(futures::future::ready(Ok(reply))).boxed(),
})
}
async fn post_chat(
client: &Client,
target: &Target,
oai_req: &mut OAIChatRequest,
) -> Result<ChatBody, AppError> {
if repeat_penalty_applies(target) {
apply_default_repeat_penalty_typed(oai_req);
} else {
oai_req.repeat_penalty = None;
}
let upstream = send_chat_completion(client, target, &*oai_req, &oai_req.model).await?;
chat_body(target, upstream).await
}
async fn chat_body(target: &Target, upstream: ChatUpstream) -> Result<ChatBody, AppError> {
if !upstream.status.is_success() {
let status = upstream.status;
let body = upstream.text().await;
let message = format!("{} {status}: {body}", target.describe());
let (error, status) = match target {
Target::Local(_) => match context_overflow_message(status, &body) {
Some(refusal) => (
anyhow::Error::new(ContextOverflow { message, refusal }),
StatusCode::BAD_REQUEST,
),
None => (anyhow!("{message}"), remote_status(target, status)),
},
_ => (anyhow!("{message}"), remote_status(target, status)),
};
return Err(AppError(error, status));
}
Ok(upstream.body)
}
#[derive(Debug)]
struct ContextOverflow {
message: String,
refusal: String,
}
impl std::fmt::Display for ContextOverflow {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.message)
}
}
impl std::error::Error for ContextOverflow {}
fn context_overflow_message(status: StatusCode, body: &str) -> Option<String> {
if status != StatusCode::BAD_REQUEST {
return None;
}
let json = &body[body.find('{')?..];
let value: serde_json::Value = serde_json::from_str(json).ok()?;
let error = &value["error"];
let message = error["message"]
.as_str()
.or_else(|| value["message"].as_str())
.unwrap_or("");
let overflow = error["type"] == "exceed_context_size_error"
|| message.contains("exceeds the available context size")
|| message.contains("maximum context length is");
overflow.then(|| message.to_string())
}
fn remote_status(target: &Target, upstream: StatusCode) -> StatusCode {
match target {
Target::Local(_) if upstream == StatusCode::BAD_REQUEST => upstream,
Target::Local(_) => StatusCode::INTERNAL_SERVER_ERROR,
Target::Peer(_) => upstream,
Target::Remote(_) if upstream.is_client_error() => upstream,
Target::Remote(_) => StatusCode::BAD_GATEWAY,
}
}
#[derive(Deserialize)]
struct PropsQuery {
#[serde(default)]
model: Option<String>,
}
const PROPS_TIMEOUT: Duration = Duration::from_secs(5);
async fn handle_props(
State(state): State<AppState>,
Query(query): Query<PropsQuery>,
) -> Result<Response, AppError> {
let Some(model) = query
.model
.as_deref()
.map(str::trim)
.filter(|m| !m.is_empty())
else {
return Ok(Json(router_props()).into_response());
};
let claimed = if crate::providers::is_remote_ref(model) {
None
} else {
claim_llama_server(&state, &running_key(&state, model)).await
};
let (port, guard) = claimed.ok_or_else(|| {
AppError::status(
StatusCode::NOT_FOUND,
format!("no props for '{model}': it is not a loaded llama-server model"),
)
})?;
let props = backend_json(&state.0.client, port, "/props", PROPS_TIMEOUT).await;
guard.release_idle();
let props = props.map_err(|e| {
AppError::status(
StatusCode::BAD_GATEWAY,
format!("inference backend /props: {e}"),
)
})?;
Ok(Json(props).into_response())
}
async fn claim_llama_server(state: &AppState, key: &str) -> Option<(u16, ActivityGuard)> {
let (port, guard) = match check_running(state, key).await {
Some(claimed) => claimed,
None => {
let digest = OciStore::open(&state.0.store_path)
.and_then(|store| store.find(key))
.ok()?
.digest;
let (_, port, guard) = check_running_by_digest(state, key, &digest, None).await?;
(port, guard)
}
};
if local_engine(state, &Target::Local(port)).await == Some(Engine::LlamaServer) {
return Some((port, guard));
}
guard.release_idle();
None
}
fn router_props() -> serde_json::Value {
serde_json::json!({
"role": "router",
"total_slots": 0,
"model_path": "",
"chat_template": "",
"bos_token": "",
"eos_token": "",
"build_info": env!("LLMMAN_VERSION"),
"modalities": { "vision": false, "audio": false },
"default_generation_settings": {
"id": 0,
"id_task": 0,
"n_ctx": 4096,
"speculative": false,
"is_processing": false,
"params": {
"n_predict": -1,
"seed": 0,
"temperature": 0.8,
"dynatemp_range": 0.0,
"dynatemp_exponent": 1.0,
"top_k": 40,
"top_p": 0.95,
"min_p": 0.05,
"top_n_sigma": 0.0,
"xtc_probability": 0.0,
"xtc_threshold": 0.1,
"typ_p": 1.0,
"repeat_last_n": 64,
"repeat_penalty": 1.0,
"presence_penalty": 0.0,
"frequency_penalty": 0.0,
"dry_multiplier": 0.0,
"dry_base": 1.75,
"dry_allowed_length": 2,
"dry_penalty_last_n": -1,
"dry_sequence_breakers": [],
"mirostat": 0,
"mirostat_tau": 5.0,
"mirostat_eta": 0.1,
"stop": [],
"max_tokens": -1,
"n_keep": 0,
"n_discard": 0,
"ignore_eos": false,
"stream": true,
"logit_bias": [],
"n_probs": 0,
"min_keep": 0,
"grammar": "",
"grammar_lazy": false,
"grammar_triggers": [],
"preserved_tokens": [],
"chat_format": "",
"reasoning_format": "",
"reasoning_in_content": false,
"generation_prompt": "",
"samplers": ["top_k", "top_p", "min_p", "temperature"],
"backend_sampling": false,
"speculative.n_max": 16,
"speculative.n_min": 5,
"speculative.p_min": 0.9,
"timings_per_token": false,
"post_sampling_probs": false,
"lora": []
},
"prompt": "",
"next_token": {
"has_next_token": false,
"has_new_line": false,
"n_remain": 0,
"n_decoded": 0,
"stopping_word": ""
}
}
})
}
#[derive(Serialize)]
struct ProviderSummary {
id: String,
name: String,
base_url: String,
#[serde(skip_serializing_if = "Option::is_none")]
key_env: Option<String>,
wire: &'static str,
key_set: bool,
key_usable: bool,
key_optional: bool,
models: usize,
}
impl ProviderSummary {
fn new(state: &AppState, p: &crate::providers::Provider) -> Self {
Self {
id: p.id.clone(),
name: p.name.clone(),
base_url: p.base_url.clone(),
key_env: p.key_env.clone(),
wire: p.wire.as_str(),
key_set: p.api_key().is_some(),
key_usable: daemon_key_usable(state, p),
key_optional: p.key_optional,
models: p.models.len(),
}
}
}
fn daemon_key_usable(state: &AppState, provider: &crate::providers::Provider) -> bool {
provider.api_key().is_some() && daemon_key_spendable(state, None)
}
#[derive(Serialize)]
struct ProvidersResponse {
providers: Vec<ProviderSummary>,
}
#[derive(Serialize)]
struct ProviderResponse {
id: String,
name: String,
base_url: String,
#[serde(skip_serializing_if = "Option::is_none")]
key_env: Option<String>,
wire: &'static str,
key_set: bool,
key_usable: bool,
key_optional: bool,
models: Vec<ProviderModelResponse>,
}
#[derive(Serialize)]
struct ProviderModelResponse {
id: String,
#[serde(skip_serializing_if = "Option::is_none")]
cost: Option<ProviderCostResponse>,
#[serde(skip_serializing_if = "Option::is_none")]
thinking: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
context: Option<u64>,
#[serde(skip_serializing_if = "Option::is_none")]
output: Option<u32>,
}
#[derive(Serialize)]
struct ProviderCostResponse {
input: f64,
output: f64,
#[serde(skip_serializing_if = "Option::is_none")]
cache_read: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
cache_write: Option<f64>,
#[serde(skip_serializing_if = "Option::is_none")]
reasoning: Option<f64>,
}
impl ProviderResponse {
fn new(state: &AppState, p: &crate::providers::Provider) -> Self {
Self {
id: p.id.clone(),
name: p.name.clone(),
base_url: p.base_url.clone(),
key_env: p.key_env.clone(),
wire: p.wire.as_str(),
key_set: p.api_key().is_some(),
key_usable: daemon_key_usable(state, p),
key_optional: p.key_optional,
models: p
.models
.iter()
.map(|m| ProviderModelResponse {
id: m.id.clone(),
cost: m.cost.as_ref().map(|c| ProviderCostResponse {
input: c.input,
output: c.output,
cache_read: c.cache_read,
cache_write: c.cache_write,
reasoning: c.reasoning,
}),
thinking: m.thinking.clone(),
context: m.max_context,
output: m.max_output,
})
.collect(),
}
}
}
#[derive(Deserialize)]
struct ProviderQuery {
#[serde(default)]
model: Option<String>,
}
impl ProviderQuery {
fn model(&self) -> Option<&str> {
self.model
.as_deref()
.map(str::trim)
.filter(|m| !m.is_empty())
}
}
async fn handle_llmman_providers(
State(state): State<AppState>,
) -> Result<impl IntoResponse, AppError> {
let catalog = provider_catalog().await?;
Ok(Json(ProvidersResponse {
providers: catalog
.iter()
.map(|p| ProviderSummary::new(&state, p))
.collect(),
}))
}
async fn handle_llmman_provider(
State(state): State<AppState>,
UrlPath(id): UrlPath<String>,
Query(query): Query<ProviderQuery>,
) -> Result<impl IntoResponse, AppError> {
let catalog = match query.model() {
Some(model) => provider_catalog_listing(&id, model).await?,
None => provider_catalog().await?,
};
let provider = catalog.get(&id).ok_or_else(|| {
AppError(
crate::providers::unknown_provider_error(&id, &catalog),
StatusCode::NOT_FOUND,
)
})?;
let mut response = ProviderResponse::new(&state, provider);
if provider.key_optional && provider.models.is_empty() {
response.models = configured_provider_models(&state, provider)
.await
.into_iter()
.map(|id| ProviderModelResponse {
id,
cost: None,
thinking: None,
context: None,
output: None,
})
.collect();
}
Ok(Json(response))
}
#[derive(Deserialize)]
struct ProviderKeyBody {
api_key: String,
}
#[derive(Serialize)]
struct ProviderKeyResponse {
id: String,
key_usable: bool,
#[serde(skip_serializing_if = "Option::is_none")]
key_env: Option<String>,
env_override: bool,
}
async fn handle_set_provider_key(
State(state): State<AppState>,
UrlPath(id): UrlPath<String>,
headers: HeaderMap,
Json(body): Json<ProviderKeyBody>,
) -> Result<Json<ProviderKeyResponse>, AppError> {
let key = body.api_key.trim();
if key.is_empty() || key.contains(['\n', '\r']) {
return Err(AppError(
anyhow!("api_key must be one non-blank line"),
StatusCode::BAD_REQUEST,
));
}
store_provider_key(state, headers, id, Some(key.to_string())).await
}
async fn handle_delete_provider_key(
State(state): State<AppState>,
UrlPath(id): UrlPath<String>,
headers: HeaderMap,
) -> Result<Json<ProviderKeyResponse>, AppError> {
store_provider_key(state, headers, id, None).await
}
async fn store_provider_key(
state: AppState,
headers: HeaderMap,
id: String,
key: Option<String>,
) -> Result<Json<ProviderKeyResponse>, AppError> {
if let Some(reason) = provider_key_write_refusal(&state, &headers) {
return Err(AppError(anyhow!(reason), StatusCode::FORBIDDEN));
}
let catalog = provider_catalog().await?;
if catalog.get(&id).is_none() {
return Err(AppError(
crate::providers::unknown_provider_error(&id, &catalog),
StatusCode::NOT_FOUND,
));
}
let path = crate::config::provider_key_file()
.ok_or_else(|| anyhow!("no home directory to hold llmman.conf"))?;
let provider_id = id.clone();
let written = tokio::task::spawn_blocking(move || {
crate::cmd::config::write_provider_key(&path, &provider_id, key.as_deref())
})
.await
.map_err(|e| anyhow!("writing llmman.conf: {e}"))?;
if let Err(e) = written {
eprintln!("[llmman] provider key for {id} not updated: {e:#}");
return Err(AppError(
anyhow!(
"could not update the provider key in llmman.conf; \
llmman serve's log says why"
),
StatusCode::INTERNAL_SERVER_ERROR,
));
}
let provider = catalog.get(&id).expect("looked up above");
let key_env = provider.key_env.clone();
let env_override = key_env
.as_deref()
.and_then(crate::providers::key_from_env)
.is_some();
Ok(Json(ProviderKeyResponse {
key_usable: daemon_key_usable(&state, provider),
id,
key_env,
env_override,
}))
}
fn provider_key_write_refusal(state: &AppState, headers: &HeaderMap) -> Option<&'static str> {
if daemon_key_spendable(state, Some(headers)) {
None
} else if is_cross_site(Some(headers)) {
Some("provider keys cannot be set from another site's page")
} else {
Some(
"llmman serve is reachable beyond loopback without LLMMAN_API_KEYS, \
so provider keys cannot be set over HTTP; use `llmman config set` on that computer",
)
}
}
const CONFIGURED_MODELS_TIMEOUT: Duration = Duration::from_secs(5);
async fn configured_provider_models(
state: &AppState,
provider: &crate::providers::Provider,
) -> Vec<String> {
#[derive(Deserialize)]
struct ModelsResponse {
#[serde(default)]
data: Vec<ModelEntry>,
}
#[derive(Deserialize)]
struct ModelEntry {
id: String,
}
if provider.wire != Wire::OpenAi {
return Vec::new();
}
let url = provider.url("/v1/models");
let mut req = state.0.client.get(&url).timeout(CONFIGURED_MODELS_TIMEOUT);
if daemon_key_usable(state, provider) {
if let Some(key) = provider.api_key() {
req = req.bearer_auth(key);
}
}
let listed = async {
let resp = req.send().await?.error_for_status()?;
resp.json::<ModelsResponse>().await
}
.await;
match listed {
Ok(models) => {
let mut ids: Vec<String> = models.data.into_iter().map(|m| m.id).collect();
ids.sort_unstable();
ids.dedup();
ids
}
Err(e) => {
crate::debug_log!("provider {}: GET {url} failed: {e}", provider.id);
Vec::new()
}
}
}
fn default_allowed_origins() -> Vec<String> {
let mut origins = Vec::new();
for host in ["localhost", "127.0.0.1", "0.0.0.0", "[::1]"] {
for scheme in ["http", "https"] {
origins.push(format!("{scheme}://{host}"));
origins.push(format!("{scheme}://{host}:*"));
}
}
origins
}
fn allowed_origins_from_env() -> Vec<String> {
let mut origins: Vec<String> = std::env::var("LLMMAN_ORIGINS")
.ok()
.map(|raw| {
raw.split(',')
.map(str::trim)
.filter(|s| !s.is_empty())
.map(str::to_string)
.collect()
})
.unwrap_or_default();
origins.extend(default_allowed_origins());
origins
}
fn origin_matches(origin: &str, pattern: &str) -> bool {
match pattern.split_once('*') {
Some((prefix, suffix)) if !suffix.contains('*') => {
origin.len() >= prefix.len() + suffix.len()
&& origin.starts_with(prefix)
&& origin.ends_with(suffix)
}
Some(_) => false,
None => origin == pattern,
}
}
fn cors_layer() -> tower_http::cors::CorsLayer {
let patterns = allowed_origins_from_env();
tower_http::cors::CorsLayer::new()
.allow_methods(tower_http::cors::AllowMethods::any())
.allow_headers(tower_http::cors::AllowHeaders::any())
.allow_origin(tower_http::cors::AllowOrigin::predicate(
move |origin, _parts| {
origin
.to_str()
.is_ok_and(|origin| patterns.iter().any(|p| origin_matches(origin, p)))
},
))
}
pub const MEDIAGEN_ENV_PASSTHROUGH_VARS: &[&str] = &["MEDIAGEN_DUMP", "MEDIAGEN_VAE_TILE"];
async fn track_metrics(req: Request, next: Next) -> Response {
let route = req.extensions().get::<MatchedPath>().cloned();
let started = Instant::now();
let response = next.run(req).await;
let Some(route) = route else {
return response;
};
metrics::record_request(
route.as_str(),
response.status().as_u16(),
started.elapsed(),
);
let (parts, body) = response.into_parts();
if body.size_hint().exact().is_some() {
metrics::record_response_body(route.as_str(), started.elapsed());
return Response::from_parts(parts, body);
}
let timer = ResponseBodyTimer {
route: route.as_str().to_string(),
started,
};
let body = Body::from_stream(body.into_data_stream().map(move |chunk| {
let _timer = &timer;
chunk
}));
Response::from_parts(parts, body)
}
async fn record_prompt(State(state): State<AppState>, req: Request, next: Next) -> Response {
use axum::extract::FromRequestParts;
let Some(log) = state.0.prompt_log.as_deref() else {
return next.run(req).await;
};
let route = req
.extensions()
.get::<MatchedPath>()
.map(|p| p.as_str().to_string())
.filter(|r| crate::promptlog::is_generation_route(r));
let Some(route) = route.filter(|r| {
req.method() == axum::http::Method::POST
&& !(r.starts_with("/api/") && forged_cross_site(req.headers()))
}) else {
return next.run(req).await;
};
let (mut parts, body) = req.into_parts();
let selected_model = if route == "/gemini/:model/*gemini_path" {
let model = UrlPath::<(String, String)>::from_request_parts(&mut parts, &())
.await
.ok()
.filter(|UrlPath((_, path))| {
gemini_method(path).is_some_and(|(_, method)| method.generates())
})
.and_then(|UrlPath((encoded_model, _))| {
base64::engine::general_purpose::URL_SAFE_NO_PAD
.decode(encoded_model)
.ok()
})
.and_then(|bytes| String::from_utf8(bytes).ok());
let Some(model) = model else {
return next.run(Request::from_parts(parts, body)).await;
};
Some(model)
} else {
None
};
let body = match Bytes::from_request(Request::from_parts(parts.clone(), body), &()).await {
Ok(body) => body,
Err(rejection) => return rejection.into_response(),
};
let client = parts
.headers
.get(axum::http::header::USER_AGENT)
.and_then(|v| v.to_str().ok());
if let Some(mut entry) = crate::promptlog::entry(&route, &body, client, now_rfc3339()) {
if let Some(model) = selected_model {
entry.model = model;
}
match crate::promptlog::append(log, &entry) {
Ok(()) => {
parts
.extensions
.insert(crate::promptlog::PromptId(entry.id));
}
Err(e) => eprintln!("[llmman] warning: prompt log {}: {e}", log.display()),
}
}
next.run(Request::from_parts(parts, Body::from(body))).await
}
async fn accept_any_content_type(mut req: Request, next: Next) -> Response {
if forged_cross_site(req.headers()) {
return cross_site_refused().into_response();
}
if !has_json_content_type(req.headers()) {
req.headers_mut().insert(
axum::http::header::CONTENT_TYPE,
axum::http::HeaderValue::from_static("application/json"),
);
}
next.run(req).await
}
fn cross_site_refused() -> AppError {
AppError::status(StatusCode::FORBIDDEN, "cross-site request refused")
}
fn upload_route<H, T>(handler: H) -> axum::routing::MethodRouter<AppState>
where
H: axum::handler::Handler<T, AppState>,
T: 'static,
{
post(handler)
.layer(DefaultBodyLimit::max(TRANSCRIPTION_BODY_LIMIT_BYTES))
.layer(middleware::from_fn(|req: Request, next: Next| async move {
if forged_cross_site(req.headers()) {
return cross_site_refused().into_response();
}
next.run(req).await
}))
}
fn has_json_content_type(headers: &HeaderMap) -> bool {
headers
.get(axum::http::header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.is_some_and(is_json_content_type)
}
fn forged_cross_site(headers: &HeaderMap) -> bool {
!has_json_content_type(headers) && foreign_origin(headers)
}
fn foreign_origin(headers: &HeaderMap) -> bool {
headers
.get(axum::http::header::ORIGIN)
.and_then(|v| v.to_str().ok())
.is_some_and(|origin| {
!allowed_origins_from_env()
.iter()
.any(|pattern| origin_matches(origin, pattern))
})
}
fn is_json_content_type(value: &str) -> bool {
let essence = value
.split(';')
.next()
.unwrap_or_default()
.trim()
.to_ascii_lowercase();
essence == "application/json"
|| (essence.starts_with("application/") && essence.ends_with("+json"))
}
struct ResponseBodyTimer {
route: String,
started: Instant,
}
impl Drop for ResponseBodyTimer {
fn drop(&mut self) {
metrics::record_response_body(&self.route, self.started.elapsed());
}
}
async fn handle_metrics(State(state): State<AppState>) -> impl IntoResponse {
let (models_loaded, models_loading, models) = {
let mut mgr = state.0.manager.lock().await;
let models = mgr
.running
.iter_mut()
.map(|(model, m)| metrics::ModelState {
model: model.clone(),
engine: m.engine_label(),
up: m.process.is_alive(),
requests_in_flight: u64::from(m.in_flight),
requests_queued: m
.parallel_slots
.map(|slots| u64::from(m.in_flight.saturating_sub(slots))),
})
.collect();
(mgr.running.len() as u64, mgr.pending_loads as u64, models)
};
let snapshot = metrics::Snapshot {
version: env!("LLMMAN_VERSION").to_string(),
start_time_seconds: metrics::process_start_seconds(),
scheduling_requests_in_flight: PENDING_REQUESTS.load(std::sync::atomic::Ordering::SeqCst)
as u64,
scheduling_capacity: state.0.max_queue.max(1) as u64,
models_loaded,
models_loading,
models,
};
(
[("content-type", "text/plain; version=0.0.4; charset=utf-8")],
metrics::render(&snapshot),
)
}
pub fn run(args: &ServeArgs) -> anyhow::Result<()> {
if args.detach {
if crate::daemon::server_alive() {
crate::daemon::check_server_identity()?;
}
crate::daemon::ensure_server_with_args(
args.model.as_deref().unwrap_or(""),
&args.detached_child_args(),
)?;
return crate::daemon::check_server_identity();
}
if let (Some(port), Some(model)) = (args.port, &args.model) {
return tokio::runtime::Runtime::new()?.block_on(model_backend(model, port, args));
}
tokio::runtime::Runtime::new()?.block_on(serve_async(args))
}
impl ServeArgs {
fn detached_child_args(&self) -> Vec<&str> {
let mut args = vec!["--runtime", self.runtime.as_str()];
if self.no_hardened {
args.push("--no-hardened");
}
for (flag, value) in [
("--llama-cpp-version", self.llama_cpp_version.as_deref()),
("--vllm-version", self.vllm_version.as_deref()),
("--vllm-image", self.vllm_image.as_deref()),
("--sglang-version", self.sglang_version.as_deref()),
] {
if let Some(value) = value {
args.extend([flag, value]);
}
}
args
}
}
pub fn llama_libs(
runtime: Runtime,
pinned_version: Option<&str>,
) -> anyhow::Result<(Runtime, PathBuf)> {
let (source, bin) = runtime::resolve_local(runtime, pinned_version)?;
let dir = crate::mediagen::ffi::lib_dir_of(&bin).ok_or_else(|| {
anyhow!(
"no ggml/llama shared libraries next to {}; media generation needs a llama.cpp release or installed build",
bin.display()
)
})?;
Ok((source, dir))
}
async fn model_backend(model_ref: &str, port: u16, args: &ServeArgs) -> anyhow::Result<()> {
let store_path = default_store()?;
let cache_path = crate::default_cache()?;
match resolve_model(&store_path, &cache_path, model_ref)? {
ModelPath::Diffusion(paths) => mediagen_backend(model_ref, paths, port, args).await,
ModelPath::Gguf(path, _) => {
if !is_speech_model(&crate::gguf::read_info(&path)?) {
anyhow::bail!(
"{model_ref} is a GGUF for llama-server, not a Canary GGUF; \
`llmman serve MODEL --port PORT` runs only a diffusion model or a Canary GGUF"
);
}
speech_backend(model_ref, &path, port, args).await
}
other => anyhow::bail!(
"{model_ref} is a {} model; `llmman serve MODEL --port PORT` runs only \
a diffusion model or a Canary GGUF",
other.format()
),
}
}
async fn load_ggml_api(
args: &ServeArgs,
) -> anyhow::Result<(&'static crate::mediagen::ffi::Api, PathBuf)> {
let pinned = runtime::llama_cpp_pin(args.llama_cpp_version.as_deref());
let runtime = args.runtime;
let lib_dir = tokio::task::spawn_blocking(move || {
llama_libs(runtime, pinned.as_deref()).map(|(_, lib_dir)| lib_dir)
})
.await??;
let api = Box::leak(Box::new(crate::mediagen::ffi::Api::load(&lib_dir)?));
Ok((api, lib_dir))
}
fn backend_threads() -> i32 {
std::env::var("LLAMA_ARG_THREADS")
.ok()
.and_then(|v| v.parse().ok())
.or_else(|| threads_from_env_or_host().map(|n| n as i32))
.unwrap_or_else(|| std::thread::available_parallelism().map_or(4, |n| n.get() as i32))
}
async fn mediagen_backend(
model_ref: &str,
paths: crate::modelpack::DiffusionPaths,
port: u16,
args: &ServeArgs,
) -> anyhow::Result<()> {
if paths.text_encoder.is_none() && crate::mediagen::needs_text_encoder(&paths.model) {
anyhow::bail!("{model_ref}: no text encoder in the model pack");
}
let (api, lib_dir) = load_ggml_api(args).await?;
let params = crate::mediagen::ContextParams {
model: paths.model.clone(),
vae: paths.vae.clone(),
audio_vae: paths.audio_vae.clone(),
text_proj: paths.text_proj.clone(),
text_model: paths.text_encoder.clone(),
files: paths.files.clone(),
use_gpu: true,
text_gpu_layers: std::env::var("LLAMA_ARG_N_GPU_LAYERS")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(999),
n_threads: backend_threads(),
flash_attn: flash_attention_from_env().as_deref() != Some("off"),
};
let ctx = tokio::task::spawn_blocking(move || crate::mediagen::Context::init(api, ¶ms))
.await?
.with_context(|| {
format!(
"loading the model with the llama.cpp libraries in {} (an old build? \
remove it from PATH or pass --llama-cpp-version to use a release)",
lib_dir.display()
)
})?;
let router = crate::mediagen::server::router(
ctx,
model_ref.to_string(),
paths.model.to_string_lossy().into_owned(),
);
crate::mediagen::http::serve("mediagen", router, (args.host, port).into()).await
}
async fn speech_backend(
model_ref: &str,
path: &Path,
port: u16,
args: &ServeArgs,
) -> anyhow::Result<()> {
let (api, lib_dir) = load_ggml_api(args).await?;
let n_threads = backend_threads();
let model_path = path.to_path_buf();
let worker = tokio::task::spawn_blocking(move || {
let info = crate::gguf::read_info(&model_path)?;
let hp = crate::speech::model::CanaryHParams::from_info(&info)?;
let max_nodes = crate::speech::graph_nodes(hp.enc_n_layers.max(hp.dec_n_layers) as usize);
let be = crate::mediagen::backend::Backend::new(api, false, n_threads, max_nodes)?;
let model = crate::speech::model::load(&model_path, &be)?;
crate::speech::server::Worker::new(be, model)
})
.await?
.with_context(|| {
format!(
"loading {} with the llama.cpp libraries in {}",
path.display(),
lib_dir.display()
)
})?;
let router = crate::speech::server::router(
worker,
model_ref.to_string(),
path.to_string_lossy().into_owned(),
);
crate::mediagen::http::serve("speech", router, (args.host, port).into()).await
}
fn mediagen_backend_args(model_ref: &str, port: u16) -> [String; 4] {
[
"serve".into(),
model_ref.into(),
"--port".into(),
port.to_string(),
]
}
async fn spawn_mediagen_backend(
model_ref: &str,
port: u16,
state: &AppState,
workload: Workload,
) -> anyhow::Result<(tokio::process::Child, OutputTail)> {
let exe = std::env::current_exe().context("locating the llmman binary")?;
let mut cmd = tokio::process::Command::new(&exe);
cmd.args(mediagen_backend_args(model_ref, port));
let runtime = state.0.runtime.resolve_for(workload).await?.runtime();
cmd.args(["--runtime", runtime.as_str()]);
cmd.args([
"--llama-cpp-version",
state.0.llama_cpp_version.as_deref().unwrap_or("latest"),
]);
for var in GPU_VISIBLE_DEVICE_VARS
.iter()
.chain(LLAMA_CPP_ENV_PASSTHROUGH_VARS)
.chain(MEDIAGEN_ENV_PASSTHROUGH_VARS)
{
if let Ok(val) = std::env::var(var) {
cmd.env(var, val);
}
}
crate::debug_log!("spawning {}: {:?}", exe.display(), cmd);
cmd.stdout(std::process::Stdio::piped());
cmd.stderr(std::process::Stdio::piped());
let mut child = cmd
.kill_on_drop(true)
.spawn()
.with_context(|| format!("spawn media generation backend from {}", exe.display()))?;
let tail = tail_child_output(&mut child);
Ok((child, tail))
}
const GENERATION_BODY_LIMIT_BYTES: usize = 32 * 1024 * 1024;
fn build_router(app_state: AppState, metrics_enabled: bool) -> Router {
let app = Router::new()
.route("/", get(webui::handle_root))
.route("/ui/*path", get(webui::handle_asset))
.fallback(webui::handle_page)
.route("/props", get(handle_props))
.route("/llmman/providers", get(handle_llmman_providers))
.route("/llmman/providers/:id", get(handle_llmman_provider))
.route(
"/llmman/providers/:id/key",
put(handle_set_provider_key).delete(handle_delete_provider_key),
)
.route("/llmman/node", get(aggregation::handle_node))
.route("/llmman/search", get(search::handle_search))
.route("/llmman/search/popular", get(search::handle_popular))
.route("/llmman/search/model", get(search::handle_model))
.route("/llmman/search/avatar", get(search::handle_avatar))
.route("/llmman/shell", get(shell::handle_shell))
.merge(ollama_router())
.route("/v1/models", get(handle_openai_models))
.route("/v1/embeddings", post(handle_openai_embeddings))
.route(
"/v1/systemone",
post(handle_openai_systemone).layer(DefaultBodyLimit::max(GENERATION_BODY_LIMIT_BYTES)),
)
.route(
"/v1/audio/transcriptions",
upload_route(handle_openai_transcriptions),
)
.route(
"/audio/transcriptions",
upload_route(handle_openai_transcriptions),
)
.route("/v1/images/generations", post(handle_openai_images))
.route("/v1/videos", post(handle_openai_videos))
.route("/v1/videos/:id", get(handle_openai_video_get))
.route("/v1/videos/:id/content", get(handle_openai_video_get))
.route("/v1/audio/speech", post(handle_openai_speech))
.route("/tts", upload_route(handle_tts))
.route(realtime::REALTIME_ROUTE, get(realtime::handle_realtime));
let log_requests = |router: Router<AppState>| {
router
.layer(middleware::from_fn_with_state(
app_state.clone(),
usage::record_usage,
))
.layer(middleware::from_fn_with_state(
app_state.clone(),
record_prompt,
))
};
let generation = Router::new()
.route("/v1/chat/completions", post(handle_openai_chat))
.route("/v1/completions", post(handle_openai_completions))
.route("/v1/messages", post(anthropic::handle_anthropic_messages))
.route("/v1/responses", post(responses::handle_openai_responses))
.route(
"/v1/responses/input_tokens",
post(responses::handle_openai_responses_input_tokens),
)
.route("/gemini/:model/*gemini_path", post(handle_pinned_gemini))
.merge(
Router::new()
.route("/api/chat", post(handle_ollama_chat))
.route("/api/generate", post(handle_ollama_generate))
.layer(middleware::from_fn(accept_any_content_type)),
);
let app = log_requests(app)
.merge(log_requests(generation).layer(DefaultBodyLimit::max(GENERATION_BODY_LIMIT_BYTES)));
let app = if metrics_enabled {
app.layer(middleware::from_fn(track_metrics))
} else {
app
};
let require_key = || middleware::from_fn_with_state(app_state.clone(), auth::require_key);
app.layer(require_key())
.layer(cors_layer())
.merge(metrics_router(metrics_enabled).layer(require_key()))
.with_state(app_state)
}
fn ollama_router() -> Router<AppState> {
Router::new()
.route("/api/version", get(handle_version))
.route("/api/tags", get(handle_tags))
.route("/api/ps", get(handle_ps))
.route("/api/show", post(handle_show))
.route("/api/pull", post(handle_pull))
.route("/api/push", post(handle_push))
.route("/api/delete", delete(handle_delete))
.route("/api/copy", post(handle_copy))
.route("/api/create", post(handle_create))
.route(
"/api/blobs/:digest",
axum::routing::head(handle_blob_head)
.post(handle_blob_upload)
.layer(DefaultBodyLimit::disable()),
)
.route("/api/embed", post(handle_embed))
.route("/api/embeddings", post(handle_embeddings))
.layer(middleware::from_fn(accept_any_content_type))
}
fn metrics_router(enabled: bool) -> Router<AppState> {
if !enabled {
return Router::new();
}
Router::new()
.route("/metrics", get(handle_metrics))
.layer(middleware::from_fn(track_metrics))
}
#[derive(Debug, PartialEq, Eq)]
enum PullOnlyStep {
Done,
ResolveCpuOnly,
PullEngine(crate::container::ContainerEngine),
}
fn pull_only_step(
resolved: &runtime::Resolved,
classify: impl FnOnce() -> anyhow::Result<crate::container::ContainerEngine>,
) -> anyhow::Result<PullOnlyStep> {
if resolved.ociman().is_none() {
return Ok(PullOnlyStep::Done);
}
Ok(match classify()? {
crate::container::ContainerEngine::Speech => PullOnlyStep::ResolveCpuOnly,
crate::container::ContainerEngine::LlamaServer => PullOnlyStep::Done,
engine => PullOnlyStep::PullEngine(engine),
})
}
fn pull_only_engine(model: Option<&str>) -> anyhow::Result<crate::container::ContainerEngine> {
let model = model
.map(crate::hybrid::local_half)
.filter(|m| !crate::providers::is_remote_ref(m));
let Some(model) = model else {
return Ok(crate::container::ContainerEngine::LlamaServer);
};
let model_ref = crate::shortnames::resolve_ollama_api(model)?;
let store_path = default_store()?;
let format = crate::modelpack::stored_format(&store_path, &model_ref).with_context(|| {
format!("--pull-only: pull {model_ref} first to learn which backend it needs")
})?;
Ok(match format {
crate::modelpack::ModelFormat::SafeTensors
if safetensors_engine_from_env() == SafetensorsEngine::Sglang =>
{
crate::container::ContainerEngine::Sglang
}
crate::modelpack::ModelFormat::SafeTensors => crate::container::ContainerEngine::Vllm,
crate::modelpack::ModelFormat::Omni => crate::container::ContainerEngine::VllmOmni,
crate::modelpack::ModelFormat::Gguf => {
let speech = match resolve_model(&store_path, &crate::default_cache()?, &model_ref)? {
ModelPath::Gguf(path, _) => {
crate::gguf::read_info(&path).is_ok_and(|info| is_speech_model(&info))
}
_ => false,
};
if speech {
crate::container::ContainerEngine::Speech
} else {
crate::container::ContainerEngine::LlamaServer
}
}
crate::modelpack::ModelFormat::Diffusion => crate::container::ContainerEngine::LlamaServer,
})
}
async fn serve_async(_args: &ServeArgs) -> anyhow::Result<()> {
let requested = _args.runtime;
let installs_mlx = match backend::mlx_source(requested) {
Some(fallback) if use_mlx_for_safetensors() && requested.ociman().is_none() => {
Some(fallback)
}
_ => None,
};
if requested == Runtime::Path && _args.pull_only {
anyhow::bail!("--pull-only: --runtime path runs what is on PATH; nothing to pull");
}
backend::check_extra_args_env()?;
let shell_token = shell::token_for(&crate::daemon::bind_addr())?;
let shell = shell::Policy::from_env(&shell_token)?;
let llama_cpp_version = runtime::llama_cpp_pin(_args.llama_cpp_version.as_deref());
let hardened = hardened_from(
_args.no_hardened,
std::env::var("LLMMAN_NO_HARDENED").ok().as_deref(),
);
let resolved = {
let pin = llama_cpp_version.clone();
tokio::task::spawn_blocking(move || runtime::resolve(requested, pin.as_deref(), hardened))
.await
.context("resolve runtime task panicked")?
};
let resolved = match resolved {
Ok(resolved) => Some(resolved),
Err(e) if _args.pull_only => return Err(e),
Err(e) => {
eprintln!(
"[llmman] warning: could not set up the llama.cpp runtime at startup ({e:#}); \
the first model load that needs it will retry"
);
None
}
};
let runtime = runtime::Lazy::new(
requested,
llama_cpp_version.clone(),
hardened,
resolved.clone(),
);
if let Some(fallback) = installs_mlx {
let installed =
tokio::task::spawn_blocking(move || crate::mlx_release::ensure_mlx_server(fallback))
.await
.context("ensure mlx_lm.server task panicked")?;
match installed {
Ok(_) => {}
Err(e) if _args.pull_only => return Err(e.context("install mlx-lm")),
Err(e) => eprintln!(
"[llmman] warning: could not install mlx-lm at startup ({e:#}); \
the first safetensors model load will retry it"
),
}
}
if _args.pull_only {
let resolved = resolved
.as_ref()
.expect("a pull-only failure returned above");
match pull_only_step(resolved, || pull_only_engine(_args.model.as_deref()))? {
PullOnlyStep::Done => {}
PullOnlyStep::ResolveCpuOnly => {
let pin = llama_cpp_version.clone();
tokio::task::spawn_blocking(move || {
runtime::resolve_for(Workload::CpuOnly, requested, pin.as_deref(), hardened)
})
.await
.context("resolve runtime task panicked")??;
}
PullOnlyStep::PullEngine(engine) => {
let ociman = resolved.ociman().expect("a container engine was picked");
let version = match engine {
crate::container::ContainerEngine::Sglang => _args.sglang_version.as_deref(),
_ => _args.vllm_version.as_deref(),
};
crate::container::pull_image(ociman, engine, version, _args.vllm_image.as_deref())?;
}
}
return Ok(());
}
let store_path = default_store()?;
let cache_path = crate::default_cache()?;
std::fs::create_dir_all(&cache_path)?;
crate::storage::repair::repair_store(&store_path)?;
if !crate::storage::gc::noprune_from_env() {
if let Ok(store) = OciStore::open(&store_path) {
if let Err(e) = crate::storage::gc::sweep(&store, &cache_path) {
eprintln!("[llmman] GC sweep failed: {e:#}");
}
}
}
let peers: Vec<String> = crate::config::peers()
.iter()
.map(|p| crate::daemon::peer_url(p))
.collect();
let ctx_size_explicit = context_length_from_env();
let vram = if !peers.is_empty() {
tokio::task::spawn_blocking(crate::hostgpu::detect_with_vram)
.await
.context("hostgpu probe task panicked")?
.1
} else {
0
};
let ctx_size = ctx_size_explicit.or(Some(DEFAULT_CTX_SIZE));
let hybrid_local_bytes_env = std::env::var("LLMMAN_HYBRID_LOCAL_BYTES").ok();
let memory = crate::hostgpu::memory_bytes(vram);
if !peers.is_empty() {
eprintln!(
"[llmman] aggregation peers: {} (this node: {} of model memory)",
peers.join(", "),
crate::fmt::human_size(memory)
);
}
let cpu_limit = container_cpu_limit();
let threads = threads_from_env_or_host();
if let Some(n) = threads {
eprintln!("[llmman] llama-server gets --threads {n} (half the available CPUs above 8, else all of them)");
} else if std::env::var_os("LLAMA_ARG_THREADS").is_some() {
eprintln!("[llmman] LLAMA_ARG_THREADS set: leaving llama-server thread count to it");
}
if let (Some(n), Some(_)) = (cpu_limit, runtime.known().and_then(|r| r.ociman())) {
eprintln!("[llmman] backend container gets --cpus {n} (this daemon's own CPU limit)");
}
if let Some(resolved) = runtime.known() {
match (resolved.ociman(), hardened) {
(Some(_), true) => eprintln!(
"[llmman] backend containers run hardened (read-only rootfs except behind --gpus, all capabilities dropped, no new privileges)"
),
(Some(_), false) => eprintln!(
"[llmman] warning: backend containers run without the hardened flags (--no-hardened)"
),
(None, true) => {}
(None, false) => eprintln!(
"[llmman] warning: --no-hardened has no effect under --runtime {}: the engine runs on the host, not in a container",
resolved.runtime().as_str()
),
}
}
let chosen = match safetensors_engine_from_env() {
SafetensorsEngine::Auto => None,
SafetensorsEngine::Vllm => Some("vllm"),
SafetensorsEngine::Sglang => Some("sglang"),
};
if let Some(engine) = chosen {
eprintln!(
"[llmman] safetensors models go to {engine} ({})",
backend::SAFETENSORS_ENGINE_VAR
);
}
match gguf_engine_from_env() {
GgufEngine::Ds4 => eprintln!(
"[llmman] GGUF models go to ds4-server ({})",
backend::GGUF_ENGINE_VAR
),
GgufEngine::LlamaServer => eprintln!(
"[llmman] GGUF models go to llama-server ({})",
backend::GGUF_ENGINE_VAR
),
GgufEngine::Auto if backend::ds4_server_on_path() => {
eprintln!("[llmman] DeepSeek V4.1 GGUFs go to ds4-server (on PATH)")
}
GgufEngine::Auto => {}
}
let auth = auth::Policy::from_env()?;
if auth.enforced() {
eprintln!("[llmman] API key required on every request (LLMMAN_API_KEYS)");
} else if !crate::daemon::reachable_only_locally() {
eprintln!(
"[llmman] warning: LLMMAN_AUTH=off — serving everyone who can reach {} without a key",
crate::daemon::bind_addr()
);
}
let tls = tls_from_env()?;
let managed_config = crate::config::managed().map_err(anyhow::Error::msg)?;
if managed_config.enabled {
if let Some((_, key)) = &tls {
crate::config::owner_readable_only(key)
.map_err(anyhow::Error::msg)
.context("managed TLS private key permissions")?;
}
}
let managed_routes = managed::routes(managed_config, auth.clone(), tls.is_some())?;
anyhow::ensure!(
tls.is_some() == crate::daemon::tls_scheme(),
"LLMMAN_HOST and LLMMAN_TLS_CERT/LLMMAN_TLS_KEY disagree: an https:// host needs the \
certificate and key, and they need an https:// host, so clients in this \
environment connect the way the daemon listens"
);
let client = crate::auth::trusted_client()?
.build()
.context("build http client")?;
let state = AppState(Arc::new(Inner {
manager: Mutex::new(ModelManager {
running: HashMap::new(),
pending_loads: 0,
}),
exe: std::env::current_exe()
.ok()
.map(|p| dunce::canonicalize(&p).unwrap_or(p)),
runtime,
llama_cpp_version,
vllm_version: _args.vllm_version.clone(),
vllm_image: _args.vllm_image.clone(),
sglang_version: _args.sglang_version.clone(),
ctx_size,
ctx_size_explicit: ctx_size_explicit.is_some(),
embedding_ctx_size: embedding_context_length_from_env(),
hybrid_local_bytes: crate::hybrid::local_budget_bytes(
ctx_size,
hybrid_local_bytes_env.as_deref(),
),
hybrid_local_bytes_explicit: crate::hybrid::local_budget_override(
hybrid_local_bytes_env.as_deref(),
)
.is_some(),
flash_attention: flash_attention_from_env(),
kv_cache_type: kv_cache_type_from_env(),
split_mode: sched_spread_from_env(),
num_parallel: num_parallel_from_env(),
threads,
cpu_limit,
hardened,
max_queue: max_queue_from_env(),
max_loaded_models: max_loaded_models_from_env(),
peers,
memory,
store_path,
cache_path,
prompt_log: crate::promptlog::enabled_from_env()
.then(crate::promptlog::path)
.transpose()?,
usage_log: crate::usage::enabled_from_env()
.then(crate::usage::path)
.transpose()?,
shell,
auth,
peer_key: crate::auth::peer_key(),
client,
}));
let app = build_router(state.clone(), metrics_enabled_from_env()).merge(managed_routes);
metrics::mark_process_start();
crate::verify::Policy::load().context("signature trust policy")?;
let addr = crate::daemon::bind_addr();
let listener = tokio::net::TcpListener::bind(&addr)
.await
.with_context(|| format!("bind {addr}"))?;
let addr = crate::daemon::bound_addr(listener.local_addr().context("listener address")?.port());
eprintln!(
"llmman serve listening on http{}://{addr}",
if tls.is_some() { "s" } else { "" }
);
shell::announce(&state.0.shell, &shell_token, tls.is_some(), &addr);
tokio::spawn(reap_idle_models(state.clone()));
if let Some(model) = _args
.model
.as_deref()
.map(crate::hybrid::local_half)
.filter(|m| !crate::providers::is_remote_ref(m))
{
match crate::shortnames::resolve_ollama_api(model) {
Ok(model) => {
let state_clone = state.clone();
tokio::spawn(async move {
match ensure_model(&state_clone, &model, None, None).await {
Ok((_, _, guard)) => refresh_activity(guard, None).await,
Err(e) => eprintln!("[llmman] pre-load failed: {:#}", e.0),
}
});
}
Err(e) => eprintln!("[llmman] pre-load failed: {e}"),
}
}
match tls {
None => {
axum::serve(
listener,
app.into_make_service_with_connect_info::<std::net::SocketAddr>(),
)
.with_graceful_shutdown(shutdown_signal())
.await?
}
Some((cert, key)) => {
let _ = rustls::crypto::ring::default_provider().install_default();
let config = axum_server::tls_rustls::RustlsConfig::from_pem_file(&cert, &key)
.await
.with_context(|| {
format!(
"load TLS certificate {} and key {}",
cert.display(),
key.display()
)
})?;
let handle = axum_server::Handle::new();
tokio::spawn({
let handle = handle.clone();
async move {
shutdown_signal().await;
handle.graceful_shutdown(Some(Duration::from_secs(30)));
}
});
axum_server::from_tcp_rustls(listener.into_std()?, config)
.handle(handle)
.serve(app.into_make_service_with_connect_info::<std::net::SocketAddr>())
.await?
}
}
state.0.manager.lock().await.running.clear();
Ok(())
}
async fn shutdown_signal() {
let ctrl_c = async {
let _ = tokio::signal::ctrl_c().await;
};
#[cfg(unix)]
let terminate = async {
match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) {
Ok(mut sig) => {
sig.recv().await;
}
Err(_) => std::future::pending().await,
}
};
#[cfg(not(unix))]
let terminate = std::future::pending::<()>();
tokio::select! {
_ = ctrl_c => {}
_ = terminate => {}
}
eprintln!("llmman serve shutting down");
}
#[cfg(test)]
mod test_support;
#[cfg(test)]
mod tests;