use std::collections::HashMap;
use std::path::PathBuf;
use std::sync::{Arc, LazyLock, Mutex as StdMutex};
use anyhow::{anyhow, Context};
use base64::Engine as _;
use axum::body::{Body, Bytes, HttpBody as _};
use axum::extract::{DefaultBodyLimit, FromRequest, MatchedPath, Path as UrlPath, Request, State};
use axum::http::{HeaderMap, StatusCode};
use axum::middleware::{self, Next};
use axum::response::{IntoResponse, Response};
use axum::routing::{delete, get, post};
use axum::{Json, Router};
use clap::Args;
use futures::StreamExt;
use reqwest::Client;
use serde::{Deserialize, Serialize};
use tokio::sync::Mutex;
use tokio::time::{sleep, Duration, Instant};
use crate::default_store;
use crate::metrics::{self, UnloadReason};
use crate::modelpack::{resolve_model, ModelPath};
use crate::providers::{Wire, PLACEHOLDER_API_KEY};
use crate::storage::OciStore;
mod aggregation;
mod anthropic;
mod auth;
mod backend;
mod config;
mod gemini;
mod messages;
mod ollama;
mod openai;
mod responses;
pub mod runtime;
mod sched;
mod shell;
mod stream;
mod types;
mod webui;
use backend::{
find_free_port, safetensors_engine_from_env, sglang_serve_args_from_env,
sglang_served_model_name, spawn_llama_server, spawn_mlx_server, spawn_sglang_server,
spawn_vllm_omni_server, spawn_vllm_server, tail_child_output, use_mlx_for_safetensors,
vllm_max_model_len, vllm_omni_serve_args_from_env, vllm_serve_args_from_env, wait_for_ready,
OutputTail, SafetensorsEngine, POLL_INTERVAL,
};
pub use backend::{GPU_VISIBLE_DEVICE_VARS, LLAMA_CPP_ENV_PASSTHROUGH_VARS};
pub use config::DEFAULT_CTX_SIZE;
use config::{
backend_ctx_size, container_cpu_limit, context_length_from_env, effective_num_parallel,
embedding_model_ctx, flash_attention_from_env, gguf_trained_ctx, initial_ctx_size,
kv_cache_type_from_env, looks_like_oom, max_loaded_models_from_env, max_queue_from_env,
metrics_enabled_from_env, next_ctx_size_after_oom, num_parallel_from_env,
sched_spread_from_env, supports_context_shift, threads_from_env_or_host, tls_from_env,
MAX_CTX_SHRINK_ATTEMPTS,
};
use gemini::{gemini_method, handle_pinned_gemini};
use ollama::{
handle_blob_head, handle_blob_upload, handle_copy, handle_create, handle_delete, handle_embed,
handle_embeddings, handle_ollama_chat, handle_ollama_generate, handle_ps, handle_pull,
handle_push, handle_show, handle_tags, handle_version,
};
use openai::{
handle_openai_chat, handle_openai_completions, handle_openai_embeddings, handle_openai_images,
handle_openai_models, handle_openai_speech, handle_openai_transcriptions,
handle_openai_video_get, handle_openai_videos, TRANSCRIPTION_BODY_LIMIT_BYTES,
};
pub use runtime::Runtime;
use sched::{
begin_activity, default_keep_alive, reap_idle_models, refresh_activity, ActivityGuard,
};
use stream::bytes_to_lines;
use types::*;
const SERVE_ENV_HELP: &str = "\
Environment Variables:
LLMMAN_DEBUG Show additional debug information (e.g. LLMMAN_DEBUG=1)
LLMMAN_HOST [scheme://][host][:port] to bind (default \"127.0.0.1:17434\"; https:// tells clients to expect TLS)
LLMMAN_API_KEYS Comma-separated API keys every request must present (overrides [auth] in llmman.conf; required off loopback)
LLMMAN_AUTH off: serve without keys even on a bind the network can reach
LLMMAN_PEER_API_KEY The key presented to aggregation peers (default: the first of LLMMAN_API_KEYS)
LLMMAN_TLS_CERT PEM certificate chain to terminate TLS with (with LLMMAN_TLS_KEY)
LLMMAN_TLS_KEY PEM private key for LLMMAN_TLS_CERT
LLMMAN_TLS_CA PEM bundle of extra roots to trust when reaching peers (and, for the CLI, the daemon)
LLMMAN_CONTEXT_LENGTH Context size for llama-server/vLLM/SGLang when set (default 262144 for llama-server)
LLMMAN_HYBRID_LOCAL_BYTES Largest request a hybrid pair serves locally, in bytes (0 disables; default: from the context length)
LLMMAN_KEEP_ALIVE The duration that models stay loaded in memory (default \"5m\")
LLMMAN_MAX_LOADED_MODELS Maximum number of loaded models (default: unbounded)
LLMMAN_MAX_TRANSFER_STREAMS Maximum parallel transfer streams for safetensors model pulls (default 4)
LLMMAN_MAX_QUEUE Maximum number of queued requests (default 512)
LLMMAN_METRICS Serve a Prometheus scrape endpoint at /metrics (default: off)
LLMMAN_MODELS The path to the models directory
LLMMAN_NUM_PARALLEL Maximum number of parallel requests per model (GGUF only)
LLMMAN_MLX_LM_VERSION Exact mlx-lm release to install on macOS (default: PyPI's current)
LLMMAN_SAFETENSORS_ENGINE Engine for safetensors models: vllm or sglang (default: mlx_lm.server on macOS, installed at startup; else vllm)
LLMMAN_VLLM_ARGS Extra whitespace-separated arguments appended to every `vllm serve` (e.g. \"--dtype bfloat16 --tp 2\")
LLMMAN_SGLANG_ARGS Extra whitespace-separated arguments appended to every sglang launch (e.g. \"--disable-cuda-graph\")
LLMMAN_NOHISTORY Do not record prompts for `llmman log`
LLMMAN_NOPRUNE Do not prune model blobs on startup
LLMMAN_ORIGINS A comma separated list of allowed CORS origins
LLMMAN_PEERS A comma separated list of peer daemons ([scheme://]host[:port]) to pool hardware with (overrides [aggregation] in llmman.conf)
LLMMAN_REGISTRY_MIRRORS A comma separated list of Docker Hub mirrors ([scheme://]host[:port]) to try before docker.io (overrides [registries.\"docker.io\"] in llmman.conf)
LLMMAN_SCHED_SPREAD Always schedule model across all GPUs
LLMMAN_FLASH_ATTENTION Enable flash attention
LLMMAN_KV_CACHE_TYPE Quantization type for the K/V cache (default: f16)
LLMMAN_RUNTIME Where the inference engine comes from: auto (default), docker, podman, bin or path — same as --runtime
LLMMAN_LLM_LIBRARY Set backend (cpu/cuda/cuda13/rocm/vulkan/metal) to bypass GPU autodetection
LLMMAN_IGPU_ENABLE Enable integrated GPUs
LLMMAN_LOAD_TIMEOUT How long to allow model loads to stall before giving up (default \"10m\")
LLMMAN_VLLM_OMNI_GUARDRAILS Keep a Diffusers-layout model's vLLM-Omni safety guardrails on (default: off)
LLMMAN_TMPDIR Staging directory for llama-server release downloads
LLAMA_ARG_FIT Enable llama.cpp automatic fit of unset memory options (default \"on\")
LLAMA_ARG_FIT_TARGET Target free VRAM margin per device for llama.cpp fit (MiB)
LLAMA_ARG_THREADS Thread count for llama-server (default: llama-server autodetection, overridden by a binding CPU quota/affinity limit)
";
#[derive(Args, Debug)]
#[command(after_help = SERVE_ENV_HELP)]
pub struct ServeArgs {
#[arg(value_name = "MODEL")]
pub model: Option<String>,
#[arg(long, value_enum, default_value = "auto", env = "LLMMAN_RUNTIME")]
pub runtime: Runtime,
#[arg(long, value_name = "TAG|latest")]
pub llama_cpp_version: Option<String>,
#[arg(long, value_name = "TAG")]
pub vllm_version: Option<String>,
#[arg(long, value_name = "TAG")]
pub sglang_version: Option<String>,
#[arg(long)]
pub pull_only: bool,
#[arg(long, hide = true, requires = "model")]
pub port: Option<u16>,
#[arg(long, hide = true, requires = "port", default_value = "127.0.0.1")]
pub host: std::net::IpAddr,
}
#[derive(Clone)]
struct AppState(Arc<Inner>);
struct Inner {
manager: Mutex<ModelManager>,
exe: Option<PathBuf>,
runtime: runtime::Lazy,
llama_cpp_version: Option<String>,
vllm_version: Option<String>,
sglang_version: Option<String>,
ctx_size: Option<u32>,
ctx_size_explicit: bool,
hybrid_local_bytes: Option<u64>,
flash_attention: Option<String>,
kv_cache_type: Option<String>,
split_mode: Option<&'static str>,
num_parallel: Option<u32>,
threads: Option<u32>,
cpu_limit: Option<f64>,
max_queue: usize,
max_loaded_models: usize,
peers: Vec<String>,
memory: u64,
store_path: PathBuf,
cache_path: PathBuf,
prompt_log: Option<PathBuf>,
shell: shell::Policy,
auth: auth::Policy,
peer_key: Option<String>,
client: Client,
}
struct ModelManager {
running: HashMap<String, RunningModel>,
pending_loads: usize,
}
struct RunningModel {
process: ModelProcess,
port: u16,
digest: String,
size: u64,
started_at: String,
last_active: Instant,
last_active_wall: chrono::DateTime<chrono::Utc>,
keep_alive: Option<Duration>,
in_flight: u32,
backend_model_path: Option<String>,
}
impl RunningModel {
fn processor(&self) -> String {
match &self.process {
ModelProcess::Local(engine, _, _) => format!("{} (local)", engine.label()),
ModelProcess::Container(ociman, engine, _) => {
format!("{} (container/{})", engine.label(), ociman.binary())
}
}
}
fn pid(&self) -> Option<u32> {
match &self.process {
ModelProcess::Local(_, child, _) => child.id(),
ModelProcess::Container(_, _, child) => child.id(),
}
}
fn engine_label(&self) -> &'static str {
match &self.process {
ModelProcess::Local(engine, _, _) | ModelProcess::Container(_, engine, _) => {
engine.label()
}
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Engine {
LlamaServer,
Vllm,
VllmOmni,
Sglang,
Mlx,
}
impl Engine {
fn label(self) -> &'static str {
match self {
Engine::LlamaServer => "llama-server",
Engine::Vllm => "vllm",
Engine::VllmOmni => "vllm-omni",
Engine::Sglang => "sglang",
Engine::Mlx => "mlx",
}
}
}
enum ModelProcess {
Local(
Engine,
tokio::process::Child,
#[cfg_attr(not(unix), allow(dead_code))] Option<u32>,
),
Container(
crate::container::ContainerManager,
Engine,
tokio::process::Child,
),
}
impl Drop for ModelProcess {
fn drop(&mut self) {
match self {
ModelProcess::Container(_, _, child) => {
if let Some(pid) = child.id() {
crate::container::stop(pid);
}
}
#[cfg(unix)]
ModelProcess::Local(
engine @ (Engine::Vllm | Engine::VllmOmni | Engine::Sglang),
_,
pid,
) => {
if let Some(pid) = pid {
let result = unsafe { libc::kill(-(*pid as libc::pid_t), libc::SIGKILL) };
if result != 0 {
let err = std::io::Error::last_os_error();
eprintln!(
"[llmman] warning: SIGKILL to {} process group {pid} failed: {err}",
engine.label()
);
}
}
}
#[cfg(not(unix))]
ModelProcess::Local(Engine::Vllm | Engine::VllmOmni | Engine::Sglang, _, _) => {}
ModelProcess::Local(Engine::Mlx, _, _) => {}
ModelProcess::Local(Engine::LlamaServer, _, _) => {}
}
}
}
impl ModelProcess {
fn engine(&self) -> Engine {
match self {
ModelProcess::Local(engine, _, _) | ModelProcess::Container(_, engine, _) => *engine,
}
}
fn is_alive(&mut self) -> bool {
let child = match self {
ModelProcess::Local(_, child, _) => child,
ModelProcess::Container(_, _, child) => child,
};
matches!(child.try_wait(), Ok(None))
}
async fn stop_and_wait(&mut self) {
match self {
ModelProcess::Container(_, _, child) => {
if let Some(pid) = child.id() {
crate::container::stop(pid);
}
}
#[cfg(unix)]
ModelProcess::Local(Engine::Vllm | Engine::VllmOmni | Engine::Sglang, _, pid) => {
if let Some(pid) = pid {
unsafe { libc::kill(-(*pid as libc::pid_t), libc::SIGKILL) };
}
}
ModelProcess::Local(_, _, _) => {}
}
let child = match self {
ModelProcess::Local(_, child, _) => child,
ModelProcess::Container(_, _, child) => child,
};
let _ = child.kill().await;
}
}
#[derive(Debug)]
struct AppError(anyhow::Error, StatusCode);
impl<E: Into<anyhow::Error>> From<E> for AppError {
fn from(e: E) -> Self {
Self(e.into(), StatusCode::INTERNAL_SERVER_ERROR)
}
}
impl AppError {
fn status(status: StatusCode, message: impl Into<String>) -> Self {
Self(anyhow!(message.into()), status)
}
fn bad_request(e: crate::shortnames::InvalidReference) -> Self {
Self(e.into(), StatusCode::BAD_REQUEST)
}
}
impl IntoResponse for AppError {
fn into_response(self) -> Response {
let body = serde_json::json!({ "error": format!("{:#}", self.0) });
(self.1, Json(body)).into_response()
}
}
fn now_rfc3339() -> String {
chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true)
}
static MODEL_LOCKS: LazyLock<StdMutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>> =
LazyLock::new(|| StdMutex::new(HashMap::new()));
static LOAD_LOCKS: LazyLock<StdMutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>> =
LazyLock::new(|| StdMutex::new(HashMap::new()));
fn keyed_lock(
registry: &StdMutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>,
key: &str,
) -> Arc<tokio::sync::Mutex<()>> {
let mut locks = registry.lock().unwrap();
locks
.entry(key.to_owned())
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
.clone()
}
fn release_keyed_lock(
registry: &StdMutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>,
key: &str,
) {
let mut locks = registry.lock().unwrap();
if let Some(arc) = locks.get(key) {
if Arc::strong_count(arc) <= 1 {
locks.remove(key);
}
}
}
fn model_lock(model: &str) -> Arc<tokio::sync::Mutex<()>> {
keyed_lock(&MODEL_LOCKS, model)
}
fn release_model_lock(model: &str) {
release_keyed_lock(&MODEL_LOCKS, model)
}
fn load_lock(model: &str) -> Arc<tokio::sync::Mutex<()>> {
keyed_lock(&LOAD_LOCKS, model)
}
fn release_load_lock(model: &str) {
release_keyed_lock(&LOAD_LOCKS, model)
}
struct LoadLockGuard {
model: String,
guard: Option<tokio::sync::OwnedMutexGuard<()>>,
}
impl Drop for LoadLockGuard {
fn drop(&mut self) {
self.guard.take(); release_load_lock(&self.model);
}
}
async fn acquire_load_lock(model: &str) -> LoadLockGuard {
let guard = load_lock(model).lock_owned().await;
LoadLockGuard {
model: model.to_owned(),
guard: Some(guard),
}
}
fn ffi_pull_ref<'a>(progress_key: &'a str, classified: &str) -> &'a str {
debug_assert!(
classified == progress_key || classified.strip_prefix(progress_key) == Some(":latest"),
"classify should differ from the progress key only by a defaulted tag",
);
progress_key
}
fn pull_serialized(store_path: &std::path::Path, model: &str) -> anyhow::Result<Vec<String>> {
let lock = model_lock(model);
let result = (|| {
let _guard = lock.blocking_lock();
let existing = OciStore::open(store_path).and_then(|s| s.find(model)).ok();
if let Some(desc) = existing {
if !crate::verify::is_registry_reference(model) {
return Ok(Vec::new());
}
if !crate::verify::is_enabled_for(model)? {
return Ok(Vec::new());
}
return verify_stored(store_path, model, &desc.digest);
}
let layout_dir = store_path
.to_str()
.ok_or_else(|| anyhow!("store path is not valid UTF-8"))?;
tokio::runtime::Handle::current().block_on(async {
match crate::hf::classify(model).await {
crate::hf::ClassifiedRef::Hf(reference) => {
if crate::verify::is_registry_reference(model)
&& crate::verify::is_enabled_for(model)?
{
return Err(anyhow!(concat!(
"is covered by a signature policy but could not be confirmed ",
"to be an OCI registry (the /v2/ probe failed); refusing to ",
"pull it as a HuggingFace repository unchecked",
))
.context(model.to_string()));
}
crate::hf::pull::pull(&reference, store_path, model)
.await
.map(|()| Vec::new())
}
crate::hf::ClassifiedRef::Source(reference) => {
crate::sources::pull(&reference, store_path, model)
.await
.map(|()| Vec::new())
}
crate::hf::ClassifiedRef::Other(normalized) => {
let guard = crate::verify::PullGuard::check(&normalized)?;
crate::ffi::pull(ffi_pull_ref(model, &normalized), layout_dir)?;
let stored = OciStore::open(store_path)?.find(&normalized)?;
match guard.confirm(&stored.digest) {
Ok(notices) => Ok(notices),
Err(e) => Err(reject_stored(store_path, &normalized, e)),
}
}
}
})
})();
drop(lock);
release_model_lock(model);
result
}
fn verify_stored(
store_path: &std::path::Path,
model: &str,
digest: &str,
) -> anyhow::Result<Vec<String>> {
debug_assert!(crate::verify::is_registry_reference(model));
match crate::verify::check(model, Some(digest)) {
Ok(verdict) => Ok(verdict.notices),
Err(e) if crate::verify::is_indeterminate(&e) => Err(e),
Err(e) => Err(reject_stored(store_path, model, e)),
}
}
fn reject_stored(
store_path: &std::path::Path,
reference: &str,
cause: anyhow::Error,
) -> anyhow::Error {
match OciStore::open(store_path).and_then(|s| s.remove(reference)) {
Ok(_) => cause,
Err(e) => cause.context(format!(
"could not remove the rejected model {reference} from the store ({e:#}); \
it will keep being refused, but `llmman rm {reference}` is needed to clear it"
)),
}
}
fn canonical_ref(store_path: &std::path::Path, model_ref: &str) -> String {
let Ok(store) = crate::storage::OciStore::open(store_path) else {
return model_ref.to_owned();
};
let Ok(desc) = store.find(model_ref) else {
return model_ref.to_owned();
};
desc.annotations
.as_ref()
.and_then(|a| a.get("org.opencontainers.image.ref.name"))
.cloned()
.unwrap_or_else(|| model_ref.to_owned())
}
fn load_identity(
store_path: &std::path::Path,
model: &str,
) -> Result<String, crate::shortnames::InvalidReference> {
if crate::providers::is_remote_ref(model) {
return Ok(model.to_string());
}
let resolved = crate::shortnames::resolve_ollama_api(model)?;
let (without_digest, digest) = crate::storage::split_ref_digest(&resolved);
if digest.is_some() {
if let Some(tag) = stored_tag_for_digest(store_path, &resolved) {
return Ok(tag);
}
}
Ok(crate::storage::default_tag(without_digest))
}
fn stored_tag_for_digest(store_path: &std::path::Path, reference: &str) -> Option<String> {
crate::storage::split_ref_digest(reference).1?;
let desc = OciStore::open(store_path).ok()?.find(reference).ok()?;
let stored = desc
.annotations
.as_ref()?
.get("org.opencontainers.image.ref.name")?;
crate::storage::split_ref_digest(stored)
.1
.is_none()
.then(|| crate::storage::default_tag(stored))
}
async fn unload_model(state: &AppState, model: &str) -> Result<(), AppError> {
let lock_key = load_identity(&state.0.store_path, model).map_err(AppError::bad_request)?;
let _guard = acquire_load_lock(&lock_key).await;
let canonical = if crate::providers::is_remote_ref(model) {
lock_key
} else {
let resolved =
crate::shortnames::resolve_ollama_api(model).map_err(AppError::bad_request)?;
canonical_ref(&state.0.store_path, &crate::storage::default_tag(&resolved))
};
if state
.0
.manager
.lock()
.await
.running
.remove(&canonical)
.is_some()
{
metrics::record_model_unload(&canonical, UnloadReason::Requested);
return Ok(());
}
if crate::providers::is_remote_ref(model) {
return Ok(());
}
let stored = OciStore::open(&state.0.store_path)?.find(&canonical).ok();
let (base, digest) = crate::storage::split_ref_digest(&canonical);
let digest = digest
.map(str::to_owned)
.or_else(|| stored.as_ref().map(|d| d.digest.clone()));
if let Some(digest) = digest {
let spelled = crate::storage::default_tag(base);
let mut mgr = state.0.manager.lock().await;
let key = mgr
.running
.iter()
.filter(|(key, m)| {
m.digest.eq_ignore_ascii_case(&digest)
&& crate::storage::repo_name(key) == crate::storage::repo_name(base)
})
.map(|(key, _)| key.clone())
.min_by_key(|key| *key != spelled);
if let Some(key) = key {
mgr.running.remove(&key);
return Ok(());
}
}
if stored.is_none() {
return Err(AppError(
anyhow!("model '{model}' not found"),
StatusCode::NOT_FOUND,
));
}
Ok(())
}
async fn forward_ollama<T: Serialize>(
state: &AppState,
target: &Target,
route: &str,
headers: &HeaderMap,
req: &T,
model: &str,
guard: ActivityGuard,
) -> Result<Response, AppError> {
let mut req = serde_json::to_value(req).context("re-serialize Ollama request")?;
if req["model"]
.as_str()
.is_some_and(crate::hybrid::is_hybrid_ref)
{
req["model"] = serde_json::Value::String(model.to_string());
}
let body = Bytes::from(serde_json::to_vec(&req).context("re-serialize Ollama request")?);
let activity = begin_activity(guard, None).await;
proxy(&state.0.client, target, route, headers, body, activity).await
}
async fn unload_everywhere(
state: &AppState,
model: &str,
headers: &HeaderMap,
) -> Result<(), AppError> {
let local = unload_model(state, model).await;
if aggregation::unload(state, model, headers).await {
return Ok(());
}
local
}
async fn check_running(state: &AppState, model_ref: &str) -> Option<(u16, ActivityGuard)> {
let mut mgr = state.0.manager.lock().await;
if let Some(m) = mgr.running.get_mut(model_ref) {
if m.process.is_alive() {
m.in_flight += 1;
return Some((m.port, ActivityGuard::new(state, model_ref)));
}
eprintln!(
"[llmman] {model_ref} was marked running on port {} but its process has exited — reloading",
m.port
);
mgr.running.remove(model_ref);
metrics::record_model_unload(model_ref, UnloadReason::Crashed);
}
None
}
async fn check_running_by_digest(
state: &AppState,
model_ref: &str,
digest: &str,
) -> Option<(String, u16, ActivityGuard)> {
let repo = crate::storage::repo_name(model_ref);
let mut mgr = state.0.manager.lock().await;
let key = mgr
.running
.iter()
.find(|(key, m)| {
crate::storage::repo_name(key) == repo && m.digest.eq_ignore_ascii_case(digest)
})
.map(|(key, _)| key.clone())?;
let m = mgr.running.get_mut(&key)?;
if !m.process.is_alive() {
eprintln!(
"[llmman] {key} was marked running on port {} but its process has exited — reloading",
m.port
);
mgr.running.remove(&key);
return None;
}
m.in_flight += 1;
let port = m.port;
Some((key.clone(), port, ActivityGuard::new(state, &key)))
}
async fn evict_other_models(state: &AppState, model_ref: &str) -> bool {
let mut mgr = state.0.manager.lock().await;
let other_keys: Vec<String> = mgr
.running
.iter()
.filter(|(k, m)| k.as_str() != model_ref && m.in_flight == 0)
.map(|(k, _)| k.clone())
.collect();
let mut evicted: Vec<(String, RunningModel)> = Vec::with_capacity(other_keys.len());
for key in other_keys {
if let Some(running) = mgr.running.remove(&key) {
metrics::record_model_unload(&key, UnloadReason::Oom);
evicted.push((key, running));
}
}
drop(mgr); let any = !evicted.is_empty();
for (name, mut running) in evicted {
eprintln!("[llmman] evicting {name} to free memory before retrying {model_ref}");
running.process.stop_and_wait().await;
}
any
}
struct PendingLoadGuard {
state: AppState,
armed: bool,
}
impl PendingLoadGuard {
fn release_into(&mut self, mgr: &mut ModelManager) {
if !std::mem::take(&mut self.armed) {
return;
}
mgr.pending_loads = mgr.pending_loads.saturating_sub(1);
}
}
impl std::fmt::Debug for PendingLoadGuard {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("PendingLoadGuard")
.field("armed", &self.armed)
.finish()
}
}
impl Drop for PendingLoadGuard {
fn drop(&mut self) {
if !self.armed {
return;
}
let state = self.state.clone();
let Ok(handle) = tokio::runtime::Handle::try_current() else {
return;
};
handle.spawn(async move {
let mut mgr = state.0.manager.lock().await;
mgr.pending_loads = mgr.pending_loads.saturating_sub(1);
});
}
}
async fn enforce_max_loaded_models(
state: &AppState,
max_loaded: usize,
) -> Result<PendingLoadGuard, AppError> {
if max_loaded == 0 {
state.0.manager.lock().await.pending_loads += 1;
return Ok(PendingLoadGuard {
state: state.clone(),
armed: true,
});
}
loop {
let mut mgr = state.0.manager.lock().await;
if mgr.running.len() + mgr.pending_loads < max_loaded {
mgr.pending_loads += 1;
return Ok(PendingLoadGuard {
state: state.clone(),
armed: true,
});
}
if mgr.running.len() < max_loaded {
drop(mgr);
sleep(POLL_INTERVAL).await;
continue;
}
let victim = mgr
.running
.iter()
.filter(|(_, m)| m.in_flight == 0)
.min_by_key(|(_, m)| m.last_active)
.map(|(k, _)| k.clone());
let Some(victim) = victim else {
return Err(AppError::status(
StatusCode::SERVICE_UNAVAILABLE,
format!(
"max loaded models ({max_loaded}) reached, and every loaded model is busy — try again"
),
));
};
mgr.pending_loads += 1;
let mut running = mgr
.running
.remove(&victim)
.expect("victim key was just looked up under this same lock");
metrics::record_model_unload(&victim, UnloadReason::Evicted);
drop(mgr); eprintln!(
"[llmman] evicting {victim} to free a loaded-model slot (LLMMAN_MAX_LOADED_MODELS={max_loaded})"
);
running.process.stop_and_wait().await;
return Ok(PendingLoadGuard {
state: state.clone(),
armed: true,
});
}
}
const MAX_QUEUE_ERROR: &str = "server busy, please try again. maximum pending requests exceeded";
static PENDING_REQUESTS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
struct QueueGuard(&'static std::sync::atomic::AtomicUsize);
impl Drop for QueueGuard {
fn drop(&mut self) {
self.0.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
}
}
fn try_admit_against(
counter: &'static std::sync::atomic::AtomicUsize,
max_queue: usize,
) -> Result<QueueGuard, AppError> {
let cap = max_queue.max(1);
let admitted = counter
.fetch_update(
std::sync::atomic::Ordering::SeqCst,
std::sync::atomic::Ordering::SeqCst,
|n| (n < cap).then_some(n + 1),
)
.is_ok();
if admitted {
Ok(QueueGuard(counter))
} else {
Err(AppError::status(
StatusCode::SERVICE_UNAVAILABLE,
MAX_QUEUE_ERROR,
))
}
}
fn try_admit(max_queue: usize) -> Result<QueueGuard, AppError> {
let admitted = try_admit_against(&PENDING_REQUESTS, max_queue);
if admitted.is_err() {
metrics::record_scheduling_rejection();
}
admitted
}
#[derive(Clone, Debug)]
enum Target {
Local(u16),
Peer(Arc<PeerTarget>),
Remote(Arc<RemoteTarget>),
}
struct PeerTarget {
origin: String,
api_key: Option<String>,
}
impl std::fmt::Debug for PeerTarget {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("PeerTarget")
.field("origin", &self.origin)
.field("api_key", &self.api_key.as_ref().map(|_| "<redacted>"))
.finish()
}
}
struct RemoteTarget {
provider: String,
base_url: String,
wire: Wire,
model: String,
max_output: Option<u32>,
api_key: Option<String>,
}
impl std::fmt::Debug for RemoteTarget {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RemoteTarget")
.field("provider", &self.provider)
.field("base_url", &self.base_url)
.field("wire", &self.wire)
.field("model", &self.model)
.field("api_key", &self.api_key.as_ref().map(|_| "<redacted>"))
.finish()
}
}
impl Target {
fn url(&self, route: &str) -> String {
match self {
Self::Local(port) => format!("http://127.0.0.1:{port}{route}"),
Self::Peer(peer) => format!("{}{route}", peer.origin),
Self::Remote(remote) => crate::providers::rebase_url(&remote.base_url, route),
}
}
fn authorize(&self, req: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
match self {
Self::Local(_) => req,
Self::Peer(peer) => aggregation::hop(req, peer.api_key.as_deref()),
Self::Remote(remote) => match (remote.wire, remote.api_key.as_deref()) {
(Wire::OpenAi, Some(key)) => req.bearer_auth(key),
(Wire::OpenAi, None) => req,
(Wire::Anthropic, key) => {
let req = req.header("anthropic-version", anthropic::VERSION);
match key {
Some(key) => req.header("x-api-key", key),
None => req,
}
}
},
}
}
fn is_remote(&self) -> bool {
matches!(self, Self::Remote(_))
}
fn wire(&self) -> Option<Wire> {
match self {
Self::Remote(remote) => Some(remote.wire),
_ => None,
}
}
fn is_anthropic(&self) -> bool {
self.wire() == Some(Wire::Anthropic)
}
fn describe(&self) -> String {
match self {
Self::Local(_) => "inference backend".to_string(),
Self::Peer(peer) => format!("peer {}", peer.origin),
Self::Remote(remote) => format!("provider {}", remote.provider),
}
}
}
const CHAT_COMPLETIONS_ROUTE: &str = "/v1/chat/completions";
fn client_api_key(headers: Option<&HeaderMap>) -> Option<String> {
let headers = headers?;
let usable = |k: &str| {
let k = k.trim();
(!k.is_empty() && k != PLACEHOLDER_API_KEY).then(|| k.to_string())
};
let bearer = auth::bearer(headers).and_then(usable);
bearer
.or_else(|| {
headers
.get("x-api-key")
.and_then(|v| v.to_str().ok())
.and_then(usable)
})
.or_else(|| {
headers
.get("x-goog-api-key")
.and_then(|v| v.to_str().ok())
.and_then(usable)
})
}
fn is_cross_site(headers: Option<&HeaderMap>) -> bool {
headers
.and_then(|h| h.get("sec-fetch-site"))
.and_then(|v| v.to_str().ok())
.is_some_and(|site| {
let site = site.trim();
site.eq_ignore_ascii_case("cross-site") || site.eq_ignore_ascii_case("same-site")
})
}
fn daemon_key_spendable(state: &AppState, headers: Option<&HeaderMap>) -> bool {
state.0.auth.enforced() || (crate::daemon::reachable_only_locally() && !is_cross_site(headers))
}
async fn provider_catalog() -> Result<Arc<crate::providers::Catalog>, AppError> {
tokio::task::spawn_blocking(crate::providers::catalog)
.await
.context("provider catalog task panicked")?
.map_err(|e| AppError(e, StatusCode::BAD_GATEWAY))
}
async fn resolve_remote_target(
state: &AppState,
model_ref: &str,
headers: Option<&HeaderMap>,
) -> Result<Option<Target>, AppError> {
let Some((provider_id, model)) = crate::providers::split_remote_ref(model_ref) else {
return Ok(None);
};
let (provider_id, model) = (provider_id.to_string(), model.to_string());
let catalog = provider_catalog().await?;
let provider = catalog.get(&provider_id).ok_or_else(|| {
AppError(
crate::providers::unknown_provider_error(&provider_id, &catalog),
StatusCode::BAD_REQUEST,
)
})?;
let trusted = daemon_key_spendable(state, headers);
let own_key = || trusted.then(|| provider.api_key()).flatten();
let api_key = match client_api_key(headers).or_else(own_key) {
Some(key) => Some(key),
None if provider.key_optional => None,
None => {
return Err(AppError(
anyhow!(
"no API key for provider {provider_id:?} — send it as an Authorization \
header{}",
if is_cross_site(headers) && !state.0.auth.enforced() {
". This request came from another site, so llmman serve's own \
environment is deliberately not used"
.to_string()
} else if trusted {
format!(
", or give llmman serve a key of its own: {}",
crate::providers::key_hint(&provider.id, provider.key_env.as_deref())
)
} else {
". llmman serve is not bound to loopback, so its own environment is \
deliberately not used"
.to_string()
}
),
StatusCode::UNAUTHORIZED,
));
}
};
let target = RemoteTarget {
provider: provider_id,
base_url: provider.base_url.clone(),
wire: provider.wire,
max_output: provider
.models
.iter()
.find(|m| m.id == model)
.and_then(|m| m.max_output),
model,
api_key,
};
eprintln!(
"[llmman] routing {} to provider {} ({}, {} wire)",
target.model,
target.provider,
target.base_url,
target.wire.as_str()
);
Ok(Some(Target::Remote(Arc::new(target))))
}
fn resolve_hybrid_side(
state: &AppState,
pair: &crate::hybrid::Pair<'_>,
headers: Option<&HeaderMap>,
) -> Result<String, AppError> {
let pin = request_pin(headers)?;
let request_bytes = headers
.and_then(|h| h.get(reqwest::header::CONTENT_LENGTH))
.and_then(|v| v.to_str().ok())
.and_then(|v| v.trim().parse::<u64>().ok());
let decision = crate::hybrid::route(pin, request_bytes, state.0.hybrid_local_bytes);
let why = match decision.reason {
crate::hybrid::Reason::Pinned => format!("pinned by {}", crate::hybrid::ROUTE_HEADER),
crate::hybrid::Reason::Overflow { bytes, budget } => format!(
"{} request exceeds the {} this host serves locally",
crate::fmt::human_size(bytes),
crate::fmt::human_size(budget)
),
crate::hybrid::Reason::LocalFirst => "no reason to leave this machine".to_string(),
};
eprintln!(
"[llmman] hybrid {:?} + {:?} -> {} ({why})",
pair.local,
pair.remote_ref(),
decision.side.as_str()
);
Ok(pair.side_ref(decision.side))
}
fn request_pin(headers: Option<&HeaderMap>) -> Result<Option<crate::hybrid::Side>, AppError> {
let mut values = headers
.map(|h| h.get_all(crate::hybrid::ROUTE_HEADER).iter())
.into_iter()
.flatten();
let value = values.next();
if values.next().is_some() {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
format!("{} given more than once", crate::hybrid::ROUTE_HEADER),
));
}
crate::hybrid::parse_pin(value.map(|v| v.as_bytes()))
.map_err(|e| AppError(e, StatusCode::BAD_REQUEST))
}
fn hybrid_fallback(
model_ref: &str,
headers: Option<&HeaderMap>,
) -> Result<Option<String>, AppError> {
let Some(pair) = crate::hybrid::split_ref(model_ref) else {
return Ok(None);
};
Ok((request_pin(headers)? != Some(crate::hybrid::Side::Local)).then(|| pair.remote_ref()))
}
async fn send_with_hybrid_fallback<F, Fut>(
state: &AppState,
model_ref: &str,
headers: Option<&HeaderMap>,
request_threads: Option<u32>,
send: F,
) -> Result<Response, AppError>
where
F: Fn(String, Target, ActivityGuard) -> Fut,
Fut: std::future::Future<Output = Result<Response, AppError>>,
{
let resolve =
|m: String| async move { ensure_model(state, &m, headers, request_threads).await };
with_hybrid_fallback(model_ref, headers, resolve, send).await
}
async fn with_hybrid_fallback<R, RFut, F, Fut>(
model_ref: &str,
headers: Option<&HeaderMap>,
resolve: R,
send: F,
) -> Result<Response, AppError>
where
R: Fn(String) -> RFut,
RFut: std::future::Future<Output = Result<(String, Target, ActivityGuard), AppError>>,
F: Fn(String, Target, ActivityGuard) -> Fut,
Fut: std::future::Future<Output = Result<Response, AppError>>,
{
let (model, target, guard) = resolve(model_ref.to_string()).await?;
let fallback = match target {
Target::Local(_) => hybrid_fallback(model_ref, headers)?,
_ => None,
};
let Some(cloud) = fallback else {
return send(model, target, guard).await;
};
let refusal = match send(model, target, guard).await {
Ok(resp) => match local_context_overflow(resp).await {
Ok(resp) => return Ok(resp),
Err(refusal) => refusal,
},
Err(err) => match err.0.downcast_ref::<ContextOverflow>() {
Some(overflow) => overflow.refusal.clone(),
None => return Err(err),
},
};
eprintln!("[llmman] hybrid {model_ref:?} -> cloud ({refusal})");
let (model, target, guard) = resolve(cloud).await?;
send(model, target, guard).await
}
const OVERFLOW_BODY_LIMIT: usize = 64 * 1024;
async fn local_context_overflow(resp: Response) -> Result<Response, String> {
if resp.status() != StatusCode::BAD_REQUEST {
return Ok(resp);
}
let (parts, body) = resp.into_parts();
let mut rest = body.into_data_stream();
let mut head = Vec::new();
while head.len() <= OVERFLOW_BODY_LIMIT {
match rest.next().await {
Some(Ok(chunk)) => head.extend_from_slice(&chunk),
Some(Err(_)) | None => break,
}
}
if head.len() <= OVERFLOW_BODY_LIMIT {
if let Some(refusal) =
context_overflow_message(parts.status, &String::from_utf8_lossy(&head))
{
return Err(refusal);
}
}
let head = futures::stream::once(futures::future::ready(Ok(Bytes::from(head))));
Ok(Response::from_parts(
parts,
Body::from_stream(head.chain(rest)),
))
}
async fn ensure_model(
state: &AppState,
model_ref: &str,
headers: Option<&HeaderMap>,
request_threads: Option<u32>,
) -> Result<(String, Target, ActivityGuard), AppError> {
let hybrid_side = crate::hybrid::split_ref(model_ref)
.map(|pair| resolve_hybrid_side(state, &pair, headers))
.transpose()?;
let model_ref = hybrid_side.as_deref().unwrap_or(model_ref);
if let Some(target) = resolve_remote_target(state, model_ref, headers).await? {
return Ok((
model_ref.to_string(),
target,
ActivityGuard::new(state, model_ref),
));
}
let load_id = load_identity(&state.0.store_path, model_ref).map_err(AppError::bad_request)?;
let model_ref =
crate::shortnames::resolve_ollama_api(model_ref).map_err(AppError::bad_request)?;
let model_ref = crate::storage::default_tag(&model_ref);
let model_ref = canonical_ref(&state.0.store_path, &model_ref);
let model_ref = model_ref.as_str();
if let Some((port, guard)) = check_running(state, model_ref).await {
return Ok((model_ref.to_string(), Target::Local(port), guard));
}
if let Some(peer) = aggregation::route(state, model_ref, headers).await {
return Ok((
model_ref.to_string(),
aggregation::target(state, peer),
ActivityGuard::new(state, model_ref),
));
}
let load_started = Instant::now();
let _queue_guard = try_admit(state.0.max_queue)?;
let _guard = acquire_load_lock(&load_id).await;
if let Some((port, guard)) = check_running(state, model_ref).await {
return Ok((model_ref.to_string(), Target::Local(port), guard));
}
{
let present = crate::storage::OciStore::open(&state.0.store_path)
.and_then(|s| s.find(model_ref))
.is_ok();
if !present {
eprintln!("[llmman] {model_ref} not in store — pulling");
}
let store_path = state.0.store_path.clone();
let model_ref_owned = model_ref.to_owned();
let notices =
tokio::task::spawn_blocking(move || pull_serialized(&store_path, &model_ref_owned))
.await
.context("pull task panicked")?
.context("pull failed")?;
for notice in notices {
eprintln!("[llmman] {notice}");
}
}
let model_ref = canonical_ref(&state.0.store_path, model_ref);
let model_ref = model_ref.as_str();
if let Some((port, guard)) = check_running(state, model_ref).await {
return Ok((model_ref.to_string(), Target::Local(port), guard));
}
let (digest, size) = OciStore::open(&state.0.store_path)
.and_then(|s| {
s.find(model_ref).map(|d| {
let size = s.total_size(&d);
(d.digest, size)
})
})
.unwrap_or_default();
if !digest.is_empty() {
if let Some((key, port, guard)) = check_running_by_digest(state, model_ref, &digest).await {
return Ok((key, Target::Local(port), guard));
}
}
let model_path = resolve_model(&state.0.store_path, &state.0.cache_path, model_ref)
.with_context(|| format!("resolve model {model_ref}"))?;
let context_shift = supports_context_shift(model_ref);
let gguf_info = match &model_path {
ModelPath::Gguf(path, _) => crate::gguf::read_info(path).ok(),
_ => None,
};
let trained_ctx = gguf_info.as_ref().and_then(gguf_trained_ctx);
let embedding_ctx = gguf_info.as_ref().and_then(embedding_model_ctx);
let mut pending_load_guard =
enforce_max_loaded_models(state, state.0.max_loaded_models).await?;
let mut ctx_size = initial_ctx_size(
state.0.ctx_size,
state.0.ctx_size_explicit,
trained_ctx,
embedding_ctx.is_some(),
);
let mut split_mode = state.0.split_mode;
let num_parallel = effective_num_parallel(ctx_size, state.0.num_parallel);
if state.0.num_parallel.is_some() && num_parallel.is_none() {
eprintln!(
"[llmman] {model_ref}: no explicit ctx-size to scale, ignoring LLMMAN_NUM_PARALLEL for this load"
);
}
let mut shrink_attempts = 0u32;
let mut evicted_others = false;
let mut split_mode_relaxed = false;
let mut process;
let mut port = find_free_port()?;
loop {
eprintln!("[llmman] loading {model_ref} on port {port}");
let scaled_ctx_size = backend_ctx_size(ctx_size, num_parallel);
let llama_opts = crate::container::LlamaOptions {
port,
ctx_size: scaled_ctx_size,
flash_attention: state.0.flash_attention.as_deref(),
kv_cache_type: state.0.kv_cache_type.as_deref(),
context_shift,
split_mode,
num_parallel,
embeddings: embedding_ctx.is_some(),
batch_size: embedding_ctx.and(ctx_size).filter(|n| *n > 0),
threads: request_threads.or(state.0.threads),
cpus: state.0.cpu_limit,
};
let mut stderr_tail: Option<OutputTail> = None;
let mut oom_retryable = false;
let max_model_len = vllm_max_model_len(ctx_size, state.0.ctx_size_explicit);
let safetensors_engine = safetensors_engine_from_env();
let ociman = state.0.runtime.ociman().await?;
process = match (&model_path, ociman) {
(ModelPath::Gguf(path, mmproj), Some(ociman)) => {
let mut child = crate::container::spawn(
ociman,
path,
mmproj.as_deref(),
state.0.llama_cpp_version.as_deref(),
llama_opts,
)?;
stderr_tail = Some(tail_child_output(&mut child));
oom_retryable = true;
ModelProcess::Container(ociman, Engine::LlamaServer, child)
}
(ModelPath::Gguf(path, mmproj), None) => {
let bin = state.0.runtime.local_llama_server_bin().await?;
let (child, tail) =
spawn_llama_server(&bin, path, mmproj.as_deref(), llama_opts).await?;
stderr_tail = Some(tail);
oom_retryable = true;
ModelProcess::Local(Engine::LlamaServer, child, None)
}
(ModelPath::Diffusion(_), Some(ociman)) => {
let mut child = crate::container::spawn_mediagen(
ociman,
model_ref,
&state.0.store_path,
&state.0.cache_path,
state.0.llama_cpp_version.as_deref(),
port,
state.0.cpu_limit,
)?;
stderr_tail = Some(tail_child_output(&mut child));
oom_retryable = true;
ModelProcess::Container(ociman, Engine::LlamaServer, child)
}
(ModelPath::Diffusion(_), None) => {
let (child, tail) = spawn_mediagen_backend(model_ref, port, state).await?;
stderr_tail = Some(tail);
oom_retryable = true;
ModelProcess::Local(Engine::LlamaServer, child, None)
}
(ModelPath::SafeTensors(dir), Some(ociman)) => {
let sglang = safetensors_engine == SafetensorsEngine::Sglang;
let (engine, container_engine, version) = if sglang {
(
Engine::Sglang,
crate::container::ContainerEngine::Sglang,
state.0.sglang_version.as_deref(),
)
} else {
(
Engine::Vllm,
crate::container::ContainerEngine::Vllm,
state.0.vllm_version.as_deref(),
)
};
let mut child = crate::container::spawn_engine(
ociman,
container_engine,
dir,
version,
port,
state.0.cpu_limit,
|model_dir, host| {
if sglang {
sglang_serve_args_from_env(
model_dir,
host,
port,
model_ref,
max_model_len,
)
} else {
vllm_serve_args_from_env(
model_dir,
host,
port,
model_ref,
max_model_len,
)
}
},
)?;
stderr_tail = Some(tail_child_output(&mut child));
ModelProcess::Container(ociman, engine, child)
}
(ModelPath::Omni(dir), Some(ociman)) => {
let mut child = crate::container::spawn_engine(
ociman,
crate::container::ContainerEngine::VllmOmni,
dir,
state.0.vllm_version.as_deref(),
port,
state.0.cpu_limit,
|model_dir, host| {
vllm_omni_serve_args_from_env(model_dir, host, port, model_ref)
},
)?;
stderr_tail = Some(tail_child_output(&mut child));
ModelProcess::Container(ociman, Engine::VllmOmni, child)
}
(ModelPath::Omni(dir), None) => {
let (child, tail) = spawn_vllm_omni_server(dir, port, model_ref).await?;
stderr_tail = Some(tail);
let pid = child.id();
ModelProcess::Local(Engine::VllmOmni, child, pid)
}
(ModelPath::SafeTensors(dir), None)
if safetensors_engine == SafetensorsEngine::Sglang =>
{
let (child, tail) =
spawn_sglang_server(dir, port, model_ref, max_model_len).await?;
stderr_tail = Some(tail);
let pid = child.id();
ModelProcess::Local(Engine::Sglang, child, pid)
}
(ModelPath::SafeTensors(_dir), None) if use_mlx_for_safetensors() => {
let child = spawn_mlx_server(state, port).await?;
let pid = child.id();
ModelProcess::Local(Engine::Mlx, child, pid)
}
(ModelPath::SafeTensors(dir), None) => {
let child = spawn_vllm_server(dir, port, model_ref, max_model_len).await?;
let pid = child.id();
ModelProcess::Local(Engine::Vllm, child, pid)
}
};
match wait_for_ready(&state.0.client, port, &mut process, stderr_tail.as_ref()).await {
Ok(()) => break,
Err(e) => {
let looks_oom = oom_retryable && looks_like_oom(&e.to_string());
if !looks_oom {
return Err(e.into());
}
process.stop_and_wait().await;
if !evicted_others {
evicted_others = true;
if evict_other_models(state, model_ref).await {
metrics::record_oom_retry(model_ref, metrics::OomRetry::EvictOthers);
eprintln!(
"[llmman] {model_ref} failed to load on port {port}, which looks like an out-of-memory error — evicted other loaded models and retrying: {:#}",
e
);
port = find_free_port()?;
continue;
}
}
if !split_mode_relaxed && split_mode == Some("none") {
split_mode_relaxed = true;
split_mode = Some("layer");
metrics::record_oom_retry(model_ref, metrics::OomRetry::SplitMode);
eprintln!(
"[llmman] {model_ref} failed to load on port {port} with --split-mode none, which looks like an out-of-memory error — retrying with --split-mode layer (spread across every GPU) instead of failing outright: {:#}",
e
);
port = find_free_port()?;
continue;
}
let can_shrink =
!state.0.ctx_size_explicit && shrink_attempts < MAX_CTX_SHRINK_ATTEMPTS;
let Some(next) = can_shrink
.then(|| ctx_size.and_then(next_ctx_size_after_oom))
.flatten()
else {
return Err(e.into());
};
eprintln!(
"[llmman] {model_ref} failed to load on port {port}, which looks like an out-of-memory error — retrying with --ctx-size {next} (was {}): {:#}",
ctx_size
.map(|n| n.to_string())
.unwrap_or_else(|| "model default".to_string()),
e
);
ctx_size = Some(next);
shrink_attempts += 1;
metrics::record_oom_retry(model_ref, metrics::OomRetry::CtxShrink);
port = find_free_port()?;
}
}
}
eprintln!("[llmman] {model_ref} ready on port {port}");
let backend_model_path = match process.engine() {
Engine::Mlx => model_path.path().to_str().map(|s| s.to_string()),
Engine::Sglang => Some(sglang_served_model_name(model_ref)),
Engine::LlamaServer | Engine::Vllm | Engine::VllmOmni => None,
};
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
model_ref.to_string(),
RunningModel {
process,
port,
digest,
size,
started_at: now_rfc3339(),
last_active: Instant::now(),
last_active_wall: chrono::Utc::now(),
backend_model_path,
keep_alive: default_keep_alive(),
in_flight: 1,
},
);
pending_load_guard.release_into(&mut mgr);
drop(mgr);
metrics::record_model_load(model_ref, load_started.elapsed());
Ok((
model_ref.to_string(),
Target::Local(port),
ActivityGuard::new(state, model_ref),
))
}
async fn backend_wire_model(state: &AppState, target: &Target, canonical_model: &str) -> String {
if let Target::Remote(remote) = target {
return remote.model.clone();
}
state
.0
.manager
.lock()
.await
.running
.get(canonical_model)
.and_then(|r| r.backend_model_path.clone())
.unwrap_or_else(|| canonical_model.to_string())
}
async fn proxy(
client: &Client,
target: &Target,
route: &str,
headers: &HeaderMap,
body: Bytes,
activity: ActivityGuard,
) -> Result<Response, AppError> {
let mut req = target.authorize(client.post(target.url(route)).body(body));
if let Some(ct) = headers.get("content-type") {
req = req.header("content-type", ct);
}
let resp = req
.send()
.await
.with_context(|| format!("proxy request to {}", target.describe()))?;
Ok(relay(resp, activity))
}
fn relay(resp: reqwest::Response, activity: ActivityGuard) -> Response {
let status = resp.status();
let resp_headers = resp.headers().clone();
let stream = resp.bytes_stream().map(move |item| {
let _activity = &activity;
item.map_err(|e| Box::new(e) as Box<dyn std::error::Error + Send + Sync>)
});
let mut builder = Response::builder().status(status.as_u16());
for (k, v) in &resp_headers {
builder = builder.header(k, v);
}
builder.body(Body::from_stream(stream)).unwrap()
}
fn set_response_model(value: &mut serde_json::Value, canonical_model: &str) {
if value.get("model").is_some() {
value["model"] = serde_json::Value::String(canonical_model.to_string());
}
for key in ["response", "message"] {
if let Some(nested) = value.get_mut(key) {
if nested.get("model").is_some() {
nested["model"] = serde_json::Value::String(canonical_model.to_string());
}
}
}
}
fn rewrite_json_response_model(raw: &Bytes, canonical_model: &str) -> Bytes {
match serde_json::from_slice::<serde_json::Value>(raw) {
Ok(mut value) => {
set_response_model(&mut value, canonical_model);
serde_json::to_vec(&value)
.map(Bytes::from)
.unwrap_or_else(|_| raw.clone())
}
Err(_) => raw.clone(),
}
}
fn rewrite_sse_line_model(line: &str, canonical_model: &str) -> String {
match line.strip_prefix("data: ") {
Some(payload) if payload != "[DONE]" => match serde_json::from_str(payload) {
Ok(mut value) => {
set_response_model(&mut value, canonical_model);
format!(
"data: {}",
serde_json::to_string(&value).unwrap_or_else(|_| payload.to_string())
)
}
Err(_) => line.to_string(),
},
_ => line.to_string(),
}
}
async fn proxy_rewriting_model(
client: &Client,
target: &Target,
route: &str,
headers: &HeaderMap,
body: Bytes,
activity: ActivityGuard,
canonical_model: &str,
) -> Result<Response, AppError> {
let mut req = target.authorize(client.post(target.url(route)).body(body));
if let Some(ct) = headers.get("content-type") {
req = req.header("content-type", ct);
}
let resp = req
.send()
.await
.with_context(|| format!("proxy request to {}", target.describe()))?;
relay_rewriting_model(resp, activity, canonical_model).await
}
async fn relay_rewriting_model(
resp: reqwest::Response,
activity: ActivityGuard,
canonical_model: &str,
) -> Result<Response, AppError> {
let status = resp.status();
let resp_headers = resp.headers().clone();
let raw = resp
.bytes()
.await
.context("read inference backend response")?;
drop(activity);
let rewritten = rewrite_json_response_model(&raw, canonical_model);
let mut builder = Response::builder().status(status.as_u16());
for (k, v) in &resp_headers {
if k == reqwest::header::CONTENT_LENGTH {
continue;
}
builder = builder.header(k, v);
}
Ok(builder.body(Body::from(rewritten)).unwrap())
}
async fn stream_rewriting_model(
client: &Client,
target: &Target,
route: &str,
headers: &HeaderMap,
body: Bytes,
activity: ActivityGuard,
canonical_model: String,
) -> Result<Response, AppError> {
let mut req = target.authorize(client.post(target.url(route)).body(body));
if let Some(ct) = headers.get("content-type") {
req = req.header("content-type", ct);
}
let resp = req
.send()
.await
.with_context(|| format!("proxy request to {}", target.describe()))?;
Ok(relay_stream_rewriting_model(
resp,
activity,
canonical_model,
))
}
fn relay_stream_rewriting_model(
resp: reqwest::Response,
activity: ActivityGuard,
canonical_model: String,
) -> Response {
let status = resp.status();
let mut resp_headers = resp.headers().clone();
resp_headers.remove(reqwest::header::CONTENT_LENGTH);
resp_headers.remove(reqwest::header::TRANSFER_ENCODING);
let stream = bytes_to_lines(resp.bytes_stream()).map(move |line| {
let _activity = &activity;
let out = rewrite_sse_line_model(&line, &canonical_model) + "\n";
Ok::<_, std::convert::Infallible>(Bytes::from(out))
});
let mut builder = Response::builder().status(status.as_u16());
for (k, v) in &resp_headers {
builder = builder.header(k, v);
}
builder.body(Body::from_stream(stream)).unwrap()
}
fn apply_default_repeat_penalty_typed(oai_req: &mut OAIChatRequest) {
if oai_req.repeat_penalty.is_none() {
oai_req.repeat_penalty = Some(DEFAULT_REPEAT_PENALTY);
}
}
fn repeat_penalty_applies(target: &Target) -> bool {
!target.is_remote()
}
const LLAMA_FIELDS: &[&str] = &[
"repeat_penalty",
"chat_template_kwargs",
"top_k",
"min_p",
"typical_p",
"n_probs",
"cache_prompt",
"samplers",
"id_slot",
"n_keep",
"n_indent",
"dynatemp_range",
"dynatemp_exponent",
"mirostat",
"mirostat_tau",
"mirostat_eta",
"dry_multiplier",
"dry_base",
"dry_allowed_length",
"dry_penalty_last_n",
"dry_sequence_breakers",
"xtc_probability",
"xtc_threshold",
"top_n_sigma",
"penalty_last_n",
"repeat_last_n",
"n_predict",
"grammar",
"grammar_lazy",
"grammar_triggers",
"preserved_tokens",
"json_schema",
"reasoning_format",
"reasoning_budget",
"thinking_forced_open",
"return_progress",
"return_tokens",
"timings_per_token",
"post_sampling_probs",
"response_fields",
"min_keep",
"t_max_predict_ms",
"t_max_prompt_ms",
"ignore_eos",
"lora",
];
fn strip_llama_fields(req: &mut serde_json::Value) {
if let Some(o) = req.as_object_mut() {
for field in LLAMA_FIELDS {
o.remove(*field);
}
}
}
fn provider_compat(remote: &RemoteTarget, req: &mut serde_json::Value) {
let Some(o) = req.as_object_mut() else {
return;
};
if let Some(kwargs) = o.remove("chat_template_kwargs") {
let level = kwargs.get("reasoning_effort").and_then(|v| v.as_str());
let enabled = kwargs.get("enable_thinking").and_then(|v| v.as_bool());
let effort = match (level, enabled, remote.wire) {
(Some(level), _, _) => Some(level.to_string()),
(None, Some(true), Wire::Anthropic) => Some("medium".to_string()),
(None, Some(false), Wire::Anthropic) => Some("none".to_string()),
_ => None,
};
if let Some(effort) = effort {
o.entry("reasoning_effort")
.or_insert(serde_json::Value::String(effort));
}
}
for field in LLAMA_FIELDS {
o.remove(*field);
}
if remote.provider == "cohere" {
o.remove("stream_options");
}
if remote.wire == Wire::Anthropic {
anthropic::sampling_compat(&remote.model, o);
}
if remote.provider != "openai" {
return;
}
if !openai_reasoning_model(&remote.model) {
o.remove("reasoning_effort");
return;
}
if let Some(max) = o.remove("max_tokens") {
o.entry("max_completion_tokens").or_insert(max);
}
if o.get("temperature").and_then(|t| t.as_f64()) != Some(1.0) {
o.remove("temperature");
}
for field in [
"top_p",
"presence_penalty",
"frequency_penalty",
"logprobs",
"top_logprobs",
] {
o.remove(field);
}
if remote.model.contains("gpt-5") || remote.model.contains("gpt-6") {
for field in ["stop", "logit_bias", "modalities", "prediction", "audio"] {
o.remove(field);
}
}
}
fn openai_reasoning_model(model: &str) -> bool {
let name = model.rsplit('/').next().unwrap_or(model);
let o_series = name
.strip_prefix('o')
.and_then(|rest| rest.chars().next())
.is_some_and(|c| c.is_ascii_digit());
let gpt = (name.contains("gpt-5") || name.contains("gpt-6")) && !name.starts_with("gpt-5-chat");
o_series || gpt
}
type ChatBody = futures::stream::BoxStream<'static, reqwest::Result<Bytes>>;
struct ChatUpstream {
status: StatusCode,
headers: HeaderMap,
body: ChatBody,
}
impl ChatUpstream {
fn relay(resp: reqwest::Response) -> Self {
Self {
status: resp.status(),
headers: resp.headers().clone(),
body: resp.bytes_stream().boxed(),
}
}
fn translated(resp: &reqwest::Response, content_type: &'static str) -> HeaderMap {
let mut headers = resp.headers().clone();
headers.remove(reqwest::header::CONTENT_LENGTH);
headers.remove(reqwest::header::TRANSFER_ENCODING);
headers.insert(
reqwest::header::CONTENT_TYPE,
reqwest::header::HeaderValue::from_static(content_type),
);
headers
}
async fn text(self) -> String {
String::from_utf8_lossy(&collect_body(self.body).await).into_owned()
}
}
async fn collect_body(mut body: ChatBody) -> Vec<u8> {
let mut out = Vec::new();
while let Some(Ok(chunk)) = body.next().await {
out.extend_from_slice(&chunk);
}
out
}
async fn send_chat_completion<T: Serialize + ?Sized>(
client: &Client,
target: &Target,
req: &T,
response_model: &str,
) -> Result<ChatUpstream, AppError> {
let Target::Remote(remote) = target else {
let resp = target
.authorize(client.post(target.url(CHAT_COMPLETIONS_ROUTE)).json(req))
.send()
.await
.with_context(|| format!("send to {}", target.describe()))?;
return Ok(ChatUpstream::relay(resp));
};
let mut req = serde_json::to_value(req).context("serialize chat request")?;
provider_compat(remote, &mut req);
if remote.wire == Wire::OpenAi {
let resp = target
.authorize(client.post(target.url(CHAT_COMPLETIONS_ROUTE)).json(&req))
.send()
.await
.with_context(|| format!("send to {}", target.describe()))?;
return Ok(ChatUpstream::relay(resp));
}
let streaming = req
.get("stream")
.and_then(serde_json::Value::as_bool)
.unwrap_or(false);
if let Some(ceiling) = remote.max_output {
for key in ["max_tokens", "max_completion_tokens"] {
if req
.get(key)
.and_then(serde_json::Value::as_u64)
.is_some_and(|n| n > u64::from(ceiling))
{
req[key] = serde_json::json!(ceiling);
}
}
}
let default_max_tokens = remote.max_output.unwrap_or(anthropic::DEFAULT_MAX_TOKENS);
let messages_req = anthropic::from_chat_request(&req, default_max_tokens)
.map_err(|e| AppError(e, StatusCode::BAD_REQUEST))?;
let mut upstream = target.authorize(client.post(target.url(anthropic::MESSAGES_ROUTE)));
if anthropic::thinks(&messages_req) {
upstream = upstream.header("anthropic-beta", anthropic::INTERLEAVED_THINKING_BETA);
}
let resp = upstream
.json(&messages_req)
.send()
.await
.with_context(|| format!("send to {}", target.describe()))?;
if !resp.status().is_success() {
return Ok(ChatUpstream::relay(resp));
}
let mut converter =
anthropic::StreamConverter::new(response_model).json_tool(anthropic::json_tool_name(&req));
if !streaming {
let headers = ChatUpstream::translated(&resp, "application/json");
let lines: Vec<String> = bytes_to_lines(resp.bytes_stream()).collect().await;
for line in &lines {
converter.line(line);
}
converter.finish();
let (status, body) = match converter.error() {
Some(error) => (
StatusCode::BAD_GATEWAY,
serde_json::json!({ "error": error }),
),
None if !converter.finished() => (
StatusCode::BAD_GATEWAY,
serde_json::json!({ "error": {
"message": format!("{} closed the stream before finishing", target.describe()),
"type": "server_error",
} }),
),
None => (StatusCode::OK, converter.completion()),
};
let body = Bytes::from(serde_json::to_vec(&body).unwrap_or_default());
return Ok(ChatUpstream {
status,
headers,
body: futures::stream::once(futures::future::ready(Ok(body))).boxed(),
});
}
let headers = ChatUpstream::translated(&resp, "text/event-stream");
let body = bytes_to_lines(resp.bytes_stream())
.map(Some)
.chain(futures::stream::once(futures::future::ready(None)))
.map(move |line| {
let out = match line {
Some(line) => converter.line(&line),
None => converter.finish(),
};
Ok(Bytes::from(out))
})
.boxed();
Ok(ChatUpstream {
status: StatusCode::OK,
headers,
body,
})
}
fn relay_chat_upstream(upstream: ChatUpstream, activity: ActivityGuard) -> Response {
let stream = upstream.body.map(move |item| {
let _activity = &activity;
item.map_err(|e| Box::new(e) as Box<dyn std::error::Error + Send + Sync>)
});
let mut builder = Response::builder().status(upstream.status.as_u16());
for (k, v) in &upstream.headers {
builder = builder.header(k, v);
}
builder.body(Body::from_stream(stream)).unwrap()
}
async fn post_chat(
client: &Client,
target: &Target,
oai_req: &mut OAIChatRequest,
) -> Result<ChatBody, AppError> {
if repeat_penalty_applies(target) {
apply_default_repeat_penalty_typed(oai_req);
} else {
oai_req.repeat_penalty = None;
}
let upstream = send_chat_completion(client, target, &*oai_req, &oai_req.model).await?;
chat_body(target, upstream).await
}
async fn chat_body(target: &Target, upstream: ChatUpstream) -> Result<ChatBody, AppError> {
if !upstream.status.is_success() {
let status = upstream.status;
let body = upstream.text().await;
let message = format!("{} {status}: {body}", target.describe());
let error = match target {
Target::Local(_) => match context_overflow_message(status, &body) {
Some(refusal) => anyhow::Error::new(ContextOverflow { message, refusal }),
None => anyhow!("{message}"),
},
_ => anyhow!("{message}"),
};
return Err(AppError(error, remote_status(target, status)));
}
Ok(upstream.body)
}
#[derive(Debug)]
struct ContextOverflow {
message: String,
refusal: String,
}
impl std::fmt::Display for ContextOverflow {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.message)
}
}
impl std::error::Error for ContextOverflow {}
fn context_overflow_message(status: StatusCode, body: &str) -> Option<String> {
if status != StatusCode::BAD_REQUEST {
return None;
}
let json = &body[body.find('{')?..];
let value: serde_json::Value = serde_json::from_str(json).ok()?;
let error = &value["error"];
let message = error["message"]
.as_str()
.or_else(|| value["message"].as_str())
.unwrap_or("");
let overflow = error["type"] == "exceed_context_size_error"
|| message.contains("exceeds the available context size")
|| message.contains("maximum context length is");
overflow.then(|| message.to_string())
}
fn remote_status(target: &Target, upstream: StatusCode) -> StatusCode {
match target {
Target::Local(_) => StatusCode::INTERNAL_SERVER_ERROR,
Target::Peer(_) => upstream,
Target::Remote(_) if upstream.is_client_error() => upstream,
Target::Remote(_) => StatusCode::BAD_GATEWAY,
}
}
async fn handle_props() -> impl IntoResponse {
Json(serde_json::json!({
"role": "router",
"total_slots": 0,
"model_path": "",
"chat_template": "",
"bos_token": "",
"eos_token": "",
"build_info": env!("LLMMAN_VERSION"),
"modalities": { "vision": false, "audio": false },
"default_generation_settings": {
"id": 0,
"id_task": 0,
"n_ctx": 4096,
"speculative": false,
"is_processing": false,
"params": {
"n_predict": -1,
"seed": 0,
"temperature": 0.8,
"dynatemp_range": 0.0,
"dynatemp_exponent": 1.0,
"top_k": 40,
"top_p": 0.95,
"min_p": 0.05,
"top_n_sigma": 0.0,
"xtc_probability": 0.0,
"xtc_threshold": 0.1,
"typ_p": 1.0,
"repeat_last_n": 64,
"repeat_penalty": 1.0,
"presence_penalty": 0.0,
"frequency_penalty": 0.0,
"dry_multiplier": 0.0,
"dry_base": 1.75,
"dry_allowed_length": 2,
"dry_penalty_last_n": -1,
"dry_sequence_breakers": [],
"mirostat": 0,
"mirostat_tau": 5.0,
"mirostat_eta": 0.1,
"stop": [],
"max_tokens": -1,
"n_keep": 0,
"n_discard": 0,
"ignore_eos": false,
"stream": true,
"logit_bias": [],
"n_probs": 0,
"min_keep": 0,
"grammar": "",
"grammar_lazy": false,
"grammar_triggers": [],
"preserved_tokens": [],
"chat_format": "",
"reasoning_format": "",
"reasoning_in_content": false,
"generation_prompt": "",
"samplers": ["top_k", "top_p", "min_p", "temperature"],
"backend_sampling": false,
"speculative.n_max": 16,
"speculative.n_min": 5,
"speculative.p_min": 0.9,
"timings_per_token": false,
"post_sampling_probs": false,
"lora": []
},
"prompt": "",
"next_token": {
"has_next_token": false,
"has_new_line": false,
"n_remain": 0,
"n_decoded": 0,
"stopping_word": ""
}
}
}))
}
#[derive(Serialize)]
struct ProviderSummary {
id: String,
name: String,
base_url: String,
#[serde(skip_serializing_if = "Option::is_none")]
key_env: Option<String>,
wire: &'static str,
key_set: bool,
key_usable: bool,
key_optional: bool,
models: usize,
}
impl ProviderSummary {
fn new(state: &AppState, p: &crate::providers::Provider) -> Self {
Self {
id: p.id.clone(),
name: p.name.clone(),
base_url: p.base_url.clone(),
key_env: p.key_env.clone(),
wire: p.wire.as_str(),
key_set: p.api_key().is_some(),
key_usable: daemon_key_usable(state, p),
key_optional: p.key_optional,
models: p.models.len(),
}
}
}
fn daemon_key_usable(state: &AppState, provider: &crate::providers::Provider) -> bool {
provider.api_key().is_some() && daemon_key_spendable(state, None)
}
#[derive(Serialize)]
struct ProvidersResponse {
providers: Vec<ProviderSummary>,
}
#[derive(Serialize)]
struct ProviderResponse {
id: String,
name: String,
base_url: String,
#[serde(skip_serializing_if = "Option::is_none")]
key_env: Option<String>,
wire: &'static str,
key_set: bool,
key_usable: bool,
key_optional: bool,
models: Vec<ProviderModelResponse>,
}
#[derive(Serialize)]
struct ProviderModelResponse {
id: String,
#[serde(skip_serializing_if = "Option::is_none")]
cost: Option<ProviderCostResponse>,
}
#[derive(Serialize)]
struct ProviderCostResponse {
input: f64,
output: f64,
}
impl ProviderResponse {
fn new(state: &AppState, p: &crate::providers::Provider) -> Self {
Self {
id: p.id.clone(),
name: p.name.clone(),
base_url: p.base_url.clone(),
key_env: p.key_env.clone(),
wire: p.wire.as_str(),
key_set: p.api_key().is_some(),
key_usable: daemon_key_usable(state, p),
key_optional: p.key_optional,
models: p
.models
.iter()
.map(|m| ProviderModelResponse {
id: m.id.clone(),
cost: m.cost.map(|c| ProviderCostResponse {
input: c.input,
output: c.output,
}),
})
.collect(),
}
}
}
async fn handle_llmman_providers(
State(state): State<AppState>,
) -> Result<impl IntoResponse, AppError> {
let catalog = provider_catalog().await?;
Ok(Json(ProvidersResponse {
providers: catalog
.iter()
.map(|p| ProviderSummary::new(&state, p))
.collect(),
}))
}
async fn handle_llmman_provider(
State(state): State<AppState>,
UrlPath(id): UrlPath<String>,
) -> Result<impl IntoResponse, AppError> {
let catalog = provider_catalog().await?;
let provider = catalog.get(&id).ok_or_else(|| {
AppError(
crate::providers::unknown_provider_error(&id, &catalog),
StatusCode::NOT_FOUND,
)
})?;
let mut response = ProviderResponse::new(&state, provider);
if provider.key_optional && provider.models.is_empty() {
response.models = configured_provider_models(&state, provider)
.await
.into_iter()
.map(|id| ProviderModelResponse { id, cost: None })
.collect();
}
Ok(Json(response))
}
const CONFIGURED_MODELS_TIMEOUT: Duration = Duration::from_secs(5);
async fn configured_provider_models(
state: &AppState,
provider: &crate::providers::Provider,
) -> Vec<String> {
#[derive(Deserialize)]
struct ModelsResponse {
#[serde(default)]
data: Vec<ModelEntry>,
}
#[derive(Deserialize)]
struct ModelEntry {
id: String,
}
if provider.wire != Wire::OpenAi {
return Vec::new();
}
let url = provider.url("/v1/models");
let mut req = state.0.client.get(&url).timeout(CONFIGURED_MODELS_TIMEOUT);
if daemon_key_usable(state, provider) {
if let Some(key) = provider.api_key() {
req = req.bearer_auth(key);
}
}
let listed = async {
let resp = req.send().await?.error_for_status()?;
resp.json::<ModelsResponse>().await
}
.await;
match listed {
Ok(models) => {
let mut ids: Vec<String> = models.data.into_iter().map(|m| m.id).collect();
ids.sort_unstable();
ids.dedup();
ids
}
Err(e) => {
crate::debug_log!("provider {}: GET {url} failed: {e}", provider.id);
Vec::new()
}
}
}
trait SseConverter: Send + 'static {
fn line(&mut self, line: &str) -> String;
fn finish(&mut self) -> String;
fn failed(&self) -> bool;
fn fold(&mut self, lines: Vec<String>) -> serde_json::Value;
}
macro_rules! sse_converter {
($($t:ty),*) => {$(
impl SseConverter for $t {
fn line(&mut self, line: &str) -> String {
Self::line(self, line)
}
fn finish(&mut self) -> String {
Self::finish(self)
}
fn failed(&self) -> bool {
Self::failed(self)
}
fn fold(&mut self, lines: Vec<String>) -> serde_json::Value {
Self::fold(self, lines)
}
}
)*};
}
sse_converter!(responses::StreamConverter, messages::StreamConverter);
async fn convert_upstream(
body: ChatBody,
activity: ActivityGuard,
mut converter: impl SseConverter,
streaming: bool,
) -> Response {
if !streaming {
let lines: Vec<String> = bytes_to_lines(body).collect().await;
drop(activity);
let response = converter.fold(lines);
let status = if converter.failed() {
StatusCode::BAD_GATEWAY
} else {
StatusCode::OK
};
return (status, Json(response)).into_response();
}
let sse_stream = bytes_to_lines(body)
.map(Some)
.chain(futures::stream::once(futures::future::ready(None)))
.map(move |line| {
let _activity = &activity;
let out = match line {
Some(line) => converter.line(&line),
None => converter.finish(),
};
Ok::<_, std::convert::Infallible>(Bytes::from(out))
});
Response::builder()
.header("content-type", "text/event-stream")
.header("cache-control", "no-cache")
.body(Body::from_stream(sse_stream))
.unwrap()
}
fn unsupported_on_wire(target: &Target, route: &str) -> Option<Response> {
let message = wire_refusal(target, route)?;
let body = serde_json::json!({
"error": { "message": message, "type": "invalid_request_error" }
});
Some((StatusCode::NOT_IMPLEMENTED, Json(body)).into_response())
}
fn wire_refusal(target: &Target, route: &str) -> Option<String> {
let Target::Remote(remote) = target else {
return None;
};
if remote.wire != Wire::Anthropic
|| matches!(route, CHAT_COMPLETIONS_ROUTE | responses::RESPONSES_ROUTE)
{
return None;
}
Some(format!(
"provider {} speaks the Anthropic Messages API, which has no equivalent of {route}; \
only chat completions, /v1/responses and /v1/messages reach it",
remote.provider
))
}
fn explain_missing_route(target: &Target, route: &str, resp: Response) -> Response {
if resp.status() != StatusCode::NOT_FOUND || !responses::is_responses_route(route) {
return resp;
}
let Target::Remote(remote) = target else {
return resp;
};
let body = serde_json::json!({
"error": {
"message": format!(
"provider {} has no {route} — it is OpenAI-compatible but does not \
implement the Responses API's token counting. Generation on \
/v1/responses is bridged; this route cannot be.",
remote.provider
),
"type": "invalid_request_error",
}
});
(StatusCode::NOT_IMPLEMENTED, Json(body)).into_response()
}
fn opt_f64(opts: &Option<serde_json::Value>, key: &str) -> Option<f32> {
opts.as_ref()?.get(key)?.as_f64().map(|f| f as f32)
}
fn opt_u32(opts: &Option<serde_json::Value>, key: &str) -> Option<u32> {
opts.as_ref()?.get(key)?.as_u64().map(|n| n as u32)
}
fn opt_num_thread(opts: &Option<serde_json::Value>) -> Option<u32> {
let n = opts.as_ref()?.get("num_thread")?.as_u64()?;
u32::try_from(n).ok().filter(|&n| n != 0)
}
fn default_allowed_origins() -> Vec<String> {
let mut origins = Vec::new();
for host in ["localhost", "127.0.0.1", "0.0.0.0", "[::1]"] {
for scheme in ["http", "https"] {
origins.push(format!("{scheme}://{host}"));
origins.push(format!("{scheme}://{host}:*"));
}
}
origins
}
fn allowed_origins_from_env() -> Vec<String> {
let mut origins: Vec<String> = std::env::var("LLMMAN_ORIGINS")
.ok()
.map(|raw| {
raw.split(',')
.map(str::trim)
.filter(|s| !s.is_empty())
.map(str::to_string)
.collect()
})
.unwrap_or_default();
origins.extend(default_allowed_origins());
origins
}
fn origin_matches(origin: &str, pattern: &str) -> bool {
match pattern.split_once('*') {
Some((prefix, suffix)) if !suffix.contains('*') => {
origin.len() >= prefix.len() + suffix.len()
&& origin.starts_with(prefix)
&& origin.ends_with(suffix)
}
Some(_) => false,
None => origin == pattern,
}
}
fn cors_layer() -> tower_http::cors::CorsLayer {
let patterns = allowed_origins_from_env();
tower_http::cors::CorsLayer::new()
.allow_methods(tower_http::cors::AllowMethods::any())
.allow_headers(tower_http::cors::AllowHeaders::any())
.allow_origin(tower_http::cors::AllowOrigin::predicate(
move |origin, _parts| {
origin
.to_str()
.is_ok_and(|origin| patterns.iter().any(|p| origin_matches(origin, p)))
},
))
}
pub const MEDIAGEN_ENV_PASSTHROUGH_VARS: &[&str] = &["MEDIAGEN_DUMP", "MEDIAGEN_VAE_TILE"];
async fn track_metrics(req: Request, next: Next) -> Response {
let route = req.extensions().get::<MatchedPath>().cloned();
let started = Instant::now();
let response = next.run(req).await;
let Some(route) = route else {
return response;
};
metrics::record_request(
route.as_str(),
response.status().as_u16(),
started.elapsed(),
);
let (parts, body) = response.into_parts();
if body.size_hint().exact().is_some() {
metrics::record_response_body(route.as_str(), started.elapsed());
return Response::from_parts(parts, body);
}
let timer = ResponseBodyTimer {
route: route.as_str().to_string(),
started,
};
let body = Body::from_stream(body.into_data_stream().map(move |chunk| {
let _timer = &timer;
chunk
}));
Response::from_parts(parts, body)
}
async fn record_prompt(State(state): State<AppState>, req: Request, next: Next) -> Response {
use axum::extract::FromRequestParts;
let Some(log) = state.0.prompt_log.as_deref() else {
return next.run(req).await;
};
let route = req
.extensions()
.get::<MatchedPath>()
.map(|p| p.as_str().to_string())
.filter(|r| crate::promptlog::is_generation_route(r));
let Some(route) = route.filter(|r| {
req.method() == axum::http::Method::POST
&& !(r.starts_with("/api/") && forged_cross_site(req.headers()))
}) else {
return next.run(req).await;
};
let (mut parts, body) = req.into_parts();
let selected_model = if route == "/gemini/:model/*gemini_path" {
let model = UrlPath::<(String, String)>::from_request_parts(&mut parts, &())
.await
.ok()
.filter(|UrlPath((_, path))| {
gemini_method(path).is_some_and(|(_, method)| method.generates())
})
.and_then(|UrlPath((encoded_model, _))| {
base64::engine::general_purpose::URL_SAFE_NO_PAD
.decode(encoded_model)
.ok()
})
.and_then(|bytes| String::from_utf8(bytes).ok());
let Some(model) = model else {
return next.run(Request::from_parts(parts, body)).await;
};
Some(model)
} else {
None
};
let body = match Bytes::from_request(Request::from_parts(parts.clone(), body), &()).await {
Ok(body) => body,
Err(rejection) => return rejection.into_response(),
};
let client = parts
.headers
.get(axum::http::header::USER_AGENT)
.and_then(|v| v.to_str().ok());
if let Some(mut entry) = crate::promptlog::entry(&route, &body, client, now_rfc3339()) {
if let Some(model) = selected_model {
entry.model = model;
}
if let Err(e) = crate::promptlog::append(log, &entry) {
eprintln!("[llmman] warning: prompt log {}: {e}", log.display());
}
}
next.run(Request::from_parts(parts, Body::from(body))).await
}
async fn accept_any_content_type(mut req: Request, next: Next) -> Response {
if !has_json_content_type(req.headers()) {
if foreign_origin(req.headers()) {
let body = serde_json::json!({ "error": "cross-site request refused" });
return (StatusCode::FORBIDDEN, Json(body)).into_response();
}
req.headers_mut().insert(
axum::http::header::CONTENT_TYPE,
axum::http::HeaderValue::from_static("application/json"),
);
}
next.run(req).await
}
fn has_json_content_type(headers: &HeaderMap) -> bool {
headers
.get(axum::http::header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.is_some_and(is_json_content_type)
}
fn forged_cross_site(headers: &HeaderMap) -> bool {
!has_json_content_type(headers) && foreign_origin(headers)
}
fn foreign_origin(headers: &HeaderMap) -> bool {
headers
.get(axum::http::header::ORIGIN)
.and_then(|v| v.to_str().ok())
.is_some_and(|origin| {
!allowed_origins_from_env()
.iter()
.any(|pattern| origin_matches(origin, pattern))
})
}
fn is_json_content_type(value: &str) -> bool {
let essence = value
.split(';')
.next()
.unwrap_or_default()
.trim()
.to_ascii_lowercase();
essence == "application/json"
|| (essence.starts_with("application/") && essence.ends_with("+json"))
}
struct ResponseBodyTimer {
route: String,
started: Instant,
}
impl Drop for ResponseBodyTimer {
fn drop(&mut self) {
metrics::record_response_body(&self.route, self.started.elapsed());
}
}
async fn handle_metrics(State(state): State<AppState>) -> impl IntoResponse {
let (models_loaded, models_loading, models) = {
let mut mgr = state.0.manager.lock().await;
let models = mgr
.running
.iter_mut()
.map(|(model, m)| metrics::ModelState {
model: model.clone(),
engine: m.engine_label(),
up: m.process.is_alive(),
})
.collect();
(mgr.running.len() as u64, mgr.pending_loads as u64, models)
};
let snapshot = metrics::Snapshot {
version: env!("LLMMAN_VERSION").to_string(),
start_time_seconds: metrics::process_start_seconds(),
scheduling_requests_in_flight: PENDING_REQUESTS.load(std::sync::atomic::Ordering::SeqCst)
as u64,
scheduling_capacity: state.0.max_queue.max(1) as u64,
models_loaded,
models_loading,
models,
};
(
[("content-type", "text/plain; version=0.0.4; charset=utf-8")],
metrics::render(&snapshot),
)
}
pub fn run(args: &ServeArgs) -> anyhow::Result<()> {
if let (Some(port), Some(model)) = (args.port, &args.model) {
return tokio::runtime::Runtime::new()?.block_on(mediagen_backend(model, port, args));
}
tokio::runtime::Runtime::new()?.block_on(serve_async(args))
}
pub fn llama_lib_dir(runtime: Runtime, pinned_version: Option<&str>) -> anyhow::Result<PathBuf> {
let bin = runtime::resolve_local(runtime, pinned_version)?;
crate::mediagen::ffi::lib_dir_of(&bin).ok_or_else(|| {
anyhow!(
"no ggml/llama shared libraries next to {}; media generation needs a llama.cpp release or installed build",
bin.display()
)
})
}
async fn mediagen_backend(model_ref: &str, port: u16, args: &ServeArgs) -> anyhow::Result<()> {
let store_path = default_store()?;
let cache_path = crate::default_cache()?;
let ModelPath::Diffusion(paths) = resolve_model(&store_path, &cache_path, model_ref)? else {
anyhow::bail!("{model_ref} is not a diffusion model");
};
if paths.text_encoder.is_none() && crate::mediagen::needs_text_encoder(&paths.model) {
anyhow::bail!("{model_ref}: no text encoder in the model pack");
}
let pinned = runtime::llama_cpp_pin(args.llama_cpp_version.as_deref());
let runtime = args.runtime;
let lib_dir =
tokio::task::spawn_blocking(move || llama_lib_dir(runtime, pinned.as_deref())).await??;
let api: &'static crate::mediagen::ffi::Api =
Box::leak(Box::new(crate::mediagen::ffi::Api::load(&lib_dir)?));
let params = crate::mediagen::ContextParams {
model: paths.model.clone(),
vae: paths.vae.clone(),
audio_vae: paths.audio_vae.clone(),
text_proj: paths.text_proj.clone(),
text_model: paths.text_encoder.clone(),
files: paths.files.clone(),
use_gpu: true,
text_gpu_layers: std::env::var("LLAMA_ARG_N_GPU_LAYERS")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(999),
n_threads: std::env::var("LLAMA_ARG_THREADS")
.ok()
.and_then(|v| v.parse().ok())
.or_else(|| threads_from_env_or_host().map(|n| n as i32))
.unwrap_or_else(|| std::thread::available_parallelism().map_or(4, |n| n.get() as i32)),
flash_attn: flash_attention_from_env().as_deref() != Some("off"),
};
let ctx = tokio::task::spawn_blocking(move || crate::mediagen::Context::init(api, ¶ms))
.await?
.with_context(|| {
format!(
"loading the model with the llama.cpp libraries in {} (an old build? \
remove it from PATH or pass --llama-cpp-version to use a release)",
lib_dir.display()
)
})?;
let router = crate::mediagen::server::router(
ctx,
model_ref.to_string(),
paths.model.to_string_lossy().into_owned(),
);
crate::mediagen::server::serve(router, (args.host, port).into()).await
}
async fn spawn_mediagen_backend(
model_ref: &str,
port: u16,
state: &AppState,
) -> anyhow::Result<(tokio::process::Child, OutputTail)> {
let exe = std::env::current_exe().context("locating the llmman binary")?;
let mut cmd = tokio::process::Command::new(&exe);
cmd.args(["serve", model_ref, "--port", &port.to_string()]);
let runtime = state.0.runtime.resolve().await?.runtime();
cmd.args(["--runtime", runtime.as_str()]);
cmd.args([
"--llama-cpp-version",
state.0.llama_cpp_version.as_deref().unwrap_or("latest"),
]);
for var in GPU_VISIBLE_DEVICE_VARS
.iter()
.chain(LLAMA_CPP_ENV_PASSTHROUGH_VARS)
.chain(MEDIAGEN_ENV_PASSTHROUGH_VARS)
{
if let Ok(val) = std::env::var(var) {
cmd.env(var, val);
}
}
crate::debug_log!("spawning {}: {:?}", exe.display(), cmd);
cmd.stdout(std::process::Stdio::piped());
cmd.stderr(std::process::Stdio::piped());
let mut child = cmd
.kill_on_drop(true)
.spawn()
.with_context(|| format!("spawn media generation backend from {}", exe.display()))?;
let tail = tail_child_output(&mut child);
Ok((child, tail))
}
fn build_router(app_state: AppState, metrics_enabled: bool) -> Router {
let app = Router::new()
.route("/", get(webui::handle_root))
.route("/ui/*path", get(webui::handle_asset))
.route("/props", get(handle_props))
.route("/llmman/providers", get(handle_llmman_providers))
.route("/llmman/providers/:id", get(handle_llmman_provider))
.route("/llmman/node", get(aggregation::handle_node))
.route("/llmman/shell", get(shell::handle_shell))
.merge(ollama_router())
.route("/v1/models", get(handle_openai_models))
.route("/v1/chat/completions", post(handle_openai_chat))
.route("/v1/completions", post(handle_openai_completions))
.route("/v1/embeddings", post(handle_openai_embeddings))
.route(
"/v1/audio/transcriptions",
post(handle_openai_transcriptions)
.layer(DefaultBodyLimit::max(TRANSCRIPTION_BODY_LIMIT_BYTES)),
)
.route(
"/audio/transcriptions",
post(handle_openai_transcriptions)
.layer(DefaultBodyLimit::max(TRANSCRIPTION_BODY_LIMIT_BYTES)),
)
.route("/v1/responses", post(responses::handle_openai_responses))
.route(
"/v1/responses/input_tokens",
post(responses::handle_openai_responses_input_tokens),
)
.route("/v1/images/generations", post(handle_openai_images))
.route("/v1/videos", post(handle_openai_videos))
.route("/v1/videos/:id", get(handle_openai_video_get))
.route("/v1/videos/:id/content", get(handle_openai_video_get))
.route("/v1/audio/speech", post(handle_openai_speech))
.route("/gemini/:model/*gemini_path", post(handle_pinned_gemini))
.route("/v1/messages", post(anthropic::handle_anthropic_messages));
let app = app.layer(middleware::from_fn_with_state(
app_state.clone(),
record_prompt,
));
let app = if metrics_enabled {
app.layer(middleware::from_fn(track_metrics))
} else {
app
};
let require_key = || middleware::from_fn_with_state(app_state.clone(), auth::require_key);
app.layer(require_key())
.layer(cors_layer())
.merge(metrics_router(metrics_enabled).layer(require_key()))
.with_state(app_state)
}
fn ollama_router() -> Router<AppState> {
Router::new()
.route("/api/version", get(handle_version))
.route("/api/tags", get(handle_tags))
.route("/api/ps", get(handle_ps))
.route("/api/show", post(handle_show))
.route("/api/pull", post(handle_pull))
.route("/api/push", post(handle_push))
.route("/api/delete", delete(handle_delete))
.route("/api/copy", post(handle_copy))
.route("/api/create", post(handle_create))
.route(
"/api/blobs/:digest",
axum::routing::head(handle_blob_head)
.post(handle_blob_upload)
.layer(DefaultBodyLimit::disable()),
)
.route("/api/chat", post(handle_ollama_chat))
.route("/api/generate", post(handle_ollama_generate))
.route("/api/embed", post(handle_embed))
.route("/api/embeddings", post(handle_embeddings))
.layer(middleware::from_fn(accept_any_content_type))
}
fn metrics_router(enabled: bool) -> Router<AppState> {
if !enabled {
return Router::new();
}
Router::new()
.route("/metrics", get(handle_metrics))
.layer(middleware::from_fn(track_metrics))
}
fn pull_only_engine(model: Option<&str>) -> anyhow::Result<crate::container::ContainerEngine> {
let model = model
.map(crate::hybrid::local_half)
.filter(|m| !crate::providers::is_remote_ref(m));
let Some(model) = model else {
return Ok(crate::container::ContainerEngine::LlamaServer);
};
let model_ref = crate::shortnames::resolve_ollama_api(model)?;
let store_path = default_store()?;
let format = crate::modelpack::stored_format(&store_path, &model_ref).with_context(|| {
format!("--pull-only: pull {model_ref} first to learn which backend it needs")
})?;
Ok(match format {
crate::modelpack::ModelFormat::SafeTensors
if safetensors_engine_from_env() == SafetensorsEngine::Sglang =>
{
crate::container::ContainerEngine::Sglang
}
crate::modelpack::ModelFormat::SafeTensors => crate::container::ContainerEngine::Vllm,
crate::modelpack::ModelFormat::Omni => crate::container::ContainerEngine::VllmOmni,
crate::modelpack::ModelFormat::Gguf | crate::modelpack::ModelFormat::Diffusion => {
crate::container::ContainerEngine::LlamaServer
}
})
}
async fn serve_async(_args: &ServeArgs) -> anyhow::Result<()> {
let requested = _args.runtime;
let installs_mlx = match backend::mlx_source(requested) {
Some(fallback) if use_mlx_for_safetensors() && requested.ociman().is_none() => {
Some(fallback)
}
_ => None,
};
if requested == Runtime::Path && _args.pull_only {
anyhow::bail!("--pull-only: --runtime path runs what is on PATH; nothing to pull");
}
let llama_cpp_version = runtime::llama_cpp_pin(_args.llama_cpp_version.as_deref());
let resolved = {
let pin = llama_cpp_version.clone();
tokio::task::spawn_blocking(move || runtime::resolve(requested, pin.as_deref()))
.await
.context("resolve runtime task panicked")?
};
let resolved = match resolved {
Ok(resolved) => Some(resolved),
Err(e) if _args.pull_only => return Err(e),
Err(e) => {
eprintln!(
"[llmman] warning: could not set up the llama.cpp runtime at startup ({e:#}); \
the first model load that needs it will retry"
);
None
}
};
let runtime = runtime::Lazy::new(requested, llama_cpp_version.clone(), resolved);
if let Some(fallback) = installs_mlx {
let installed =
tokio::task::spawn_blocking(move || crate::mlx_release::ensure_mlx_server(fallback))
.await
.context("ensure mlx_lm.server task panicked")?;
match installed {
Ok(_) => {}
Err(e) if _args.pull_only => return Err(e.context("install mlx-lm")),
Err(e) => eprintln!(
"[llmman] warning: could not install mlx-lm at startup ({e:#}); \
the first safetensors model load will retry it"
),
}
}
if _args.pull_only {
if let Some(ociman) = runtime.known().and_then(|r| r.ociman()) {
let engine = pull_only_engine(_args.model.as_deref())?;
let version = match engine {
crate::container::ContainerEngine::LlamaServer => None,
crate::container::ContainerEngine::Sglang => Some(_args.sglang_version.as_deref()),
crate::container::ContainerEngine::Vllm
| crate::container::ContainerEngine::VllmOmni => {
Some(_args.vllm_version.as_deref())
}
};
if let Some(version) = version {
crate::container::pull_image(ociman, engine, version)?;
}
}
return Ok(());
}
let store_path = default_store()?;
let cache_path = crate::default_cache()?;
std::fs::create_dir_all(&cache_path)?;
crate::storage::repair::repair_store(&store_path)?;
if !crate::storage::gc::noprune_from_env() {
if let Ok(store) = OciStore::open(&store_path) {
if let Ok(live) = crate::storage::gc::referenced_digests(&store) {
let grace = crate::storage::gc::GC_GRACE_PERIOD;
if let Err(e) = crate::storage::gc::prune_blobs(&store_path, &live, grace) {
eprintln!("[llmman] blob GC sweep failed: {e:#}");
}
if let Err(e) = crate::storage::gc::prune_cache(&cache_path, &live, grace) {
eprintln!("[llmman] cache GC sweep failed: {e:#}");
}
}
}
}
let peers: Vec<String> = crate::config::peers()
.iter()
.map(|p| crate::daemon::peer_url(p))
.collect();
let ctx_size_explicit = context_length_from_env();
let vram = if !peers.is_empty() {
tokio::task::spawn_blocking(crate::hostgpu::detect_with_vram)
.await
.context("hostgpu probe task panicked")?
.1
} else {
0
};
let ctx_size = ctx_size_explicit.or(Some(DEFAULT_CTX_SIZE));
let memory = crate::hostgpu::memory_bytes(vram);
if !peers.is_empty() {
eprintln!(
"[llmman] aggregation peers: {} (this node: {} of model memory)",
peers.join(", "),
crate::fmt::human_size(memory)
);
}
let cpu_limit = container_cpu_limit();
let threads = threads_from_env_or_host();
if let Some(n) = threads {
eprintln!("[llmman] llama-server gets --threads {n} (CPU quota/affinity limit below the online CPU count)");
} else if std::env::var_os("LLAMA_ARG_THREADS").is_some() {
eprintln!("[llmman] LLAMA_ARG_THREADS set: leaving llama-server thread count to it");
}
if let (Some(n), Some(_)) = (cpu_limit, runtime.known().and_then(|r| r.ociman())) {
eprintln!("[llmman] backend container gets --cpus {n} (this daemon's own CPU limit)");
}
let chosen = match safetensors_engine_from_env() {
SafetensorsEngine::Auto => None,
SafetensorsEngine::Vllm => Some("vllm"),
SafetensorsEngine::Sglang => Some("sglang"),
};
if let Some(engine) = chosen {
eprintln!(
"[llmman] safetensors models go to {engine} ({})",
backend::SAFETENSORS_ENGINE_VAR
);
}
let auth = auth::Policy::from_env()?;
if auth.enforced() {
eprintln!("[llmman] API key required on every request (LLMMAN_API_KEYS)");
} else if !crate::daemon::reachable_only_locally() {
eprintln!(
"[llmman] warning: LLMMAN_AUTH=off — serving everyone who can reach {} without a key",
crate::daemon::bind_addr()
);
}
let tls = tls_from_env()?;
anyhow::ensure!(
tls.is_some() == crate::daemon::tls_scheme(),
"LLMMAN_HOST and LLMMAN_TLS_CERT/LLMMAN_TLS_KEY disagree: an https:// host needs the \
certificate and key, and they need an https:// host, so clients in this \
environment connect the way the daemon listens"
);
let client = crate::auth::trusted_client()?
.build()
.context("build http client")?;
let state = AppState(Arc::new(Inner {
manager: Mutex::new(ModelManager {
running: HashMap::new(),
pending_loads: 0,
}),
exe: std::env::current_exe()
.ok()
.map(|p| dunce::canonicalize(&p).unwrap_or(p)),
runtime,
llama_cpp_version,
vllm_version: _args.vllm_version.clone(),
sglang_version: _args.sglang_version.clone(),
ctx_size,
ctx_size_explicit: ctx_size_explicit.is_some(),
hybrid_local_bytes: crate::hybrid::local_budget_bytes_from_env(ctx_size),
flash_attention: flash_attention_from_env(),
kv_cache_type: kv_cache_type_from_env(),
split_mode: sched_spread_from_env(),
num_parallel: num_parallel_from_env(),
threads,
cpu_limit,
max_queue: max_queue_from_env(),
max_loaded_models: max_loaded_models_from_env(),
peers,
memory,
store_path,
cache_path,
prompt_log: crate::promptlog::enabled_from_env()
.then(crate::promptlog::path)
.transpose()?,
shell: shell::Policy::from_env(),
auth,
peer_key: crate::auth::peer_key(),
client,
}));
let app = build_router(state.clone(), metrics_enabled_from_env());
metrics::mark_process_start();
crate::verify::Policy::load().context("signature trust policy")?;
let addr = crate::daemon::bind_addr();
let listener = tokio::net::TcpListener::bind(&addr)
.await
.with_context(|| format!("bind {addr}"))?;
eprintln!(
"llmman serve listening on {addr}{}",
if tls.is_some() { " (TLS)" } else { "" }
);
tokio::spawn(reap_idle_models(state.clone()));
if let Some(model) = _args
.model
.as_deref()
.map(crate::hybrid::local_half)
.filter(|m| !crate::providers::is_remote_ref(m))
{
match crate::shortnames::resolve_ollama_api(model) {
Ok(model) => {
let state_clone = state.clone();
tokio::spawn(async move {
match ensure_model(&state_clone, &model, None, None).await {
Ok((_, _, guard)) => refresh_activity(guard, None).await,
Err(e) => eprintln!("[llmman] pre-load failed: {:#}", e.0),
}
});
}
Err(e) => eprintln!("[llmman] pre-load failed: {e}"),
}
}
match tls {
None => {
axum::serve(listener, app)
.with_graceful_shutdown(shutdown_signal())
.await?
}
Some((cert, key)) => {
let _ = rustls::crypto::ring::default_provider().install_default();
let config = axum_server::tls_rustls::RustlsConfig::from_pem_file(&cert, &key)
.await
.with_context(|| {
format!(
"load TLS certificate {} and key {}",
cert.display(),
key.display()
)
})?;
let handle = axum_server::Handle::new();
tokio::spawn({
let handle = handle.clone();
async move {
shutdown_signal().await;
handle.graceful_shutdown(Some(Duration::from_secs(30)));
}
});
axum_server::from_tcp_rustls(listener.into_std()?, config)
.handle(handle)
.serve(app.into_make_service())
.await?
}
}
state.0.manager.lock().await.running.clear();
Ok(())
}
async fn shutdown_signal() {
let ctrl_c = async {
let _ = tokio::signal::ctrl_c().await;
};
#[cfg(unix)]
let terminate = async {
match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) {
Ok(mut sig) => {
sig.recv().await;
}
Err(_) => std::future::pending().await,
}
};
#[cfg(not(unix))]
let terminate = std::future::pending::<()>();
tokio::select! {
_ = ctrl_c => {}
_ = terminate => {}
}
eprintln!("llmman serve shutting down");
}
#[cfg(test)]
mod tests;