use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::{Arc, LazyLock, Mutex as StdMutex};
use anyhow::{anyhow, Context};
use axum::body::{Body, Bytes, HttpBody as _};
use axum::extract::{DefaultBodyLimit, FromRequest, MatchedPath, Path as UrlPath, Request, State};
use axum::http::{HeaderMap, StatusCode};
use axum::middleware::{self, Next};
use axum::response::{IntoResponse, Response};
use axum::routing::{delete, get, post};
use axum::{Json, Router};
use clap::Args;
use futures::StreamExt;
use reqwest::Client;
use serde::{Deserialize, Serialize};
use tokio::sync::Mutex;
use tokio::time::{sleep, Duration, Instant};
use crate::default_store;
use crate::metrics::{self, UnloadReason};
use crate::modelpack::{resolve_model, ModelPath};
use crate::providers::{Wire, PLACEHOLDER_API_KEY};
use crate::storage::OciStore;
mod aggregation;
mod anthropic;
mod backend;
mod config;
mod messages;
mod responses;
mod sched;
mod shell;
mod stream;
mod types;
mod webui;
use backend::{
find_free_port, local_llama_server_bin, resolve_llama_server, spawn_llama_server,
spawn_mlx_server, spawn_vllm_omni_server, spawn_vllm_server, tail_child_output,
use_mlx_for_safetensors, vllm_max_model_len, vllm_omni_serve_args_from_env, vllm_serve_args,
wait_for_ready, would_use_mlx, OutputTail, POLL_INTERVAL,
};
pub use backend::{GPU_VISIBLE_DEVICE_VARS, LLAMA_CPP_ENV_PASSTHROUGH_VARS};
pub use config::DEFAULT_CTX_SIZE;
use config::{
backend_ctx_size, context_length_from_env, effective_num_parallel, embedding_model_ctx,
flash_attention_from_env, gguf_trained_ctx, initial_ctx_size, kv_cache_type_from_env,
looks_like_oom, max_loaded_models_from_env, max_queue_from_env, metrics_enabled_from_env,
next_ctx_size_after_oom, num_parallel_from_env, sched_spread_from_env, supports_context_shift,
threads_from_env_or_host, MAX_CTX_SHRINK_ATTEMPTS,
};
use sched::{
begin_activity, default_keep_alive, is_explicit_unload, reap_idle_models, refresh_activity,
resolve_keep_alive, ActivityGuard,
};
use stream::{bytes_to_lines, stream_ollama};
use types::*;
const SERVE_ENV_HELP: &str = "\
Environment Variables:
LLMMAN_DEBUG Show additional debug information (e.g. LLMMAN_DEBUG=1)
LLMMAN_HOST [host][:port] to bind (default \"127.0.0.1:17434\")
LLMMAN_CONTEXT_LENGTH Context size for llama-server/vLLM when set (default 262144 for llama-server)
LLMMAN_HYBRID_LOCAL_BYTES Largest request a hybrid pair serves locally, in bytes (0 disables; default: from the context length)
LLMMAN_KEEP_ALIVE The duration that models stay loaded in memory (default \"5m\")
LLMMAN_MAX_LOADED_MODELS Maximum number of loaded models (default: unbounded)
LLMMAN_MAX_TRANSFER_STREAMS Maximum parallel transfer streams for safetensors model pulls (default 4)
LLMMAN_MAX_QUEUE Maximum number of queued requests (default 512)
LLMMAN_METRICS Serve a Prometheus scrape endpoint at /metrics (default: off)
LLMMAN_MODELS The path to the models directory
LLMMAN_NUM_PARALLEL Maximum number of parallel requests per model (GGUF only)
LLMMAN_NOHISTORY Do not record prompts for `llmman log`
LLMMAN_NOPRUNE Do not prune model blobs on startup
LLMMAN_ORIGINS A comma separated list of allowed CORS origins
LLMMAN_PEERS A comma separated list of peer daemons ([scheme://]host[:port]) to pool hardware with (overrides [aggregation] in llmman.conf)
LLMMAN_SCHED_SPREAD Always schedule model across all GPUs
LLMMAN_FLASH_ATTENTION Enable flash attention
LLMMAN_KV_CACHE_TYPE Quantization type for the K/V cache (default: f16)
LLMMAN_LLM_LIBRARY Set backend (cpu/cuda/cuda13/rocm/vulkan/metal) to bypass GPU autodetection
LLMMAN_IGPU_ENABLE Enable integrated GPUs
LLMMAN_LOAD_TIMEOUT How long to allow model loads to stall before giving up (default \"10m\")
LLMMAN_VLLM_OMNI_GUARDRAILS Keep a Diffusers-layout model's vLLM-Omni safety guardrails on (default: off)
LLMMAN_TMPDIR Staging directory for llama-server release downloads
LLAMA_ARG_FIT Enable llama.cpp automatic fit of unset memory options (default \"on\")
LLAMA_ARG_FIT_TARGET Target free VRAM margin per device for llama.cpp fit (MiB)
LLAMA_ARG_THREADS Thread count for llama-server (default: llama-server autodetection, overridden by a binding CPU quota/affinity limit)
";
#[derive(Args, Debug)]
#[command(after_help = SERVE_ENV_HELP)]
pub struct ServeArgs {
#[arg(value_name = "MODEL")]
pub model: Option<String>,
#[arg(long, value_name = "docker|podman")]
pub ociman: Option<crate::container::ContainerManager>,
#[arg(long, value_name = "TAG")]
pub llama_cpp_version: Option<String>,
#[arg(long, value_name = "TAG", requires = "ociman")]
pub vllm_version: Option<String>,
#[arg(long, requires = "ociman")]
pub pull_oci: bool,
#[arg(long, conflicts_with_all = ["ociman", "pull_oci"])]
pub pull_bin: bool,
#[arg(long, hide = true, requires = "model")]
pub port: Option<u16>,
#[arg(long, hide = true, requires = "port", default_value = "127.0.0.1")]
pub host: std::net::IpAddr,
}
#[derive(Clone)]
struct AppState(Arc<Inner>);
struct Inner {
manager: Mutex<ModelManager>,
llama_server_bin: StdMutex<Option<PathBuf>>,
exe: Option<PathBuf>,
ociman: Option<crate::container::ContainerManager>,
llama_cpp_version: Option<String>,
vllm_version: Option<String>,
ctx_size: Option<u32>,
ctx_size_explicit: bool,
hybrid_local_bytes: Option<u64>,
flash_attention: Option<String>,
kv_cache_type: Option<String>,
split_mode: Option<&'static str>,
num_parallel: Option<u32>,
threads: Option<u32>,
max_queue: usize,
max_loaded_models: usize,
peers: Vec<String>,
memory: u64,
store_path: PathBuf,
cache_path: PathBuf,
prompt_log: Option<PathBuf>,
shell: shell::Policy,
client: Client,
}
struct ModelManager {
running: HashMap<String, RunningModel>,
pending_loads: usize,
}
struct RunningModel {
process: ModelProcess,
port: u16,
digest: String,
size: u64,
started_at: String,
last_active: Instant,
last_active_wall: chrono::DateTime<chrono::Utc>,
keep_alive: Option<Duration>,
in_flight: u32,
backend_model_path: Option<String>,
}
impl RunningModel {
fn processor(&self) -> String {
match &self.process {
ModelProcess::Local(engine, _, _) => format!("{} (local)", engine.label()),
ModelProcess::Container(ociman, engine, _) => {
format!("{} (container/{})", engine.label(), ociman.binary())
}
}
}
fn pid(&self) -> Option<u32> {
match &self.process {
ModelProcess::Local(_, child, _) => child.id(),
ModelProcess::Container(_, _, child) => child.id(),
}
}
fn engine_label(&self) -> &'static str {
match &self.process {
ModelProcess::Local(engine, _, _) | ModelProcess::Container(_, engine, _) => {
engine.label()
}
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum Engine {
LlamaServer,
Vllm,
VllmOmni,
Mlx,
}
impl Engine {
fn label(self) -> &'static str {
match self {
Engine::LlamaServer => "llama-server",
Engine::Vllm => "vllm",
Engine::VllmOmni => "vllm-omni",
Engine::Mlx => "mlx",
}
}
}
enum ModelProcess {
Local(
Engine,
tokio::process::Child,
#[cfg_attr(not(unix), allow(dead_code))] Option<u32>,
),
Container(
crate::container::ContainerManager,
Engine,
tokio::process::Child,
),
}
impl Drop for ModelProcess {
fn drop(&mut self) {
match self {
ModelProcess::Container(_, _, child) => {
if let Some(pid) = child.id() {
crate::container::stop(pid);
}
}
#[cfg(unix)]
ModelProcess::Local(Engine::Vllm | Engine::VllmOmni, _, pid) => {
if let Some(pid) = pid {
let result = unsafe { libc::kill(-(*pid as libc::pid_t), libc::SIGKILL) };
if result != 0 {
let err = std::io::Error::last_os_error();
eprintln!(
"[llmman] warning: SIGKILL to vllm process group {pid} failed: {err}"
);
}
}
}
#[cfg(not(unix))]
ModelProcess::Local(Engine::Vllm | Engine::VllmOmni, _, _) => {}
ModelProcess::Local(Engine::Mlx, _, _) => {}
ModelProcess::Local(Engine::LlamaServer, _, _) => {}
}
}
}
impl ModelProcess {
fn engine(&self) -> Engine {
match self {
ModelProcess::Local(engine, _, _) | ModelProcess::Container(_, engine, _) => *engine,
}
}
fn is_alive(&mut self) -> bool {
let child = match self {
ModelProcess::Local(_, child, _) => child,
ModelProcess::Container(_, _, child) => child,
};
matches!(child.try_wait(), Ok(None))
}
async fn stop_and_wait(&mut self) {
match self {
ModelProcess::Container(_, _, child) => {
if let Some(pid) = child.id() {
crate::container::stop(pid);
}
}
#[cfg(unix)]
ModelProcess::Local(Engine::Vllm | Engine::VllmOmni, _, pid) => {
if let Some(pid) = pid {
unsafe { libc::kill(-(*pid as libc::pid_t), libc::SIGKILL) };
}
}
ModelProcess::Local(_, _, _) => {}
}
let child = match self {
ModelProcess::Local(_, child, _) => child,
ModelProcess::Container(_, _, child) => child,
};
let _ = child.kill().await;
}
}
#[derive(Debug)]
struct AppError(anyhow::Error, StatusCode);
impl<E: Into<anyhow::Error>> From<E> for AppError {
fn from(e: E) -> Self {
Self(e.into(), StatusCode::INTERNAL_SERVER_ERROR)
}
}
impl AppError {
fn status(status: StatusCode, message: impl Into<String>) -> Self {
Self(anyhow!(message.into()), status)
}
fn bad_request(e: crate::shortnames::InvalidReference) -> Self {
Self(e.into(), StatusCode::BAD_REQUEST)
}
}
impl IntoResponse for AppError {
fn into_response(self) -> Response {
let body = serde_json::json!({ "error": format!("{:#}", self.0) });
(self.1, Json(body)).into_response()
}
}
fn now_rfc3339() -> String {
chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Secs, true)
}
static MODEL_LOCKS: LazyLock<StdMutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>> =
LazyLock::new(|| StdMutex::new(HashMap::new()));
static LOAD_LOCKS: LazyLock<StdMutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>> =
LazyLock::new(|| StdMutex::new(HashMap::new()));
fn keyed_lock(
registry: &StdMutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>,
key: &str,
) -> Arc<tokio::sync::Mutex<()>> {
let mut locks = registry.lock().unwrap();
locks
.entry(key.to_owned())
.or_insert_with(|| Arc::new(tokio::sync::Mutex::new(())))
.clone()
}
fn release_keyed_lock(
registry: &StdMutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>>,
key: &str,
) {
let mut locks = registry.lock().unwrap();
if let Some(arc) = locks.get(key) {
if Arc::strong_count(arc) <= 1 {
locks.remove(key);
}
}
}
fn model_lock(model: &str) -> Arc<tokio::sync::Mutex<()>> {
keyed_lock(&MODEL_LOCKS, model)
}
fn release_model_lock(model: &str) {
release_keyed_lock(&MODEL_LOCKS, model)
}
fn load_lock(model: &str) -> Arc<tokio::sync::Mutex<()>> {
keyed_lock(&LOAD_LOCKS, model)
}
fn release_load_lock(model: &str) {
release_keyed_lock(&LOAD_LOCKS, model)
}
struct LoadLockGuard {
model: String,
guard: Option<tokio::sync::OwnedMutexGuard<()>>,
}
impl Drop for LoadLockGuard {
fn drop(&mut self) {
self.guard.take(); release_load_lock(&self.model);
}
}
async fn acquire_load_lock(model: &str) -> LoadLockGuard {
let guard = load_lock(model).lock_owned().await;
LoadLockGuard {
model: model.to_owned(),
guard: Some(guard),
}
}
fn ffi_pull_ref<'a>(progress_key: &'a str, classified: &str) -> &'a str {
debug_assert!(
classified == progress_key || classified.strip_prefix(progress_key) == Some(":latest"),
"classify should differ from the progress key only by a defaulted tag",
);
progress_key
}
fn pull_serialized(store_path: &std::path::Path, model: &str) -> anyhow::Result<Vec<String>> {
let lock = model_lock(model);
let result = (|| {
let _guard = lock.blocking_lock();
let existing = OciStore::open(store_path).and_then(|s| s.find(model)).ok();
if let Some(desc) = existing {
if !crate::verify::is_registry_reference(model) {
return Ok(Vec::new());
}
if !crate::verify::is_enabled_for(model)? {
return Ok(Vec::new());
}
return verify_stored(store_path, model, &desc.digest);
}
let layout_dir = store_path
.to_str()
.ok_or_else(|| anyhow!("store path is not valid UTF-8"))?;
tokio::runtime::Handle::current().block_on(async {
match crate::hf::classify(model).await {
crate::hf::ClassifiedRef::Hf(reference) => {
if crate::verify::is_registry_reference(model)
&& crate::verify::is_enabled_for(model)?
{
return Err(anyhow!(concat!(
"is covered by a signature policy but could not be confirmed ",
"to be an OCI registry (the /v2/ probe failed); refusing to ",
"pull it as a HuggingFace repository unchecked",
))
.context(model.to_string()));
}
crate::hf::pull::pull(&reference, store_path, model)
.await
.map(|()| Vec::new())
}
crate::hf::ClassifiedRef::Source(reference) => {
crate::sources::pull(&reference, store_path, model)
.await
.map(|()| Vec::new())
}
crate::hf::ClassifiedRef::Other(normalized) => {
let guard = crate::verify::PullGuard::check(&normalized)?;
crate::ffi::pull(ffi_pull_ref(model, &normalized), layout_dir)?;
let stored = OciStore::open(store_path)?.find(&normalized)?;
match guard.confirm(&stored.digest) {
Ok(notices) => Ok(notices),
Err(e) => Err(reject_stored(store_path, &normalized, e)),
}
}
}
})
})();
drop(lock);
release_model_lock(model);
result
}
fn verify_stored(
store_path: &std::path::Path,
model: &str,
digest: &str,
) -> anyhow::Result<Vec<String>> {
debug_assert!(crate::verify::is_registry_reference(model));
match crate::verify::check(model, Some(digest)) {
Ok(verdict) => Ok(verdict.notices),
Err(e) if crate::verify::is_indeterminate(&e) => Err(e),
Err(e) => Err(reject_stored(store_path, model, e)),
}
}
fn reject_stored(
store_path: &std::path::Path,
reference: &str,
cause: anyhow::Error,
) -> anyhow::Error {
match OciStore::open(store_path).and_then(|s| s.remove(reference)) {
Ok(_) => cause,
Err(e) => cause.context(format!(
"could not remove the rejected model {reference} from the store ({e:#}); \
it will keep being refused, but `llmman rm {reference}` is needed to clear it"
)),
}
}
fn canonical_ref(store_path: &std::path::Path, model_ref: &str) -> String {
let Ok(store) = crate::storage::OciStore::open(store_path) else {
return model_ref.to_owned();
};
let Ok(desc) = store.find(model_ref) else {
return model_ref.to_owned();
};
desc.annotations
.as_ref()
.and_then(|a| a.get("org.opencontainers.image.ref.name"))
.cloned()
.unwrap_or_else(|| model_ref.to_owned())
}
fn load_identity(
store_path: &std::path::Path,
model: &str,
) -> Result<String, crate::shortnames::InvalidReference> {
if crate::providers::is_remote_ref(model) {
return Ok(model.to_string());
}
let resolved = crate::shortnames::resolve_ollama_api(model)?;
let (without_digest, digest) = crate::storage::split_ref_digest(&resolved);
if digest.is_some() {
if let Some(tag) = stored_tag_for_digest(store_path, &resolved) {
return Ok(tag);
}
}
Ok(crate::storage::default_tag(without_digest))
}
fn stored_tag_for_digest(store_path: &std::path::Path, reference: &str) -> Option<String> {
crate::storage::split_ref_digest(reference).1?;
let desc = OciStore::open(store_path).ok()?.find(reference).ok()?;
let stored = desc
.annotations
.as_ref()?
.get("org.opencontainers.image.ref.name")?;
crate::storage::split_ref_digest(stored)
.1
.is_none()
.then(|| crate::storage::default_tag(stored))
}
async fn unload_model(state: &AppState, model: &str) -> Result<(), AppError> {
let lock_key = load_identity(&state.0.store_path, model).map_err(AppError::bad_request)?;
let _guard = acquire_load_lock(&lock_key).await;
let canonical = if crate::providers::is_remote_ref(model) {
lock_key
} else {
let resolved =
crate::shortnames::resolve_ollama_api(model).map_err(AppError::bad_request)?;
canonical_ref(&state.0.store_path, &crate::storage::default_tag(&resolved))
};
if state
.0
.manager
.lock()
.await
.running
.remove(&canonical)
.is_some()
{
metrics::record_model_unload(&canonical, UnloadReason::Requested);
return Ok(());
}
if crate::providers::is_remote_ref(model) {
return Ok(());
}
let stored = OciStore::open(&state.0.store_path)?.find(&canonical).ok();
let (base, digest) = crate::storage::split_ref_digest(&canonical);
let digest = digest
.map(str::to_owned)
.or_else(|| stored.as_ref().map(|d| d.digest.clone()));
if let Some(digest) = digest {
let spelled = crate::storage::default_tag(base);
let mut mgr = state.0.manager.lock().await;
let key = mgr
.running
.iter()
.filter(|(key, m)| {
m.digest.eq_ignore_ascii_case(&digest)
&& crate::storage::repo_name(key) == crate::storage::repo_name(base)
})
.map(|(key, _)| key.clone())
.min_by_key(|key| *key != spelled);
if let Some(key) = key {
mgr.running.remove(&key);
return Ok(());
}
}
if stored.is_none() {
return Err(AppError(
anyhow!("model '{model}' not found"),
StatusCode::NOT_FOUND,
));
}
Ok(())
}
async fn forward_ollama<T: Serialize>(
state: &AppState,
target: &Target,
route: &str,
headers: &HeaderMap,
req: &T,
model: &str,
guard: ActivityGuard,
) -> Result<Response, AppError> {
let mut req = serde_json::to_value(req).context("re-serialize Ollama request")?;
if req["model"]
.as_str()
.is_some_and(crate::hybrid::is_hybrid_ref)
{
req["model"] = serde_json::Value::String(model.to_string());
}
let body = Bytes::from(serde_json::to_vec(&req).context("re-serialize Ollama request")?);
let activity = begin_activity(guard, None).await;
proxy(&state.0.client, target, route, headers, body, activity).await
}
async fn unload_everywhere(
state: &AppState,
model: &str,
headers: &HeaderMap,
) -> Result<(), AppError> {
let local = unload_model(state, model).await;
if aggregation::unload(state, model, headers).await {
return Ok(());
}
local
}
async fn check_running(state: &AppState, model_ref: &str) -> Option<(u16, ActivityGuard)> {
let mut mgr = state.0.manager.lock().await;
if let Some(m) = mgr.running.get_mut(model_ref) {
if m.process.is_alive() {
m.in_flight += 1;
return Some((m.port, ActivityGuard::new(state, model_ref)));
}
eprintln!(
"[llmman] {model_ref} was marked running on port {} but its process has exited — reloading",
m.port
);
mgr.running.remove(model_ref);
metrics::record_model_unload(model_ref, UnloadReason::Crashed);
}
None
}
async fn check_running_by_digest(
state: &AppState,
model_ref: &str,
digest: &str,
) -> Option<(String, u16, ActivityGuard)> {
let repo = crate::storage::repo_name(model_ref);
let mut mgr = state.0.manager.lock().await;
let key = mgr
.running
.iter()
.find(|(key, m)| {
crate::storage::repo_name(key) == repo && m.digest.eq_ignore_ascii_case(digest)
})
.map(|(key, _)| key.clone())?;
let m = mgr.running.get_mut(&key)?;
if !m.process.is_alive() {
eprintln!(
"[llmman] {key} was marked running on port {} but its process has exited — reloading",
m.port
);
mgr.running.remove(&key);
return None;
}
m.in_flight += 1;
let port = m.port;
Some((key.clone(), port, ActivityGuard::new(state, &key)))
}
async fn evict_other_models(state: &AppState, model_ref: &str) -> bool {
let mut mgr = state.0.manager.lock().await;
let other_keys: Vec<String> = mgr
.running
.iter()
.filter(|(k, m)| k.as_str() != model_ref && m.in_flight == 0)
.map(|(k, _)| k.clone())
.collect();
let mut evicted: Vec<(String, RunningModel)> = Vec::with_capacity(other_keys.len());
for key in other_keys {
if let Some(running) = mgr.running.remove(&key) {
metrics::record_model_unload(&key, UnloadReason::Oom);
evicted.push((key, running));
}
}
drop(mgr); let any = !evicted.is_empty();
for (name, mut running) in evicted {
eprintln!("[llmman] evicting {name} to free memory before retrying {model_ref}");
running.process.stop_and_wait().await;
}
any
}
struct PendingLoadGuard {
state: AppState,
armed: bool,
}
impl PendingLoadGuard {
fn release_into(&mut self, mgr: &mut ModelManager) {
if !std::mem::take(&mut self.armed) {
return;
}
mgr.pending_loads = mgr.pending_loads.saturating_sub(1);
}
}
impl std::fmt::Debug for PendingLoadGuard {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("PendingLoadGuard")
.field("armed", &self.armed)
.finish()
}
}
impl Drop for PendingLoadGuard {
fn drop(&mut self) {
if !self.armed {
return;
}
let state = self.state.clone();
let Ok(handle) = tokio::runtime::Handle::try_current() else {
return;
};
handle.spawn(async move {
let mut mgr = state.0.manager.lock().await;
mgr.pending_loads = mgr.pending_loads.saturating_sub(1);
});
}
}
async fn enforce_max_loaded_models(
state: &AppState,
max_loaded: usize,
) -> Result<PendingLoadGuard, AppError> {
if max_loaded == 0 {
state.0.manager.lock().await.pending_loads += 1;
return Ok(PendingLoadGuard {
state: state.clone(),
armed: true,
});
}
loop {
let mut mgr = state.0.manager.lock().await;
if mgr.running.len() + mgr.pending_loads < max_loaded {
mgr.pending_loads += 1;
return Ok(PendingLoadGuard {
state: state.clone(),
armed: true,
});
}
if mgr.running.len() < max_loaded {
drop(mgr);
sleep(POLL_INTERVAL).await;
continue;
}
let victim = mgr
.running
.iter()
.filter(|(_, m)| m.in_flight == 0)
.min_by_key(|(_, m)| m.last_active)
.map(|(k, _)| k.clone());
let Some(victim) = victim else {
return Err(AppError::status(
StatusCode::SERVICE_UNAVAILABLE,
format!(
"max loaded models ({max_loaded}) reached, and every loaded model is busy — try again"
),
));
};
mgr.pending_loads += 1;
let mut running = mgr
.running
.remove(&victim)
.expect("victim key was just looked up under this same lock");
metrics::record_model_unload(&victim, UnloadReason::Evicted);
drop(mgr); eprintln!(
"[llmman] evicting {victim} to free a loaded-model slot (LLMMAN_MAX_LOADED_MODELS={max_loaded})"
);
running.process.stop_and_wait().await;
return Ok(PendingLoadGuard {
state: state.clone(),
armed: true,
});
}
}
const MAX_QUEUE_ERROR: &str = "server busy, please try again. maximum pending requests exceeded";
static PENDING_REQUESTS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
struct QueueGuard(&'static std::sync::atomic::AtomicUsize);
impl Drop for QueueGuard {
fn drop(&mut self) {
self.0.fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
}
}
fn try_admit_against(
counter: &'static std::sync::atomic::AtomicUsize,
max_queue: usize,
) -> Result<QueueGuard, AppError> {
let cap = max_queue.max(1);
let admitted = counter
.fetch_update(
std::sync::atomic::Ordering::SeqCst,
std::sync::atomic::Ordering::SeqCst,
|n| (n < cap).then_some(n + 1),
)
.is_ok();
if admitted {
Ok(QueueGuard(counter))
} else {
Err(AppError::status(
StatusCode::SERVICE_UNAVAILABLE,
MAX_QUEUE_ERROR,
))
}
}
fn try_admit(max_queue: usize) -> Result<QueueGuard, AppError> {
let admitted = try_admit_against(&PENDING_REQUESTS, max_queue);
if admitted.is_err() {
metrics::record_scheduling_rejection();
}
admitted
}
#[derive(Clone, Debug)]
enum Target {
Local(u16),
Peer(String),
Remote(Arc<RemoteTarget>),
}
struct RemoteTarget {
provider: String,
base_url: String,
wire: Wire,
model: String,
max_output: Option<u32>,
api_key: Option<String>,
}
impl std::fmt::Debug for RemoteTarget {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RemoteTarget")
.field("provider", &self.provider)
.field("base_url", &self.base_url)
.field("wire", &self.wire)
.field("model", &self.model)
.field("api_key", &self.api_key.as_ref().map(|_| "<redacted>"))
.finish()
}
}
impl Target {
fn url(&self, route: &str) -> String {
match self {
Self::Local(port) => format!("http://127.0.0.1:{port}{route}"),
Self::Peer(origin) => format!("{origin}{route}"),
Self::Remote(remote) => crate::providers::rebase_url(&remote.base_url, route),
}
}
fn authorize(&self, req: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
match self {
Self::Local(_) => req,
Self::Peer(_) => req.header(aggregation::HOP, "1"),
Self::Remote(remote) => match (remote.wire, remote.api_key.as_deref()) {
(Wire::OpenAi, Some(key)) => req.bearer_auth(key),
(Wire::OpenAi, None) => req,
(Wire::Anthropic, key) => {
let req = req.header("anthropic-version", anthropic::VERSION);
match key {
Some(key) => req.header("x-api-key", key),
None => req,
}
}
},
}
}
fn is_remote(&self) -> bool {
matches!(self, Self::Remote(_))
}
fn wire(&self) -> Option<Wire> {
match self {
Self::Remote(remote) => Some(remote.wire),
_ => None,
}
}
fn is_anthropic(&self) -> bool {
self.wire() == Some(Wire::Anthropic)
}
fn describe(&self) -> String {
match self {
Self::Local(_) => "inference backend".to_string(),
Self::Peer(origin) => format!("peer {origin}"),
Self::Remote(remote) => format!("provider {}", remote.provider),
}
}
}
const CHAT_COMPLETIONS_ROUTE: &str = "/v1/chat/completions";
fn client_api_key(headers: Option<&HeaderMap>) -> Option<String> {
let headers = headers?;
let usable = |k: &str| {
let k = k.trim();
(!k.is_empty() && k != PLACEHOLDER_API_KEY).then(|| k.to_string())
};
let bearer = headers
.get(reqwest::header::AUTHORIZATION)
.and_then(|v| v.to_str().ok())
.and_then(|v| {
let (scheme, token) = v.trim().split_once(' ')?;
scheme.eq_ignore_ascii_case("bearer").then_some(token)
})
.and_then(usable);
bearer.or_else(|| {
headers
.get("x-api-key")
.and_then(|v| v.to_str().ok())
.and_then(usable)
})
}
fn is_cross_site(headers: Option<&HeaderMap>) -> bool {
headers
.and_then(|h| h.get("sec-fetch-site"))
.and_then(|v| v.to_str().ok())
.is_some_and(|site| {
let site = site.trim();
site.eq_ignore_ascii_case("cross-site") || site.eq_ignore_ascii_case("same-site")
})
}
async fn provider_catalog() -> Result<Arc<crate::providers::Catalog>, AppError> {
tokio::task::spawn_blocking(crate::providers::catalog)
.await
.context("provider catalog task panicked")?
.map_err(|e| AppError(e, StatusCode::BAD_GATEWAY))
}
async fn resolve_remote_target(
model_ref: &str,
headers: Option<&HeaderMap>,
) -> Result<Option<Target>, AppError> {
let Some((provider_id, model)) = crate::providers::split_remote_ref(model_ref) else {
return Ok(None);
};
let (provider_id, model) = (provider_id.to_string(), model.to_string());
let catalog = provider_catalog().await?;
let provider = catalog.get(&provider_id).ok_or_else(|| {
AppError(
crate::providers::unknown_provider_error(&provider_id, &catalog),
StatusCode::BAD_REQUEST,
)
})?;
let own_key = || {
(crate::daemon::reachable_only_locally() && !is_cross_site(headers))
.then(|| provider.api_key())
.flatten()
};
let api_key = match client_api_key(headers).or_else(own_key) {
Some(key) => Some(key),
None if provider.key_optional => None,
None => {
return Err(AppError(
anyhow!(
"no API key for provider {provider_id:?} — send it as an Authorization \
header{}",
if is_cross_site(headers) {
". This request came from another site, so llmman serve's own \
environment is deliberately not used"
.to_string()
} else if crate::daemon::reachable_only_locally() {
format!(
", or give llmman serve a key of its own: {}",
crate::providers::key_hint(&provider.id, provider.key_env.as_deref())
)
} else {
". llmman serve is not bound to loopback, so its own environment is \
deliberately not used"
.to_string()
}
),
StatusCode::UNAUTHORIZED,
));
}
};
let target = RemoteTarget {
provider: provider_id,
base_url: provider.base_url.clone(),
wire: provider.wire,
max_output: provider
.models
.iter()
.find(|m| m.id == model)
.and_then(|m| m.max_output),
model,
api_key,
};
eprintln!(
"[llmman] routing {} to provider {} ({}, {} wire)",
target.model,
target.provider,
target.base_url,
target.wire.as_str()
);
Ok(Some(Target::Remote(Arc::new(target))))
}
fn resolve_hybrid_side(
state: &AppState,
pair: &crate::hybrid::Pair<'_>,
headers: Option<&HeaderMap>,
) -> Result<String, AppError> {
let pin = request_pin(headers)?;
let request_bytes = headers
.and_then(|h| h.get(reqwest::header::CONTENT_LENGTH))
.and_then(|v| v.to_str().ok())
.and_then(|v| v.trim().parse::<u64>().ok());
let decision = crate::hybrid::route(pin, request_bytes, state.0.hybrid_local_bytes);
let why = match decision.reason {
crate::hybrid::Reason::Pinned => format!("pinned by {}", crate::hybrid::ROUTE_HEADER),
crate::hybrid::Reason::Overflow { bytes, budget } => format!(
"{} request exceeds the {} this host serves locally",
crate::fmt::human_size(bytes),
crate::fmt::human_size(budget)
),
crate::hybrid::Reason::LocalFirst => "no reason to leave this machine".to_string(),
};
eprintln!(
"[llmman] hybrid {:?} + {:?} -> {} ({why})",
pair.local,
pair.remote_ref(),
decision.side.as_str()
);
Ok(pair.side_ref(decision.side))
}
fn request_pin(headers: Option<&HeaderMap>) -> Result<Option<crate::hybrid::Side>, AppError> {
let mut values = headers
.map(|h| h.get_all(crate::hybrid::ROUTE_HEADER).iter())
.into_iter()
.flatten();
let value = values.next();
if values.next().is_some() {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
format!("{} given more than once", crate::hybrid::ROUTE_HEADER),
));
}
crate::hybrid::parse_pin(value.map(|v| v.as_bytes()))
.map_err(|e| AppError(e, StatusCode::BAD_REQUEST))
}
fn hybrid_fallback(
model_ref: &str,
headers: Option<&HeaderMap>,
) -> Result<Option<String>, AppError> {
let Some(pair) = crate::hybrid::split_ref(model_ref) else {
return Ok(None);
};
Ok((request_pin(headers)? != Some(crate::hybrid::Side::Local)).then(|| pair.remote_ref()))
}
async fn send_with_hybrid_fallback<F, Fut>(
state: &AppState,
model_ref: &str,
headers: Option<&HeaderMap>,
request_threads: Option<u32>,
send: F,
) -> Result<Response, AppError>
where
F: Fn(String, Target, ActivityGuard) -> Fut,
Fut: std::future::Future<Output = Result<Response, AppError>>,
{
let resolve =
|m: String| async move { ensure_model(state, &m, headers, request_threads).await };
with_hybrid_fallback(model_ref, headers, resolve, send).await
}
async fn with_hybrid_fallback<R, RFut, F, Fut>(
model_ref: &str,
headers: Option<&HeaderMap>,
resolve: R,
send: F,
) -> Result<Response, AppError>
where
R: Fn(String) -> RFut,
RFut: std::future::Future<Output = Result<(String, Target, ActivityGuard), AppError>>,
F: Fn(String, Target, ActivityGuard) -> Fut,
Fut: std::future::Future<Output = Result<Response, AppError>>,
{
let (model, target, guard) = resolve(model_ref.to_string()).await?;
let fallback = match target {
Target::Local(_) => hybrid_fallback(model_ref, headers)?,
_ => None,
};
let Some(cloud) = fallback else {
return send(model, target, guard).await;
};
let refusal = match send(model, target, guard).await {
Ok(resp) => match local_context_overflow(resp).await {
Ok(resp) => return Ok(resp),
Err(refusal) => refusal,
},
Err(err) => match err.0.downcast_ref::<ContextOverflow>() {
Some(overflow) => overflow.refusal.clone(),
None => return Err(err),
},
};
eprintln!("[llmman] hybrid {model_ref:?} -> cloud ({refusal})");
let (model, target, guard) = resolve(cloud).await?;
send(model, target, guard).await
}
const OVERFLOW_BODY_LIMIT: usize = 64 * 1024;
async fn local_context_overflow(resp: Response) -> Result<Response, String> {
if resp.status() != StatusCode::BAD_REQUEST {
return Ok(resp);
}
let (parts, body) = resp.into_parts();
let mut rest = body.into_data_stream();
let mut head = Vec::new();
while head.len() <= OVERFLOW_BODY_LIMIT {
match rest.next().await {
Some(Ok(chunk)) => head.extend_from_slice(&chunk),
Some(Err(_)) | None => break,
}
}
if head.len() <= OVERFLOW_BODY_LIMIT {
if let Some(refusal) =
context_overflow_message(parts.status, &String::from_utf8_lossy(&head))
{
return Err(refusal);
}
}
let head = futures::stream::once(futures::future::ready(Ok(Bytes::from(head))));
Ok(Response::from_parts(
parts,
Body::from_stream(head.chain(rest)),
))
}
async fn ensure_model(
state: &AppState,
model_ref: &str,
headers: Option<&HeaderMap>,
request_threads: Option<u32>,
) -> Result<(String, Target, ActivityGuard), AppError> {
let hybrid_side = crate::hybrid::split_ref(model_ref)
.map(|pair| resolve_hybrid_side(state, &pair, headers))
.transpose()?;
let model_ref = hybrid_side.as_deref().unwrap_or(model_ref);
if let Some(target) = resolve_remote_target(model_ref, headers).await? {
return Ok((
model_ref.to_string(),
target,
ActivityGuard::new(state, model_ref),
));
}
let load_id = load_identity(&state.0.store_path, model_ref).map_err(AppError::bad_request)?;
let model_ref =
crate::shortnames::resolve_ollama_api(model_ref).map_err(AppError::bad_request)?;
let model_ref = crate::storage::default_tag(&model_ref);
let model_ref = canonical_ref(&state.0.store_path, &model_ref);
let model_ref = model_ref.as_str();
if let Some((port, guard)) = check_running(state, model_ref).await {
return Ok((model_ref.to_string(), Target::Local(port), guard));
}
if let Some(peer) = aggregation::route(state, model_ref, headers).await {
return Ok((
model_ref.to_string(),
Target::Peer(peer),
ActivityGuard::new(state, model_ref),
));
}
let load_started = Instant::now();
let _queue_guard = try_admit(state.0.max_queue)?;
let _guard = acquire_load_lock(&load_id).await;
if let Some((port, guard)) = check_running(state, model_ref).await {
return Ok((model_ref.to_string(), Target::Local(port), guard));
}
{
let present = crate::storage::OciStore::open(&state.0.store_path)
.and_then(|s| s.find(model_ref))
.is_ok();
if !present {
eprintln!("[llmman] {model_ref} not in store — pulling");
}
let store_path = state.0.store_path.clone();
let model_ref_owned = model_ref.to_owned();
let notices =
tokio::task::spawn_blocking(move || pull_serialized(&store_path, &model_ref_owned))
.await
.context("pull task panicked")?
.context("pull failed")?;
for notice in notices {
eprintln!("[llmman] {notice}");
}
}
let model_ref = canonical_ref(&state.0.store_path, model_ref);
let model_ref = model_ref.as_str();
if let Some((port, guard)) = check_running(state, model_ref).await {
return Ok((model_ref.to_string(), Target::Local(port), guard));
}
let (digest, size) = OciStore::open(&state.0.store_path)
.and_then(|s| {
s.find(model_ref).map(|d| {
let size = s.total_size(&d);
(d.digest, size)
})
})
.unwrap_or_default();
if !digest.is_empty() {
if let Some((key, port, guard)) = check_running_by_digest(state, model_ref, &digest).await {
return Ok((key, Target::Local(port), guard));
}
}
let model_path = resolve_model(&state.0.store_path, &state.0.cache_path, model_ref)
.with_context(|| format!("resolve model {model_ref}"))?;
let context_shift = supports_context_shift(model_ref);
let gguf_info = match &model_path {
ModelPath::Gguf(path, _) => crate::gguf::read_info(path).ok(),
_ => None,
};
let trained_ctx = gguf_info.as_ref().and_then(gguf_trained_ctx);
let embedding_ctx = gguf_info.as_ref().and_then(embedding_model_ctx);
let mut pending_load_guard =
enforce_max_loaded_models(state, state.0.max_loaded_models).await?;
let mut ctx_size = initial_ctx_size(
state.0.ctx_size,
state.0.ctx_size_explicit,
trained_ctx,
embedding_ctx.is_some(),
);
let mut split_mode = state.0.split_mode;
let num_parallel = effective_num_parallel(ctx_size, state.0.num_parallel);
if state.0.num_parallel.is_some() && num_parallel.is_none() {
eprintln!(
"[llmman] {model_ref}: no explicit ctx-size to scale, ignoring LLMMAN_NUM_PARALLEL for this load"
);
}
let mut shrink_attempts = 0u32;
let mut evicted_others = false;
let mut split_mode_relaxed = false;
let mut process;
let mut port = find_free_port()?;
loop {
eprintln!("[llmman] loading {model_ref} on port {port}");
let scaled_ctx_size = backend_ctx_size(ctx_size, num_parallel);
let llama_opts = crate::container::LlamaOptions {
port,
ctx_size: scaled_ctx_size,
flash_attention: state.0.flash_attention.as_deref(),
kv_cache_type: state.0.kv_cache_type.as_deref(),
context_shift,
split_mode,
num_parallel,
embeddings: embedding_ctx.is_some(),
batch_size: embedding_ctx.and(ctx_size).filter(|n| *n > 0),
threads: request_threads.or(state.0.threads),
};
let mut stderr_tail: Option<OutputTail> = None;
let mut oom_retryable = false;
let max_model_len = vllm_max_model_len(ctx_size, state.0.ctx_size_explicit);
process = match (&model_path, state.0.ociman) {
(ModelPath::Gguf(path, mmproj), Some(ociman)) => {
let mut child = crate::container::spawn(
ociman,
path,
mmproj.as_deref(),
state.0.llama_cpp_version.as_deref(),
llama_opts,
)?;
stderr_tail = Some(tail_child_output(&mut child));
oom_retryable = true;
ModelProcess::Container(ociman, Engine::LlamaServer, child)
}
(ModelPath::Gguf(path, mmproj), None) => {
let bin = local_llama_server_bin(state).await?;
let (child, tail) =
spawn_llama_server(&bin, path, mmproj.as_deref(), llama_opts).await?;
stderr_tail = Some(tail);
oom_retryable = true;
ModelProcess::Local(Engine::LlamaServer, child, None)
}
(ModelPath::Diffusion(_), Some(ociman)) => {
let mut child = crate::container::spawn_mediagen(
ociman,
model_ref,
&state.0.store_path,
&state.0.cache_path,
state.0.llama_cpp_version.as_deref(),
port,
)?;
stderr_tail = Some(tail_child_output(&mut child));
oom_retryable = true;
ModelProcess::Container(ociman, Engine::LlamaServer, child)
}
(ModelPath::Diffusion(_), None) => {
let (child, tail) = spawn_mediagen_backend(model_ref, port, state).await?;
stderr_tail = Some(tail);
oom_retryable = true;
ModelProcess::Local(Engine::LlamaServer, child, None)
}
(ModelPath::SafeTensors(dir), Some(ociman)) => {
let mut child = crate::container::spawn_vllm(
ociman,
crate::container::ContainerEngine::Vllm,
dir,
state.0.vllm_version.as_deref(),
port,
|model_dir, host| {
vllm_serve_args(model_dir, host, port, model_ref, max_model_len)
},
)?;
stderr_tail = Some(tail_child_output(&mut child));
ModelProcess::Container(ociman, Engine::Vllm, child)
}
(ModelPath::Omni(dir), Some(ociman)) => {
let mut child = crate::container::spawn_vllm(
ociman,
crate::container::ContainerEngine::VllmOmni,
dir,
state.0.vllm_version.as_deref(),
port,
|model_dir, host| {
vllm_omni_serve_args_from_env(model_dir, host, port, model_ref)
},
)?;
stderr_tail = Some(tail_child_output(&mut child));
ModelProcess::Container(ociman, Engine::VllmOmni, child)
}
(ModelPath::Omni(dir), None) => {
let (child, tail) = spawn_vllm_omni_server(dir, port, model_ref).await?;
stderr_tail = Some(tail);
let pid = child.id();
ModelProcess::Local(Engine::VllmOmni, child, pid)
}
(ModelPath::SafeTensors(_dir), None) if use_mlx_for_safetensors() => {
let child = spawn_mlx_server(port).await?;
let pid = child.id();
ModelProcess::Local(Engine::Mlx, child, pid)
}
(ModelPath::SafeTensors(dir), None) => {
let child = spawn_vllm_server(dir, port, model_ref, max_model_len).await?;
let pid = child.id();
ModelProcess::Local(Engine::Vllm, child, pid)
}
};
match wait_for_ready(&state.0.client, port, &mut process, stderr_tail.as_ref()).await {
Ok(()) => break,
Err(e) => {
let looks_oom = oom_retryable && looks_like_oom(&e.to_string());
if !looks_oom {
return Err(e.into());
}
process.stop_and_wait().await;
if !evicted_others {
evicted_others = true;
if evict_other_models(state, model_ref).await {
metrics::record_oom_retry(model_ref, metrics::OomRetry::EvictOthers);
eprintln!(
"[llmman] {model_ref} failed to load on port {port}, which looks like an out-of-memory error — evicted other loaded models and retrying: {:#}",
e
);
port = find_free_port()?;
continue;
}
}
if !split_mode_relaxed && split_mode == Some("none") {
split_mode_relaxed = true;
split_mode = Some("layer");
metrics::record_oom_retry(model_ref, metrics::OomRetry::SplitMode);
eprintln!(
"[llmman] {model_ref} failed to load on port {port} with --split-mode none, which looks like an out-of-memory error — retrying with --split-mode layer (spread across every GPU) instead of failing outright: {:#}",
e
);
port = find_free_port()?;
continue;
}
let can_shrink =
!state.0.ctx_size_explicit && shrink_attempts < MAX_CTX_SHRINK_ATTEMPTS;
let Some(next) = can_shrink
.then(|| ctx_size.and_then(next_ctx_size_after_oom))
.flatten()
else {
return Err(e.into());
};
eprintln!(
"[llmman] {model_ref} failed to load on port {port}, which looks like an out-of-memory error — retrying with --ctx-size {next} (was {}): {:#}",
ctx_size
.map(|n| n.to_string())
.unwrap_or_else(|| "model default".to_string()),
e
);
ctx_size = Some(next);
shrink_attempts += 1;
metrics::record_oom_retry(model_ref, metrics::OomRetry::CtxShrink);
port = find_free_port()?;
}
}
}
eprintln!("[llmman] {model_ref} ready on port {port}");
let backend_model_path = match &process {
ModelProcess::Local(Engine::Mlx, _, _) => model_path.path().to_str().map(|s| s.to_string()),
_ => None,
};
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
model_ref.to_string(),
RunningModel {
process,
port,
digest,
size,
started_at: now_rfc3339(),
last_active: Instant::now(),
last_active_wall: chrono::Utc::now(),
backend_model_path,
keep_alive: default_keep_alive(),
in_flight: 1,
},
);
pending_load_guard.release_into(&mut mgr);
drop(mgr);
metrics::record_model_load(model_ref, load_started.elapsed());
Ok((
model_ref.to_string(),
Target::Local(port),
ActivityGuard::new(state, model_ref),
))
}
async fn backend_wire_model(state: &AppState, target: &Target, canonical_model: &str) -> String {
if let Target::Remote(remote) = target {
return remote.model.clone();
}
state
.0
.manager
.lock()
.await
.running
.get(canonical_model)
.and_then(|r| r.backend_model_path.clone())
.unwrap_or_else(|| canonical_model.to_string())
}
async fn proxy(
client: &Client,
target: &Target,
route: &str,
headers: &HeaderMap,
body: Bytes,
activity: ActivityGuard,
) -> Result<Response, AppError> {
let mut req = target.authorize(client.post(target.url(route)).body(body));
if let Some(ct) = headers.get("content-type") {
req = req.header("content-type", ct);
}
let resp = req
.send()
.await
.with_context(|| format!("proxy request to {}", target.describe()))?;
Ok(relay(resp, activity))
}
fn relay(resp: reqwest::Response, activity: ActivityGuard) -> Response {
let status = resp.status();
let resp_headers = resp.headers().clone();
let stream = resp.bytes_stream().map(move |item| {
let _activity = &activity;
item.map_err(|e| Box::new(e) as Box<dyn std::error::Error + Send + Sync>)
});
let mut builder = Response::builder().status(status.as_u16());
for (k, v) in &resp_headers {
builder = builder.header(k, v);
}
builder.body(Body::from_stream(stream)).unwrap()
}
fn set_response_model(value: &mut serde_json::Value, canonical_model: &str) {
if value.get("model").is_some() {
value["model"] = serde_json::Value::String(canonical_model.to_string());
}
for key in ["response", "message"] {
if let Some(nested) = value.get_mut(key) {
if nested.get("model").is_some() {
nested["model"] = serde_json::Value::String(canonical_model.to_string());
}
}
}
}
fn rewrite_json_response_model(raw: &Bytes, canonical_model: &str) -> Bytes {
match serde_json::from_slice::<serde_json::Value>(raw) {
Ok(mut value) => {
set_response_model(&mut value, canonical_model);
serde_json::to_vec(&value)
.map(Bytes::from)
.unwrap_or_else(|_| raw.clone())
}
Err(_) => raw.clone(),
}
}
fn rewrite_sse_line_model(line: &str, canonical_model: &str) -> String {
match line.strip_prefix("data: ") {
Some(payload) if payload != "[DONE]" => match serde_json::from_str(payload) {
Ok(mut value) => {
set_response_model(&mut value, canonical_model);
format!(
"data: {}",
serde_json::to_string(&value).unwrap_or_else(|_| payload.to_string())
)
}
Err(_) => line.to_string(),
},
_ => line.to_string(),
}
}
async fn proxy_rewriting_model(
client: &Client,
target: &Target,
route: &str,
headers: &HeaderMap,
body: Bytes,
activity: ActivityGuard,
canonical_model: &str,
) -> Result<Response, AppError> {
let mut req = target.authorize(client.post(target.url(route)).body(body));
if let Some(ct) = headers.get("content-type") {
req = req.header("content-type", ct);
}
let resp = req
.send()
.await
.with_context(|| format!("proxy request to {}", target.describe()))?;
relay_rewriting_model(resp, activity, canonical_model).await
}
async fn relay_rewriting_model(
resp: reqwest::Response,
activity: ActivityGuard,
canonical_model: &str,
) -> Result<Response, AppError> {
let status = resp.status();
let resp_headers = resp.headers().clone();
let raw = resp
.bytes()
.await
.context("read inference backend response")?;
drop(activity);
let rewritten = rewrite_json_response_model(&raw, canonical_model);
let mut builder = Response::builder().status(status.as_u16());
for (k, v) in &resp_headers {
if k == reqwest::header::CONTENT_LENGTH {
continue;
}
builder = builder.header(k, v);
}
Ok(builder.body(Body::from(rewritten)).unwrap())
}
async fn stream_rewriting_model(
client: &Client,
target: &Target,
route: &str,
headers: &HeaderMap,
body: Bytes,
activity: ActivityGuard,
canonical_model: String,
) -> Result<Response, AppError> {
let mut req = target.authorize(client.post(target.url(route)).body(body));
if let Some(ct) = headers.get("content-type") {
req = req.header("content-type", ct);
}
let resp = req
.send()
.await
.with_context(|| format!("proxy request to {}", target.describe()))?;
Ok(relay_stream_rewriting_model(
resp,
activity,
canonical_model,
))
}
fn relay_stream_rewriting_model(
resp: reqwest::Response,
activity: ActivityGuard,
canonical_model: String,
) -> Response {
let status = resp.status();
let mut resp_headers = resp.headers().clone();
resp_headers.remove(reqwest::header::CONTENT_LENGTH);
resp_headers.remove(reqwest::header::TRANSFER_ENCODING);
let stream = bytes_to_lines(resp.bytes_stream()).map(move |line| {
let _activity = &activity;
let out = rewrite_sse_line_model(&line, &canonical_model) + "\n";
Ok::<_, std::convert::Infallible>(Bytes::from(out))
});
let mut builder = Response::builder().status(status.as_u16());
for (k, v) in &resp_headers {
builder = builder.header(k, v);
}
builder.body(Body::from_stream(stream)).unwrap()
}
fn apply_default_repeat_penalty_typed(oai_req: &mut OAIChatRequest) {
if oai_req.repeat_penalty.is_none() {
oai_req.repeat_penalty = Some(DEFAULT_REPEAT_PENALTY);
}
}
fn repeat_penalty_applies(target: &Target) -> bool {
!target.is_remote()
}
const LLAMA_FIELDS: &[&str] = &[
"repeat_penalty",
"chat_template_kwargs",
"top_k",
"min_p",
"typical_p",
"n_probs",
"cache_prompt",
"samplers",
"id_slot",
"n_keep",
"n_indent",
"dynatemp_range",
"dynatemp_exponent",
"mirostat",
"mirostat_tau",
"mirostat_eta",
"dry_multiplier",
"dry_base",
"dry_allowed_length",
"dry_penalty_last_n",
"dry_sequence_breakers",
"xtc_probability",
"xtc_threshold",
"top_n_sigma",
"penalty_last_n",
"repeat_last_n",
"n_predict",
"grammar",
"grammar_lazy",
"grammar_triggers",
"preserved_tokens",
"json_schema",
"reasoning_format",
"reasoning_budget",
"thinking_forced_open",
"return_progress",
"return_tokens",
"timings_per_token",
"post_sampling_probs",
"response_fields",
"min_keep",
"t_max_predict_ms",
"t_max_prompt_ms",
"ignore_eos",
"lora",
];
fn strip_llama_fields(req: &mut serde_json::Value) {
if let Some(o) = req.as_object_mut() {
for field in LLAMA_FIELDS {
o.remove(*field);
}
}
}
fn provider_compat(remote: &RemoteTarget, req: &mut serde_json::Value) {
let Some(o) = req.as_object_mut() else {
return;
};
if let Some(kwargs) = o.remove("chat_template_kwargs") {
let level = kwargs.get("reasoning_effort").and_then(|v| v.as_str());
let enabled = kwargs.get("enable_thinking").and_then(|v| v.as_bool());
let effort = match (level, enabled, remote.wire) {
(Some(level), _, _) => Some(level.to_string()),
(None, Some(true), Wire::Anthropic) => Some("medium".to_string()),
(None, Some(false), Wire::Anthropic) => Some("none".to_string()),
_ => None,
};
if let Some(effort) = effort {
o.entry("reasoning_effort")
.or_insert(serde_json::Value::String(effort));
}
}
for field in LLAMA_FIELDS {
o.remove(*field);
}
if remote.provider == "cohere" {
o.remove("stream_options");
}
if remote.provider != "openai" {
return;
}
if !openai_reasoning_model(&remote.model) {
o.remove("reasoning_effort");
return;
}
if let Some(max) = o.remove("max_tokens") {
o.entry("max_completion_tokens").or_insert(max);
}
if o.get("temperature").and_then(|t| t.as_f64()) != Some(1.0) {
o.remove("temperature");
}
for field in [
"top_p",
"presence_penalty",
"frequency_penalty",
"logprobs",
"top_logprobs",
] {
o.remove(field);
}
if remote.model.contains("gpt-5") || remote.model.contains("gpt-6") {
for field in ["stop", "logit_bias", "modalities", "prediction", "audio"] {
o.remove(field);
}
}
}
fn openai_reasoning_model(model: &str) -> bool {
let name = model.rsplit('/').next().unwrap_or(model);
let o_series = name
.strip_prefix('o')
.and_then(|rest| rest.chars().next())
.is_some_and(|c| c.is_ascii_digit());
let gpt = (name.contains("gpt-5") || name.contains("gpt-6")) && !name.starts_with("gpt-5-chat");
o_series || gpt
}
type ChatBody = futures::stream::BoxStream<'static, reqwest::Result<Bytes>>;
struct ChatUpstream {
status: StatusCode,
headers: HeaderMap,
body: ChatBody,
}
impl ChatUpstream {
fn relay(resp: reqwest::Response) -> Self {
Self {
status: resp.status(),
headers: resp.headers().clone(),
body: resp.bytes_stream().boxed(),
}
}
fn translated(resp: &reqwest::Response, content_type: &'static str) -> HeaderMap {
let mut headers = resp.headers().clone();
headers.remove(reqwest::header::CONTENT_LENGTH);
headers.remove(reqwest::header::TRANSFER_ENCODING);
headers.insert(
reqwest::header::CONTENT_TYPE,
reqwest::header::HeaderValue::from_static(content_type),
);
headers
}
async fn text(self) -> String {
String::from_utf8_lossy(&collect_body(self.body).await).into_owned()
}
}
async fn collect_body(mut body: ChatBody) -> Vec<u8> {
let mut out = Vec::new();
while let Some(Ok(chunk)) = body.next().await {
out.extend_from_slice(&chunk);
}
out
}
async fn send_chat_completion<T: Serialize + ?Sized>(
client: &Client,
target: &Target,
req: &T,
response_model: &str,
) -> Result<ChatUpstream, AppError> {
let Target::Remote(remote) = target else {
let resp = target
.authorize(client.post(target.url(CHAT_COMPLETIONS_ROUTE)).json(req))
.send()
.await
.with_context(|| format!("send to {}", target.describe()))?;
return Ok(ChatUpstream::relay(resp));
};
let mut req = serde_json::to_value(req).context("serialize chat request")?;
provider_compat(remote, &mut req);
if remote.wire == Wire::OpenAi {
let resp = target
.authorize(client.post(target.url(CHAT_COMPLETIONS_ROUTE)).json(&req))
.send()
.await
.with_context(|| format!("send to {}", target.describe()))?;
return Ok(ChatUpstream::relay(resp));
}
let streaming = req
.get("stream")
.and_then(serde_json::Value::as_bool)
.unwrap_or(false);
let default_max_tokens = match target {
Target::Remote(remote) => remote.max_output,
_ => None,
}
.unwrap_or(anthropic::DEFAULT_MAX_TOKENS);
let messages_req = anthropic::from_chat_request(&req, default_max_tokens)
.map_err(|e| AppError(e, StatusCode::BAD_REQUEST))?;
let mut upstream = target.authorize(client.post(target.url(anthropic::MESSAGES_ROUTE)));
if anthropic::thinks(&messages_req) {
upstream = upstream.header("anthropic-beta", anthropic::INTERLEAVED_THINKING_BETA);
}
let resp = upstream
.json(&messages_req)
.send()
.await
.with_context(|| format!("send to {}", target.describe()))?;
if !resp.status().is_success() {
return Ok(ChatUpstream::relay(resp));
}
let mut converter =
anthropic::StreamConverter::new(response_model).json_tool(anthropic::json_tool_name(&req));
if !streaming {
let headers = ChatUpstream::translated(&resp, "application/json");
let lines: Vec<String> = bytes_to_lines(resp.bytes_stream()).collect().await;
for line in &lines {
converter.line(line);
}
converter.finish();
let (status, body) = match converter.error() {
Some(error) => (
StatusCode::BAD_GATEWAY,
serde_json::json!({ "error": error }),
),
None if !converter.finished() => (
StatusCode::BAD_GATEWAY,
serde_json::json!({ "error": {
"message": format!("{} closed the stream before finishing", target.describe()),
"type": "server_error",
} }),
),
None => (StatusCode::OK, converter.completion()),
};
let body = Bytes::from(serde_json::to_vec(&body).unwrap_or_default());
return Ok(ChatUpstream {
status,
headers,
body: futures::stream::once(futures::future::ready(Ok(body))).boxed(),
});
}
let headers = ChatUpstream::translated(&resp, "text/event-stream");
let body = bytes_to_lines(resp.bytes_stream())
.map(Some)
.chain(futures::stream::once(futures::future::ready(None)))
.map(move |line| {
let out = match line {
Some(line) => converter.line(&line),
None => converter.finish(),
};
Ok(Bytes::from(out))
})
.boxed();
Ok(ChatUpstream {
status: StatusCode::OK,
headers,
body,
})
}
fn relay_chat_upstream(upstream: ChatUpstream, activity: ActivityGuard) -> Response {
let stream = upstream.body.map(move |item| {
let _activity = &activity;
item.map_err(|e| Box::new(e) as Box<dyn std::error::Error + Send + Sync>)
});
let mut builder = Response::builder().status(upstream.status.as_u16());
for (k, v) in &upstream.headers {
builder = builder.header(k, v);
}
builder.body(Body::from_stream(stream)).unwrap()
}
async fn post_chat(
client: &Client,
target: &Target,
oai_req: &mut OAIChatRequest,
) -> Result<ChatBody, AppError> {
if repeat_penalty_applies(target) {
apply_default_repeat_penalty_typed(oai_req);
} else {
oai_req.repeat_penalty = None;
}
let upstream = send_chat_completion(client, target, &*oai_req, &oai_req.model).await?;
chat_body(target, upstream).await
}
async fn chat_body(target: &Target, upstream: ChatUpstream) -> Result<ChatBody, AppError> {
if !upstream.status.is_success() {
let status = upstream.status;
let body = upstream.text().await;
let message = format!("{} {status}: {body}", target.describe());
let error = match target {
Target::Local(_) => match context_overflow_message(status, &body) {
Some(refusal) => anyhow::Error::new(ContextOverflow { message, refusal }),
None => anyhow!("{message}"),
},
_ => anyhow!("{message}"),
};
return Err(AppError(error, remote_status(target, status)));
}
Ok(upstream.body)
}
#[derive(Debug)]
struct ContextOverflow {
message: String,
refusal: String,
}
impl std::fmt::Display for ContextOverflow {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.message)
}
}
impl std::error::Error for ContextOverflow {}
fn context_overflow_message(status: StatusCode, body: &str) -> Option<String> {
if status != StatusCode::BAD_REQUEST {
return None;
}
let json = &body[body.find('{')?..];
let value: serde_json::Value = serde_json::from_str(json).ok()?;
let error = &value["error"];
let message = error["message"]
.as_str()
.or_else(|| value["message"].as_str())
.unwrap_or("");
let overflow = error["type"] == "exceed_context_size_error"
|| message.contains("exceeds the available context size")
|| message.contains("maximum context length is");
overflow.then(|| message.to_string())
}
fn remote_status(target: &Target, upstream: StatusCode) -> StatusCode {
match target {
Target::Local(_) => StatusCode::INTERNAL_SERVER_ERROR,
Target::Peer(_) => upstream,
Target::Remote(_) if upstream.is_client_error() => upstream,
Target::Remote(_) => StatusCode::BAD_GATEWAY,
}
}
async fn handle_props() -> impl IntoResponse {
Json(serde_json::json!({
"role": "router",
"total_slots": 0,
"model_path": "",
"chat_template": "",
"bos_token": "",
"eos_token": "",
"build_info": env!("LLMMAN_VERSION"),
"modalities": { "vision": false, "audio": false },
"default_generation_settings": {
"id": 0,
"id_task": 0,
"n_ctx": 4096,
"speculative": false,
"is_processing": false,
"params": {
"n_predict": -1,
"seed": 0,
"temperature": 0.8,
"dynatemp_range": 0.0,
"dynatemp_exponent": 1.0,
"top_k": 40,
"top_p": 0.95,
"min_p": 0.05,
"top_n_sigma": 0.0,
"xtc_probability": 0.0,
"xtc_threshold": 0.1,
"typ_p": 1.0,
"repeat_last_n": 64,
"repeat_penalty": 1.0,
"presence_penalty": 0.0,
"frequency_penalty": 0.0,
"dry_multiplier": 0.0,
"dry_base": 1.75,
"dry_allowed_length": 2,
"dry_penalty_last_n": -1,
"dry_sequence_breakers": [],
"mirostat": 0,
"mirostat_tau": 5.0,
"mirostat_eta": 0.1,
"stop": [],
"max_tokens": -1,
"n_keep": 0,
"n_discard": 0,
"ignore_eos": false,
"stream": true,
"logit_bias": [],
"n_probs": 0,
"min_keep": 0,
"grammar": "",
"grammar_lazy": false,
"grammar_triggers": [],
"preserved_tokens": [],
"chat_format": "",
"reasoning_format": "",
"reasoning_in_content": false,
"generation_prompt": "",
"samplers": ["top_k", "top_p", "min_p", "temperature"],
"backend_sampling": false,
"speculative.n_max": 16,
"speculative.n_min": 5,
"speculative.p_min": 0.9,
"timings_per_token": false,
"post_sampling_probs": false,
"lora": []
},
"prompt": "",
"next_token": {
"has_next_token": false,
"has_new_line": false,
"n_remain": 0,
"n_decoded": 0,
"stopping_word": ""
}
}
}))
}
#[derive(Serialize)]
struct ProviderSummary {
id: String,
name: String,
base_url: String,
#[serde(skip_serializing_if = "Option::is_none")]
key_env: Option<String>,
wire: &'static str,
key_set: bool,
key_usable: bool,
key_optional: bool,
models: usize,
}
impl From<&crate::providers::Provider> for ProviderSummary {
fn from(p: &crate::providers::Provider) -> Self {
Self {
id: p.id.clone(),
name: p.name.clone(),
base_url: p.base_url.clone(),
key_env: p.key_env.clone(),
wire: p.wire.as_str(),
key_set: p.api_key().is_some(),
key_usable: daemon_key_usable(p),
key_optional: p.key_optional,
models: p.models.len(),
}
}
}
fn daemon_key_usable(provider: &crate::providers::Provider) -> bool {
provider.api_key().is_some() && crate::daemon::reachable_only_locally()
}
#[derive(Serialize)]
struct ProvidersResponse {
providers: Vec<ProviderSummary>,
}
#[derive(Serialize)]
struct ProviderResponse {
id: String,
name: String,
base_url: String,
#[serde(skip_serializing_if = "Option::is_none")]
key_env: Option<String>,
wire: &'static str,
key_set: bool,
key_usable: bool,
key_optional: bool,
models: Vec<ProviderModelResponse>,
}
#[derive(Serialize)]
struct ProviderModelResponse {
id: String,
#[serde(skip_serializing_if = "Option::is_none")]
cost: Option<ProviderCostResponse>,
}
#[derive(Serialize)]
struct ProviderCostResponse {
input: f64,
output: f64,
}
impl From<&crate::providers::Provider> for ProviderResponse {
fn from(p: &crate::providers::Provider) -> Self {
Self {
id: p.id.clone(),
name: p.name.clone(),
base_url: p.base_url.clone(),
key_env: p.key_env.clone(),
wire: p.wire.as_str(),
key_set: p.api_key().is_some(),
key_usable: daemon_key_usable(p),
key_optional: p.key_optional,
models: p
.models
.iter()
.map(|m| ProviderModelResponse {
id: m.id.clone(),
cost: m.cost.map(|c| ProviderCostResponse {
input: c.input,
output: c.output,
}),
})
.collect(),
}
}
}
async fn handle_llmman_providers() -> Result<impl IntoResponse, AppError> {
let catalog = provider_catalog().await?;
Ok(Json(ProvidersResponse {
providers: catalog.iter().map(ProviderSummary::from).collect(),
}))
}
async fn handle_llmman_provider(
State(state): State<AppState>,
UrlPath(id): UrlPath<String>,
) -> Result<impl IntoResponse, AppError> {
let catalog = provider_catalog().await?;
let provider = catalog.get(&id).ok_or_else(|| {
AppError(
crate::providers::unknown_provider_error(&id, &catalog),
StatusCode::NOT_FOUND,
)
})?;
let mut response = ProviderResponse::from(provider);
if provider.key_optional && provider.models.is_empty() {
response.models = configured_provider_models(&state.0.client, provider)
.await
.into_iter()
.map(|id| ProviderModelResponse { id, cost: None })
.collect();
}
Ok(Json(response))
}
const CONFIGURED_MODELS_TIMEOUT: Duration = Duration::from_secs(5);
async fn configured_provider_models(
client: &Client,
provider: &crate::providers::Provider,
) -> Vec<String> {
#[derive(Deserialize)]
struct ModelsResponse {
#[serde(default)]
data: Vec<ModelEntry>,
}
#[derive(Deserialize)]
struct ModelEntry {
id: String,
}
if provider.wire != Wire::OpenAi {
return Vec::new();
}
let url = provider.url("/v1/models");
let mut req = client.get(&url).timeout(CONFIGURED_MODELS_TIMEOUT);
if daemon_key_usable(provider) {
if let Some(key) = provider.api_key() {
req = req.bearer_auth(key);
}
}
let listed = async {
let resp = req.send().await?.error_for_status()?;
resp.json::<ModelsResponse>().await
}
.await;
match listed {
Ok(models) => {
let mut ids: Vec<String> = models.data.into_iter().map(|m| m.id).collect();
ids.sort_unstable();
ids.dedup();
ids
}
Err(e) => {
crate::debug_log!("provider {}: GET {url} failed: {e}", provider.id);
Vec::new()
}
}
}
async fn handle_version(State(state): State<AppState>) -> impl IntoResponse {
Json(serde_json::json!({
"version": env!("LLMMAN_VERSION"),
"exe": state.0.exe.as_ref().map(|p| p.to_string_lossy()),
"pid": std::process::id(),
}))
}
async fn handle_tags(
State(state): State<AppState>,
headers: HeaderMap,
) -> Result<impl IntoResponse, AppError> {
let store = OciStore::open(&state.0.store_path)?;
let list = store.list()?;
let mut models: Vec<OllamaModelInfo> = list
.into_iter()
.map(|img| OllamaModelInfo {
name: img.reference.clone(),
model: img.reference,
size: img.size,
digest: img.digest,
modified_at: img
.modified_at
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
.and_then(|d| chrono::DateTime::from_timestamp(d.as_secs() as i64, 0))
.map(|dt| dt.to_rfc3339())
.unwrap_or_else(now_rfc3339),
details: OllamaModelDetails {
format: "gguf".into(),
family: String::new(),
parameter_size: String::new(),
quantization_level: String::new(),
},
})
.collect();
if aggregation::aggregates(&state, &headers) {
for (_, peer) in aggregation::poll::<OllamaTagsResponse>(&state, "/api/tags").await {
for m in peer.models {
if !models.iter().any(|have| have.name == m.name) {
models.push(m);
}
}
}
}
Ok(Json(OllamaTagsResponse { models }))
}
struct PsEntry {
name: String,
digest: String,
size: u64,
port: u16,
pid: Option<u32>,
processor: String,
started_at: String,
expires_at: Option<chrono::DateTime<chrono::Utc>>,
}
async fn handle_ps(State(state): State<AppState>, headers: HeaderMap) -> impl IntoResponse {
let entries: Vec<PsEntry> = {
let mgr = state.0.manager.lock().await;
mgr.running
.iter()
.map(|(name, m)| PsEntry {
name: name.clone(),
digest: m.digest.clone(),
size: m.size,
port: m.port,
pid: m.pid(),
processor: m.processor(),
started_at: m.started_at.clone(),
expires_at: m
.keep_alive
.and_then(|d| chrono::Duration::from_std(d).ok())
.map(|d| m.last_active_wall + d),
})
.collect()
};
let mut models = Vec::with_capacity(entries.len());
for entry in entries {
let context_length = query_context_length(&state.0.client, entry.port).await;
models.push(OllamaRunningModelInfo {
name: entry.name.clone(),
model: entry.name,
digest: entry.digest,
size: entry.size,
size_vram: 0, pid: entry.pid,
port: entry.port,
processor: entry.processor,
context_length,
started_at: entry.started_at,
expires_at: entry
.expires_at
.map(|t| t.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)),
node: None,
});
}
if aggregation::aggregates(&state, &headers) {
for (origin, peer) in aggregation::poll::<OllamaPsResponse>(&state, "/api/ps").await {
models.extend(peer.models.into_iter().map(|m| OllamaRunningModelInfo {
node: Some(origin.clone()),
..m
}));
}
}
Json(OllamaPsResponse { models })
}
async fn query_context_length(client: &Client, port: u16) -> Option<u64> {
let url = format!("http://127.0.0.1:{port}/props");
let resp = client
.get(&url)
.timeout(Duration::from_millis(500))
.send()
.await
.ok()?;
let body: serde_json::Value = resp.json().await.ok()?;
body.get("default_generation_settings")?
.get("n_ctx")?
.as_u64()
}
async fn handle_show(
State(state): State<AppState>,
Json(req): Json<OllamaShowRequest>,
) -> Result<impl IntoResponse, AppError> {
let model_ref = req
.name
.as_deref()
.filter(|s| !s.is_empty())
.unwrap_or(&req.model);
if crate::providers::is_remote_ref(model_ref) {
eprintln!("[llmman] /api/show model={model_ref:?} (provider-routed)");
return Ok(Json(OllamaShowResponse {
model_info: serde_json::json!({ "digest": "", "size": 0 }),
details: OllamaModelDetails {
format: String::new(),
family: String::new(),
parameter_size: String::new(),
quantization_level: String::new(),
},
capabilities: Vec::new(),
}));
}
let model_ref =
crate::shortnames::resolve_ollama_api(model_ref).map_err(AppError::bad_request)?;
let model_ref = model_ref.as_str();
eprintln!("[llmman] /api/show model={model_ref:?}");
let store = OciStore::open(&state.0.store_path)?;
let desc = store.find(model_ref).map_err(|e| {
if e.downcast_ref::<crate::storage::oci::NotFound>().is_some() {
AppError::status(
StatusCode::NOT_FOUND,
format!("model not found: {model_ref}"),
)
} else {
AppError::from(e)
}
})?;
let manifest = store.read_manifest(&desc.digest)?;
let capabilities = crate::modelpack::capabilities(&store, &manifest);
Ok(Json(OllamaShowResponse {
model_info: serde_json::json!({ "digest": desc.digest, "size": desc.size }),
details: OllamaModelDetails {
format: "gguf".into(),
family: String::new(),
parameter_size: String::new(),
quantization_level: String::new(),
},
capabilities,
}))
}
#[derive(Debug, Deserialize)]
struct OllamaPullRequest {
#[serde(default)]
model: String,
#[serde(default)]
name: String,
}
async fn handle_pull(
State(state): State<AppState>,
Json(req): Json<OllamaPullRequest>,
) -> Result<Response, AppError> {
let model_ref = if req.model.is_empty() {
req.name.as_str()
} else {
req.model.as_str()
};
if model_ref.is_empty() {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
"model is required",
));
}
let model = crate::shortnames::resolve_ollama_api(model_ref).map_err(AppError::bad_request)?;
eprintln!("[llmman] /api/pull model={model:?}");
let store_path = state.0.store_path.clone();
let model_for_task = model.clone();
let pull_task =
tokio::task::spawn_blocking(move || pull_serialized(&store_path, &model_for_task));
Ok(stream_ffi_progress(
model,
"pull",
"pulling manifest",
pull_task,
))
}
#[derive(Debug, Deserialize)]
struct OllamaPushRequest {
#[serde(default)]
model: String,
#[serde(default)]
name: String,
}
async fn handle_push(
State(state): State<AppState>,
Json(req): Json<OllamaPushRequest>,
) -> Result<Response, AppError> {
let model_ref = if req.model.is_empty() {
req.name.as_str()
} else {
req.model.as_str()
};
push_impl(state, model_ref, "/api/push").await
}
async fn push_impl(
state: AppState,
model_ref: &str,
route: &'static str,
) -> Result<Response, AppError> {
if model_ref.is_empty() {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
"model is required",
));
}
let model = crate::shortnames::resolve_ollama_api(model_ref).map_err(AppError::bad_request)?;
eprintln!("[llmman] {route} model={model:?}");
let store_path = state.0.store_path.clone();
if OciStore::open(&store_path)
.and_then(|s| s.find(&model))
.is_err()
{
return Err(AppError::status(
StatusCode::NOT_FOUND,
format!("model not found: {model}"),
));
}
let model_for_task = model.clone();
let push_task = tokio::task::spawn_blocking(move || {
let lock = model_lock(&model_for_task);
let result = (|| {
let _guard = lock.blocking_lock();
let layout_dir = store_path
.to_str()
.ok_or_else(|| anyhow!("store path is not valid UTF-8"))?;
crate::ffi::push(layout_dir, &model_for_task)?;
let desc = OciStore::open(&store_path)?.find(&model_for_task)?;
Ok(PushOutcome {
digest: desc.digest,
})
})();
drop(lock);
release_model_lock(&model_for_task);
result
});
Ok(stream_ffi_progress(
model,
"push",
"retrieving manifest",
push_task,
))
}
struct PushOutcome {
digest: String,
}
trait StreamedOutcome {
fn into_lines(self) -> Vec<serde_json::Value>;
}
impl StreamedOutcome for Vec<String> {
fn into_lines(self) -> Vec<serde_json::Value> {
self.into_iter()
.map(|notice| serde_json::json!({"notice": notice}))
.collect()
}
}
impl StreamedOutcome for PushOutcome {
fn into_lines(self) -> Vec<serde_json::Value> {
vec![serde_json::json!({"digest": self.digest})]
}
}
fn progress_line(
verb: &str,
model: &str,
snap: (String, i64, i64),
saw_bytes: &mut bool,
) -> Option<serde_json::Value> {
let (status, total, completed) = snap;
if total > 0 {
*saw_bytes = true;
return Some(serde_json::json!({
"status": if status.is_empty() { format!("{verb}ing {model}") } else { status },
"total": total,
"completed": completed.clamp(0, total),
}));
}
if !status.is_empty() {
return Some(serde_json::json!({"status": status}));
}
if *saw_bytes {
return None;
}
Some(serde_json::json!({"status": format!("{verb}ing {model}")}))
}
fn stream_ffi_progress<T: StreamedOutcome + Send + 'static>(
model: String,
verb: &'static str,
first_status: &'static str,
task: tokio::task::JoinHandle<anyhow::Result<T>>,
) -> Response {
let first_line = serde_json::json!({"status": first_status}).to_string() + "\n";
let stream = futures::stream::once(futures::future::ready(Bytes::from(first_line)))
.chain(futures::stream::unfold((Some(task), false), move |state| {
let (task, mut saw_bytes) = state;
let model = model.clone();
async move {
let mut task = task?;
tokio::select! {
result = &mut task => {
let mut out = String::new();
let line = match result {
Ok(Ok(outcome)) => {
for field in outcome.into_lines() {
out.push_str(&field.to_string());
out.push('\n');
}
serde_json::json!({"status": "success"}).to_string()
}
Ok(Err(e)) => serde_json::json!({"error": format!("{e:#}")}).to_string(),
Err(e) => serde_json::json!({"error": format!("{verb} task panicked: {e}")}).to_string(),
};
out.push_str(&line);
out.push('\n');
Some((Bytes::from(out), (None, saw_bytes)))
}
_ = sleep(Duration::from_millis(200)) => {
let rust_snap = crate::hf::progress::poll(&model);
let go_snap = (rust_snap.total == 0).then(|| crate::ffi::progress(&model).ok()).flatten();
let (status, total, completed) = if rust_snap.total > 0 {
(rust_snap.status, rust_snap.total, rust_snap.completed)
} else if !rust_snap.status.is_empty() {
(rust_snap.status, 0, 0)
} else if let Some(p) = &go_snap {
(p.status.clone(), p.total, p.completed)
} else {
(String::new(), 0, 0)
};
let line = progress_line(verb, &model, (status, total, completed), &mut saw_bytes);
let out = line.map(|l| l.to_string() + "\n").unwrap_or_default();
Some((Bytes::from(out), (Some(task), saw_bytes)))
}
}
}
}))
.map(Ok::<_, std::convert::Infallible>);
Response::builder()
.status(StatusCode::OK)
.header("content-type", "application/x-ndjson")
.body(Body::from_stream(stream))
.unwrap()
}
async fn handle_delete(
State(state): State<AppState>,
Json(req): Json<OllamaDeleteRequest>,
) -> Result<impl IntoResponse, AppError> {
let model_ref = req
.name
.as_deref()
.filter(|s| !s.is_empty())
.unwrap_or(&req.model);
let model_ref =
crate::shortnames::resolve_ollama_api(model_ref).map_err(AppError::bad_request)?;
let store = OciStore::open(&state.0.store_path)?;
store.remove(&model_ref)?;
Ok(StatusCode::OK)
}
async fn handle_copy(
State(state): State<AppState>,
Json(req): Json<OllamaCopyRequest>,
) -> Result<impl IntoResponse, AppError> {
if req.source.is_empty() || req.destination.is_empty() {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
"source and destination are required",
));
}
let source =
crate::shortnames::resolve_ollama_api(&req.source).map_err(AppError::bad_request)?;
let destination =
crate::shortnames::resolve_ollama_api(&req.destination).map_err(AppError::bad_request)?;
eprintln!("[llmman] /api/copy {source:?} -> {destination:?}");
let store = OciStore::open(&state.0.store_path)?;
if store.find(&source).is_err() {
return Err(AppError::status(
StatusCode::NOT_FOUND,
format!("model '{}' not found", req.source),
));
}
let digest = crate::cmd::cp::copy(&store, &source, &destination)?;
evict_if_retagged(&state, &destination, &digest).await;
Ok(StatusCode::OK)
}
async fn evict_if_retagged(state: &AppState, reference: &str, digest: &str) {
let want = crate::storage::default_tag(reference);
let mut mgr = state.0.manager.lock().await;
let stale: Vec<String> = mgr
.running
.iter()
.filter(|(k, m)| m.digest != digest && crate::storage::default_tag(k) == want)
.map(|(k, _)| k.clone())
.collect();
for key in stale {
mgr.running.remove(&key);
metrics::record_model_unload(&key, UnloadReason::Requested);
}
}
fn blob_staging_dir(state: &AppState) -> PathBuf {
state.0.cache_path.join("blobs")
}
static STAGING_COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
fn staging_temp_name(prefix: &str) -> String {
let n = STAGING_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
format!("{prefix}-{}-{n}", std::process::id())
}
fn staged_blob_path(state: &AppState, digest: &str) -> Result<PathBuf, AppError> {
let hex = digest.strip_prefix("sha256:").unwrap_or_default();
if hex.len() != 64 || !hex.bytes().all(|b| b.is_ascii_hexdigit()) {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
format!("invalid digest {digest:?}: expected sha256:<64 hex digits>"),
));
}
Ok(blob_staging_dir(state).join(hex.to_ascii_lowercase()))
}
async fn handle_blob_head(
State(state): State<AppState>,
UrlPath(digest): UrlPath<String>,
) -> Result<StatusCode, AppError> {
let path = staged_blob_path(&state, &digest)?;
Ok(if path.is_file() {
StatusCode::OK
} else {
StatusCode::NOT_FOUND
})
}
async fn handle_blob_upload(
State(state): State<AppState>,
UrlPath(digest): UrlPath<String>,
body: Body,
) -> Result<StatusCode, AppError> {
use sha2::Digest as _;
use tokio::io::AsyncWriteExt as _;
let dest = staged_blob_path(&state, &digest)?;
if dest.is_file() {
return Ok(StatusCode::OK);
}
let dir = blob_staging_dir(&state);
tokio::fs::create_dir_all(&dir).await?;
let tmp = dir.join(staging_temp_name("tmp"));
let mut file = tokio::fs::File::create(&tmp).await?;
let mut hasher = sha2::Sha256::new();
let mut stream = body.into_data_stream();
let written = async {
while let Some(chunk) = stream.next().await {
let chunk = chunk.context("read upload body")?;
hasher.update(&chunk);
file.write_all(&chunk).await?;
}
file.flush().await?;
anyhow::Ok(())
}
.await;
if let Err(e) = written {
let _ = tokio::fs::remove_file(&tmp).await;
return Err(e.into());
}
let actual = format!("sha256:{}", hex::encode(hasher.finalize()));
if !actual.eq_ignore_ascii_case(&digest) {
let _ = tokio::fs::remove_file(&tmp).await;
return Err(AppError::status(
StatusCode::BAD_REQUEST,
format!("digest mismatch, expected {digest:?}, got {actual:?}"),
));
}
tokio::fs::rename(&tmp, &dest).await?;
eprintln!("[llmman] /api/blobs stored {digest} ({})", dest.display());
Ok(StatusCode::CREATED)
}
async fn handle_create(
State(state): State<AppState>,
Json(req): Json<OllamaCreateRequest>,
) -> Result<Response, AppError> {
let model = if req.model.is_empty() {
req.name.as_str()
} else {
req.model.as_str()
};
if model.is_empty() {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
"model is required",
));
}
let refused: Vec<&str> = req
.unsupported
.iter()
.filter(|(_, v)| !is_empty_json(v))
.map(|(k, _)| k.as_str())
.collect();
if !refused.is_empty() {
let mut refused = refused;
refused.sort_unstable();
return Err(AppError::status(
StatusCode::BAD_REQUEST,
format!(
"/api/create: {} not supported by llmman — models are plain OCI artifacts \
whose GGUF carries its own chat template; only `from` (alias an existing \
model) and `files` (package uploaded blobs) are honoured",
refused.join(", ")
),
));
}
let model = &crate::shortnames::resolve_ollama_api(model).map_err(AppError::bad_request)?;
if crate::storage::split_ref_digest(model).1.is_some() {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
"/api/create: the model name can't be a digest reference",
));
}
let files = req.files.unwrap_or_default();
let from = req.from.filter(|f| !f.is_empty());
let statuses: Vec<String> = match (from, files.is_empty()) {
(Some(_), false) => {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
"/api/create: `from` and `files` are mutually exclusive here",
))
}
(None, true) => {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
"/api/create: one of `from` or `files` is required",
))
}
(Some(from), true) => {
let source =
crate::shortnames::resolve_ollama_api(&from).map_err(AppError::bad_request)?;
eprintln!("[llmman] /api/create {model:?} from {source:?}");
let store = OciStore::open(&state.0.store_path)?;
if store.find(&source).is_err() {
return Err(AppError::status(
StatusCode::NOT_FOUND,
format!("model '{from}' not found"),
));
}
let digest = crate::cmd::cp::copy(&store, &source, model)?;
evict_if_retagged(&state, model, &digest).await;
vec![format!("using existing layer {source}")]
}
(None, false) => {
eprintln!(
"[llmman] /api/create {model:?} from {} uploaded file(s)",
files.len()
);
let staged = files
.iter()
.map(|(name, digest)| Ok((name.clone(), staged_file(&state, name, digest)?)))
.collect::<Result<Vec<_>, AppError>>()?;
let store_path = state.0.store_path.clone();
let staging = blob_staging_dir(&state);
let model_owned = model.to_string();
let (digest, statuses) = tokio::task::spawn_blocking(move || {
create_from_staged_blobs(&store_path, &staging, &model_owned, &staged)
})
.await
.context("create task panicked")??;
evict_if_retagged(&state, model, &digest).await;
statuses
}
};
let mut lines: Vec<serde_json::Value> = statuses
.into_iter()
.map(|s| serde_json::json!({"status": s}))
.collect();
lines.push(serde_json::json!({"status": "success"}));
Ok(if req.stream {
let body: String = lines.iter().map(|l| l.to_string() + "\n").collect();
([("content-type", "application/x-ndjson")], body).into_response()
} else {
Json(lines.pop().unwrap_or_default()).into_response()
})
}
fn is_empty_json(v: &serde_json::Value) -> bool {
match v {
serde_json::Value::Null => true,
serde_json::Value::String(s) => s.is_empty(),
serde_json::Value::Object(m) => m.is_empty(),
serde_json::Value::Array(a) => a.is_empty(),
_ => false,
}
}
fn staged_file(state: &AppState, name: &str, digest: &str) -> Result<PathBuf, AppError> {
let bare = Path::new(name)
.file_name()
.and_then(|n| n.to_str())
.is_some_and(|n| n == name);
if !bare {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
format!("files: {name:?} must be a bare file name"),
));
}
let src = staged_blob_path(state, digest)?;
if !src.is_file() {
return Err(AppError::status(
StatusCode::NOT_FOUND,
format!("files: {digest} for {name:?} was never uploaded to /api/blobs"),
));
}
Ok(src)
}
fn create_from_staged_blobs(
store_path: &Path,
staging: &Path,
model: &str,
files: &[(String, PathBuf)],
) -> anyhow::Result<(String, Vec<String>)> {
let tmp = (0..3)
.map(|_| staging.join(staging_temp_name("create")))
.find_map(|dir| std::fs::create_dir(&dir).ok().map(|()| dir))
.ok_or_else(|| {
anyhow!(
"couldn't create a fresh build directory under {}",
staging.display()
)
})?;
let result = (|| {
let mut statuses = Vec::new();
for (name, src) in files {
let dst = tmp.join(name);
if std::fs::hard_link(src, &dst).is_err() {
std::fs::copy(src, &dst)
.with_context(|| format!("stage {name} from {}", src.display()))?;
}
let digest = src.file_name().unwrap_or_default().to_string_lossy();
statuses.push(format!("using sha256:{digest} as {name}"));
}
let store = OciStore::open(store_path)?;
let desc = store.build(&tmp, model, &HashMap::new())?;
statuses.push(format!("writing manifest {}", desc.digest));
Ok((desc.digest, statuses))
})();
let _ = std::fs::remove_dir_all(&tmp);
result
}
fn empty_chat_chunk(model: String, done_reason: &str) -> OllamaChatChunk {
OllamaChatChunk {
model,
created_at: now_rfc3339(),
message: OllamaMessage {
role: "assistant".into(),
..Default::default()
},
done: true,
done_reason: Some(done_reason.into()),
metrics: OllamaMetrics::default(),
}
}
async fn handle_ollama_chat(
State(state): State<AppState>,
headers: HeaderMap,
Json(req): Json<OllamaChatRequest>,
) -> Result<Response, AppError> {
eprintln!(
"[llmman] /api/chat model={:?} messages={}",
req.model,
req.messages.len()
);
if req.messages.is_empty() && is_explicit_unload(&req.keep_alive) {
unload_everywhere(&state, crate::hybrid::local_half(&req.model), &headers).await?;
return Ok(Json(empty_chat_chunk(req.model, "unload")).into_response());
}
let model_ref = req.model.clone();
let request_threads = opt_num_thread(&req.options);
let started = Instant::now();
send_with_hybrid_fallback(
&state,
&model_ref,
Some(&headers),
request_threads,
|model, target, guard| {
ollama_chat_to(&state, &headers, &req, model, target, guard, started)
},
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn ollama_chat_to(
state: &AppState,
headers: &HeaderMap,
req: &OllamaChatRequest,
model: String,
target: Target,
guard: ActivityGuard,
started: Instant,
) -> Result<Response, AppError> {
let load_duration = if target.is_remote() {
Duration::ZERO
} else {
started.elapsed()
};
if matches!(target, Target::Peer(_)) {
return forward_ollama(state, &target, "/api/chat", headers, req, &model, guard).await;
}
if req.messages.is_empty() {
refresh_activity(guard, resolve_keep_alive(&req.keep_alive)).await;
return Ok(Json(empty_chat_chunk(req.model.clone(), "load")).into_response());
}
let keep_alive = resolve_keep_alive(&req.keep_alive);
let activity = begin_activity(guard, Some(keep_alive)).await;
let wire_model = backend_wire_model(state, &target, &model).await;
let oai = OAIChatRequest {
model: wire_model,
messages: req.messages.iter().map(ollama_message_to_oai).collect(),
stream: true,
chat_template_kwargs: think_to_chat_template_kwargs(&req.think),
tools: req.tools.clone(),
response_format: format_to_response_format(&req.format),
..options_to_oai(&req.options)
};
stream_ollama(
req.stream,
state.0.client.clone(),
target,
oai,
activity,
started,
load_duration,
move |delta| OllamaChatChunk {
model: model.clone(),
created_at: now_rfc3339(),
message: OllamaMessage {
role: "assistant".into(),
content: delta.content,
thinking: delta.thinking,
tool_calls: delta.tool_calls,
..Default::default()
},
done: delta.done,
done_reason: delta.done_reason,
metrics: delta.metrics,
},
)
.await
}
fn options_to_oai(options: &Option<serde_json::Value>) -> OAIChatRequest {
OAIChatRequest {
temperature: opt_f64(options, "temperature"),
top_p: opt_f64(options, "top_p"),
max_tokens: opt_u32(options, "num_predict"),
repeat_penalty: opt_f64(options, "repeat_penalty"),
seed: options.as_ref().and_then(|o| o.get("seed")?.as_u64()),
stop: opt_stop(options),
presence_penalty: opt_f64(options, "presence_penalty"),
frequency_penalty: opt_f64(options, "frequency_penalty"),
top_k: opt_u32(options, "top_k"),
min_p: opt_f64(options, "min_p"),
..Default::default()
}
}
fn opt_stop(options: &Option<serde_json::Value>) -> Option<Vec<String>> {
match options.as_ref()?.get("stop")? {
serde_json::Value::String(s) => Some(vec![s.clone()]),
serde_json::Value::Array(a) => Some(
a.iter()
.filter_map(|v| v.as_str().map(str::to_string))
.collect(),
),
_ => None,
}
}
async fn handle_ollama_generate(
State(state): State<AppState>,
headers: HeaderMap,
Json(req): Json<OllamaGenerateRequest>,
) -> Result<Response, AppError> {
eprintln!(
"[llmman] /api/generate model={:?} prompt_len={}",
req.model,
req.prompt.len()
);
let is_unload = req.prompt.is_empty() && is_explicit_unload(&req.keep_alive);
if is_unload {
unload_everywhere(&state, crate::hybrid::local_half(&req.model), &headers).await?;
return Ok(Json(OllamaGenerateChunk {
model: req.model,
created_at: now_rfc3339(),
response: String::new(),
thinking: None,
done: true,
done_reason: Some("unload".into()),
metrics: OllamaMetrics::default(),
})
.into_response());
}
let unsupported = if req.raw {
Some("raw")
} else if req.suffix.as_deref().is_some_and(|s| !s.is_empty()) {
Some("suffix")
} else if req.template.as_deref().is_some_and(|t| !t.is_empty()) {
Some("template")
} else {
None
};
if let Some(field) = unsupported {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
format!("llmman does not support the {field:?} field of /api/generate"),
));
}
let model_ref = req.model.clone();
let request_threads = opt_num_thread(&req.options);
let started = Instant::now();
send_with_hybrid_fallback(
&state,
&model_ref,
Some(&headers),
request_threads,
|model, target, guard| {
ollama_generate_to(&state, &headers, &req, model, target, guard, started)
},
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn ollama_generate_to(
state: &AppState,
headers: &HeaderMap,
req: &OllamaGenerateRequest,
model: String,
target: Target,
guard: ActivityGuard,
started: Instant,
) -> Result<Response, AppError> {
let load_duration = if target.is_remote() {
Duration::ZERO
} else {
started.elapsed()
};
if matches!(target, Target::Peer(_)) {
return forward_ollama(state, &target, "/api/generate", headers, req, &model, guard).await;
}
if req.prompt.is_empty() {
refresh_activity(guard, resolve_keep_alive(&req.keep_alive)).await;
return Ok(Json(OllamaGenerateChunk {
model: req.model.clone(),
created_at: now_rfc3339(),
response: String::new(),
thinking: None,
done: true,
done_reason: Some("load".into()),
metrics: OllamaMetrics::default(),
})
.into_response());
}
let keep_alive = resolve_keep_alive(&req.keep_alive);
let activity = begin_activity(guard, Some(keep_alive)).await;
let wire_model = backend_wire_model(state, &target, &model).await;
let user = OllamaMessage {
role: "user".into(),
content: req.prompt.clone(),
images: req.images.clone(),
..Default::default()
};
let messages = req
.system
.as_deref()
.filter(|s| !s.is_empty())
.map(|s| OAIMessage::text("system", s))
.into_iter()
.chain(std::iter::once(ollama_message_to_oai(&user)))
.collect();
let oai = OAIChatRequest {
model: wire_model,
messages,
stream: true,
chat_template_kwargs: think_to_chat_template_kwargs(&req.think),
tools: None,
response_format: format_to_response_format(&req.format),
..options_to_oai(&req.options)
};
stream_ollama(
req.stream,
state.0.client.clone(),
target,
oai,
activity,
started,
load_duration,
move |delta| OllamaGenerateChunk {
model: model.clone(),
created_at: now_rfc3339(),
response: delta.content,
thinking: delta.thinking,
done: delta.done,
done_reason: delta.done_reason,
metrics: delta.metrics,
},
)
.await
}
fn embed_inputs(input: &serde_json::Value) -> Result<Vec<String>, AppError> {
let invalid = || AppError::status(StatusCode::BAD_REQUEST, "invalid input type");
match input {
serde_json::Value::Null => Ok(Vec::new()),
serde_json::Value::String(s) if s.is_empty() => Ok(Vec::new()),
serde_json::Value::String(s) => Ok(vec![s.clone()]),
serde_json::Value::Array(items) => items
.iter()
.map(|v| v.as_str().map(str::to_owned).ok_or_else(invalid))
.collect(),
_ => Err(invalid()),
}
}
async fn embed_via_backend(
state: &AppState,
headers: &HeaderMap,
model_ref: &str,
inputs: &[String],
truncate: bool,
keep_alive: &Option<serde_json::Value>,
) -> Result<(Vec<Vec<f32>>, u64, Duration), AppError> {
let mlx_unsupported = |model: &str| {
AppError::status(
StatusCode::NOT_IMPLEMENTED,
format!(
"{model} is served by mlx_lm.server, which llmman never starts with \
--embedding-model; use a GGUF or vllm-served model for embeddings"
),
)
};
if !crate::providers::is_remote_ref(model_ref) {
if let Some(canonical) = would_use_mlx(state, model_ref).await {
return Err(mlx_unsupported(&canonical));
}
}
let started = Instant::now();
let (model, target, guard) = ensure_model(state, model_ref, Some(headers), None).await?;
let loaded = started.elapsed();
if !target.is_remote() && would_use_mlx(state, &model).await.is_some() {
return Err(mlx_unsupported(&model));
}
if let Some(refusal) = wire_refusal(&target, "/v1/embeddings") {
return Err(AppError::status(StatusCode::NOT_IMPLEMENTED, refusal));
}
let keep_alive = resolve_keep_alive(keep_alive);
if inputs.is_empty() {
refresh_activity(guard, keep_alive).await;
return Ok((Vec::new(), 0, loaded));
}
let _activity = begin_activity(guard, Some(keep_alive)).await;
let wire_model = backend_wire_model(state, &target, &model).await;
use futures::TryStreamExt as _;
let calls: Vec<_> = inputs
.iter()
.map(|text| embed_one(state, &target, &wire_model, text, truncate))
.collect();
let results: Vec<(Vec<f32>, u64)> = futures::stream::iter(calls)
.buffered(EMBED_CONCURRENCY)
.try_collect()
.await?;
let total_tokens = results.iter().map(|(_, n)| n).sum();
let embeddings = results
.into_iter()
.map(|(mut v, _)| {
normalize_in_place(&mut v)?;
Ok(v)
})
.collect::<Result<_, AppError>>()?;
Ok((embeddings, total_tokens, loaded))
}
const EMBED_CONCURRENCY: usize = 8;
async fn embed_one(
state: &AppState,
target: &Target,
wire_model: &str,
text: &str,
truncate: bool,
) -> Result<(Vec<f32>, u64), AppError> {
match post_embeddings(&state.0.client, target, wire_model, text).await {
Ok(ok) => Ok(ok),
Err((status, body)) if truncate && !target.is_remote() && status.is_server_error() => {
let Target::Local(port) = target else {
unreachable!("guarded by !is_remote")
};
let limit = query_context_length(&state.0.client, *port)
.await
.ok_or_else(|| anyhow!("{} {status}: {body}", target.describe()))?;
let limit = usize::try_from(limit)
.unwrap_or(usize::MAX)
.saturating_sub(2);
let truncated = truncate_to_tokens(&state.0.client, *port, text, limit).await?;
if truncated.len() >= text.len() {
return Err(AppError(
anyhow!("{} {status}: {body}", target.describe()),
StatusCode::INTERNAL_SERVER_ERROR,
));
}
post_embeddings(&state.0.client, target, wire_model, &truncated)
.await
.map_err(|(status, body)| {
AppError(
anyhow!("{} {status}: {body}", target.describe()),
remote_status(target, status),
)
})
}
Err((status, body))
if !truncate
&& !target.is_remote()
&& status.is_server_error()
&& body.contains("too large") =>
{
Err(AppError(
anyhow!("the input length exceeds the context length: {body}"),
StatusCode::BAD_REQUEST,
))
}
Err((status, body)) => Err(AppError(
anyhow!("{} {status}: {body}", target.describe()),
remote_status(target, status),
)),
}
}
async fn post_embeddings(
client: &Client,
target: &Target,
wire_model: &str,
text: &str,
) -> Result<(Vec<f32>, u64), (StatusCode, String)> {
let resp = target
.authorize(
client
.post(target.url("/v1/embeddings"))
.timeout(EMBEDDINGS_TIMEOUT)
.json(&serde_json::json!({ "model": wire_model, "input": text })),
)
.send()
.await
.map_err(|e| {
(
StatusCode::BAD_GATEWAY,
format!("send to {}: {e}", target.describe()),
)
})?;
let status = resp.status();
let body = resp.bytes().await.unwrap_or_default();
if !status.is_success() {
return Err((status, String::from_utf8_lossy(&body).into_owned()));
}
let mut parsed: OAIEmbeddingsResponse = serde_json::from_slice(&body).map_err(|e| {
(
StatusCode::BAD_GATEWAY,
format!(
"{} returned an unexpected embeddings body: {e}",
target.describe()
),
)
})?;
parsed.data.sort_by_key(|d| d.index);
let vector = parsed
.data
.into_iter()
.next()
.map(|d| d.embedding)
.ok_or_else(|| {
(
StatusCode::BAD_GATEWAY,
format!("{} returned no embedding", target.describe()),
)
})?;
Ok((vector, parsed.usage.prompt_tokens))
}
const TOKENIZE_TIMEOUT: Duration = Duration::from_secs(30);
const EMBEDDINGS_TIMEOUT: Duration = Duration::from_secs(600);
async fn truncate_to_tokens(
client: &Client,
port: u16,
text: &str,
limit: usize,
) -> anyhow::Result<String> {
#[derive(Deserialize)]
struct Tokens {
tokens: Vec<i64>,
}
#[derive(Deserialize)]
struct Content {
content: String,
}
let base = format!("http://127.0.0.1:{port}");
let Tokens { mut tokens } = client
.post(format!("{base}/tokenize"))
.timeout(TOKENIZE_TIMEOUT)
.json(&serde_json::json!({ "content": text }))
.send()
.await
.context("tokenize for truncation")?
.error_for_status()?
.json()
.await?;
if limit == 0 {
anyhow::bail!("input after truncation exceeds maximum context length");
}
if tokens.len() <= limit {
return Ok(text.to_string());
}
tokens.truncate(limit);
let Content { content } = client
.post(format!("{base}/detokenize"))
.timeout(TOKENIZE_TIMEOUT)
.json(&serde_json::json!({ "tokens": tokens }))
.send()
.await
.context("detokenize for truncation")?
.error_for_status()?
.json()
.await?;
Ok(content)
}
fn normalize_in_place(v: &mut [f32]) -> Result<(), AppError> {
if v.iter().any(|x| !x.is_finite()) {
return Err(AppError::status(
StatusCode::BAD_GATEWAY,
"embedding contains NaN or Inf values",
));
}
let norm = v.iter().map(|&x| f64::from(x).powi(2)).sum::<f64>().sqrt();
if norm > 0.0 {
for x in v.iter_mut() {
*x = (f64::from(*x) / norm) as f32;
}
}
Ok(())
}
async fn handle_embed(
State(state): State<AppState>,
headers: HeaderMap,
Json(req): Json<OllamaEmbedRequest>,
) -> Result<Response, AppError> {
let started = Instant::now();
let inputs = embed_inputs(&req.input)?;
eprintln!(
"[llmman] /api/embed model={:?} inputs={}",
req.model,
inputs.len()
);
let (mut embeddings, tokens, loaded) = embed_via_backend(
&state,
&headers,
&req.model,
&inputs,
req.truncate != Some(false),
&req.keep_alive,
)
.await?;
if let Some(dims) = req.dimensions.filter(|d| *d > 0) {
for v in &mut embeddings {
if dims < v.len() {
v.truncate(dims);
normalize_in_place(v)?;
}
}
}
Ok(Json(OllamaEmbedResponse {
model: req.model,
embeddings,
total_duration: started.elapsed().as_nanos() as u64,
load_duration: loaded.as_nanos() as u64,
prompt_eval_count: tokens,
})
.into_response())
}
async fn handle_embeddings(
State(state): State<AppState>,
headers: HeaderMap,
Json(req): Json<OllamaEmbeddingsRequest>,
) -> Result<Response, AppError> {
eprintln!(
"[llmman] /api/embeddings model={:?} prompt_len={}",
req.model,
req.prompt.len()
);
let inputs = if req.prompt.is_empty() {
Vec::new()
} else {
vec![req.prompt.clone()]
};
let (embeddings, _tokens, _loaded) =
embed_via_backend(&state, &headers, &req.model, &inputs, true, &req.keep_alive).await?;
let embedding = embeddings
.into_iter()
.next()
.unwrap_or_default()
.into_iter()
.map(f64::from)
.collect();
Ok(Json(OllamaEmbeddingsResponse { embedding }).into_response())
}
async fn handle_openai_models(
State(state): State<AppState>,
headers: HeaderMap,
) -> Result<impl IntoResponse, AppError> {
let store = OciStore::open(&state.0.store_path)?;
let list = store.list()?;
let mut data: Vec<serde_json::Value> = {
let mgr = state.0.manager.lock().await;
list.into_iter()
.map(|img| {
let loaded = mgr.running.contains_key(&img.reference);
serde_json::json!({
"id": img.reference,
"object": "model",
"created": 0,
"owned_by": "llmman",
"status": { "value": if loaded { "loaded" } else { "unloaded" } },
})
})
.collect()
};
if aggregation::aggregates(&state, &headers) {
for (_, peer) in aggregation::poll::<serde_json::Value>(&state, "/v1/models").await {
for m in peer["data"].as_array().into_iter().flatten() {
match data.iter_mut().find(|have| have["id"] == m["id"]) {
Some(have) if m["status"]["value"] == "loaded" => {
have["status"] = m["status"].clone();
}
Some(_) => {}
None => data.push(m.clone()),
}
}
}
}
Ok(Json(serde_json::json!({ "object": "list", "data": data })))
}
fn apply_default_repeat_penalty(req: &mut serde_json::Value) {
if req.get("repeat_penalty").is_none() {
req["repeat_penalty"] = serde_json::json!(DEFAULT_REPEAT_PENALTY);
}
}
async fn resolve_openai_request(
state: &AppState,
headers: &HeaderMap,
body: Bytes,
) -> Result<(serde_json::Value, Target, ActivityGuard, Option<String>), AppError> {
let req = parse_openai_request(&body)?;
let model = req["model"].as_str().unwrap_or("").to_string();
let (model, target, guard) = ensure_model(state, &model, Some(headers), None).await?;
prepare_openai_request(state, req, model, target, guard).await
}
fn parse_openai_request(body: &Bytes) -> Result<serde_json::Value, AppError> {
Ok(serde_json::from_slice(body).context("parse OpenAI request body")?)
}
async fn prepare_openai_request(
state: &AppState,
mut req: serde_json::Value,
model: String,
target: Target,
guard: ActivityGuard,
) -> Result<(serde_json::Value, Target, ActivityGuard, Option<String>), AppError> {
let activity = begin_activity(guard, None).await;
let wire_model = backend_wire_model(state, &target, &model).await;
let response_model_override = (wire_model != model).then_some(model);
req["model"] = serde_json::Value::String(wire_model);
Ok((req, target, activity, response_model_override))
}
async fn proxy_openai_generation(
state: &AppState,
headers: &HeaderMap,
body: Bytes,
llama_path: &str,
) -> Result<Response, AppError> {
let req = parse_openai_request(&body)?;
let model_ref = req["model"].as_str().unwrap_or("").to_string();
send_with_hybrid_fallback(
state,
&model_ref,
Some(headers),
None,
|model, target, guard| {
proxy_openai_generation_to(
state,
headers,
req.clone(),
llama_path,
model,
target,
guard,
)
},
)
.await
}
async fn proxy_openai_generation_to(
state: &AppState,
headers: &HeaderMap,
req: serde_json::Value,
llama_path: &str,
model: String,
target: Target,
guard: ActivityGuard,
) -> Result<Response, AppError> {
let (mut req, target, activity, response_model_override) =
prepare_openai_request(state, req, model, target, guard).await?;
if let Some(refusal) = unsupported_on_wire(&target, llama_path) {
drop(activity);
return Ok(refusal);
}
if llama_path == RESPONSES_ROUTE && target.is_remote() {
strip_llama_fields(&mut req);
let canonical = response_model_override
.unwrap_or_else(|| req["model"].as_str().unwrap_or_default().to_string());
return remote_responses(&state.0.client, &target, headers, req, activity, canonical).await;
}
if llama_path == CHAT_COMPLETIONS_ROUTE && target.is_anthropic() {
let canonical = response_model_override
.unwrap_or_else(|| req["model"].as_str().unwrap_or_default().to_string());
let upstream = send_chat_completion(&state.0.client, &target, &req, &canonical).await?;
return Ok(relay_chat_upstream(upstream, activity));
}
if is_responses_route(llama_path) && !target.is_remote() {
sanitize_responses_request(&mut req);
}
match &target {
Target::Remote(remote) if llama_path == CHAT_COMPLETIONS_ROUTE => {
provider_compat(remote, &mut req)
}
Target::Remote(_) => strip_llama_fields(&mut req),
_ => apply_default_repeat_penalty(&mut req),
}
let streaming = req.get("stream").and_then(|v| v.as_bool()).unwrap_or(false);
let body = Bytes::from(serde_json::to_vec(&req).context("re-serialize OpenAI request body")?);
let resp = match response_model_override {
Some(canonical) if streaming => {
stream_rewriting_model(
&state.0.client,
&target,
llama_path,
headers,
body,
activity,
canonical,
)
.await
}
Some(canonical) => {
proxy_rewriting_model(
&state.0.client,
&target,
llama_path,
headers,
body,
activity,
&canonical,
)
.await
}
None => {
proxy(
&state.0.client,
&target,
llama_path,
headers,
body,
activity,
)
.await
}
}?;
Ok(explain_missing_route(&target, llama_path, resp))
}
async fn proxy_openai_passthrough(
state: &AppState,
headers: &HeaderMap,
body: Bytes,
llama_path: &str,
) -> Result<Response, AppError> {
let embeddings = llama_path == "/v1/embeddings";
if embeddings {
if let Ok(peek) = serde_json::from_slice::<serde_json::Value>(&body) {
if let Some(model_ref) = peek["model"].as_str() {
if !crate::providers::is_remote_ref(model_ref) {
if let Some(canonical) = would_use_mlx(state, model_ref).await {
return Ok(mlx_embeddings_unsupported_response(&canonical));
}
}
}
}
}
let (req, target, activity, response_model_override) =
resolve_openai_request(state, headers, body).await?;
forward_openai_request(
state,
headers,
req,
target,
activity,
response_model_override,
llama_path,
)
.await
}
async fn forward_openai_request(
state: &AppState,
headers: &HeaderMap,
mut req: serde_json::Value,
target: Target,
activity: ActivityGuard,
response_model_override: Option<String>,
llama_path: &str,
) -> Result<Response, AppError> {
let embeddings = llama_path == "/v1/embeddings";
if let Some(refusal) = unsupported_on_wire(&target, llama_path) {
drop(activity);
return Ok(refusal);
}
if is_responses_route(llama_path) && !target.is_remote() {
sanitize_responses_request(&mut req);
}
if embeddings && !target.is_remote() {
if let Some(canonical) = &response_model_override {
drop(activity);
return Ok(mlx_embeddings_unsupported_response(canonical));
}
}
let body = Bytes::from(serde_json::to_vec(&req).context("re-serialize OpenAI request body")?);
let resp = match response_model_override {
Some(canonical) => {
proxy_rewriting_model(
&state.0.client,
&target,
llama_path,
headers,
body,
activity,
&canonical,
)
.await
}
None => {
proxy(
&state.0.client,
&target,
llama_path,
headers,
body,
activity,
)
.await
}
}?;
Ok(explain_missing_route(&target, llama_path, resp))
}
fn mlx_embeddings_unsupported_response(canonical_model: &str) -> Response {
let body = serde_json::json!({
"error": {
"message": format!(
"{canonical_model} is served by mlx_lm.server, which llmman never starts with \
--embedding-model — /v1/embeddings isn't supported for it; use a GGUF or \
vllm-served model for embeddings instead"
),
"type": "invalid_request_error",
}
});
(StatusCode::NOT_IMPLEMENTED, Json(body)).into_response()
}
async fn handle_openai_chat(
State(state): State<AppState>,
headers: HeaderMap,
body: Bytes,
) -> Result<Response, AppError> {
proxy_openai_generation(&state, &headers, body, "/v1/chat/completions").await
}
async fn handle_openai_completions(
State(state): State<AppState>,
headers: HeaderMap,
body: Bytes,
) -> Result<Response, AppError> {
proxy_openai_generation(&state, &headers, body, "/v1/completions").await
}
async fn handle_openai_embeddings(
State(state): State<AppState>,
headers: HeaderMap,
body: Bytes,
) -> Result<Response, AppError> {
proxy_openai_passthrough(&state, &headers, body, "/v1/embeddings").await
}
async fn handle_openai_images(
State(state): State<AppState>,
headers: HeaderMap,
body: Bytes,
) -> Result<Response, AppError> {
handle_openai_media(&state, &headers, body, "/v1/images/generations").await
}
async fn handle_openai_videos(
State(state): State<AppState>,
headers: HeaderMap,
body: Bytes,
) -> Result<Response, AppError> {
handle_openai_media(&state, &headers, body, "/v1/videos").await
}
async fn handle_openai_media(
state: &AppState,
headers: &HeaderMap,
body: Bytes,
route: &str,
) -> Result<Response, AppError> {
let (req, target, activity, override_) = resolve_openai_request(state, headers, body).await?;
if local_engine(state, &target).await == Some(Engine::VllmOmni) {
return if route == "/v1/videos" {
omni_videos(state, &target, req, activity).await
} else {
omni_images(state, &target, req, activity).await
};
}
forward_openai_request(state, headers, req, target, activity, override_, route).await
}
async fn local_engine(state: &AppState, target: &Target) -> Option<Engine> {
let Target::Local(port) = target else {
return None;
};
let mgr = state.0.manager.lock().await;
mgr.running
.values()
.find(|m| m.port == *port)
.map(|m| m.process.engine())
}
fn omni_image_request(mut req: serde_json::Value) -> Result<(serde_json::Value, bool), String> {
let stream = req["stream"].as_bool().unwrap_or(false);
let Some(obj) = req.as_object_mut() else {
return Ok((req, stream));
};
obj.remove("stream");
let dim = |v: Option<serde_json::Value>| v.and_then(|v| v.as_u64()).filter(|n| *n > 0);
let (width, height) = (dim(obj.remove("width")), dim(obj.remove("height")));
match (width, height) {
(Some(w), Some(h)) if !obj.contains_key("size") => {
obj.insert("size".into(), serde_json::json!(format!("{w}x{h}")));
}
(Some(_), None) | (None, Some(_)) => {
return Err("vLLM-Omni needs both width and height, or neither".into());
}
_ => {}
}
for (llama, omni) in [
("steps", "num_inference_steps"),
("cfg_scale", "guidance_scale"),
] {
if let Some(v) = obj.remove(llama) {
if !obj.contains_key(omni) && v.as_f64().is_some_and(|n| n > 0.0) {
obj.insert(omni.into(), v);
}
}
}
if !obj.contains_key("response_format") {
obj.insert("response_format".into(), "b64_json".into());
}
Ok((req, stream))
}
async fn omni_images(
state: &AppState,
target: &Target,
req: serde_json::Value,
activity: ActivityGuard,
) -> Result<Response, AppError> {
let (req, stream) = match omni_image_request(req) {
Ok(ok) => ok,
Err(m) => {
drop(activity);
return Err(AppError::status(StatusCode::BAD_REQUEST, m));
}
};
let resp = state
.0
.client
.post(target.url("/v1/images/generations"))
.json(&req)
.send()
.await
.with_context(|| format!("proxy request to {}", target.describe()))?;
if !stream || !resp.status().is_success() {
return Ok(relay(resp, activity));
}
let body: serde_json::Value = resp
.json()
.await
.context("decode vllm-omni image response")?;
let created = body["created"]
.as_i64()
.unwrap_or_else(|| chrono::Utc::now().timestamp());
let mut events = String::new();
for image in body["data"].as_array().into_iter().flatten() {
let ev = serde_json::json!({
"type": "image_generation.completed",
"b64_json": image["b64_json"],
"revised_prompt": image["revised_prompt"],
"created_at": created,
});
events.push_str(&format!("data: {ev}\n\n"));
}
drop(activity);
Ok(Response::builder()
.status(StatusCode::OK)
.header("content-type", "text/event-stream")
.header("cache-control", "no-cache")
.body(Body::from(events))
.context("build image event stream")?)
}
fn omni_video_fields(req: &serde_json::Value) -> Vec<(String, String)> {
let Some(obj) = req.as_object() else {
return Vec::new();
};
let renamed = |name: &str| match name {
"steps" => Some("num_inference_steps"),
"cfg_scale" => Some("guidance_scale"),
"frames" => Some("num_frames"),
"audio" => Some("generate_sound"),
_ => None,
};
let text = |value: &serde_json::Value| match value {
serde_json::Value::String(s) => Some(s.clone()),
serde_json::Value::Null => None,
other => Some(other.to_string()),
};
let mut fields: Vec<(String, String)> = Vec::new();
for (name, value) in obj {
if matches!(name.as_str(), "stream" | "response_format") || renamed(name).is_some() {
continue;
}
let value = if name == "seconds" {
match value.as_f64().or_else(|| value.as_str()?.parse().ok()) {
Some(s) if s > 0.0 => {
serde_json::Value::String((s.round().max(1.0) as u64).to_string())
}
_ => continue,
}
} else {
value.clone()
};
if let Some(text) = text(&value) {
fields.push((name.clone(), text));
}
}
for (name, value) in obj {
let Some(native) = renamed(name) else {
continue;
};
if fields.iter().any(|(existing, _)| existing == native) {
continue;
}
if let Some(text) = text(value) {
fields.push((native.to_string(), text));
}
}
fields
}
fn multipart_form(fields: &[(String, String)]) -> (Vec<u8>, String) {
let fields: Vec<&(String, String)> = fields
.iter()
.filter(|(name, _)| {
!name.is_empty() && name.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_')
})
.collect();
let stamp = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0);
let boundary = (0u32..)
.map(|salt| format!("----llmman{}{stamp:x}{salt:x}", std::process::id()))
.find(|b| !fields.iter().any(|(_, v)| v.contains(b.as_str())))
.unwrap_or_default();
let mut body = Vec::new();
for (name, value) in fields {
body.extend_from_slice(
format!("--{boundary}\r\nContent-Disposition: form-data; name=\"{name}\"\r\n\r\n")
.as_bytes(),
);
body.extend_from_slice(value.as_bytes());
body.extend_from_slice(b"\r\n");
}
body.extend_from_slice(format!("--{boundary}--\r\n").as_bytes());
(body, format!("multipart/form-data; boundary={boundary}"))
}
const OMNI_VIDEO_POLL: Duration = Duration::from_secs(1);
const OMNI_VIDEO_MAX_WAIT: Duration = Duration::from_secs(60 * 60);
async fn omni_videos(
state: &AppState,
target: &Target,
req: serde_json::Value,
activity: ActivityGuard,
) -> Result<Response, AppError> {
let (body, content_type) = multipart_form(&omni_video_fields(&req));
let resp = state
.0
.client
.post(target.url("/v1/videos"))
.header("content-type", content_type)
.body(body)
.send()
.await
.with_context(|| format!("proxy request to {}", target.describe()))?;
if !resp.status().is_success() {
return Ok(relay(resp, activity));
}
let mut job: serde_json::Value = resp.json().await.context("decode vllm-omni video job")?;
let id = job["id"]
.as_str()
.context("vllm-omni video job has no id")?
.to_string();
let poll_url = target.url(&format!("/v1/videos/{id}"));
let deadline = Instant::now() + OMNI_VIDEO_MAX_WAIT;
while !matches!(job["status"].as_str(), Some("completed" | "failed")) {
if Instant::now() >= deadline {
drop(activity);
return Err(AppError::status(
StatusCode::GATEWAY_TIMEOUT,
format!("video job {id} did not finish within {OMNI_VIDEO_MAX_WAIT:?}"),
));
}
sleep(OMNI_VIDEO_POLL).await;
let resp = state
.0
.client
.get(&poll_url)
.timeout(Duration::from_secs(30))
.send()
.await
.with_context(|| format!("poll video job {id} on {}", target.describe()))?;
if !resp.status().is_success() {
return Ok(relay(resp, activity));
}
job = resp.json().await.context("decode vllm-omni video job")?;
}
drop(activity);
if job["status"] == "failed" {
let message = job["error"]["message"]
.as_str()
.unwrap_or("video generation failed")
.to_string();
let body = serde_json::json!({
"error": { "message": message, "type": "server_error" }
});
return Ok((StatusCode::INTERNAL_SERVER_ERROR, Json(body)).into_response());
}
job["content_url"] = serde_json::json!(format!("/v1/videos/{id}/content"));
Ok(Json(job).into_response())
}
async fn handle_openai_video_get(
State(state): State<AppState>,
axum::extract::OriginalUri(uri): axum::extract::OriginalUri,
) -> Result<Response, AppError> {
let ports: Vec<u16> = {
let mgr = state.0.manager.lock().await;
mgr.running.values().map(|m| m.port).collect()
};
let path = uri.path();
let probes = ports.into_iter().map(|port| {
state
.0
.client
.get(format!("http://127.0.0.1:{port}{path}"))
.timeout(Duration::from_secs(30))
.send()
});
let found = futures::future::join_all(probes)
.await
.into_iter()
.flatten()
.find(|r| r.status() != reqwest::StatusCode::NOT_FOUND);
if let Some(resp) = found {
let status = resp.status();
let mut builder = Response::builder().status(status);
for name in ["content-type", "content-disposition"] {
if let Some(v) = resp.headers().get(name) {
builder = builder.header(name, v);
}
}
let stream = resp
.bytes_stream()
.map(|item| item.map_err(|e| Box::new(e) as Box<dyn std::error::Error + Send + Sync>));
return Ok(builder
.body(Body::from_stream(stream))
.context("build video response")?);
}
let body = serde_json::json!({
"error": { "message": "video not found", "type": "not_found_error" }
});
Ok((StatusCode::NOT_FOUND, Json(body)).into_response())
}
async fn handle_openai_speech(
State(state): State<AppState>,
headers: HeaderMap,
body: Bytes,
) -> Result<Response, AppError> {
proxy_openai_passthrough(&state, &headers, body, "/v1/audio/speech").await
}
const TRANSCRIPTION_BODY_LIMIT_BYTES: usize = 200 * 1024 * 1024;
async fn multipart_text_field(
body: &Bytes,
headers: &HeaderMap,
field_name: &str,
) -> Option<String> {
let content_type = headers.get("content-type")?.to_str().ok()?;
let boundary = multer::parse_boundary(content_type).ok()?;
let stream = futures::stream::once(async { Ok::<_, std::io::Error>(body.clone()) });
let mut multipart = multer::Multipart::new(stream, boundary);
while let Ok(Some(field)) = multipart.next_field().await {
if field.name() == Some(field_name) {
return field.text().await.ok();
}
}
None
}
async fn handle_openai_transcriptions(
State(state): State<AppState>,
headers: HeaderMap,
body: Bytes,
) -> Result<Response, AppError> {
let Some(model) = multipart_text_field(&body, &headers, "model")
.await
.filter(|m| !m.is_empty())
else {
let body = serde_json::json!({
"error": "transcription request is missing a required \"model\" form field"
});
return Ok((StatusCode::BAD_REQUEST, Json(body)).into_response());
};
let model = match crate::hybrid::split_ref(&model) {
Some(pair) => {
let side = match request_pin(Some(&headers))? {
Some(crate::hybrid::Side::Cloud) => crate::hybrid::Side::Cloud,
_ => crate::hybrid::Side::Local,
};
eprintln!(
"[llmman] hybrid {:?} + {:?} -> {} (transcription)",
pair.local,
pair.remote_ref(),
side.as_str()
);
pair.side_ref(side)
}
None => model,
};
if crate::providers::is_remote_ref(&model) {
let body = serde_json::json!({
"error": "llmman does not route /v1/audio/transcriptions to a provider — \
use a locally served model with audio support"
});
return Ok((StatusCode::BAD_REQUEST, Json(body)).into_response());
}
let (_, target, guard) = ensure_model(&state, &model, Some(&headers), None).await?;
let activity = begin_activity(guard, None).await;
proxy(
&state.0.client,
&target,
"/v1/audio/transcriptions",
&headers,
body,
activity,
)
.await
}
async fn handle_openai_responses(
State(state): State<AppState>,
headers: HeaderMap,
body: Bytes,
) -> Result<Response, AppError> {
proxy_openai_generation(&state, &headers, body, RESPONSES_ROUTE).await
}
const RESPONSES_ROUTE: &str = "/v1/responses";
trait SseConverter: Send + 'static {
fn line(&mut self, line: &str) -> String;
fn finish(&mut self) -> String;
fn failed(&self) -> bool;
fn fold(&mut self, lines: Vec<String>) -> serde_json::Value;
}
macro_rules! sse_converter {
($($t:ty),*) => {$(
impl SseConverter for $t {
fn line(&mut self, line: &str) -> String {
Self::line(self, line)
}
fn finish(&mut self) -> String {
Self::finish(self)
}
fn failed(&self) -> bool {
Self::failed(self)
}
fn fold(&mut self, lines: Vec<String>) -> serde_json::Value {
Self::fold(self, lines)
}
}
)*};
}
sse_converter!(responses::StreamConverter, messages::StreamConverter);
async fn convert_upstream(
body: ChatBody,
activity: ActivityGuard,
mut converter: impl SseConverter,
streaming: bool,
) -> Response {
if !streaming {
let lines: Vec<String> = bytes_to_lines(body).collect().await;
drop(activity);
let response = converter.fold(lines);
let status = if converter.failed() {
StatusCode::BAD_GATEWAY
} else {
StatusCode::OK
};
return (status, Json(response)).into_response();
}
let sse_stream = bytes_to_lines(body)
.map(Some)
.chain(futures::stream::once(futures::future::ready(None)))
.map(move |line| {
let _activity = &activity;
let out = match line {
Some(line) => converter.line(&line),
None => converter.finish(),
};
Ok::<_, std::convert::Infallible>(Bytes::from(out))
});
Response::builder()
.header("content-type", "text/event-stream")
.header("cache-control", "no-cache")
.body(Body::from_stream(sse_stream))
.unwrap()
}
async fn remote_responses(
client: &Client,
target: &Target,
headers: &HeaderMap,
req: serde_json::Value,
activity: ActivityGuard,
canonical_model: String,
) -> Result<Response, AppError> {
let streaming = req.get("stream").and_then(|v| v.as_bool()).unwrap_or(false);
if !target.is_anthropic() {
let body =
Bytes::from(serde_json::to_vec(&req).context("re-serialize OpenAI request body")?);
let mut native = target.authorize(client.post(target.url(RESPONSES_ROUTE)).body(body));
if let Some(ct) = headers.get("content-type") {
native = native.header("content-type", ct);
}
let resp = native
.send()
.await
.with_context(|| format!("proxy request to {}", target.describe()))?;
let status = resp.status();
if status.is_success() {
return if streaming {
Ok(relay_stream_rewriting_model(
resp,
activity,
canonical_model,
))
} else {
relay_rewriting_model(resp, activity, &canonical_model).await
};
}
if !responses::falls_back(status) {
return Ok(relay(resp, activity));
}
eprintln!(
"[llmman] {} answered {RESPONSES_ROUTE} with {status}; retrying as a chat completion",
target.describe()
);
}
let chat_req = responses::from_responses_request(&req)
.map_err(|e| AppError(e, StatusCode::BAD_REQUEST))?;
let upstream = send_chat_completion(client, target, &chat_req, &canonical_model).await?;
let status = upstream.status;
if status.is_client_error() {
return Ok(relay_chat_upstream(upstream, activity));
}
if !status.is_success() {
let body = upstream.text().await;
return Err(AppError(
anyhow!("{} {status}: {body}", target.describe()),
remote_status(target, status),
));
}
let converter = responses::StreamConverter::new(&canonical_model, &req);
Ok(convert_upstream(upstream.body, activity, converter, streaming).await)
}
async fn handle_openai_responses_input_tokens(
State(state): State<AppState>,
headers: HeaderMap,
body: Bytes,
) -> Result<Response, AppError> {
proxy_openai_passthrough(&state, &headers, body, "/v1/responses/input_tokens").await
}
fn sanitize_responses_request(req: &mut serde_json::Value) {
filter_non_function_tools(req);
consolidate_responses_instructions(req);
}
fn is_responses_route(llama_path: &str) -> bool {
llama_path.starts_with("/v1/responses")
}
fn unsupported_on_wire(target: &Target, route: &str) -> Option<Response> {
let message = wire_refusal(target, route)?;
let body = serde_json::json!({
"error": { "message": message, "type": "invalid_request_error" }
});
Some((StatusCode::NOT_IMPLEMENTED, Json(body)).into_response())
}
fn wire_refusal(target: &Target, route: &str) -> Option<String> {
let Target::Remote(remote) = target else {
return None;
};
if remote.wire != Wire::Anthropic || matches!(route, CHAT_COMPLETIONS_ROUTE | RESPONSES_ROUTE) {
return None;
}
Some(format!(
"provider {} speaks the Anthropic Messages API, which has no equivalent of {route}; \
only chat completions, /v1/responses and /v1/messages reach it",
remote.provider
))
}
fn explain_missing_route(target: &Target, route: &str, resp: Response) -> Response {
if resp.status() != StatusCode::NOT_FOUND || !is_responses_route(route) {
return resp;
}
let Target::Remote(remote) = target else {
return resp;
};
let body = serde_json::json!({
"error": {
"message": format!(
"provider {} has no {route} — it is OpenAI-compatible but does not \
implement the Responses API's token counting. Generation on \
/v1/responses is bridged; this route cannot be.",
remote.provider
),
"type": "invalid_request_error",
}
});
(StatusCode::NOT_IMPLEMENTED, Json(body)).into_response()
}
fn filter_non_function_tools(req: &mut serde_json::Value) {
if let Some(tools) = req.get_mut("tools").and_then(|t| t.as_array_mut()) {
tools.retain(|t| t.get("type").and_then(|v| v.as_str()) == Some("function"));
}
}
fn consolidate_responses_instructions(req: &mut serde_json::Value) {
let mut instructions = req
.get("instructions")
.and_then(|v| v.as_str())
.unwrap_or("")
.to_string();
if let Some(input) = req.get_mut("input").and_then(|v| v.as_array_mut()) {
input.retain(|item| {
let role = item.get("role").and_then(|v| v.as_str()).unwrap_or("");
if role != "developer" && role != "system" {
return true;
}
if let Some(text) = responses_input_item_text(item) {
if !text.is_empty() {
if !instructions.is_empty() {
instructions.push_str("\n\n");
}
instructions.push_str(&text);
}
}
false
});
}
if !instructions.is_empty() {
req["instructions"] = serde_json::Value::String(instructions);
}
}
fn responses_input_item_text(item: &serde_json::Value) -> Option<String> {
match item.get("content")? {
serde_json::Value::String(s) => Some(s.clone()),
serde_json::Value::Array(blocks) => Some(
blocks
.iter()
.filter_map(|b| b.get("text").and_then(|t| t.as_str()))
.collect::<Vec<_>>()
.join(""),
),
_ => None,
}
}
async fn handle_anthropic_messages(
State(state): State<AppState>,
headers: HeaderMap,
body: Bytes,
) -> Result<Response, AppError> {
let raw: serde_json::Value = serde_json::from_slice(&body).map_err(|e| {
AppError(
anyhow!("parse Anthropic request: {e}"),
StatusCode::BAD_REQUEST,
)
})?;
let model_ref = raw["model"].as_str().unwrap_or("").to_string();
send_with_hybrid_fallback(
&state,
&model_ref,
Some(&headers),
None,
|model, target, guard| anthropic_messages_to(&state, &headers, &raw, model, target, guard),
)
.await
}
async fn anthropic_messages_to(
state: &AppState,
headers: &HeaderMap,
raw: &serde_json::Value,
canonical_model: String,
target: Target,
guard: ActivityGuard,
) -> Result<Response, AppError> {
let activity = begin_activity(guard, None).await;
let wire_model = backend_wire_model(state, &target, &canonical_model).await;
if target.is_anthropic() {
return relay_anthropic_messages(
&state.0.client,
&target,
headers,
raw,
&wire_model,
activity,
)
.await;
}
let client_model = raw["model"].as_str().unwrap_or_default().to_string();
let bad_request = |e| AppError(e, StatusCode::BAD_REQUEST);
let streaming = messages::streaming(raw).map_err(bad_request)?;
let (mut chat_req, tool_names) =
messages::from_messages_request(raw, &wire_model).map_err(bad_request)?;
if repeat_penalty_applies(&target) {
apply_default_repeat_penalty(&mut chat_req);
}
let upstream = send_chat_completion(&state.0.client, &target, &chat_req, &wire_model).await?;
let body = chat_body(&target, upstream).await?;
let converter = messages::StreamConverter::new(&client_model, tool_names);
Ok(convert_upstream(body, activity, converter, streaming).await)
}
const ANTHROPIC_PASSTHROUGH_HEADERS: [&str; 2] = ["anthropic-beta", "anthropic-version"];
async fn relay_anthropic_messages(
client: &Client,
target: &Target,
headers: &HeaderMap,
raw: &serde_json::Value,
wire_model: &str,
activity: ActivityGuard,
) -> Result<Response, AppError> {
let client_model = raw["model"].as_str().unwrap_or_default().to_string();
let streaming = raw["stream"].as_bool().unwrap_or(false);
let mut body = raw.clone();
body["model"] = serde_json::Value::String(wire_model.to_string());
let mut passthrough = HeaderMap::new();
for name in ANTHROPIC_PASSTHROUGH_HEADERS {
if let Some(value) = headers.get(name) {
passthrough.insert(name, value.clone());
}
}
let resp = target
.authorize(client.post(target.url(anthropic::MESSAGES_ROUTE)))
.headers(passthrough)
.json(&body)
.send()
.await
.with_context(|| format!("proxy request to {}", target.describe()))?;
if streaming {
return Ok(relay_stream_rewriting_model(resp, activity, client_model));
}
relay_rewriting_model(resp, activity, &client_model).await
}
fn opt_f64(opts: &Option<serde_json::Value>, key: &str) -> Option<f32> {
opts.as_ref()?.get(key)?.as_f64().map(|f| f as f32)
}
fn opt_u32(opts: &Option<serde_json::Value>, key: &str) -> Option<u32> {
opts.as_ref()?.get(key)?.as_u64().map(|n| n as u32)
}
fn opt_num_thread(opts: &Option<serde_json::Value>) -> Option<u32> {
let n = opts.as_ref()?.get("num_thread")?.as_u64()?;
u32::try_from(n).ok().filter(|&n| n != 0)
}
fn default_allowed_origins() -> Vec<String> {
let mut origins = Vec::new();
for host in ["localhost", "127.0.0.1", "0.0.0.0", "[::1]"] {
for scheme in ["http", "https"] {
origins.push(format!("{scheme}://{host}"));
origins.push(format!("{scheme}://{host}:*"));
}
}
origins
}
fn allowed_origins_from_env() -> Vec<String> {
let mut origins: Vec<String> = std::env::var("LLMMAN_ORIGINS")
.ok()
.map(|raw| {
raw.split(',')
.map(str::trim)
.filter(|s| !s.is_empty())
.map(str::to_string)
.collect()
})
.unwrap_or_default();
origins.extend(default_allowed_origins());
origins
}
fn origin_matches(origin: &str, pattern: &str) -> bool {
match pattern.split_once('*') {
Some((prefix, suffix)) if !suffix.contains('*') => {
origin.len() >= prefix.len() + suffix.len()
&& origin.starts_with(prefix)
&& origin.ends_with(suffix)
}
Some(_) => false,
None => origin == pattern,
}
}
fn cors_layer() -> tower_http::cors::CorsLayer {
let patterns = allowed_origins_from_env();
tower_http::cors::CorsLayer::new()
.allow_methods(tower_http::cors::AllowMethods::any())
.allow_headers(tower_http::cors::AllowHeaders::any())
.allow_origin(tower_http::cors::AllowOrigin::predicate(
move |origin, _parts| {
origin
.to_str()
.is_ok_and(|origin| patterns.iter().any(|p| origin_matches(origin, p)))
},
))
}
pub const MEDIAGEN_ENV_PASSTHROUGH_VARS: &[&str] = &["MEDIAGEN_DUMP", "MEDIAGEN_VAE_TILE"];
async fn track_metrics(req: Request, next: Next) -> Response {
let route = req.extensions().get::<MatchedPath>().cloned();
let started = Instant::now();
let response = next.run(req).await;
let Some(route) = route else {
return response;
};
metrics::record_request(
route.as_str(),
response.status().as_u16(),
started.elapsed(),
);
let (parts, body) = response.into_parts();
if body.size_hint().exact().is_some() {
metrics::record_response_body(route.as_str(), started.elapsed());
return Response::from_parts(parts, body);
}
let timer = ResponseBodyTimer {
route: route.as_str().to_string(),
started,
};
let body = Body::from_stream(body.into_data_stream().map(move |chunk| {
let _timer = &timer;
chunk
}));
Response::from_parts(parts, body)
}
async fn record_prompt(State(state): State<AppState>, req: Request, next: Next) -> Response {
let Some(log) = state.0.prompt_log.as_deref() else {
return next.run(req).await;
};
let route = req
.extensions()
.get::<MatchedPath>()
.map(|p| p.as_str().to_string())
.filter(|r| crate::promptlog::is_generation_route(r));
let Some(route) = route.filter(|r| {
req.method() == axum::http::Method::POST
&& !(r.starts_with("/api/") && forged_cross_site(req.headers()))
}) else {
return next.run(req).await;
};
let (parts, body) = req.into_parts();
let body = match Bytes::from_request(Request::from_parts(parts.clone(), body), &()).await {
Ok(body) => body,
Err(rejection) => return rejection.into_response(),
};
let client = parts
.headers
.get(axum::http::header::USER_AGENT)
.and_then(|v| v.to_str().ok());
if let Some(entry) = crate::promptlog::entry(&route, &body, client, now_rfc3339()) {
if let Err(e) = crate::promptlog::append(log, &entry) {
eprintln!("[llmman] warning: prompt log {}: {e}", log.display());
}
}
next.run(Request::from_parts(parts, Body::from(body))).await
}
async fn accept_any_content_type(mut req: Request, next: Next) -> Response {
if !has_json_content_type(req.headers()) {
if foreign_origin(req.headers()) {
let body = serde_json::json!({ "error": "cross-site request refused" });
return (StatusCode::FORBIDDEN, Json(body)).into_response();
}
req.headers_mut().insert(
axum::http::header::CONTENT_TYPE,
axum::http::HeaderValue::from_static("application/json"),
);
}
next.run(req).await
}
fn has_json_content_type(headers: &HeaderMap) -> bool {
headers
.get(axum::http::header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.is_some_and(is_json_content_type)
}
fn forged_cross_site(headers: &HeaderMap) -> bool {
!has_json_content_type(headers) && foreign_origin(headers)
}
fn foreign_origin(headers: &HeaderMap) -> bool {
headers
.get(axum::http::header::ORIGIN)
.and_then(|v| v.to_str().ok())
.is_some_and(|origin| {
!allowed_origins_from_env()
.iter()
.any(|pattern| origin_matches(origin, pattern))
})
}
fn is_json_content_type(value: &str) -> bool {
let essence = value
.split(';')
.next()
.unwrap_or_default()
.trim()
.to_ascii_lowercase();
essence == "application/json"
|| (essence.starts_with("application/") && essence.ends_with("+json"))
}
struct ResponseBodyTimer {
route: String,
started: Instant,
}
impl Drop for ResponseBodyTimer {
fn drop(&mut self) {
metrics::record_response_body(&self.route, self.started.elapsed());
}
}
async fn handle_metrics(State(state): State<AppState>) -> impl IntoResponse {
let (models_loaded, models_loading, models) = {
let mut mgr = state.0.manager.lock().await;
let models = mgr
.running
.iter_mut()
.map(|(model, m)| metrics::ModelState {
model: model.clone(),
engine: m.engine_label(),
up: m.process.is_alive(),
})
.collect();
(mgr.running.len() as u64, mgr.pending_loads as u64, models)
};
let snapshot = metrics::Snapshot {
version: env!("LLMMAN_VERSION").to_string(),
start_time_seconds: metrics::process_start_seconds(),
scheduling_requests_in_flight: PENDING_REQUESTS.load(std::sync::atomic::Ordering::SeqCst)
as u64,
scheduling_capacity: state.0.max_queue.max(1) as u64,
models_loaded,
models_loading,
models,
};
(
[("content-type", "text/plain; version=0.0.4; charset=utf-8")],
metrics::render(&snapshot),
)
}
pub fn run(args: &ServeArgs) -> anyhow::Result<()> {
if let (Some(port), Some(model)) = (args.port, &args.model) {
return tokio::runtime::Runtime::new()?.block_on(mediagen_backend(model, port, args));
}
tokio::runtime::Runtime::new()?.block_on(serve_async(args))
}
fn llama_lib_dir(pinned_version: Option<&str>) -> anyhow::Result<PathBuf> {
let bin = resolve_llama_server(pinned_version)?;
let dir = bin
.parent()
.ok_or_else(|| anyhow!("{} has no parent directory", bin.display()))?;
[dir.to_path_buf(), dir.join("../lib"), dir.join("../lib64")]
.into_iter()
.find(|d| crate::mediagen::ffi::has_libs(d))
.ok_or_else(|| {
anyhow!(
"no ggml/llama shared libraries next to {}; media generation needs a llama.cpp release or installed build",
bin.display()
)
})
}
async fn mediagen_backend(model_ref: &str, port: u16, args: &ServeArgs) -> anyhow::Result<()> {
let store_path = default_store()?;
let cache_path = crate::default_cache()?;
let ModelPath::Diffusion(paths) = resolve_model(&store_path, &cache_path, model_ref)? else {
anyhow::bail!("{model_ref} is not a diffusion model");
};
if paths.text_encoder.is_none() && crate::mediagen::needs_text_encoder(&paths.model) {
anyhow::bail!("{model_ref}: no text encoder in the model pack");
}
let pinned = args.llama_cpp_version.clone();
let lib_dir = tokio::task::spawn_blocking(move || llama_lib_dir(pinned.as_deref())).await??;
let api: &'static crate::mediagen::ffi::Api =
Box::leak(Box::new(crate::mediagen::ffi::Api::load(&lib_dir)?));
let params = crate::mediagen::ContextParams {
model: paths.model.clone(),
vae: paths.vae.clone(),
audio_vae: paths.audio_vae.clone(),
text_proj: paths.text_proj.clone(),
text_model: paths.text_encoder.clone(),
files: paths.files.clone(),
use_gpu: true,
text_gpu_layers: std::env::var("LLAMA_ARG_N_GPU_LAYERS")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(999),
n_threads: std::env::var("LLAMA_ARG_THREADS")
.ok()
.and_then(|v| v.parse().ok())
.or_else(|| threads_from_env_or_host().map(|n| n as i32))
.unwrap_or_else(|| std::thread::available_parallelism().map_or(4, |n| n.get() as i32)),
flash_attn: flash_attention_from_env().as_deref() != Some("off"),
};
let ctx = tokio::task::spawn_blocking(move || crate::mediagen::Context::init(api, ¶ms))
.await?
.with_context(|| {
format!(
"loading the model with the llama.cpp libraries in {} (an old build? \
remove it from PATH or pass --llama-cpp-version to use a release)",
lib_dir.display()
)
})?;
let router = crate::mediagen::server::router(
ctx,
model_ref.to_string(),
paths.model.to_string_lossy().into_owned(),
);
crate::mediagen::server::serve(router, (args.host, port).into()).await
}
async fn spawn_mediagen_backend(
model_ref: &str,
port: u16,
state: &AppState,
) -> anyhow::Result<(tokio::process::Child, OutputTail)> {
let exe = std::env::current_exe().context("locating the llmman binary")?;
let mut cmd = tokio::process::Command::new(&exe);
cmd.args(["serve", model_ref, "--port", &port.to_string()]);
if let Some(v) = &state.0.llama_cpp_version {
cmd.args(["--llama-cpp-version", v]);
}
for var in GPU_VISIBLE_DEVICE_VARS
.iter()
.chain(LLAMA_CPP_ENV_PASSTHROUGH_VARS)
.chain(MEDIAGEN_ENV_PASSTHROUGH_VARS)
{
if let Ok(val) = std::env::var(var) {
cmd.env(var, val);
}
}
crate::debug_log!("spawning {}: {:?}", exe.display(), cmd);
cmd.stdout(std::process::Stdio::piped());
cmd.stderr(std::process::Stdio::piped());
let mut child = cmd
.kill_on_drop(true)
.spawn()
.with_context(|| format!("spawn media generation backend from {}", exe.display()))?;
let tail = tail_child_output(&mut child);
Ok((child, tail))
}
fn build_router(app_state: AppState, metrics_enabled: bool) -> Router {
let app = Router::new()
.route("/", get(webui::handle_root))
.route("/ui/*path", get(webui::handle_asset))
.route("/props", get(handle_props))
.route("/llmman/providers", get(handle_llmman_providers))
.route("/llmman/providers/:id", get(handle_llmman_provider))
.route("/llmman/node", get(aggregation::handle_node))
.route("/llmman/shell", get(shell::handle_shell))
.merge(ollama_router())
.route("/v1/models", get(handle_openai_models))
.route("/v1/chat/completions", post(handle_openai_chat))
.route("/v1/completions", post(handle_openai_completions))
.route("/v1/embeddings", post(handle_openai_embeddings))
.route(
"/v1/audio/transcriptions",
post(handle_openai_transcriptions)
.layer(DefaultBodyLimit::max(TRANSCRIPTION_BODY_LIMIT_BYTES)),
)
.route(
"/audio/transcriptions",
post(handle_openai_transcriptions)
.layer(DefaultBodyLimit::max(TRANSCRIPTION_BODY_LIMIT_BYTES)),
)
.route("/v1/responses", post(handle_openai_responses))
.route(
"/v1/responses/input_tokens",
post(handle_openai_responses_input_tokens),
)
.route("/v1/images/generations", post(handle_openai_images))
.route("/v1/videos", post(handle_openai_videos))
.route("/v1/videos/:id", get(handle_openai_video_get))
.route("/v1/videos/:id/content", get(handle_openai_video_get))
.route("/v1/audio/speech", post(handle_openai_speech))
.route("/v1/messages", post(handle_anthropic_messages));
let app = app.layer(middleware::from_fn_with_state(
app_state.clone(),
record_prompt,
));
let app = if metrics_enabled {
app.layer(middleware::from_fn(track_metrics))
} else {
app
};
app.layer(cors_layer())
.merge(metrics_router(metrics_enabled))
.with_state(app_state)
}
fn ollama_router() -> Router<AppState> {
Router::new()
.route("/api/version", get(handle_version))
.route("/api/tags", get(handle_tags))
.route("/api/ps", get(handle_ps))
.route("/api/show", post(handle_show))
.route("/api/pull", post(handle_pull))
.route("/api/push", post(handle_push))
.route("/api/delete", delete(handle_delete))
.route("/api/copy", post(handle_copy))
.route("/api/create", post(handle_create))
.route(
"/api/blobs/:digest",
axum::routing::head(handle_blob_head)
.post(handle_blob_upload)
.layer(DefaultBodyLimit::disable()),
)
.route("/api/chat", post(handle_ollama_chat))
.route("/api/generate", post(handle_ollama_generate))
.route("/api/embed", post(handle_embed))
.route("/api/embeddings", post(handle_embeddings))
.layer(middleware::from_fn(accept_any_content_type))
}
fn metrics_router(enabled: bool) -> Router<AppState> {
if !enabled {
return Router::new();
}
Router::new()
.route("/metrics", get(handle_metrics))
.layer(middleware::from_fn(track_metrics))
}
fn pull_oci_engine(model: Option<&str>) -> anyhow::Result<crate::container::ContainerEngine> {
let model = model
.map(crate::hybrid::local_half)
.filter(|m| !crate::providers::is_remote_ref(m));
let Some(model) = model else {
return Ok(crate::container::ContainerEngine::LlamaServer);
};
let model_ref = crate::shortnames::resolve_ollama_api(model)?;
let store_path = default_store()?;
let format = crate::modelpack::stored_format(&store_path, &model_ref).with_context(|| {
format!("--pull-oci: pull {model_ref} first to learn which image it needs")
})?;
Ok(match format {
crate::modelpack::ModelFormat::SafeTensors => crate::container::ContainerEngine::Vllm,
crate::modelpack::ModelFormat::Omni => crate::container::ContainerEngine::VllmOmni,
crate::modelpack::ModelFormat::Gguf | crate::modelpack::ModelFormat::Diffusion => {
crate::container::ContainerEngine::LlamaServer
}
})
}
async fn serve_async(_args: &ServeArgs) -> anyhow::Result<()> {
if _args.ociman.is_some() && !cfg!(target_os = "linux") {
anyhow::bail!("--ociman is only supported on Linux");
}
if _args.pull_oci {
let ociman = _args.ociman.context("--pull-oci requires --ociman")?;
let engine = pull_oci_engine(_args.model.as_deref())?;
let version = match engine {
crate::container::ContainerEngine::Vllm
| crate::container::ContainerEngine::VllmOmni => _args.vllm_version.as_deref(),
crate::container::ContainerEngine::LlamaServer => _args.llama_cpp_version.as_deref(),
};
crate::container::pull_image(ociman, engine, version)?;
return Ok(());
}
if _args.pull_bin {
let pinned_version = _args.llama_cpp_version.clone();
tokio::task::spawn_blocking(move || resolve_llama_server(pinned_version.as_deref()))
.await
.context("resolve llama-server task panicked")??;
return Ok(());
}
let llama_server_bin = if _args.ociman.is_none() {
let pinned_version = _args.llama_cpp_version.clone();
Some(
tokio::task::spawn_blocking(move || resolve_llama_server(pinned_version.as_deref()))
.await
.context("resolve llama-server task panicked")??,
)
} else {
None
};
let store_path = default_store()?;
let cache_path = crate::default_cache()?;
std::fs::create_dir_all(&cache_path)?;
crate::storage::repair::repair_store(&store_path)?;
if !crate::storage::gc::noprune_from_env() {
if let Ok(store) = OciStore::open(&store_path) {
if let Ok(live) = crate::storage::gc::referenced_digests(&store) {
let grace = crate::storage::gc::GC_GRACE_PERIOD;
if let Err(e) = crate::storage::gc::prune_blobs(&store_path, &live, grace) {
eprintln!("[llmman] blob GC sweep failed: {e:#}");
}
if let Err(e) = crate::storage::gc::prune_cache(&cache_path, &live, grace) {
eprintln!("[llmman] cache GC sweep failed: {e:#}");
}
}
}
}
let peers: Vec<String> = crate::config::peers()
.iter()
.map(|p| crate::daemon::peer_url(p))
.collect();
let ctx_size_explicit = context_length_from_env();
let vram = if !peers.is_empty() {
tokio::task::spawn_blocking(crate::hostgpu::detect_with_vram)
.await
.context("hostgpu probe task panicked")?
.1
} else {
0
};
let ctx_size = ctx_size_explicit.or(Some(DEFAULT_CTX_SIZE));
let memory = crate::hostgpu::memory_bytes(vram);
if !peers.is_empty() {
eprintln!(
"[llmman] aggregation peers: {} (this node: {} of model memory)",
peers.join(", "),
crate::fmt::human_size(memory)
);
}
let threads = threads_from_env_or_host();
if let Some(n) = threads {
if _args.ociman.is_none() {
eprintln!("[llmman] llama-server gets --threads {n} (CPU quota/affinity limit below the online CPU count)");
}
} else if std::env::var_os("LLAMA_ARG_THREADS").is_some() {
eprintln!("[llmman] LLAMA_ARG_THREADS set: leaving llama-server thread count to it");
}
let state = AppState(Arc::new(Inner {
manager: Mutex::new(ModelManager {
running: HashMap::new(),
pending_loads: 0,
}),
llama_server_bin: StdMutex::new(llama_server_bin),
exe: std::env::current_exe()
.ok()
.map(|p| p.canonicalize().unwrap_or(p)),
ociman: _args.ociman,
llama_cpp_version: _args.llama_cpp_version.clone(),
vllm_version: _args.vllm_version.clone(),
ctx_size,
ctx_size_explicit: ctx_size_explicit.is_some(),
hybrid_local_bytes: crate::hybrid::local_budget_bytes_from_env(ctx_size),
flash_attention: flash_attention_from_env(),
kv_cache_type: kv_cache_type_from_env(),
split_mode: sched_spread_from_env(),
num_parallel: num_parallel_from_env(),
threads,
max_queue: max_queue_from_env(),
max_loaded_models: max_loaded_models_from_env(),
peers,
memory,
store_path,
cache_path,
prompt_log: crate::promptlog::enabled_from_env()
.then(crate::promptlog::path)
.transpose()?,
shell: shell::Policy::from_env(),
client: Client::new(),
}));
let app = build_router(state.clone(), metrics_enabled_from_env());
metrics::mark_process_start();
crate::verify::Policy::load().context("signature trust policy")?;
let addr = crate::daemon::bind_addr();
let listener = tokio::net::TcpListener::bind(&addr)
.await
.with_context(|| format!("bind {addr}"))?;
eprintln!("llmman serve listening on {addr}");
tokio::spawn(reap_idle_models(state.clone()));
if let Some(model) = _args
.model
.as_deref()
.map(crate::hybrid::local_half)
.filter(|m| !crate::providers::is_remote_ref(m))
{
match crate::shortnames::resolve_ollama_api(model) {
Ok(model) => {
let state_clone = state.clone();
tokio::spawn(async move {
match ensure_model(&state_clone, &model, None, None).await {
Ok((_, _, guard)) => refresh_activity(guard, None).await,
Err(e) => eprintln!("[llmman] pre-load failed: {:#}", e.0),
}
});
}
Err(e) => eprintln!("[llmman] pre-load failed: {e}"),
}
}
axum::serve(listener, app)
.with_graceful_shutdown(shutdown_signal())
.await?;
state.0.manager.lock().await.running.clear();
Ok(())
}
async fn shutdown_signal() {
let ctrl_c = async {
let _ = tokio::signal::ctrl_c().await;
};
#[cfg(unix)]
let terminate = async {
match tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) {
Ok(mut sig) => {
sig.recv().await;
}
Err(_) => std::future::pending().await,
}
};
#[cfg(not(unix))]
let terminate = std::future::pending::<()>();
tokio::select! {
_ = ctrl_c => {}
_ = terminate => {}
}
eprintln!("llmman serve shutting down");
}
#[cfg(test)]
mod tests {
use super::sched::{reap_idle_models_once, DEFAULT_KEEP_ALIVE};
use super::stream::fold_ollama_lines;
use super::*;
#[test]
fn ffi_pull_ref_is_the_polled_key_not_the_tag_normalized_ref() {
assert_eq!(
ffi_pull_ref("docker.io/ai/qwen3.8", "docker.io/ai/qwen3.8:latest"),
"docker.io/ai/qwen3.8"
);
assert_eq!(
ffi_pull_ref("docker.io/ai/qwen3.8:0.8b", "docker.io/ai/qwen3.8:0.8b"),
"docker.io/ai/qwen3.8:0.8b"
);
}
#[test]
fn progress_line_stops_heartbeating_once_byte_counts_have_been_seen() {
let mut saw_bytes = false;
assert_eq!(
progress_line("pull", "m", (String::new(), 0, 0), &mut saw_bytes),
Some(serde_json::json!({"status": "pulling m"}))
);
assert!(!saw_bytes);
assert_eq!(
progress_line("pull", "m", ("pulling".into(), 100, 40), &mut saw_bytes),
Some(serde_json::json!({"status": "pulling", "total": 100, "completed": 40}))
);
assert!(saw_bytes);
assert_eq!(
progress_line("pull", "m", (String::new(), 0, 0), &mut saw_bytes),
None
);
}
#[test]
fn progress_line_reports_status_only_snapshots_and_clamps_completed() {
let mut saw_bytes = false;
assert_eq!(
progress_line("pull", "m", ("verifying".into(), 0, 0), &mut saw_bytes),
Some(serde_json::json!({"status": "verifying"}))
);
assert!(!saw_bytes);
assert_eq!(
progress_line("push", "m", (String::new(), 100, 999), &mut saw_bytes),
Some(serde_json::json!({"status": "pushing m", "total": 100, "completed": 100}))
);
}
fn remote_target(base_url: &str) -> Target {
remote_target_on(base_url, Wire::OpenAi)
}
fn remote_target_on(base_url: &str, wire: Wire) -> Target {
Target::Remote(Arc::new(RemoteTarget {
provider: "mockprov".into(),
base_url: base_url.into(),
wire,
model: "mock-model".into(),
max_output: None,
api_key: Some("sk-test".into()),
}))
}
#[test]
fn a_local_target_addresses_loopback_unchanged() {
let target = Target::Local(17434);
assert_eq!(
target.url("/v1/chat/completions"),
"http://127.0.0.1:17434/v1/chat/completions"
);
assert_eq!(
target.url("/v1/audio/transcriptions"),
"http://127.0.0.1:17434/v1/audio/transcriptions"
);
assert_eq!(
target.url("/v1/responses/input_tokens"),
"http://127.0.0.1:17434/v1/responses/input_tokens"
);
assert!(!target.is_remote());
}
#[test]
fn a_remote_target_rebases_routes_onto_the_provider_url() {
assert_eq!(
remote_target("https://openrouter.ai/api/v1").url("/v1/chat/completions"),
"https://openrouter.ai/api/v1/chat/completions"
);
assert_eq!(
remote_target("https://generativelanguage.googleapis.com/v1beta/openai")
.url("/v1/chat/completions"),
"https://generativelanguage.googleapis.com/v1beta/openai/chat/completions"
);
assert_eq!(
remote_target("https://api.perplexity.ai").url("/v1/chat/completions"),
"https://api.perplexity.ai/chat/completions"
);
assert!(remote_target("https://example.invalid/v1").is_remote());
}
#[test]
fn client_api_key_reads_bearer_and_x_api_key() {
let mut headers = HeaderMap::new();
headers.insert("authorization", "Bearer sk-openai-style".parse().unwrap());
assert_eq!(
client_api_key(Some(&headers)),
Some("sk-openai-style".to_string())
);
let mut headers = HeaderMap::new();
headers.insert("x-api-key", "sk-anthropic-style".parse().unwrap());
assert_eq!(
client_api_key(Some(&headers)),
Some("sk-anthropic-style".to_string())
);
let mut headers = HeaderMap::new();
headers.insert("authorization", "Bearer sk-bearer".parse().unwrap());
headers.insert("x-api-key", "sk-other".parse().unwrap());
assert_eq!(
client_api_key(Some(&headers)),
Some("sk-bearer".to_string())
);
}
#[test]
fn client_api_key_ignores_the_local_placeholder_and_empty_values() {
for header in ["Bearer llmman", "Bearer ", "Bearer ", ""] {
let mut headers = HeaderMap::new();
headers.insert("authorization", header.parse().unwrap());
assert_eq!(
client_api_key(Some(&headers)),
None,
"{header:?} was treated as a credential"
);
}
let mut headers = HeaderMap::new();
headers.insert("x-api-key", PLACEHOLDER_API_KEY.parse().unwrap());
assert_eq!(client_api_key(Some(&headers)), None);
assert_eq!(client_api_key(None), None);
assert_eq!(client_api_key(Some(&HeaderMap::new())), None);
}
#[test]
fn client_api_key_ignores_non_bearer_authorization() {
for header in ["Basic dXNlcjpwYXNz", "sk-no-scheme", "Bearer"] {
let mut headers = HeaderMap::new();
headers.insert("authorization", header.parse().unwrap());
assert_eq!(
client_api_key(Some(&headers)),
None,
"{header:?} was treated as a bearer token"
);
}
}
#[test]
fn client_api_key_accepts_any_spelling_of_bearer() {
for header in ["Bearer sk-real", "bearer sk-real", "BEARER sk-real"] {
let mut headers = HeaderMap::new();
headers.insert("authorization", header.parse().unwrap());
assert_eq!(
client_api_key(Some(&headers)),
Some("sk-real".to_string()),
"{header:?}"
);
}
}
#[test]
fn client_api_key_falls_through_an_unusable_authorization() {
for header in ["Bearer llmman", "Bearer ", "Basic dXNlcjpwYXNz"] {
let mut headers = HeaderMap::new();
headers.insert("authorization", header.parse().unwrap());
headers.insert("x-api-key", "sk-real".parse().unwrap());
assert_eq!(
client_api_key(Some(&headers)),
Some("sk-real".to_string()),
"{header:?} shadowed a real x-api-key"
);
}
}
#[tokio::test]
async fn a_providers_missing_responses_route_is_explained() {
let remote = remote_target("https://example.invalid/v1");
let not_found = (StatusCode::NOT_FOUND, "{}").into_response();
let explained = explain_missing_route(&remote, "/v1/responses", not_found);
assert_eq!(explained.status(), StatusCode::NOT_IMPLEMENTED);
let body = axum::body::to_bytes(explained.into_body(), usize::MAX)
.await
.unwrap();
let body = String::from_utf8_lossy(&body);
assert!(body.contains("mockprov"), "{body}");
assert!(body.contains("Responses API"), "{body}");
let counted = explain_missing_route(
&remote,
"/v1/responses/input_tokens",
(StatusCode::NOT_FOUND, "{}").into_response(),
);
assert_eq!(counted.status(), StatusCode::NOT_IMPLEMENTED);
for (target, route, status) in [
(&remote, "/v1/chat/completions", StatusCode::NOT_FOUND),
(&remote, "/v1/embeddings", StatusCode::NOT_FOUND),
(
&Target::Local(17434),
"/v1/responses",
StatusCode::NOT_FOUND,
),
(&remote, "/v1/responses", StatusCode::OK),
(&remote, "/v1/responses", StatusCode::UNAUTHORIZED),
] {
let resp = explain_missing_route(target, route, (status, "{}").into_response());
assert_eq!(resp.status(), status, "{route} {status}");
}
}
async fn fake_provider(native: StatusCode) -> String {
use axum::routing::post;
let app = Router::new()
.route(
"/v1/responses",
post(move || async move {
(
native,
[("x-provider", "native")],
r#"{"error":{"message":"native"}}"#,
)
}),
)
.route(
"/v1/chat/completions",
post(|| async {
(
[("content-type", "text/event-stream")],
"data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"OK\"},\"finish_reason\":null}]}\n\n\
data: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n\
data: {\"choices\":[],\"usage\":{\"prompt_tokens\":3,\"completion_tokens\":1}}\n\n\
data: [DONE]\n\n",
)
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
format!("http://{addr}/v1")
}
async fn call_remote_responses(
base_url: &str,
stream: bool,
) -> (StatusCode, HeaderMap, String) {
let state = test_state();
let req = serde_json::json!({ "model": "mock-model", "input": "hi", "stream": stream });
let resp = remote_responses(
&state.0.client,
&remote_target(base_url),
&HeaderMap::new(),
req,
ActivityGuard::new(&state, "m"),
"llmman.provider/mockprov/mock-model".to_string(),
)
.await
.unwrap();
let status = resp.status();
let headers = resp.headers().clone();
let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
.await
.unwrap();
(status, headers, String::from_utf8_lossy(&body).into_owned())
}
#[tokio::test]
async fn remote_responses_falls_back_only_on_a_missing_or_broken_route() {
for native in [StatusCode::NOT_FOUND, StatusCode::INTERNAL_SERVER_ERROR] {
let base = fake_provider(native).await;
let (status, headers, body) = call_remote_responses(&base, true).await;
assert_eq!(status, StatusCode::OK, "{native}");
assert_eq!(headers["content-type"], "text/event-stream");
assert!(body.contains("event: response.created"), "{body}");
assert!(body.contains("\"delta\":\"OK\""), "{body}");
assert!(body.contains("event: response.completed"), "{body}");
assert!(
body.contains("\"model\":\"llmman.provider/mockprov/mock-model\""),
"{body}"
);
let (status, headers, body) = call_remote_responses(&base, false).await;
assert_eq!(status, StatusCode::OK, "{native}");
assert_eq!(headers["content-type"], "application/json");
let response: serde_json::Value = serde_json::from_str(&body).unwrap();
assert_eq!(response["status"], "completed");
assert_eq!(response["output"][0]["content"][0]["text"], "OK");
assert_eq!(response["usage"]["total_tokens"], 4);
}
for native in [
StatusCode::OK,
StatusCode::UNAUTHORIZED,
StatusCode::TOO_MANY_REQUESTS,
] {
let base = fake_provider(native).await;
let (status, headers, body) = call_remote_responses(&base, true).await;
assert_eq!(status, native);
assert!(body.contains("\"native\""), "{native}: {body}");
assert!(headers.contains_key("x-provider"), "{native}");
}
}
#[test]
fn only_a_browser_acting_for_another_site_is_refused_the_daemons_key() {
for site in ["cross-site", "same-site", "Cross-Site", " cross-site "] {
let mut headers = HeaderMap::new();
headers.insert("sec-fetch-site", site.parse().unwrap());
assert!(is_cross_site(Some(&headers)), "{site:?}");
}
for site in ["same-origin", "none"] {
let mut headers = HeaderMap::new();
headers.insert("sec-fetch-site", site.parse().unwrap());
assert!(!is_cross_site(Some(&headers)), "{site:?}");
}
assert!(!is_cross_site(Some(&HeaderMap::new())));
assert!(!is_cross_site(None));
}
#[test]
fn a_remote_targets_debug_output_omits_the_key() {
let rendered = format!("{:?}", remote_target("https://example.invalid/v1"));
assert!(!rendered.contains("sk-test"), "{rendered}");
assert!(rendered.contains("mockprov"), "{rendered}");
}
#[test]
fn repeat_penalty_is_local_only() {
assert!(repeat_penalty_applies(&Target::Local(17434)));
assert!(!repeat_penalty_applies(&remote_target(
"https://example.invalid/v1"
)));
}
#[test]
fn a_providers_own_status_is_not_buried_in_a_500() {
let remote = remote_target("https://example.invalid/v1");
assert_eq!(
remote_status(&remote, StatusCode::UNAUTHORIZED),
StatusCode::UNAUTHORIZED
);
assert_eq!(
remote_status(&remote, StatusCode::TOO_MANY_REQUESTS),
StatusCode::TOO_MANY_REQUESTS
);
assert_eq!(
remote_status(&remote, StatusCode::BAD_GATEWAY),
StatusCode::BAD_GATEWAY
);
assert_eq!(
remote_status(&remote, StatusCode::INTERNAL_SERVER_ERROR),
StatusCode::BAD_GATEWAY
);
let local = Target::Local(17434);
for upstream in [StatusCode::UNAUTHORIZED, StatusCode::INTERNAL_SERVER_ERROR] {
assert_eq!(
remote_status(&local, upstream),
StatusCode::INTERNAL_SERVER_ERROR,
"a local backend's status changed"
);
}
assert_eq!(local.describe(), "inference backend");
assert_eq!(remote.describe(), "provider mockprov");
}
#[tokio::test]
async fn a_remote_request_reaches_the_provider_as_plain_openai() {
let seen = Arc::new(tokio::sync::Mutex::new(None));
let captured = seen.clone();
let app = Router::new().route(
"/v1/chat/completions",
post(|headers: HeaderMap, body: Bytes| async move {
let body: serde_json::Value = serde_json::from_slice(&body).unwrap();
let auth = headers["authorization"].to_str().unwrap().to_string();
*captured.lock().await = Some((auth, body));
([("content-type", "application/json")], "{}")
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let target = remote_target(&format!("http://127.0.0.1:{}/v1", addr.port()));
let mut req = OAIChatRequest {
model: "mock-model".into(),
stream: false,
repeat_penalty: Some(1.1),
top_k: Some(40),
min_p: Some(0.05),
chat_template_kwargs: Some(serde_json::json!({ "enable_thinking": true })),
..Default::default()
};
let _body = post_chat(&Client::new(), &target, &mut req).await.unwrap();
let (auth, body) = seen.lock().await.take().expect("provider was not called");
assert_eq!(auth, "Bearer sk-test");
assert_eq!(body["model"], "mock-model");
for field in ["repeat_penalty", "top_k", "min_p", "chat_template_kwargs"] {
assert!(
body.get(field).is_none(),
"llama.cpp-only field {field} sent to a provider: {body}"
);
}
assert!(body.get("reasoning_effort").is_none(), "{body}");
}
async fn mock_anthropic(
reply: &'static str,
) -> (
String,
Arc<tokio::sync::Mutex<Vec<(String, HeaderMap, serde_json::Value)>>>,
) {
let seen = Arc::new(tokio::sync::Mutex::new(Vec::new()));
let captured = seen.clone();
let app = Router::new()
.route(
"/v1/messages",
post(move |headers: HeaderMap, body: Bytes| async move {
let body: serde_json::Value = serde_json::from_slice(&body).unwrap();
captured
.lock()
.await
.push(("/v1/messages".to_string(), headers, body));
(
[
("content-type", "text/event-stream"),
("request-id", "req_mock"),
],
reply,
)
}),
)
.fallback(|req: Request| async move {
(
StatusCode::NOT_FOUND,
format!("no such route: {}", req.uri().path()),
)
});
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let base = format!(
"http://127.0.0.1:{}/v1",
listener.local_addr().unwrap().port()
);
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
(base, seen)
}
const MOCK_MESSAGES_STREAM: &str = "event: message_start\n\
data: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_1\",\"model\":\"claude-x\",\"usage\":{\"input_tokens\":3,\"output_tokens\":1}}}\n\n\
event: content_block_start\n\
data: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n\
event: content_block_delta\n\
data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"hi there\"}}\n\n\
event: message_delta\n\
data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":2}}\n\n\
event: message_stop\n\
data: {\"type\":\"message_stop\"}\n\n";
#[tokio::test]
async fn a_typed_request_to_an_anthropic_provider_speaks_messages() {
let (base, seen) = mock_anthropic(MOCK_MESSAGES_STREAM).await;
let target = remote_target_on(&base, Wire::Anthropic);
let mut req = OAIChatRequest {
model: "mock-model".into(),
messages: vec![
OAIMessage::text("system", "be brief"),
OAIMessage::text("user", "hello"),
],
stream: true,
repeat_penalty: Some(1.1),
..Default::default()
};
let body = post_chat(&Client::new(), &target, &mut req).await.unwrap();
let lines: Vec<String> = bytes_to_lines(body).collect().await;
let calls = seen.lock().await;
let (path, headers, sent) = &calls[0];
assert_eq!(path, "/v1/messages");
assert_eq!(headers["x-api-key"], "sk-test");
assert_eq!(headers["anthropic-version"], anthropic::VERSION);
assert!(
headers.get("authorization").is_none(),
"bearer sent to Anthropic"
);
assert_eq!(sent["model"], "mock-model");
assert_eq!(sent["system"][0]["text"], "be brief");
assert_eq!(sent["messages"][0]["role"], "user");
assert_eq!(sent["messages"][0]["content"][0]["text"], "hello");
assert_eq!(sent["system"][0]["cache_control"]["type"], "ephemeral");
assert!(headers.get("anthropic-beta").is_none());
assert_eq!(sent["max_tokens"], anthropic::DEFAULT_MAX_TOKENS);
assert_eq!(sent["stream"], true);
assert!(sent.get("repeat_penalty").is_none(), "{sent}");
let fold = fold_ollama_lines(lines);
assert_eq!(fold.content, "hi there");
assert!(fold.done);
}
#[tokio::test]
async fn a_non_streaming_request_to_an_anthropic_provider_is_folded() {
let (base, seen) = mock_anthropic(MOCK_MESSAGES_STREAM).await;
let target = remote_target_on(&base, Wire::Anthropic);
let req = serde_json::json!({
"model": "mock-model",
"messages": [{ "role": "user", "content": "hello" }],
"stream": false
});
let upstream = send_chat_completion(
&Client::new(),
&target,
&req,
"llmman.provider/mockprov/mock-model",
)
.await
.unwrap();
assert_eq!(upstream.status, StatusCode::OK);
assert_eq!(upstream.headers["content-type"], "application/json");
assert!(upstream.headers.get("content-length").is_none());
assert_eq!(upstream.headers["request-id"], "req_mock");
let body: serde_json::Value = serde_json::from_str(&upstream.text().await).unwrap();
assert_eq!(body["object"], "chat.completion");
assert_eq!(body["model"], "llmman.provider/mockprov/mock-model");
assert_eq!(body["choices"][0]["message"]["content"], "hi there");
assert_eq!(body["choices"][0]["finish_reason"], "stop");
assert_eq!(body["usage"]["total_tokens"], 5);
assert_eq!(seen.lock().await[0].2["stream"], true);
}
#[tokio::test]
async fn an_inbound_messages_request_is_relayed_to_an_anthropic_provider_intact() {
let (base, seen) = mock_anthropic(MOCK_MESSAGES_STREAM).await;
let target = remote_target_on(&base, Wire::Anthropic);
let state = test_state();
let raw = serde_json::json!({
"model": "llmman.provider/mockprov/mock-model",
"max_tokens": 32,
"stream": true,
"system": [{ "type": "text", "text": "sys", "cache_control": { "type": "ephemeral" } }],
"messages": [{ "role": "user", "content": "hi" }],
"thinking": { "type": "enabled", "budget_tokens": 1024 },
"tools": [{ "name": "f", "input_schema": { "type": "object" } }]
});
let mut headers = HeaderMap::new();
headers.insert(
"anthropic-beta",
"interleaved-thinking-2025-05-14".parse().unwrap(),
);
headers.insert("anthropic-version", "2024-01-01".parse().unwrap());
headers.insert("x-api-key", "the-clients-own-key".parse().unwrap());
let activity = begin_activity(ActivityGuard::new(&state, "m"), None).await;
let resp = relay_anthropic_messages(
&Client::new(),
&target,
&headers,
&raw,
"mock-model",
activity,
)
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
assert_eq!(resp.headers()["request-id"], "req_mock");
let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
.await
.unwrap();
let body = String::from_utf8_lossy(&body);
assert!(
body.contains("\"model\":\"llmman.provider/mockprov/mock-model\""),
"{body}"
);
assert!(!body.contains("claude-x"), "{body}");
let calls = seen.lock().await;
let (_, sent_headers, sent) = &calls[0];
assert_eq!(
sent_headers["anthropic-beta"],
"interleaved-thinking-2025-05-14"
);
let versions: Vec<_> = sent_headers.get_all("anthropic-version").iter().collect();
assert_eq!(versions, ["2024-01-01"]);
assert_eq!(sent_headers["x-api-key"], "sk-test");
let mut expected = raw.clone();
expected["model"] = serde_json::json!("mock-model");
assert_eq!(*sent, expected);
}
#[test]
fn routes_without_a_messages_equivalent_are_refused_up_front() {
let anthropic = remote_target_on("https://api.anthropic.com/v1", Wire::Anthropic);
for route in [
"/v1/completions",
"/v1/embeddings",
"/v1/responses/input_tokens",
] {
let refusal = unsupported_on_wire(&anthropic, route)
.unwrap_or_else(|| panic!("{route} was not refused"));
assert_eq!(refusal.status(), StatusCode::NOT_IMPLEMENTED);
}
for route in [CHAT_COMPLETIONS_ROUTE, RESPONSES_ROUTE] {
assert!(unsupported_on_wire(&anthropic, route).is_none(), "{route}");
}
let openai = remote_target("https://api.openai.com/v1");
assert!(unsupported_on_wire(&openai, "/v1/embeddings").is_none());
assert!(unsupported_on_wire(&Target::Local(1), "/v1/embeddings").is_none());
}
#[test]
fn set_response_model_reaches_a_messages_stream_event() {
let mut event = serde_json::json!({
"type": "message_start",
"message": { "id": "msg_1", "model": "claude-x" }
});
set_response_model(&mut event, "mine");
assert_eq!(event["message"]["model"], "mine");
}
fn remote(provider: &str, model: &str, wire: Wire) -> RemoteTarget {
RemoteTarget {
provider: provider.into(),
base_url: "https://example.invalid/v1".into(),
wire,
model: model.into(),
max_output: None,
api_key: Some("k".into()),
}
}
#[test]
fn provider_compat_translates_think_per_wire() {
let with = |remote: &RemoteTarget, kwargs: serde_json::Value| {
let mut req = serde_json::json!({ "model": "m", "chat_template_kwargs": kwargs });
provider_compat(remote, &mut req);
assert!(req.get("chat_template_kwargs").is_none());
req.get("reasoning_effort").cloned()
};
let openrouter = remote("openrouter", "some/model", Wire::OpenAi);
let anthropic = remote("anthropic", "claude", Wire::Anthropic);
let level = serde_json::json!({ "enable_thinking": true, "reasoning_effort": "high" });
assert_eq!(with(&openrouter, level.clone()), Some("high".into()));
assert_eq!(with(&anthropic, level), Some("high".into()));
let on = serde_json::json!({ "enable_thinking": true });
assert_eq!(with(&openrouter, on.clone()), None);
assert_eq!(with(&anthropic, on), Some("medium".into()));
let off = serde_json::json!({ "enable_thinking": false });
assert_eq!(with(&anthropic, off), Some("none".into()));
}
#[test]
fn provider_compat_applies_openais_reasoning_model_rules() {
let mut req = serde_json::json!({
"model": "o3", "max_tokens": 100, "temperature": 0.2, "top_p": 0.9,
"presence_penalty": 0.1, "stop": ["x"], "reasoning_effort": "high",
"n_probs": 5, "cache_prompt": true, "json_schema": {}, "repeat_last_n": 64,
"n_predict": 8, "grammar_lazy": true, "grammar_triggers": [], "preserved_tokens": []
});
provider_compat(&remote("openai", "o3", Wire::OpenAi), &mut req);
assert_eq!(
req,
serde_json::json!({
"model": "o3", "max_completion_tokens": 100, "stop": ["x"], "reasoning_effort": "high"
})
);
let mut gpt5 = serde_json::json!({ "model": "gpt-5", "temperature": 1, "stop": ["x"], "logit_bias": {}, "max_completion_tokens": 7, "max_tokens": 9 });
provider_compat(&remote("openai", "gpt-5", Wire::OpenAi), &mut gpt5);
assert_eq!(
gpt5,
serde_json::json!({ "model": "gpt-5", "temperature": 1, "max_completion_tokens": 7 })
);
let mut plain = serde_json::json!({ "model": "gpt-4o", "temperature": 0.2, "reasoning_effort": "high", "max_tokens": 5 });
provider_compat(&remote("openai", "gpt-4o", Wire::OpenAi), &mut plain);
assert_eq!(
plain,
serde_json::json!({ "model": "gpt-4o", "temperature": 0.2, "max_tokens": 5 })
);
let mut other = serde_json::json!({ "model": "o3-mini", "temperature": 0.2, "max_tokens": 5, "top_k": 3, "stream_options": { "include_usage": true } });
provider_compat(&remote("groq", "o3-mini", Wire::OpenAi), &mut other);
assert_eq!(
other,
serde_json::json!({ "model": "o3-mini", "temperature": 0.2, "max_tokens": 5, "stream_options": { "include_usage": true } })
);
let mut cohere =
serde_json::json!({ "model": "c", "stream_options": { "include_usage": true } });
provider_compat(&remote("cohere", "c", Wire::OpenAi), &mut cohere);
assert_eq!(cohere, serde_json::json!({ "model": "c" }));
for (model, reasoning) in [
("o1", true),
("o4-mini", true),
("openai/o3", true),
("gpt-5-mini", true),
("gpt-5-chat-latest", false),
("gpt-4o", false),
("omni", false),
] {
assert_eq!(openai_reasoning_model(model), reasoning, "{model}");
}
}
#[test]
fn a_peer_target_is_a_local_backend_in_all_but_address() {
let peer = Target::Peer("http://spark:17434".into());
assert_eq!(
peer.url("/v1/chat/completions"),
"http://spark:17434/v1/chat/completions"
);
assert!(!peer.is_remote(), "a peer accepts llama.cpp extensions");
assert!(repeat_penalty_applies(&peer));
assert_eq!(peer.describe(), "peer http://spark:17434");
for upstream in [
StatusCode::NOT_FOUND,
StatusCode::SERVICE_UNAVAILABLE,
StatusCode::INTERNAL_SERVER_ERROR,
] {
assert_eq!(remote_status(&peer, upstream), upstream);
}
}
async fn mock_peer(
node: aggregation::Node,
) -> (
String,
Arc<tokio::sync::Mutex<Vec<(String, HeaderMap, Bytes)>>>,
) {
let seen = Arc::new(tokio::sync::Mutex::new(Vec::new()));
let captured = seen.clone();
let app = Router::new()
.route("/llmman/node", get(move || async move { Json(node) }))
.fallback(move |req: Request| async move {
let (parts, body) = req.into_parts();
let body = axum::body::to_bytes(body, usize::MAX).await.unwrap();
captured
.lock()
.await
.push((parts.uri.path().to_string(), parts.headers, body));
([("content-type", "application/json")], "{}")
});
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let origin = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port());
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
(origin, seen)
}
fn state_with_peers(peers: Vec<String>, memory: u64) -> AppState {
let dir = std::env::temp_dir().join(format!(
"llmman-aggregation-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let desc = crate::storage::oci::Descriptor {
media_type: "application/vnd.oci.image.manifest.v1+json".into(),
digest: "sha256:a1b2".into(),
size: 0,
annotations: None,
};
OciStore::open(&dir)
.unwrap()
.tag(desc, "docker.io/ai/m:latest")
.unwrap();
let mut inner = test_inner(dir);
inner.peers = peers;
inner.memory = memory;
AppState(Arc::new(inner))
}
fn node(memory: u64, loaded: &[&str], stored: &[&str]) -> aggregation::Node {
let map = |names: &[&str]| names.iter().map(|n| (n.to_string(), 1 << 30)).collect();
aggregation::Node {
memory,
loaded: map(loaded),
stored: map(stored),
}
}
#[tokio::test]
async fn a_peer_request_carries_the_hop_marker_and_nothing_else() {
let (origin, seen) = mock_peer(node(0, &[], &[])).await;
let mut req = OAIChatRequest {
model: "docker.io/ai/m:latest".into(),
..Default::default()
};
let _body = post_chat(&Client::new(), &Target::Peer(origin), &mut req)
.await
.unwrap();
let calls = seen.lock().await;
let (path, headers, body) = &calls[0];
assert_eq!(path, "/v1/chat/completions");
assert_eq!(headers[aggregation::HOP], "1");
assert!(headers.get("authorization").is_none());
let body: serde_json::Value = serde_json::from_slice(body).unwrap();
assert_eq!(body["repeat_penalty"], DEFAULT_REPEAT_PENALTY);
}
#[tokio::test]
async fn ensure_model_forwards_to_the_peer_that_has_the_model_loaded() {
let (origin, _) = mock_peer(node(8 << 30, &["docker.io/ai/m:latest"], &[])).await;
let state = state_with_peers(vec![origin.clone()], 0);
let (name, target, _) =
ensure_model(&state, "docker.io/ai/m", Some(&HeaderMap::new()), None)
.await
.unwrap();
assert_eq!(name, "docker.io/ai/m:latest");
assert!(
matches!(&target, Target::Peer(o) if *o == origin),
"{target:?}"
);
let mut hopped = HeaderMap::new();
hopped.insert(aggregation::HOP, "1".parse().unwrap());
let err = ensure_model(&state, "docker.io/ai/m", Some(&hopped), None)
.await
.err()
.expect("the fixture has no blobs to load");
assert!(
format!("{:#}", err.0).contains("resolve model"),
"{:#}",
err.0
);
let err = ensure_model(&state, "docker.io/ai/m", None, None)
.await
.err()
.unwrap();
assert!(
format!("{:#}", err.0).contains("resolve model"),
"{:#}",
err.0
);
}
#[tokio::test]
async fn ensure_model_places_a_cold_model_on_the_roomiest_reachable_node() {
let (roomy, _) = mock_peer(node(128 << 30, &[], &["docker.io/ai/m:latest"])).await;
let state = state_with_peers(vec![roomy.clone()], 8 << 30);
let (_, target, _) = ensure_model(&state, "docker.io/ai/m", Some(&HeaderMap::new()), None)
.await
.unwrap();
assert!(
matches!(&target, Target::Peer(o) if *o == roomy),
"{target:?}"
);
let (bare, _) = mock_peer(node(128 << 30, &[], &[])).await;
let dead = "http://127.0.0.1:9".to_string();
let state = state_with_peers(vec![dead, bare], 8 << 30);
let err = ensure_model(&state, "docker.io/ai/m", Some(&HeaderMap::new()), None)
.await
.err()
.expect("loads (and fails on the fixture) locally");
assert!(
format!("{:#}", err.0).contains("resolve model"),
"{:#}",
err.0
);
}
#[tokio::test]
async fn listings_cover_the_aggregation_except_for_a_peer_asking() {
let ps = serde_json::json!({ "models": [{
"name": "docker.io/ai/m:latest", "model": "docker.io/ai/m:latest",
"expires_at": null, "digest": "sha256:aa", "size": 1, "size_vram": 0,
"pid": null, "port": 1, "processor": "GPU", "context_length": null,
"started_at": "2026-01-01T00:00:00Z"
}]});
let tags = serde_json::json!({ "models": [{
"name": "docker.io/ai/m:latest", "model": "docker.io/ai/m:latest",
"size": 1, "digest": "sha256:aa", "modified_at": "2026-01-01T00:00:00Z",
"details": { "format": "gguf", "family": "", "parameter_size": "", "quantization_level": "" }
}]});
let models = serde_json::json!({ "object": "list", "data": [
{ "id": "docker.io/ai/m:latest", "object": "model", "created": 0,
"owned_by": "llmman", "status": { "value": "loaded" } }
]});
let app = Router::new()
.route("/api/ps", get(move || async move { Json(ps) }))
.route("/api/tags", get(move || async move { Json(tags) }))
.route("/v1/models", get(move || async move { Json(models) }));
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let origin = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port());
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let dir = std::env::temp_dir().join(format!(
"llmman-aggregation-listings-{}",
std::process::id()
));
let _ = std::fs::remove_dir_all(&dir);
OciStore::open(&dir).unwrap();
let mut inner = test_inner(dir);
inner.peers = vec![origin.clone()];
let state = AppState(Arc::new(inner));
async fn body(resp: Response) -> serde_json::Value {
let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX)
.await
.unwrap();
serde_json::from_slice(&bytes).unwrap()
}
let mut hopped = HeaderMap::new();
hopped.insert(aggregation::HOP, "1".parse().unwrap());
let ps = body(
handle_ps(State(state.clone()), HeaderMap::new())
.await
.into_response(),
)
.await;
assert_eq!(ps["models"][0]["name"], "docker.io/ai/m:latest");
assert_eq!(ps["models"][0]["node"], origin);
let own = body(
handle_ps(State(state.clone()), hopped.clone())
.await
.into_response(),
)
.await;
assert_eq!(own["models"].as_array().unwrap().len(), 0);
let tags = body(
handle_tags(State(state.clone()), HeaderMap::new())
.await
.unwrap()
.into_response(),
)
.await;
assert_eq!(tags["models"][0]["name"], "docker.io/ai/m:latest");
assert!(!tags.to_string().contains("\"node\""));
let own = body(
handle_tags(State(state.clone()), hopped.clone())
.await
.unwrap()
.into_response(),
)
.await;
assert_eq!(own["models"].as_array().unwrap().len(), 0);
let models = body(
handle_openai_models(State(state.clone()), HeaderMap::new())
.await
.unwrap()
.into_response(),
)
.await;
assert_eq!(models["data"][0]["id"], "docker.io/ai/m:latest");
assert_eq!(models["data"][0]["status"]["value"], "loaded");
let own = body(
handle_openai_models(State(state), hopped)
.await
.unwrap()
.into_response(),
)
.await;
assert_eq!(own["data"].as_array().unwrap().len(), 0);
}
#[tokio::test]
async fn an_ollama_request_reaches_a_peer_natively() {
let (origin, seen) = mock_peer(node(8 << 30, &["docker.io/ai/m:latest"], &[])).await;
let state = state_with_peers(vec![origin], 0);
let req: OllamaGenerateRequest = serde_json::from_value(
serde_json::json!({ "model": "docker.io/ai/m", "keep_alive": "1h" }),
)
.unwrap();
let resp = handle_ollama_generate(State(state), HeaderMap::new(), Json(req))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let calls = seen.lock().await;
let (path, headers, body) = &calls[0];
assert_eq!(path, "/api/generate");
assert_eq!(headers[aggregation::HOP], "1");
let body: serde_json::Value = serde_json::from_slice(body).unwrap();
assert_eq!(body["model"], "docker.io/ai/m");
assert_eq!(body["keep_alive"], "1h");
assert_eq!(body["prompt"], "");
assert!(body.get("think").is_none(), "{body}");
}
#[tokio::test]
async fn a_pair_reaches_a_peer_as_its_chosen_half() {
let (origin, seen) = mock_peer(node(8 << 30, &["docker.io/ai/m:latest"], &[])).await;
let state = state_with_peers(vec![origin], 0);
let req: OllamaGenerateRequest = serde_json::from_value(serde_json::json!({
"model": "llmman.hybrid/docker.io/ai/m,anthropic/claude-sonnet-5",
}))
.unwrap();
let headers = headers_with(&[("x-llmman-route", "local")]);
let resp = handle_ollama_generate(State(state), headers, Json(req))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let calls = seen.lock().await;
let body: serde_json::Value = serde_json::from_slice(&calls[0].2).unwrap();
assert_eq!(body["model"], "docker.io/ai/m:latest");
}
#[tokio::test]
async fn an_unload_is_forwarded_to_every_peer() {
let (origin, seen) = mock_peer(node(0, &[], &[])).await;
let state = state_with_peers(vec![origin], 0);
unload_everywhere(&state, "docker.io/ai/m", &HeaderMap::new())
.await
.unwrap();
let calls = seen.lock().await;
let (path, headers, body) = &calls[0];
assert_eq!(path, "/api/generate");
assert_eq!(headers[aggregation::HOP], "1");
let body: serde_json::Value = serde_json::from_slice(body).unwrap();
assert_eq!(body["model"], "docker.io/ai/m");
assert_eq!(body["keep_alive"], 0);
drop(calls);
let mut hopped = HeaderMap::new();
hopped.insert(aggregation::HOP, "1".parse().unwrap());
unload_everywhere(&state, "docker.io/ai/m", &hopped)
.await
.unwrap();
assert_eq!(seen.lock().await.len(), 1, "forwarded a forwarded unload");
}
fn headers_with(pairs: &[(&str, &str)]) -> HeaderMap {
let mut headers = HeaderMap::new();
for (name, value) in pairs {
headers.insert(
reqwest::header::HeaderName::from_bytes(name.as_bytes()).unwrap(),
value.parse().unwrap(),
);
}
headers
}
fn pair(reference: &'static str) -> crate::hybrid::Pair<'static> {
crate::hybrid::split_ref(reference).expect("test reference must be a pair")
}
const PAIR: &str = "llmman.hybrid/gemma4,anthropic/claude-sonnet-4-5";
const HOSTED: &str = "llmman.provider/anthropic/claude-sonnet-4-5";
#[test]
fn a_pair_resolves_to_an_ordinary_reference_for_the_half_it_picks() {
let state = test_state();
let local = resolve_hybrid_side(&state, &pair(PAIR), Some(&headers_with(&[]))).unwrap();
assert_eq!(local, "gemma4");
let cloud = resolve_hybrid_side(
&state,
&pair(PAIR),
Some(&headers_with(&[("x-llmman-route", "cloud")])),
)
.unwrap();
assert_eq!(cloud, HOSTED);
assert!(crate::providers::is_remote_ref(&cloud));
}
#[test]
fn a_pair_without_headers_stays_local() {
let state = test_state();
assert_eq!(
resolve_hybrid_side(&state, &pair(PAIR), None).unwrap(),
"gemma4"
);
}
#[test]
fn a_request_too_large_for_this_host_goes_to_the_hosted_half() {
let state = test_state_with_budget(262_144);
let fits = headers_with(&[("content-length", "262144")]);
assert_eq!(
resolve_hybrid_side(&state, &pair(PAIR), Some(&fits)).unwrap(),
"gemma4"
);
let does_not = headers_with(&[("content-length", "262145")]);
assert_eq!(
resolve_hybrid_side(&state, &pair(PAIR), Some(&does_not)).unwrap(),
HOSTED
);
}
#[test]
fn without_a_budget_size_never_routes_a_pair_away() {
let state = test_state();
let huge = headers_with(&[("content-length", "999999999")]);
assert_eq!(
resolve_hybrid_side(&state, &pair(PAIR), Some(&huge)).unwrap(),
"gemma4"
);
}
#[test]
fn a_transcription_pair_takes_its_local_half_whatever_its_size() {
let state = test_state_with_budget(1);
let huge = headers_with(&[("content-length", "99999999")]);
assert_eq!(
resolve_hybrid_side(&state, &pair(PAIR), Some(&huge)).unwrap(),
HOSTED,
"the generic path still routes on size"
);
assert_eq!(request_pin(Some(&huge)).unwrap(), None);
let pinned = headers_with(&[("x-llmman-route", "cloud")]);
assert_eq!(
request_pin(Some(&pinned)).unwrap(),
Some(crate::hybrid::Side::Cloud),
"an explicit cloud pin is still refused by that handler"
);
}
#[test]
fn an_unreadable_route_header_is_rejected_rather_than_guessed() {
let state = test_state();
let headers = headers_with(&[("x-llmman-route", "on-device")]);
let err = resolve_hybrid_side(&state, &pair(PAIR), Some(&headers))
.expect_err("an unknown side must not be guessed at");
assert_eq!(err.into_response().status(), StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn ensure_model_validates_the_local_half_of_a_pair() {
let state = test_state();
let headers = headers_with(&[("x-llmman-route", "local")]);
let err = ensure_model(
&state,
"llmman.hybrid/hf.co/../x,anthropic/claude-sonnet-4-5",
Some(&headers),
None,
)
.await
.err()
.expect("an invalid local half must be rejected");
assert_eq!(err.into_response().status(), StatusCode::BAD_REQUEST);
}
#[test]
fn a_local_context_refusal_is_recognised_in_every_shape_it_arrives_in() {
let llama = r#"{"error":{"code":400,"message":"request (5213 tokens) exceeds the available context size (2048 tokens), try increasing it","type":"exceed_context_size_error","n_prompt_tokens":5213,"n_ctx":2048}}"#;
let expected = "request (5213 tokens) exceeds the available context size (2048 tokens), try increasing it";
assert_eq!(
context_overflow_message(StatusCode::BAD_REQUEST, llama).as_deref(),
Some(expected)
);
assert_eq!(
context_overflow_message(
StatusCode::BAD_REQUEST,
&format!("inference backend 400 Bad Request: {llama}")
)
.as_deref(),
Some(expected)
);
let vllm = r#"{"object":"error","message":"This model's maximum context length is 2048 tokens. However, you requested 5213 tokens.","type":"BadRequestError","code":400}"#;
assert!(context_overflow_message(StatusCode::BAD_REQUEST, vllm).is_some());
for (status, body) in [
(
StatusCode::BAD_REQUEST,
r#"{"error":{"code":400,"message":"invalid grammar","type":"invalid_request_error"}}"#,
),
(StatusCode::INTERNAL_SERVER_ERROR, llama),
(StatusCode::BAD_REQUEST, "not json"),
] {
assert_eq!(
context_overflow_message(status, body),
None,
"{status} {body}"
);
}
}
#[test]
fn only_an_unpinned_pair_has_a_hosted_half_to_fall_back_to() {
assert_eq!(
hybrid_fallback(PAIR, Some(&headers_with(&[]))).unwrap(),
Some(HOSTED.to_string())
);
assert_eq!(
hybrid_fallback(PAIR, None).unwrap(),
Some(HOSTED.to_string())
);
assert_eq!(
hybrid_fallback(PAIR, Some(&headers_with(&[("x-llmman-route", "local")]))).unwrap(),
None
);
assert_eq!(hybrid_fallback("gemma4", None).unwrap(), None);
assert_eq!(hybrid_fallback(HOSTED, None).unwrap(), None);
}
#[test]
fn a_repeated_route_header_is_rejected() {
let mut headers = headers_with(&[("x-llmman-route", "local")]);
headers.append("x-llmman-route", "cloud".parse().unwrap());
let err = request_pin(Some(&headers)).expect_err("two pins must not resolve");
assert_eq!(err.into_response().status(), StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn a_local_refusal_is_retried_on_the_hosted_half_unless_pinned() {
use std::sync::atomic::{AtomicUsize, Ordering};
let state = test_state();
let refusal = r#"{"error":{"code":400,"message":"request (9 tokens) exceeds the available context size (8 tokens), try increasing it","type":"exceed_context_size_error"}}"#;
let resolve = |m: String| {
let state = state.clone();
async move {
let m = crate::hybrid::local_half(&m).to_string();
let target = if crate::providers::is_remote_ref(&m) {
Target::Remote(Arc::new(RemoteTarget {
provider: "mockprov".into(),
base_url: "http://provider".into(),
wire: Wire::OpenAi,
model: "claude".into(),
max_output: None,
api_key: Some("k".into()),
}))
} else {
Target::Local(1)
};
Ok((m.clone(), target, ActivityGuard::new(&state, &m)))
}
};
enum Local {
Error,
Relayed,
Fine,
OtherError,
}
let run = |local: Local, headers: HeaderMap| async move {
let calls = AtomicUsize::new(0);
let remote_seen = AtomicUsize::new(0);
let (calls, remote_seen, local) = (&calls, &remote_seen, &local);
let result = with_hybrid_fallback(PAIR, Some(&headers), resolve, |model, target, _| {
calls.fetch_add(1, Ordering::SeqCst);
async move {
if target.is_remote() {
remote_seen.fetch_add(1, Ordering::SeqCst);
assert_eq!(model, HOSTED);
return Ok(StatusCode::OK.into_response());
}
assert_eq!(model, "gemma4");
match local {
Local::Error => Err(AppError(
anyhow::Error::new(ContextOverflow {
message: refusal.into(),
refusal: refusal.into(),
}),
StatusCode::INTERNAL_SERVER_ERROR,
)),
Local::Relayed => Ok(Response::builder()
.status(StatusCode::BAD_REQUEST)
.body(Body::from(refusal))
.unwrap()),
Local::Fine => Ok(StatusCode::OK.into_response()),
Local::OtherError => Err(AppError::status(
StatusCode::INTERNAL_SERVER_ERROR,
"backend died",
)),
}
}
})
.await;
(
result.map(|r| r.status()).map_err(|e| e.1),
calls.load(Ordering::SeqCst),
remote_seen.load(Ordering::SeqCst),
)
};
let none = HeaderMap::new();
assert_eq!(
run(Local::Error, none.clone()).await,
(Ok(StatusCode::OK), 2, 1)
);
assert_eq!(
run(Local::Relayed, none.clone()).await,
(Ok(StatusCode::OK), 2, 1)
);
assert_eq!(
run(Local::Fine, none.clone()).await,
(Ok(StatusCode::OK), 1, 0)
);
assert_eq!(
run(Local::OtherError, none).await,
(Err(StatusCode::INTERNAL_SERVER_ERROR), 1, 0)
);
let pinned = headers_with(&[("x-llmman-route", "local")]);
assert_eq!(
run(Local::Error, pinned.clone()).await,
(Err(StatusCode::INTERNAL_SERVER_ERROR), 1, 0)
);
assert_eq!(
run(Local::Relayed, pinned).await,
(Ok(StatusCode::BAD_REQUEST), 1, 0)
);
}
#[tokio::test]
async fn a_relayed_response_is_only_intercepted_when_it_is_the_refusal() {
let llama = r#"{"error":{"code":400,"message":"request (9 tokens) exceeds the available context size (8 tokens), try increasing it","type":"exceed_context_size_error"}}"#;
let resp = |status: StatusCode, body: &'static str| {
Response::builder()
.status(status)
.body(Body::from(body))
.unwrap()
};
let refusal = local_context_overflow(resp(StatusCode::BAD_REQUEST, llama))
.await
.expect_err("the refusal must be intercepted");
assert!(
refusal.contains("exceeds the available context size"),
"{refusal}"
);
let other = r#"{"error":{"code":400,"message":"invalid grammar"}}"#;
let passed = local_context_overflow(resp(StatusCode::BAD_REQUEST, other))
.await
.expect("another 400 passes through");
assert_eq!(passed.status(), StatusCode::BAD_REQUEST);
let body = axum::body::to_bytes(passed.into_body(), usize::MAX)
.await
.unwrap();
assert_eq!(body, other.as_bytes(), "body reattached intact");
let ok = local_context_overflow(resp(StatusCode::OK, "data: {}"))
.await
.expect("a success is never read");
assert_eq!(ok.status(), StatusCode::OK);
let big: &'static str = String::from_utf8(vec![b'x'; OVERFLOW_BODY_LIMIT + 10])
.unwrap()
.leak();
let passed = local_context_overflow(resp(StatusCode::BAD_REQUEST, big))
.await
.expect("an oversized 400 passes through");
let body = axum::body::to_bytes(passed.into_body(), usize::MAX)
.await
.unwrap();
assert_eq!(body.len(), big.len());
}
#[test]
fn resolve_keep_alive_treats_every_zero_form_as_the_unload_sentinel() {
assert_eq!(
resolve_keep_alive(&Some(serde_json::json!(0))),
Some(Duration::ZERO)
);
assert_eq!(
resolve_keep_alive(&Some(serde_json::json!("0"))),
Some(Duration::ZERO)
);
assert_eq!(
resolve_keep_alive(&Some(serde_json::json!("0s"))),
Some(Duration::ZERO)
);
assert_eq!(
resolve_keep_alive(&Some(serde_json::json!(0.0))),
Some(Duration::ZERO)
);
}
#[test]
fn apply_default_repeat_penalty_sets_default_when_absent() {
let mut req = serde_json::json!({"model": "qwen3.5:0.8b", "messages": []});
apply_default_repeat_penalty(&mut req);
assert_eq!(
req["repeat_penalty"],
serde_json::json!(DEFAULT_REPEAT_PENALTY)
);
}
#[test]
fn apply_default_repeat_penalty_preserves_an_explicit_value() {
let mut req = serde_json::json!({"model": "qwen3.5:0.8b", "repeat_penalty": 1.3});
apply_default_repeat_penalty(&mut req);
assert_eq!(req["repeat_penalty"], serde_json::json!(1.3));
}
fn oai_chat_request_with_repeat_penalty(repeat_penalty: Option<f32>) -> OAIChatRequest {
OAIChatRequest {
model: "qwen3.5:0.8b".into(),
stream: true,
repeat_penalty,
..Default::default()
}
}
#[test]
fn apply_default_repeat_penalty_typed_sets_default_when_absent() {
let mut oai = oai_chat_request_with_repeat_penalty(None);
apply_default_repeat_penalty_typed(&mut oai);
assert_eq!(oai.repeat_penalty, Some(DEFAULT_REPEAT_PENALTY));
}
#[test]
fn apply_default_repeat_penalty_typed_preserves_an_explicit_value() {
let mut oai = oai_chat_request_with_repeat_penalty(Some(1.3));
apply_default_repeat_penalty_typed(&mut oai);
assert_eq!(oai.repeat_penalty, Some(1.3));
}
#[test]
fn ollama_message_to_oai_gives_each_tool_call_a_distinct_id_even_with_identical_names() {
let m = OllamaMessage {
role: "assistant".into(),
tool_calls: Some(vec![
OllamaToolCall {
id: None,
function: OllamaToolCallFunction {
index: 0,
name: "get_weather".into(),
arguments: serde_json::json!({ "city": "nyc" }),
},
},
OllamaToolCall {
id: None,
function: OllamaToolCallFunction {
index: 0,
name: "get_weather".into(),
arguments: serde_json::json!({ "city": "sf" }),
},
},
]),
..Default::default()
};
let oai = ollama_message_to_oai(&m);
let calls = oai.tool_calls.expect("tool_calls must be carried over");
assert_eq!(calls.len(), 2);
assert_ne!(
calls[0].id, calls[1].id,
"two tool calls in one message must never share an id"
);
}
#[test]
fn options_to_oai_maps_the_documented_sampling_options() {
let oai = options_to_oai(&Some(serde_json::json!({
"temperature": 0.5, "top_p": 0.9, "top_k": 40, "min_p": 0.05,
"seed": 42, "num_predict": -1, "repeat_penalty": 1.1,
"presence_penalty": 0.1, "frequency_penalty": 0.2,
"stop": ["a", "b"], "num_ctx": 8192
})));
assert_eq!(oai.temperature, Some(0.5));
assert_eq!(oai.top_k, Some(40));
assert_eq!(oai.min_p, Some(0.05));
assert_eq!(oai.seed, Some(42));
assert_eq!(oai.max_tokens, None);
assert_eq!(oai.presence_penalty, Some(0.1));
assert_eq!(oai.frequency_penalty, Some(0.2));
assert_eq!(
oai.stop.as_deref(),
Some(&["a".to_string(), "b".to_string()][..])
);
let one = options_to_oai(&Some(
serde_json::json!({ "stop": "END", "num_predict": 8 }),
));
assert_eq!(one.stop.as_deref(), Some(&["END".to_string()][..]));
assert_eq!(one.max_tokens, Some(8));
assert_eq!(options_to_oai(&None).seed, None);
}
#[tokio::test]
async fn generate_refuses_what_it_cannot_render() {
for (field, value) in [
("raw", serde_json::json!(true)),
("suffix", serde_json::json!("}")),
("template", serde_json::json!("{{ .Prompt }}")),
] {
let mut body = serde_json::json!({ "model": "docker.io/ai/m", "prompt": "x" });
body[field] = value;
let req: OllamaGenerateRequest = serde_json::from_value(body).unwrap();
let err = handle_ollama_generate(State(test_state()), HeaderMap::new(), Json(req))
.await
.err()
.unwrap_or_else(|| panic!("{field} accepted"));
assert_eq!(err.1, StatusCode::BAD_REQUEST, "{field}");
assert!(err.0.to_string().contains(field), "{field}: {}", err.0);
}
let req: OllamaGenerateRequest = serde_json::from_value(
serde_json::json!({ "model": "docker.io/ai/m", "prompt": "", "raw": false, "suffix": "" }),
)
.unwrap();
assert!(!req.raw);
}
#[test]
fn ollama_message_to_oai_sends_wav_as_input_audio() {
use base64::Engine as _;
let mut wav = b"RIFF\x58\x02\x00\x00WAVEfmt ".to_vec();
wav.resize(64, 0);
let b64 = base64::engine::general_purpose::STANDARD.encode(&wav);
let m = OllamaMessage {
role: "user".into(),
content: "transcribe".into(),
images: Some(vec![b64.clone()]),
..Default::default()
};
let oai = ollama_message_to_oai(&m);
assert_eq!(
oai.content,
serde_json::json!([
{ "type": "text", "text": "transcribe" },
{ "type": "input_audio", "input_audio": { "data": b64, "format": "wav" } }
])
);
assert!(!is_wav_base64("Zm9v"));
assert!(!is_wav_base64("data:image/png;base64,Zm9v"));
}
fn fixture_catalog() -> crate::providers::Catalog {
crate::providers::Catalog::from_json(
br#"{
"openrouter": {
"id": "openrouter", "name": "OpenRouter",
"api": "https://openrouter.ai/api/v1",
"npm": "@openrouter/ai-sdk-provider",
"env": ["LLMMAN_TEST_PROVIDER_KEY_UNSET"],
"models": {
"z-model": { "cost": { "input": 2.5, "output": 10 } },
"a-model": {}
}
}
}"#,
)
.expect("fixture parses")
}
#[test]
fn the_provider_listing_reports_model_counts_not_model_ids() {
let catalog = fixture_catalog();
let summaries: Vec<ProviderSummary> = catalog.iter().map(ProviderSummary::from).collect();
let json = serde_json::to_value(&summaries).unwrap();
assert_eq!(json[0]["id"], "openrouter");
assert_eq!(json[0]["name"], "OpenRouter");
assert_eq!(json[0]["key_env"], "LLMMAN_TEST_PROVIDER_KEY_UNSET");
assert_eq!(json[0]["models"], 2);
assert_eq!(json[0]["key_set"], false);
assert_eq!(json[0]["key_usable"], false);
}
#[test]
fn a_single_provider_carries_its_models_and_their_prices() {
let catalog = fixture_catalog();
let provider = catalog.get("openrouter").unwrap();
let json = serde_json::to_value(ProviderResponse::from(provider)).unwrap();
assert_eq!(json["base_url"], "https://openrouter.ai/api/v1");
assert_eq!(
json["models"],
serde_json::json!([
{ "id": "a-model" },
{ "id": "z-model", "cost": { "input": 2.5, "output": 10.0 } },
])
);
}
#[test]
fn provider_responses_carry_no_api_key() {
let catalog = fixture_catalog();
let provider = catalog.get("openrouter").unwrap();
for json in [
serde_json::to_value(ProviderSummary::from(provider)).unwrap(),
serde_json::to_value(ProviderResponse::from(provider)).unwrap(),
] {
let fields: Vec<&String> = json.as_object().unwrap().keys().collect();
assert!(
!fields.iter().any(|f| f.contains("api_key")),
"{fields:?} carries a key"
);
}
}
#[test]
fn a_configured_provider_reports_its_key_as_optional() {
let catalog = fixture_catalog().with_configured(&[crate::config::ConfiguredProvider {
id: "gpubox".into(),
name: "GPU box".into(),
base_url: "http://gpubox:8000/v1".into(),
wire: Wire::OpenAi,
key_env: None,
}]);
let provider = catalog.get("gpubox").unwrap();
let json = serde_json::to_value(ProviderSummary::from(provider)).unwrap();
assert_eq!(json["key_optional"], true);
assert_eq!(json["key_set"], false);
assert!(json.get("key_env").is_none(), "{json}");
assert_eq!(json["base_url"], "http://gpubox:8000/v1");
let json = serde_json::to_value(ProviderSummary::from(catalog.get("openrouter").unwrap()))
.unwrap();
assert_eq!(json["key_optional"], false);
}
#[test]
fn a_keyless_remote_target_sends_no_credential_header() {
let client = Client::new();
let keyless = |wire: Wire| {
Target::Remote(Arc::new(RemoteTarget {
provider: "gpubox".into(),
base_url: "http://gpubox:8000/v1".into(),
wire,
model: "m".into(),
max_output: None,
api_key: None,
}))
};
let headers = |target: &Target| {
target
.authorize(client.post("http://gpubox:8000/v1/x"))
.build()
.unwrap()
.headers()
.clone()
};
let openai = headers(&keyless(Wire::OpenAi));
assert!(
openai.get(reqwest::header::AUTHORIZATION).is_none(),
"{openai:?}"
);
let anthropic = headers(&keyless(Wire::Anthropic));
assert!(anthropic.get("x-api-key").is_none(), "{anthropic:?}");
assert_eq!(
anthropic.get("anthropic-version").unwrap(),
anthropic::VERSION
);
let keyed = headers(&remote_target_on("http://gpubox:8000/v1", Wire::OpenAi));
assert_eq!(
keyed.get(reqwest::header::AUTHORIZATION).unwrap(),
"Bearer sk-test"
);
}
#[tokio::test]
async fn a_configured_providers_models_are_asked_of_its_endpoint() {
let seen: Arc<tokio::sync::Mutex<Vec<HeaderMap>>> = Arc::default();
let captured = seen.clone();
let app = Router::new().route(
"/v1/models",
get(move |headers: HeaderMap| async move {
captured.lock().await.push(headers);
Json(serde_json::json!({ "object": "list", "data": [
{ "id": "qwen3-coder", "object": "model" },
{ "id": "gemma4", "object": "model" },
{ "id": "gemma4", "object": "model" }
]}))
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let base = format!(
"http://127.0.0.1:{}/v1",
listener.local_addr().unwrap().port()
);
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let provider =
crate::providers::Provider::from_configured(&crate::config::ConfiguredProvider {
id: "gpubox".into(),
name: "gpubox".into(),
base_url: base.clone(),
wire: Wire::OpenAi,
key_env: None,
});
let client = Client::new();
let models = configured_provider_models(&client, &provider).await;
assert_eq!(
models,
vec!["gemma4".to_string(), "qwen3-coder".to_string()]
);
let calls = seen.lock().await;
assert!(
calls[0].get("authorization").is_none(),
"a keyless provider was sent a bearer: {:?}",
calls[0]
);
drop(calls);
let mut down = provider.clone();
down.base_url = "http://127.0.0.1:9/v1".into();
assert!(configured_provider_models(&client, &down).await.is_empty());
let mut anthropic = provider.clone();
anthropic.wire = Wire::Anthropic;
assert!(configured_provider_models(&client, &anthropic)
.await
.is_empty());
assert_eq!(seen.lock().await.len(), 1, "no second request was made");
}
fn test_state() -> AppState {
test_state_at(std::env::temp_dir())
}
fn test_state_at(store_path: PathBuf) -> AppState {
AppState(Arc::new(test_inner(store_path)))
}
fn test_state_with_budget(hybrid_local_bytes: u64) -> AppState {
let mut inner = test_inner(std::env::temp_dir());
inner.hybrid_local_bytes = Some(hybrid_local_bytes);
AppState(Arc::new(inner))
}
fn test_inner(store_path: PathBuf) -> Inner {
Inner {
manager: Mutex::new(ModelManager {
running: HashMap::new(),
pending_loads: 0,
}),
llama_server_bin: StdMutex::new(None),
exe: None,
ociman: None,
llama_cpp_version: None,
vllm_version: None,
ctx_size: None,
ctx_size_explicit: false,
hybrid_local_bytes: None,
flash_attention: None,
kv_cache_type: None,
split_mode: None,
num_parallel: None,
threads: None,
max_queue: usize::MAX,
max_loaded_models: 0,
peers: Vec::new(),
memory: 0,
store_path,
cache_path: std::env::temp_dir(),
prompt_log: None,
shell: shell::Policy {
disabled: None,
origins: default_allowed_origins(),
command: Vec::new(),
},
client: Client::new(),
}
}
#[cfg(unix)]
fn spawn_placeholder_process() -> tokio::process::Child {
tokio::process::Command::new("sleep")
.arg("60")
.process_group(0)
.kill_on_drop(true)
.spawn()
.expect("spawn placeholder `sleep` process")
}
#[cfg(windows)]
fn spawn_placeholder_process() -> tokio::process::Child {
tokio::process::Command::new("cmd")
.args(["/C", "timeout", "/T", "60", "/NOBREAK"])
.kill_on_drop(true)
.spawn()
.expect("spawn placeholder `cmd /C timeout` process")
}
fn running_model_fixture(
keep_alive: Option<Duration>,
idle_for: Duration,
in_flight: u32,
) -> RunningModel {
RunningModel {
process: ModelProcess::Local(Engine::LlamaServer, spawn_placeholder_process(), None),
port: 0,
digest: String::new(),
size: 0,
started_at: now_rfc3339(),
last_active: Instant::now() - idle_for,
last_active_wall: chrono::Utc::now(),
backend_model_path: None,
keep_alive,
in_flight,
}
}
fn running_model_fixture_with_engine(
engine: Engine,
backend_model_path: Option<&str>,
) -> RunningModel {
RunningModel {
process: ModelProcess::Local(engine, spawn_placeholder_process(), None),
port: 0,
digest: String::new(),
size: 0,
started_at: now_rfc3339(),
last_active: Instant::now(),
last_active_wall: chrono::Utc::now(),
backend_model_path: backend_model_path.map(|s| s.to_string()),
keep_alive: None,
in_flight: 0,
}
}
#[tokio::test]
async fn the_engine_label_names_the_engine_not_the_runtime() {
for (engine, expected) in [
(Engine::LlamaServer, "llama-server"),
(Engine::Vllm, "vllm"),
(Engine::Mlx, "mlx"),
] {
assert_eq!(
running_model_fixture_with_engine(engine, None).engine_label(),
expected
);
}
}
#[tokio::test(flavor = "multi_thread")]
async fn backend_wire_model_is_the_canonical_name_for_every_engine_except_mlx() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"llama-model".into(),
running_model_fixture_with_engine(Engine::LlamaServer, None),
);
mgr.running.insert(
"vllm-model".into(),
running_model_fixture_with_engine(Engine::Vllm, None),
);
mgr.running.insert(
"mlx-model".into(),
running_model_fixture_with_engine(Engine::Mlx, Some("/cache/mlx-model/abcd")),
);
}
assert_eq!(
backend_wire_model(&state, &Target::Local(0), "llama-model").await,
"llama-model"
);
assert_eq!(
backend_wire_model(&state, &Target::Local(0), "vllm-model").await,
"vllm-model"
);
assert_eq!(
backend_wire_model(&state, &Target::Local(0), "mlx-model").await,
"/cache/mlx-model/abcd",
"an Engine::Mlx backend must be addressed by its real directory path, not its human-readable name"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn backend_wire_model_falls_back_to_the_canonical_name_when_not_running() {
let state = test_state();
assert_eq!(
backend_wire_model(&state, &Target::Local(0), "not-running").await,
"not-running"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn backend_wire_model_strips_the_routing_prefix_for_a_remote_target() {
let state = test_state();
let target = remote_target("https://example.invalid/v1");
assert_eq!(
backend_wire_model(&state, &target, "llmman.provider/mockprov/mock-model").await,
"mock-model"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn would_use_mlx_finds_an_already_running_mlx_model_without_touching_disk_or_a_process() {
let state = test_state();
let model_ref = "hf.co/mlx-community/foo:latest";
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
model_ref.to_string(),
running_model_fixture_with_engine(Engine::Mlx, Some("/cache/foo/abcd")),
);
}
assert_eq!(
would_use_mlx(&state, model_ref).await,
Some(model_ref.to_string())
);
}
#[tokio::test(flavor = "multi_thread")]
async fn would_use_mlx_is_none_for_an_already_running_non_mlx_model() {
let state = test_state();
let model_ref = "hf.co/mlx-community/foo:latest";
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
model_ref.to_string(),
running_model_fixture_with_engine(Engine::LlamaServer, None),
);
}
assert_eq!(would_use_mlx(&state, model_ref).await, None);
}
#[tokio::test(flavor = "multi_thread")]
async fn would_use_mlx_is_none_when_not_running_and_this_host_never_uses_mlx() {
let state = test_state();
assert_eq!(
would_use_mlx(&state, "hf.co/mlx-community/not-loaded-yet:latest").await,
None
);
}
#[tokio::test(flavor = "multi_thread")]
async fn reap_idle_models_unloads_only_idle_expired_models_not_in_flight_or_forever() {
let _serialised = metrics::GLOBAL_COUNTER_TEST_LOCK.lock().await;
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"expired-and-idle".into(),
running_model_fixture(Some(Duration::from_secs(1)), Duration::from_secs(10), 0),
);
mgr.running.insert(
"expired-but-in-flight".into(),
running_model_fixture(Some(Duration::from_secs(1)), Duration::from_secs(10), 1),
);
mgr.running.insert(
"expired-but-forever".into(),
running_model_fixture(None, Duration::from_secs(10), 0),
);
mgr.running.insert(
"not-yet-expired".into(),
running_model_fixture(Some(Duration::from_secs(300)), Duration::from_secs(1), 0),
);
}
reap_idle_models_once(&state).await;
let mgr = state.0.manager.lock().await;
assert!(
!mgr.running.contains_key("expired-and-idle"),
"an idle model past its keep_alive deadline must be unloaded"
);
assert!(
mgr.running.contains_key("expired-but-in-flight"),
"a model with an in-flight request must survive regardless of its deadline"
);
assert!(
mgr.running.contains_key("expired-but-forever"),
"keep_alive: None (forever) must never be reaped"
);
assert!(
mgr.running.contains_key("not-yet-expired"),
"a model whose keep_alive deadline hasn't passed yet must survive"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn evict_other_models_evicts_everything_except_the_target_and_in_flight_models() {
let _serialised = metrics::GLOBAL_COUNTER_TEST_LOCK.lock().await;
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"the-model-being-loaded".into(),
running_model_fixture(None, Duration::from_secs(0), 0),
);
mgr.running.insert(
"idle-other-model".into(),
running_model_fixture(None, Duration::from_secs(0), 0),
);
mgr.running.insert(
"busy-other-model".into(),
running_model_fixture(None, Duration::from_secs(0), 1),
);
}
let evicted_anything = evict_other_models(&state, "the-model-being-loaded").await;
assert!(evicted_anything);
let mgr = state.0.manager.lock().await;
assert!(
mgr.running.contains_key("the-model-being-loaded"),
"the model ensure_model is trying to load isn't itself an eviction target"
);
assert!(
!mgr.running.contains_key("idle-other-model"),
"an idle other model should be evicted to free memory"
);
assert!(
mgr.running.contains_key("busy-other-model"),
"a model with an in-flight request must survive eviction, same as reap_idle_models"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn evict_other_models_reports_nothing_evicted_when_nothing_is_evictable() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"the-model-being-loaded".into(),
running_model_fixture(None, Duration::from_secs(0), 0),
);
}
assert!(!evict_other_models(&state, "the-model-being-loaded").await);
}
#[tokio::test(flavor = "multi_thread")]
async fn check_running_claims_in_flight_on_a_live_hit() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running
.insert("m".into(), running_model_fixture(None, Duration::ZERO, 0));
}
let (_, _guard) = check_running(&state, "m").await.unwrap();
assert_eq!(
state.0.manager.lock().await.running["m"].in_flight,
1,
"a cache hit must claim in_flight so it can't be evicted before the caller claims it"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn an_unclaimed_activity_guard_still_releases_on_drop() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running
.insert("m".into(), running_model_fixture(None, Duration::ZERO, 0));
}
let (_, guard) = check_running(&state, "m").await.unwrap();
assert_eq!(state.0.manager.lock().await.running["m"].in_flight, 1);
drop(guard); tokio::time::sleep(Duration::from_millis(100)).await;
assert_eq!(
state.0.manager.lock().await.running["m"].in_flight,
0,
"the claim must still be released even if never handed off"
);
}
#[test]
fn try_admit_rejects_once_the_cap_is_reached_and_releases_on_drop() {
static COUNTER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
let first =
try_admit_against(&COUNTER, 1).expect("first admission under the cap must succeed");
assert!(
try_admit_against(&COUNTER, 1).is_err(),
"a second admission at the cap must be rejected"
);
drop(first);
assert!(
try_admit_against(&COUNTER, 1).is_ok(),
"dropping an admitted guard must free its slot for the next caller"
);
}
#[test]
fn try_admit_with_a_zero_cap_admits_one_at_a_time_not_none_or_unbounded() {
static COUNTER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
let first = try_admit_against(&COUNTER, 0).expect("one admission must still succeed");
assert!(
try_admit_against(&COUNTER, 0).is_err(),
"a second concurrent admission must still be rejected"
);
drop(first);
assert!(
try_admit_against(&COUNTER, 0).is_ok(),
"dropping the first must free the slot for the next caller"
);
}
#[test]
fn default_allowed_origins_covers_every_scheme_and_localhost_spelling() {
let origins = default_allowed_origins();
for expected in [
"http://localhost:*",
"https://localhost:*",
"http://127.0.0.1:*",
"https://127.0.0.1:*",
"http://0.0.0.0:*",
"https://0.0.0.0:*",
"http://[::1]:*",
"https://[::1]:*",
] {
assert!(
origins.iter().any(|o| o == expected),
"missing default origin pattern {expected:?}"
);
}
}
#[test]
fn origin_matches_a_trailing_wildcard_port_pattern() {
assert!(origin_matches(
"http://localhost:3000",
"http://localhost:*"
));
assert!(origin_matches("http://localhost:1", "http://localhost:*"));
assert!(origin_matches("http://localhost:", "http://localhost:*"));
assert!(!origin_matches(
"http://evil.example:3000",
"http://localhost:*"
));
assert!(!origin_matches(
"http://localhost.evil.example",
"http://localhost:*"
));
}
#[test]
fn origin_matches_a_wildcard_anywhere_in_the_pattern() {
assert!(origin_matches(
"https://foo.example.com",
"https://*.example.com"
));
assert!(!origin_matches(
"https://example.com",
"https://*.example.com"
));
assert!(origin_matches("https://anything.at.all", "*"));
assert!(!origin_matches("https://example.com", "https://*.*.com"));
}
#[test]
fn origin_matches_a_plain_pattern_only_byte_for_byte() {
assert!(origin_matches("https://example.com", "https://example.com"));
assert!(!origin_matches(
"https://example.com:8080",
"https://example.com"
));
assert!(!origin_matches("http://example.com", "https://example.com"));
}
#[test]
fn allowed_origins_from_env_always_includes_the_localhost_defaults() {
let origins = allowed_origins_from_env();
assert!(origins.iter().any(|o| o == "http://localhost:*"));
}
#[tokio::test(flavor = "multi_thread")]
async fn enforce_max_loaded_models_evicts_nothing_but_still_reserves_when_unbounded() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"m1".into(),
running_model_fixture(None, Duration::from_secs(0), 0),
);
}
let mut guard = enforce_max_loaded_models(&state, 0)
.await
.expect("unbounded admits every load");
{
let mut mgr = state.0.manager.lock().await;
assert_eq!(mgr.running.len(), 1, "nothing was evicted");
assert_eq!(mgr.pending_loads, 1, "the load is still counted as loading");
guard.release_into(&mut mgr);
assert_eq!(mgr.pending_loads, 0);
}
}
#[tokio::test(flavor = "multi_thread")]
async fn releasing_a_reservation_into_the_lock_leaves_no_double_counted_window() {
let state = test_state();
let mut guard = enforce_max_loaded_models(&state, 0).await.unwrap();
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"just-loaded".into(),
running_model_fixture(None, Duration::from_secs(0), 1),
);
guard.release_into(&mut mgr);
assert_eq!(mgr.running.len(), 1);
assert_eq!(mgr.pending_loads, 0);
drop(mgr);
drop(guard);
tokio::time::sleep(Duration::from_millis(50)).await;
assert_eq!(state.0.manager.lock().await.pending_loads, 0);
}
#[tokio::test(flavor = "multi_thread")]
async fn enforce_max_loaded_models_evicts_the_least_recently_active_idle_model_first() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"oldest-idle".into(),
running_model_fixture(None, Duration::from_secs(300), 0),
);
mgr.running.insert(
"newest-idle".into(),
running_model_fixture(None, Duration::from_secs(1), 0),
);
}
assert!(enforce_max_loaded_models(&state, 2).await.is_ok());
let mgr = state.0.manager.lock().await;
assert_eq!(mgr.running.len(), 1);
assert!(
!mgr.running.contains_key("oldest-idle"),
"the least-recently-active idle model must be evicted first"
);
assert!(mgr.running.contains_key("newest-idle"));
}
#[tokio::test(flavor = "multi_thread")]
async fn enforce_max_loaded_models_reserves_its_own_slot_when_evicting() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"victim".into(),
running_model_fixture(None, Duration::from_secs(300), 0),
);
}
let guard = enforce_max_loaded_models(&state, 1).await.unwrap();
let mgr = state.0.manager.lock().await;
assert_eq!(mgr.running.len(), 0, "the sole idle model must be evicted");
assert_eq!(
mgr.pending_loads, 1,
"the freed slot must already be reserved for this caller"
);
drop(mgr);
drop(guard);
}
#[tokio::test(flavor = "multi_thread")]
async fn enforce_max_loaded_models_rejects_with_503_when_every_loaded_model_is_busy() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"busy-1".into(),
running_model_fixture(None, Duration::from_secs(0), 1),
);
}
let err = enforce_max_loaded_models(&state, 1)
.await
.expect_err("every model at/over the cap is busy, so this must reject");
assert_eq!(err.1, StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(state.0.manager.lock().await.running.len(), 1);
}
#[tokio::test(flavor = "multi_thread")]
async fn enforce_max_loaded_models_reserves_a_pending_slot_for_an_in_flight_load() {
let state = test_state();
let guard1 = enforce_max_loaded_models(&state, 1).await.unwrap();
assert_eq!(state.0.manager.lock().await.pending_loads, 1);
let state2 = state.clone();
let second = tokio::spawn(async move { enforce_max_loaded_models(&state2, 1).await });
tokio::time::sleep(Duration::from_millis(50)).await;
assert!(
!second.is_finished(),
"a second load must wait for the first reservation, not double up on it"
);
drop(guard1);
tokio::time::timeout(Duration::from_secs(2), second)
.await
.expect("second load must proceed once the first reservation is released")
.unwrap()
.expect("second load must succeed once a slot is actually free");
}
#[tokio::test(flavor = "multi_thread")]
async fn begin_activity_marks_in_flight_and_its_drop_releases_it_and_updates_keep_alive() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"m".into(),
running_model_fixture(Some(DEFAULT_KEEP_ALIVE), Duration::ZERO, 1),
);
}
let claim = ActivityGuard::new(&state, "m");
let guard = begin_activity(claim, Some(Some(Duration::from_secs(42)))).await;
{
let mgr = state.0.manager.lock().await;
let m = &mgr.running["m"];
assert_eq!(
m.in_flight, 1,
"begin_activity must not add a second claim on top of ensure_model's own"
);
assert_eq!(m.keep_alive, Some(Duration::from_secs(42)));
}
drop(guard);
tokio::time::sleep(Duration::from_millis(100)).await;
let mgr = state.0.manager.lock().await;
let m = &mgr.running["m"];
assert_eq!(
m.in_flight, 0,
"dropping the guard must release the in-flight count"
);
assert_eq!(m.keep_alive, Some(Duration::from_secs(42)));
}
#[tokio::test(flavor = "multi_thread")]
async fn begin_activity_with_no_override_never_touches_an_existing_keep_alive() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"m".into(),
running_model_fixture(None, Duration::from_secs(600), 1),
);
}
let claim = ActivityGuard::new(&state, "m");
let guard = begin_activity(claim, None).await;
{
let mgr = state.0.manager.lock().await;
let m = &mgr.running["m"];
assert_eq!(m.in_flight, 1);
assert_eq!(
m.keep_alive, None,
"a None override must not touch the model's existing keep_alive"
);
assert!(
m.last_active.elapsed() < Duration::from_secs(600),
"the idle clock must still refresh even without a keep_alive override"
);
}
drop(guard);
tokio::time::sleep(Duration::from_millis(100)).await;
let mgr = state.0.manager.lock().await;
assert_eq!(
mgr.running["m"].keep_alive, None,
"dropping the guard must still leave keep_alive untouched"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn refresh_activity_releases_ensure_models_own_claim() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"m".into(),
running_model_fixture(Some(DEFAULT_KEEP_ALIVE), Duration::ZERO, 1),
);
}
refresh_activity(ActivityGuard::new(&state, "m"), None).await;
tokio::time::sleep(Duration::from_millis(100)).await;
assert_eq!(
state.0.manager.lock().await.running["m"].in_flight,
0,
"refresh_activity must release ensure_model's claim, with no guard to do it later"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn refresh_activity_with_none_pins_a_model_to_never_unload() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"preloaded".into(),
running_model_fixture(Some(DEFAULT_KEEP_ALIVE), Duration::ZERO, 1),
);
}
refresh_activity(ActivityGuard::new(&state, "preloaded"), None).await;
let mgr = state.0.manager.lock().await;
assert_eq!(
mgr.running["preloaded"].keep_alive, None,
"a pre-loaded model must be pinned to never unload, not left at the daemon default"
);
}
fn chat_request(body: serde_json::Value) -> OllamaChatRequest {
serde_json::from_value(body).expect("valid OllamaChatRequest")
}
async fn chat_response_json(resp: Response) -> serde_json::Value {
let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
.await
.unwrap();
serde_json::from_slice(&body).unwrap()
}
#[test]
fn empty_chat_chunk_matches_ollamas_message_less_reply() {
let value = serde_json::to_value(empty_chat_chunk("m:latest".into(), "load")).unwrap();
assert_eq!(value["model"], "m:latest");
assert_eq!(value["done"], true);
assert_eq!(value["done_reason"], "load");
assert_eq!(value["message"]["role"], "assistant");
assert_eq!(value["message"]["content"], "");
assert_eq!(
value["message"].as_object().unwrap().len(),
2,
"ollama's message-less reply carries only role and content"
);
assert!(value.get("created_at").is_some());
}
#[tokio::test(flavor = "multi_thread")]
async fn ollama_chat_with_no_messages_and_keep_alive_zero_unloads_the_model() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"docker.io/ai/m:latest".into(),
running_model_fixture(Some(DEFAULT_KEEP_ALIVE), Duration::ZERO, 0),
);
}
let resp = handle_ollama_chat(
State(state.clone()),
HeaderMap::new(),
Json(chat_request(serde_json::json!({
"model": "docker.io/ai/m:latest",
"messages": [],
"keep_alive": 0,
}))),
)
.await
.expect("unload must not error");
let value = chat_response_json(resp).await;
assert_eq!(value["done_reason"], "unload");
assert_eq!(value["message"]["content"], "");
assert!(
!state
.0
.manager
.lock()
.await
.running
.contains_key("docker.io/ai/m:latest"),
"keep_alive: 0 with no messages must actually unload the model"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn a_tagless_unload_still_finds_a_model_running_under_latest() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"docker.io/ai/m:latest".into(),
running_model_fixture(Some(DEFAULT_KEEP_ALIVE), Duration::ZERO, 0),
);
}
let resp = handle_ollama_chat(
State(state.clone()),
HeaderMap::new(),
Json(chat_request(serde_json::json!({
"model": "docker.io/ai/m",
"messages": [],
"keep_alive": 0,
}))),
)
.await
.expect("unload must not error");
assert_eq!(chat_response_json(resp).await["done_reason"], "unload");
assert!(
!state
.0
.manager
.lock()
.await
.running
.contains_key("docker.io/ai/m:latest"),
"a tagless unload must reach the model stored under :latest"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn unloading_a_model_llmman_does_not_have_is_a_404() {
let state = test_state();
let err = unload_model(&state, "docker.io/ai/nothing-here")
.await
.expect_err("an unknown model must not report a successful unload");
assert_eq!(err.1, StatusCode::NOT_FOUND);
let body = axum::body::to_bytes(err.into_response().into_body(), usize::MAX)
.await
.unwrap();
assert!(
crate::daemon::is_model_not_found_body(
std::str::from_utf8(&body).unwrap(),
"docker.io/ai/nothing-here"
),
"daemon::is_model_not_found_body must accept the body unload_model sends"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn unloading_a_provider_routed_model_is_not_a_404() {
let state = test_state();
unload_model(&state, "llmman.provider/openrouter/qwen/qwen3-coder")
.await
.expect("a provider-routed unload must succeed");
}
#[tokio::test(flavor = "multi_thread")]
async fn a_model_gone_from_the_store_but_still_loaded_unloads_without_a_404() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"docker.io/ai/orphan:latest".into(),
running_model_fixture(Some(DEFAULT_KEEP_ALIVE), Duration::ZERO, 0),
);
}
unload_model(&state, "docker.io/ai/orphan")
.await
.expect("a loaded model must unload even with nothing in the store");
assert!(
!state
.0
.manager
.lock()
.await
.running
.contains_key("docker.io/ai/orphan:latest"),
"the running entry must be gone"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn an_unload_by_digest_reaches_the_entry_stored_under_another_tag() {
let dir = std::env::temp_dir().join(format!(
"llmman-unload-digest-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let _ = std::fs::remove_dir_all(&dir);
let state = test_state_at(dir.clone());
let desc = crate::storage::oci::Descriptor {
media_type: "application/vnd.oci.image.manifest.v1+json".into(),
digest: "sha256:a1b2".into(),
size: 0,
annotations: None,
};
OciStore::open(&dir)
.unwrap()
.tag(desc, "docker.io/ai/m-tagged:v9")
.unwrap();
state.0.manager.lock().await.running.insert(
"docker.io/ai/m-tagged:v9".into(),
running_model_fixture(Some(DEFAULT_KEEP_ALIVE), Duration::ZERO, 0),
);
unload_model(&state, "docker.io/ai/m-tagged@sha256:a1b2")
.await
.expect("the digest spelling must unload the model loaded under its tag");
assert!(
!state
.0
.manager
.lock()
.await
.running
.contains_key("docker.io/ai/m-tagged:v9"),
"the entry under the tag must be gone"
);
unload_model(&state, "docker.io/ai/m-tagged@sha256:a1b2")
.await
.expect("a stored model must unload without a 404 whether or not it is loaded");
let err = unload_model(&state, "docker.io/ai/m-tagged@sha256:ffff")
.await
.expect_err("a digest the store does not hold must be a 404");
assert_eq!(err.1, StatusCode::NOT_FOUND);
let _ = std::fs::remove_dir_all(&dir);
}
#[tokio::test(flavor = "multi_thread")]
async fn an_unload_by_digest_reaches_a_loaded_model_the_store_has_lost() {
let state = test_state();
let running = |digest: &str| {
let mut m = running_model_fixture(Some(DEFAULT_KEEP_ALIVE), Duration::ZERO, 0);
m.digest = digest.into();
m
};
{
let mut mgr = state.0.manager.lock().await;
mgr.running
.insert("docker.io/ai/m-gone:v8".into(), running("sha256:a1b2"));
mgr.running
.insert("docker.io/ai/m-gone:v9".into(), running("sha256:a1b2"));
mgr.running
.insert("docker.io/ai/other:v9".into(), running("sha256:a1b2"));
}
unload_model(&state, "docker.io/ai/m-gone:v9@sha256:A1B2")
.await
.expect("the digest must reach the model running under its lost tag");
let keys = |mgr: &ModelManager| {
let mut k: Vec<String> = mgr.running.keys().cloned().collect();
k.sort();
k
};
assert_eq!(
keys(&*state.0.manager.lock().await),
["docker.io/ai/m-gone:v8", "docker.io/ai/other:v9"],
"the spelled tag goes first; the other tag and the other repository stay"
);
unload_model(&state, "docker.io/ai/m-gone@sha256:a1b2")
.await
.expect("with no spelled tag, the remaining entry with the digest is it");
assert_eq!(
keys(&*state.0.manager.lock().await),
["docker.io/ai/other:v9"]
);
let err = unload_model(&state, "docker.io/ai/m-gone@sha256:a1b2")
.await
.expect_err("nothing running or stored with it is the 404 case");
assert_eq!(err.1, StatusCode::NOT_FOUND);
}
#[tokio::test(flavor = "multi_thread")]
async fn a_tag_reaches_the_process_a_load_by_digest_registered() {
let dir = std::env::temp_dir().join(format!(
"llmman-digest-first-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let _ = std::fs::remove_dir_all(&dir);
let state = test_state_at(dir.clone());
let desc = |digest: &str| crate::storage::oci::Descriptor {
media_type: "application/vnd.oci.image.manifest.v1+json".into(),
digest: digest.into(),
size: 0,
annotations: None,
};
let store = OciStore::open(&dir).unwrap();
store
.tag(desc("sha256:df01"), "docker.io/ai/m-df@sha256:df01")
.unwrap();
store
.tag(desc("sha256:df01"), "docker.io/ai/m-df:latest")
.unwrap();
{
let mut running = running_model_fixture(Some(DEFAULT_KEEP_ALIVE), Duration::ZERO, 0);
running.digest = "sha256:df01".into();
state
.0
.manager
.lock()
.await
.running
.insert("docker.io/ai/m-df@sha256:df01".into(), running);
}
let (key, target, guard) = ensure_model(&state, "docker.io/ai/m-df", None, None)
.await
.expect("the tag must find the running content rather than load again");
assert_eq!(key, "docker.io/ai/m-df@sha256:df01");
assert!(matches!(target, Target::Local(0)));
drop(guard);
unload_model(&state, "docker.io/ai/m-df:latest")
.await
.expect("the tag must unload the process running under the digest key");
assert!(
state.0.manager.lock().await.running.is_empty(),
"the digest-keyed entry must be gone"
);
unload_model(&state, "docker.io/ai/m-df").await.unwrap();
let _ = std::fs::remove_dir_all(&dir);
}
#[tokio::test(flavor = "multi_thread")]
async fn an_unload_waiting_on_a_load_removes_the_key_that_load_inserted() {
let dir = std::env::temp_dir().join(format!(
"llmman-unload-race-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let _ = std::fs::remove_dir_all(&dir);
let state = test_state_at(dir.clone());
let pre_pull_key = "docker.io/ai/m:latest";
let loading = acquire_load_lock(pre_pull_key).await;
let unloader = state.clone();
let unload = tokio::spawn(async move { unload_model(&unloader, "docker.io/ai/m").await });
tokio::time::sleep(Duration::from_millis(100)).await;
let desc = crate::storage::oci::Descriptor {
media_type: "application/vnd.oci.image.manifest.v1+json".into(),
digest: "sha256:0000".into(),
size: 0,
annotations: None,
};
OciStore::open(&dir)
.unwrap()
.tag(desc, "docker.io/ai/m")
.unwrap();
state.0.manager.lock().await.running.insert(
"docker.io/ai/m".into(),
running_model_fixture(Some(DEFAULT_KEEP_ALIVE), Duration::ZERO, 0),
);
drop(loading);
unload
.await
.unwrap()
.expect("the unload must not error once the load releases the lock");
assert!(
!state
.0
.manager
.lock()
.await
.running
.contains_key("docker.io/ai/m"),
"the unload must remove the key the load inserted, not the one it resolved before waiting"
);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn load_identity_folds_the_digest_spelling_onto_the_tag_spelling() {
let store = std::env::temp_dir();
let latest = load_identity(&store, "docker.io/ai/m:latest").unwrap();
assert_eq!(load_identity(&store, "docker.io/ai/m").unwrap(), latest);
assert_eq!(
load_identity(&store, "docker.io/ai/m@sha256:0000000000000000").unwrap(),
latest
);
assert_eq!(
load_identity(&store, "docker.io/ai/m:v9@sha256:0000000000000000").unwrap(),
"docker.io/ai/m:v9"
);
}
#[test]
fn load_identity_takes_the_tag_the_store_holds_a_digest_under() {
let dir = std::env::temp_dir().join(format!(
"llmman-identity-store-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let store = OciStore::open(&dir).unwrap();
let desc = |digest: &str| crate::storage::oci::Descriptor {
media_type: "application/vnd.oci.image.manifest.v1+json".into(),
digest: digest.into(),
size: 0,
annotations: None,
};
store
.tag(desc("sha256:aaaa"), "docker.io/ai/m-key:v9")
.unwrap();
store
.tag(desc("sha256:bbbb"), "docker.io/ai/m-key@sha256:bbbb")
.unwrap();
assert_eq!(
load_identity(&dir, "docker.io/ai/m-key@sha256:aaaa").unwrap(),
"docker.io/ai/m-key:v9"
);
assert_eq!(
load_identity(&dir, "docker.io/ai/m-key:latest@sha256:aaaa").unwrap(),
"docker.io/ai/m-key:v9",
"the tag the content is under wins over the one spelled"
);
assert_eq!(
load_identity(&dir, "docker.io/ai/m-key@sha256:bbbb").unwrap(),
"docker.io/ai/m-key:latest",
"a digest-named entry is what a pull by digest writes; it folds"
);
assert_eq!(
load_identity(&dir, "docker.io/ai/m-key@sha256:ffff").unwrap(),
"docker.io/ai/m-key:latest"
);
assert_eq!(
load_identity(&dir, "docker.io/ai/m-key:v9").unwrap(),
"docker.io/ai/m-key:v9"
);
let _ = std::fs::remove_dir_all(&dir);
}
#[tokio::test(flavor = "multi_thread")]
async fn a_load_by_digest_waits_on_the_tag_spellings_lock_and_takes_its_process() {
let dir = std::env::temp_dir().join(format!(
"llmman-load-digest-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let _ = std::fs::remove_dir_all(&dir);
let state = test_state_at(dir.clone());
let desc = crate::storage::oci::Descriptor {
media_type: "application/vnd.oci.image.manifest.v1+json".into(),
digest: "sha256:d16e".into(),
size: 0,
annotations: None,
};
OciStore::open(&dir)
.unwrap()
.tag(desc, "docker.io/ai/m-digest:v9")
.unwrap();
let loading =
acquire_load_lock(&load_identity(&dir, "docker.io/ai/m-digest:v9").unwrap()).await;
let loader = state.clone();
let load = tokio::spawn(async move {
ensure_model(&loader, "docker.io/ai/m-digest@sha256:d16e", None, None).await
});
tokio::time::sleep(Duration::from_millis(100)).await;
assert!(
!load.is_finished(),
"a load by digest must wait on the lock the tag spelling holds"
);
state.0.manager.lock().await.running.insert(
"docker.io/ai/m-digest:v9".into(),
running_model_fixture(Some(DEFAULT_KEEP_ALIVE), Duration::ZERO, 0),
);
drop(loading);
let (model_ref, target, _guard) = load
.await
.unwrap()
.expect("the load by digest must succeed once the lock releases");
assert_eq!(model_ref, "docker.io/ai/m-digest:v9");
assert!(
matches!(target, Target::Local(0)),
"the process the tag spelling started must be the one handed back"
);
let _ = std::fs::remove_dir_all(&dir);
}
#[tokio::test(flavor = "multi_thread")]
async fn an_unload_after_the_pull_but_before_the_insert_still_waits_for_the_load() {
let dir = std::env::temp_dir().join(format!(
"llmman-unload-postpull-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let _ = std::fs::remove_dir_all(&dir);
let state = test_state_at(dir.clone());
let desc = crate::storage::oci::Descriptor {
media_type: "application/vnd.oci.image.manifest.v1+json".into(),
digest: "sha256:0000".into(),
size: 0,
annotations: None,
};
OciStore::open(&dir)
.unwrap()
.tag(desc, "docker.io/ai/m")
.unwrap();
let loading = acquire_load_lock(&load_identity(&dir, "docker.io/ai/m").unwrap()).await;
let unloader = state.clone();
let unload = tokio::spawn(async move { unload_model(&unloader, "docker.io/ai/m").await });
tokio::time::sleep(Duration::from_millis(100)).await;
assert!(
!unload.is_finished(),
"the unload must block on the load in flight, not resolve past it"
);
state.0.manager.lock().await.running.insert(
"docker.io/ai/m".into(),
running_model_fixture(Some(DEFAULT_KEEP_ALIVE), Duration::ZERO, 0),
);
drop(loading);
unload
.await
.unwrap()
.expect("the unload must succeed once the load releases");
assert!(
!state
.0
.manager
.lock()
.await
.running
.contains_key("docker.io/ai/m"),
"the entry the load inserted must be gone"
);
let _ = std::fs::remove_dir_all(&dir);
}
#[tokio::test(flavor = "multi_thread")]
async fn ollama_chat_with_no_messages_loads_without_generating() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"docker.io/ai/m:latest".into(),
running_model_fixture(Some(DEFAULT_KEEP_ALIVE), Duration::ZERO, 0),
);
}
let resp = handle_ollama_chat(
State(state.clone()),
HeaderMap::new(),
Json(chat_request(serde_json::json!({
"model": "docker.io/ai/m:latest",
"messages": [],
}))),
)
.await
.expect("pre-load must not error");
let value = chat_response_json(resp).await;
assert_eq!(value["done_reason"], "load");
assert_eq!(value["message"]["content"], "");
assert!(
state
.0
.manager
.lock()
.await
.running
.contains_key("docker.io/ai/m:latest"),
"a pre-load must leave the model loaded"
);
}
#[test]
fn vllm_omni_engine_has_its_own_label() {
assert_eq!(Engine::VllmOmni.label(), "vllm-omni");
}
#[test]
fn omni_image_request_translates_llama_server_fields() {
let req = serde_json::json!({
"model": "nvidia/Cosmos3-Edge",
"prompt": "a manatee",
"stream": true,
"response_format": "b64_json",
"width": 640,
"height": 480,
"steps": 8,
"seed": 42,
"cfg_scale": 5.0,
"negative_prompt": "blurry"
});
let (out, stream) = omni_image_request(req).unwrap();
assert!(stream);
assert_eq!(
out,
serde_json::json!({
"model": "nvidia/Cosmos3-Edge",
"prompt": "a manatee",
"response_format": "b64_json",
"size": "640x480",
"num_inference_steps": 8,
"seed": 42,
"guidance_scale": 5.0,
"negative_prompt": "blurry"
})
);
}
#[test]
fn omni_image_request_leaves_a_native_request_alone_and_invents_nothing() {
let req = serde_json::json!({
"model": "m", "prompt": "p", "size": "1024x1024",
"num_inference_steps": 50, "guidance_scale": 7.0
});
let (out, stream) = omni_image_request(req.clone()).unwrap();
assert!(!stream);
let mut expected = req;
expected["response_format"] = "b64_json".into();
assert_eq!(out, expected);
let (out, _) =
omni_image_request(serde_json::json!({"model": "m", "prompt": "p"})).unwrap();
assert!(out.get("size").is_none());
assert!(out.get("num_inference_steps").is_none());
let (out, _) = omni_image_request(
serde_json::json!({"prompt": "p", "width": 0, "height": 0, "steps": 0}),
)
.unwrap();
assert!(out.get("size").is_none());
assert!(out.get("num_inference_steps").is_none());
let (out, _) = omni_image_request(
serde_json::json!({"prompt": "p", "steps": 8, "num_inference_steps": 30}),
)
.unwrap();
assert_eq!(out["num_inference_steps"], 30);
let err = omni_image_request(serde_json::json!({"prompt": "p", "width": 768})).unwrap_err();
assert!(err.contains("both width and height"), "{err}");
}
#[test]
fn omni_video_fields_translate_and_round_seconds() {
let req = serde_json::json!({
"model": "nvidia/Cosmos3-Edge",
"prompt": "waves",
"stream": false,
"seconds": 2.4,
"fps": 24,
"steps": 20,
"cfg_scale": 5.0,
"audio": false,
"seed": 7,
"extra_params": {"guardrails": false}
});
let fields: std::collections::HashMap<String, String> =
omni_video_fields(&req).into_iter().collect();
assert_eq!(fields["model"], "nvidia/Cosmos3-Edge");
assert_eq!(fields["prompt"], "waves");
assert_eq!(fields["seconds"], "2", "SecondStr is a whole number");
assert_eq!(fields["fps"], "24");
assert_eq!(fields["num_inference_steps"], "20");
assert_eq!(fields["guidance_scale"], "5.0");
assert_eq!(fields["generate_sound"], "false");
assert_eq!(fields["seed"], "7");
assert_eq!(fields["extra_params"], r#"{"guardrails":false}"#);
assert!(!fields.contains_key("stream"));
assert!(!fields.contains_key("steps"));
let fields: std::collections::HashMap<String, String> =
omni_video_fields(&serde_json::json!({"prompt": "p", "seconds": 0.3}))
.into_iter()
.collect();
assert_eq!(fields["seconds"], "1");
let fields: std::collections::HashMap<String, String> =
omni_video_fields(&serde_json::json!({"prompt": "p", "frames": 33, "num_frames": 49}))
.into_iter()
.collect();
assert_eq!(fields["num_frames"], "49");
}
#[test]
fn multipart_form_is_well_formed_and_readable_back() {
let fields = vec![
("model".to_string(), "m".to_string()),
("prompt".to_string(), "a b\r\nc".to_string()),
];
let (body, content_type) = multipart_form(&fields);
let boundary = content_type
.strip_prefix("multipart/form-data; boundary=")
.unwrap();
let text = String::from_utf8(body.clone()).unwrap();
assert!(text.starts_with(&format!("--{boundary}\r\n")));
assert!(text.ends_with(&format!("--{boundary}--\r\n")));
let (filtered, _) = multipart_form(&[
("ok_1".to_string(), "v".to_string()),
("bad\"\r\nX: y".to_string(), "v".to_string()),
]);
let filtered = String::from_utf8(filtered).unwrap();
assert!(filtered.contains("name=\"ok_1\""));
assert!(!filtered.contains("name=\"bad\""), "{filtered}");
assert!(!filtered.contains("X: y"), "{filtered}");
let (_, ct2) = multipart_form(&[("prompt".to_string(), format!("x{boundary}y"))]);
assert_ne!(ct2, content_type);
let mut headers = HeaderMap::new();
headers.insert("content-type", content_type.parse().unwrap());
let body = Bytes::from(body);
let rt = tokio::runtime::Runtime::new().unwrap();
assert_eq!(
rt.block_on(multipart_text_field(&body, &headers, "prompt")),
Some("a b\r\nc".to_string())
);
assert_eq!(
rt.block_on(multipart_text_field(&body, &headers, "model")),
Some("m".to_string())
);
}
#[test]
fn embed_inputs_accepts_a_string_or_string_array_only() {
assert_eq!(embed_inputs(&serde_json::json!("hi")).unwrap(), vec!["hi"]);
assert_eq!(
embed_inputs(&serde_json::json!(["a", "b"])).unwrap(),
vec!["a", "b"]
);
assert!(embed_inputs(&serde_json::Value::Null).unwrap().is_empty());
assert!(embed_inputs(&serde_json::json!("")).unwrap().is_empty());
for bad in [
serde_json::json!(1),
serde_json::json!(["a", 1]),
serde_json::json!({}),
] {
let err = embed_inputs(&bad).unwrap_err();
assert_eq!(err.1, StatusCode::BAD_REQUEST);
}
}
#[test]
fn normalize_in_place_yields_a_unit_vector_and_rejects_non_finite() {
let mut v = vec![3.0f32, 4.0];
normalize_in_place(&mut v).unwrap();
assert!((v[0] - 0.6).abs() < 1e-6 && (v[1] - 0.8).abs() < 1e-6);
let mut zero = vec![0.0f32, 0.0];
normalize_in_place(&mut zero).unwrap();
assert_eq!(zero, vec![0.0, 0.0]);
assert!(normalize_in_place(&mut [1.0, f32::NAN]).is_err());
assert!(normalize_in_place(&mut [f32::INFINITY]).is_err());
let mut huge = vec![f32::MAX, f32::MAX];
normalize_in_place(&mut huge).unwrap();
assert!((huge[0] - std::f32::consts::FRAC_1_SQRT_2).abs() < 1e-6);
let mut tiny = vec![1e-20f32, 0.0];
normalize_in_place(&mut tiny).unwrap();
assert_eq!(tiny, vec![1.0, 0.0]);
}
#[test]
fn is_json_content_type_matches_json_and_json_suffixed_types_only() {
assert!(is_json_content_type("application/json"));
assert!(is_json_content_type("Application/JSON; charset=utf-8"));
assert!(is_json_content_type("application/vnd.api+json"));
assert!(!is_json_content_type("text/plain;charset=UTF-8"));
assert!(!is_json_content_type("application/x-www-form-urlencoded"));
assert!(!is_json_content_type(""));
}
#[tokio::test]
async fn ollama_routes_accept_json_without_a_json_content_type() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let app = build_router(test_state(), false);
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let url = format!("http://127.0.0.1:{}/api/show", addr.port());
let post = |content_type: Option<&str>, origin: Option<&str>| {
let mut req = Client::new()
.post(&url)
.body(r#"{"model":"hf:///bad ref"}"#);
if let Some(ct) = content_type {
req = req.header("content-type", ct);
}
if let Some(origin) = origin {
req = req.header("origin", origin);
}
req.send()
};
for content_type in [None, Some("text/plain;charset=UTF-8")] {
let resp = post(content_type, None).await.unwrap();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST, "{content_type:?}");
}
let resp = post(Some("text/plain"), Some("http://localhost:3000"))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
let resp = post(Some("text/plain"), Some("https://evil.example"))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
let resp = post(None, Some("https://evil.example")).await.unwrap();
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
}
#[tokio::test]
async fn create_refuses_unsupported_modelfile_fields_by_name() {
let state = test_state();
let req: OllamaCreateRequest = serde_json::from_value(serde_json::json!({
"model": "mine", "from": "gemma4",
"system": "be terse", "quantize": "q4_K_M", "template": "", "adapters": null
}))
.unwrap();
let resp = handle_create(State(state), Json(req)).await.into_response();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
.await
.unwrap();
let body = String::from_utf8_lossy(&body);
assert!(
body.contains("/api/create: quantize, system not supported"),
"{body}"
);
}
#[tokio::test]
async fn create_needs_exactly_one_of_from_or_files() {
let state = test_state();
for body in [
serde_json::json!({"model": "mine"}),
serde_json::json!({"model": format!("mine@sha256:{}", "a".repeat(64)), "from": "a"}),
serde_json::json!({"model": "mine", "from": "a", "files": {"m.gguf": "sha256:00"}}),
] {
let req: OllamaCreateRequest = serde_json::from_value(body).unwrap();
let resp = handle_create(State(state.clone()), Json(req))
.await
.into_response();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
}
}
#[tokio::test]
async fn evict_if_retagged_drops_only_a_runner_serving_stale_content() {
let state = test_state();
let key = "hf.co/o/m:latest";
let mut fixture = running_model_fixture(None, Duration::ZERO, 0);
fixture.digest = "sha256:old".into();
state
.0
.manager
.lock()
.await
.running
.insert(key.into(), fixture);
evict_if_retagged(&state, key, "sha256:old").await;
assert!(state.0.manager.lock().await.running.contains_key(key));
evict_if_retagged(&state, "hf.co/o/m", "sha256:new").await;
assert!(!state.0.manager.lock().await.running.contains_key(key));
}
#[test]
fn staged_file_refuses_a_name_that_is_not_a_bare_file_name() {
let state = test_state();
let digest = format!("sha256:{}", "a".repeat(64));
for name in ["../escape.gguf", "sub/dir.gguf", ".", ""] {
let err = staged_file(&state, name, &digest).unwrap_err();
assert_eq!(err.1, StatusCode::BAD_REQUEST, "{name:?}");
}
}
#[test]
fn staged_blob_path_requires_a_well_formed_sha256_digest() {
let state = test_state();
let ok = staged_blob_path(&state, &format!("sha256:{}", "a".repeat(64))).unwrap();
assert_eq!(ok, state.0.cache_path.join("blobs").join("a".repeat(64)));
for bad in [
"sha256:abc",
"md5:0000",
&format!("sha256:{}", "g".repeat(64)),
"../x",
] {
assert_eq!(
staged_blob_path(&state, bad).unwrap_err().1,
StatusCode::BAD_REQUEST
);
}
}
#[test]
fn filter_non_function_tools_drops_non_function_entries_only() {
let mut req = serde_json::json!({
"tools": [
{"type": "function", "name": "exec_command"},
{"type": "namespace", "name": "multi_agent_v1", "tools": [{"type": "function", "name": "close_agent"}]},
{"type": "web_search"},
{"type": "function", "name": "update_plan"}
]
});
filter_non_function_tools(&mut req);
assert_eq!(
req["tools"],
serde_json::json!([
{"type": "function", "name": "exec_command"},
{"type": "function", "name": "update_plan"}
])
);
}
#[test]
fn filter_non_function_tools_is_a_no_op_without_a_tools_field() {
let mut req = serde_json::json!({"model": "x"});
filter_non_function_tools(&mut req);
assert_eq!(req, serde_json::json!({"model": "x"}));
}
#[test]
fn consolidate_responses_instructions_folds_developer_and_system_input_items() {
let mut req = serde_json::json!({
"model": "docker.io/ai/qwen3.5:0.8b",
"instructions": "top-level instructions",
"input": [
{"type": "message", "role": "developer", "content": [
{"type": "input_text", "text": "permissions instructions"}
]},
{"type": "message", "role": "user", "content": [
{"type": "input_text", "text": "hi"}
]},
{"type": "message", "role": "system", "content": "a plain-string system item"}
]
});
consolidate_responses_instructions(&mut req);
assert_eq!(
req["instructions"],
"top-level instructions\n\npermissions instructions\n\na plain-string system item"
);
assert_eq!(
req["input"],
serde_json::json!([
{"type": "message", "role": "user", "content": [
{"type": "input_text", "text": "hi"}
]}
])
);
}
#[test]
fn consolidate_responses_instructions_is_a_no_op_without_developer_or_system_items() {
let mut req = serde_json::json!({
"instructions": "top-level instructions",
"input": [{"type": "message", "role": "user", "content": "hi"}]
});
let before = req.clone();
consolidate_responses_instructions(&mut req);
assert_eq!(req, before);
}
#[test]
fn set_response_model_overwrites_an_existing_model_field_and_leaves_a_missing_one_alone() {
let mut with_model = serde_json::json!({"model": "/abs/path/to/model", "id": "x"});
set_response_model(&mut with_model, "gemma4:latest");
assert_eq!(
with_model,
serde_json::json!({"model": "gemma4:latest", "id": "x"})
);
let mut without_model = serde_json::json!({"id": "x"});
set_response_model(&mut without_model, "gemma4:latest");
assert_eq!(without_model, serde_json::json!({"id": "x"}));
let mut event =
serde_json::json!({"type": "response.created", "response": {"model": "wire"}});
set_response_model(&mut event, "canonical");
assert_eq!(event["response"]["model"], "canonical");
}
#[test]
fn rewrite_json_response_model_rewrites_a_json_body_and_leaves_every_other_field_alone() {
let raw = Bytes::from(
serde_json::to_vec(&serde_json::json!({
"id": "chatcmpl-1",
"model": "/home/user/.local/share/llmman/cache/abcd/model-dir",
"choices": [{"message": {"content": "hi"}}]
}))
.unwrap(),
);
let rewritten = rewrite_json_response_model(&raw, "gemma4:latest");
let value: serde_json::Value = serde_json::from_slice(&rewritten).unwrap();
assert_eq!(value["model"], "gemma4:latest");
assert_eq!(value["id"], "chatcmpl-1");
assert_eq!(value["choices"][0]["message"]["content"], "hi");
}
#[test]
fn rewrite_json_response_model_passes_non_json_bodies_through_unchanged() {
let raw = Bytes::from_static(b"not json at all");
assert_eq!(rewrite_json_response_model(&raw, "gemma4:latest"), raw);
}
#[test]
fn rewrite_sse_line_model_rewrites_only_the_model_field_of_a_data_line() {
let line = r#"data: {"id":"1","model":"/abs/path","choices":[{"delta":{"content":"h"}}]}"#;
let rewritten = rewrite_sse_line_model(line, "gemma4:latest");
let payload = rewritten.strip_prefix("data: ").expect("data: prefix");
let value: serde_json::Value = serde_json::from_str(payload).unwrap();
assert_eq!(value["model"], "gemma4:latest");
assert_eq!(value["id"], "1");
assert_eq!(value["choices"][0]["delta"]["content"], "h");
}
#[test]
fn rewrite_sse_line_model_leaves_the_done_sentinel_and_blank_separators_untouched() {
assert_eq!(
rewrite_sse_line_model("data: [DONE]", "gemma4:latest"),
"data: [DONE]"
);
assert_eq!(rewrite_sse_line_model("", "gemma4:latest"), "");
}
#[test]
fn rewrite_sse_line_model_passes_a_non_json_data_line_through_unchanged() {
assert_eq!(
rewrite_sse_line_model("data: not json", "gemma4:latest"),
"data: not json"
);
}
#[tokio::test]
async fn mlx_embeddings_unsupported_response_explains_why_and_names_the_model() {
let resp = mlx_embeddings_unsupported_response("gemma4:latest");
assert_eq!(resp.status(), StatusCode::NOT_IMPLEMENTED);
let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
.await
.unwrap();
let value: serde_json::Value = serde_json::from_slice(&body).unwrap();
let message = value["error"]["message"].as_str().unwrap();
assert!(message.contains("gemma4:latest"));
assert!(message.contains("mlx_lm.server"));
assert!(message.contains("/v1/embeddings"));
}
#[test]
fn responses_input_item_text_ported_ollama_polymorphic_input_cases() {
assert_eq!(
responses_input_item_text(&serde_json::json!({"role": "user", "content": "plain"})),
Some("plain".into())
);
assert_eq!(
responses_input_item_text(&serde_json::json!({"role": "user", "content": [
{"type": "input_text", "text": "a"},
{"type": "output_text", "text": "b"}
]})),
Some("ab".into())
);
assert_eq!(
responses_input_item_text(&serde_json::json!({"content": [{"type": "input_image"}]})),
Some(String::new())
);
assert_eq!(
responses_input_item_text(&serde_json::json!({"type": "function_call", "name": "f"})),
None
);
assert_eq!(
responses_input_item_text(&serde_json::json!({"content": 42})),
None
);
}
#[test]
fn option_extractors_ported_ollama_options_blob_cases() {
let opts = Some(serde_json::json!({
"temperature": 0.5,
"top_p": 0.9,
"num_predict": 128,
"stop": ["### User:"]
}));
assert_eq!(opt_f64(&opts, "temperature"), Some(0.5));
assert_eq!(opt_f64(&opts, "top_p"), Some(0.9));
assert_eq!(opt_u32(&opts, "num_predict"), Some(128));
assert_eq!(opt_f64(&opts, "repeat_penalty"), None);
assert_eq!(opt_u32(&opts, "stop"), None);
assert_eq!(opt_f64(&None, "temperature"), None);
assert_eq!(opt_u32(&None, "num_predict"), None);
}
#[test]
fn opt_num_thread_accepts_a_positive_integer() {
assert_eq!(
opt_num_thread(&Some(serde_json::json!({"num_thread": 4}))),
Some(4)
);
assert_eq!(
opt_num_thread(&Some(serde_json::json!({"num_thread": 1}))),
Some(1)
);
}
#[test]
fn opt_num_thread_rejects_zero_negative_and_non_integer_values() {
let cases = [
(
serde_json::json!(0),
"zero: llama-server rejects --threads 0",
),
(serde_json::json!(-1), "negative"),
(serde_json::json!(2.5), "fractional"),
(serde_json::json!("4"), "string, not a JSON number"),
(
serde_json::json!(u64::from(u32::MAX) + 1),
"above u32: must not truncate to --threads 0",
),
];
for (value, why) in cases {
let opts = Some(serde_json::json!({ "num_thread": value }));
assert_eq!(opt_num_thread(&opts), None, "{why}");
}
assert_eq!(opt_num_thread(&Some(serde_json::json!({}))), None);
assert_eq!(opt_num_thread(&None), None);
}
#[test]
fn keyed_lock_is_per_key_and_release_only_drops_unreferenced_entries() {
let registry: StdMutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>> =
StdMutex::new(HashMap::new());
let a1 = keyed_lock(®istry, "model-a");
let a2 = keyed_lock(®istry, "model-a");
assert!(Arc::ptr_eq(&a1, &a2), "same key must return the same lock");
let b = keyed_lock(®istry, "model-b");
assert!(
!Arc::ptr_eq(&a1, &b),
"different keys must not share a lock"
);
drop(a1);
release_keyed_lock(®istry, "model-a");
assert!(registry.lock().unwrap().contains_key("model-a"));
drop(a2);
release_keyed_lock(®istry, "model-a");
assert!(!registry.lock().unwrap().contains_key("model-a"));
drop(b);
}
#[tokio::test(flavor = "multi_thread")]
async fn load_lock_serializes_same_model_but_not_different_models() {
let slow = load_lock("test-load-lock-slow-model");
let guard = slow.lock().await;
let other = load_lock("test-load-lock-other-model");
let _other_guard =
tokio::time::timeout(std::time::Duration::from_millis(200), other.lock())
.await
.expect("a different model's load must not block on an unrelated one");
let same = load_lock("test-load-lock-slow-model");
assert!(
tokio::time::timeout(std::time::Duration::from_millis(50), same.lock())
.await
.is_err(),
"a second load of the same model must block while the first is in flight"
);
drop(guard);
let same_guard = tokio::time::timeout(std::time::Duration::from_millis(200), same.lock())
.await
.expect("must acquire promptly once the first load releases");
drop(same_guard);
drop(_other_guard);
drop(same);
drop(other);
drop(slow);
release_load_lock("test-load-lock-slow-model");
release_load_lock("test-load-lock-other-model");
}
#[test]
fn ensure_model_key_pipeline_converges_aliases_before_the_lock() {
let store = std::env::temp_dir();
let tagless = load_identity(&store, "regression-test-model").unwrap();
let tagged = load_identity(&store, "regression-test-model:latest").unwrap();
let digest = load_identity(&store, "regression-test-model@sha256:0000").unwrap();
assert_eq!(
tagless, tagged,
"tagless and :latest aliases must resolve to one key"
);
assert_eq!(
tagless, digest,
"the digest spelling must resolve to the same key"
);
let a = load_lock(&tagless);
let b = load_lock(&tagged);
let c = load_lock(&digest);
assert!(
Arc::ptr_eq(&a, &b) && Arc::ptr_eq(&a, &c),
"all three aliases must take the same load lock"
);
drop(a);
drop(b);
drop(c);
release_load_lock(&tagless);
}
#[tokio::test]
async fn ensure_model_rejects_an_invalid_ref_with_400() {
let state = test_state();
let err = ensure_model(&state, "hf.co/../x", None, None)
.await
.err()
.expect("invalid ref must be rejected");
assert_eq!(err.into_response().status(), StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn handle_push_rejects_an_invalid_ref_with_400() {
let state = test_state();
let req = OllamaPushRequest {
model: "hf.co/../x".to_string(),
name: String::new(),
};
let resp = handle_push(State(state), Json(req)).await.into_response();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn handle_push_returns_404_for_a_model_not_in_the_store() {
let state = test_state();
let req = OllamaPushRequest {
model: "hf.co/does-not-exist/nowhere".to_string(),
name: String::new(),
};
let resp = handle_push(State(state), Json(req)).await.into_response();
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
}
#[test]
fn a_push_outcome_reports_its_digest_on_the_stream() {
let lines = PushOutcome {
digest: "sha256:abc".into(),
}
.into_lines();
assert_eq!(lines, vec![serde_json::json!({"digest": "sha256:abc"})]);
}
#[test]
fn pull_notices_go_out_as_notice_lines() {
let lines = vec!["warning: unsigned".to_string()].into_lines();
assert_eq!(
lines,
vec![serde_json::json!({"notice": "warning: unsigned"})]
);
}
#[test]
fn an_ollama_push_body_still_deserializes() {
let req: OllamaPushRequest =
serde_json::from_str(r#"{"model":"docker.io/org/model:v1"}"#).unwrap();
assert_eq!(req.model, "docker.io/org/model:v1");
let req: OllamaPushRequest = serde_json::from_str(r#"{"name":"x"}"#).unwrap();
assert_eq!(req.name, "x");
}
#[tokio::test]
async fn handle_delete_rejects_an_invalid_ref_with_400() {
let state = test_state();
let req = OllamaDeleteRequest {
model: "hf.co//foo".to_string(),
name: None,
};
let resp = handle_delete(State(state), Json(req)).await.into_response();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn handle_show_rejects_an_invalid_ref_with_400() {
let state = test_state();
let req = OllamaShowRequest {
model: "hf:///foo".to_string(),
name: None,
};
let resp = handle_show(State(state), Json(req)).await.into_response();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn handle_show_answers_404_for_a_model_not_in_the_store() {
let state = test_state();
let req = OllamaShowRequest {
model: "docker.io/ai/nothing-here".to_string(),
name: None,
};
let resp = handle_show(State(state), Json(req)).await.into_response();
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
}
#[tokio::test]
async fn load_lock_release_actually_removes_the_entry_once_unused() {
let key = "test-load-lock-release-cleanup";
let lock = load_lock(key);
let guard = lock.lock().await;
drop(guard);
drop(lock);
release_load_lock(key);
assert!(
!LOAD_LOCKS.lock().unwrap().contains_key(key),
"release_load_lock must drop the registry entry once nothing else references it"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn load_lock_guard_releases_on_task_cancellation() {
let key = "test-load-lock-guard-cancel";
let started = Arc::new(tokio::sync::Notify::new());
let started_tx = started.clone();
let handle = tokio::spawn(async move {
let _guard = acquire_load_lock("test-load-lock-guard-cancel").await;
started_tx.notify_one();
std::future::pending::<()>().await;
});
started.notified().await;
handle.abort();
let _ = handle.await;
assert!(
!LOAD_LOCKS.lock().unwrap().contains_key(key),
"aborting a task holding LoadLockGuard must still release the registry entry"
);
}
#[test]
fn ollama_pull_request_accepts_a_name_only_body() {
let req: OllamaPullRequest =
serde_json::from_value(serde_json::json!({"name": "docker.io/ai/gemma4:E2B"}))
.expect("a name-only body must still deserialize");
assert_eq!(req.model, "");
assert_eq!(req.name, "docker.io/ai/gemma4:E2B");
}
#[test]
fn ollama_pull_request_accepts_a_model_only_body() {
let req: OllamaPullRequest =
serde_json::from_value(serde_json::json!({"model": "docker.io/ai/gemma4:E2B"}))
.expect("a model-only body must still deserialize");
assert_eq!(req.model, "docker.io/ai/gemma4:E2B");
assert_eq!(req.name, "");
}
#[test]
fn ollama_push_request_accepts_a_name_only_body() {
let req: OllamaPushRequest =
serde_json::from_value(serde_json::json!({"name": "docker.io/ai/gemma4:E2B"}))
.expect("a name-only body must still deserialize");
assert_eq!(req.model, "");
assert_eq!(req.name, "docker.io/ai/gemma4:E2B");
}
fn multipart_body(fields: &[(&str, &str)]) -> (Bytes, HeaderMap) {
let boundary = "llmman-test-boundary";
let mut body = String::new();
for (name, value) in fields {
body.push_str(&format!(
"--{boundary}\r\nContent-Disposition: form-data; name=\"{name}\"\r\n\r\n{value}\r\n"
));
}
body.push_str(&format!("--{boundary}--\r\n"));
let mut headers = HeaderMap::new();
headers.insert(
"content-type",
format!("multipart/form-data; boundary={boundary}")
.parse()
.unwrap(),
);
(Bytes::from(body), headers)
}
#[tokio::test]
async fn multipart_text_field_finds_a_named_field_among_several() {
let (body, headers) = multipart_body(&[
("language", "en"),
("model", "docker.io/ai/whisper:latest"),
("response_format", "json"),
]);
assert_eq!(
multipart_text_field(&body, &headers, "model").await,
Some("docker.io/ai/whisper:latest".to_string())
);
assert_eq!(
multipart_text_field(&body, &headers, "language").await,
Some("en".to_string())
);
}
#[tokio::test]
async fn multipart_text_field_leaves_the_original_body_untouched() {
let (body, headers) = multipart_body(&[("model", "m"), ("prompt", "hello")]);
let before = body.clone();
let _ = multipart_text_field(&body, &headers, "model").await;
assert_eq!(body, before);
}
#[tokio::test]
async fn multipart_text_field_is_none_for_a_missing_field_or_non_multipart_body() {
let (body, headers) = multipart_body(&[("language", "en")]);
assert_eq!(multipart_text_field(&body, &headers, "model").await, None);
let plain_body = Bytes::from_static(b"{\"model\":\"m\"}");
let mut json_headers = HeaderMap::new();
json_headers.insert("content-type", "application/json".parse().unwrap());
assert_eq!(
multipart_text_field(&plain_body, &json_headers, "model").await,
None
);
assert_eq!(
multipart_text_field(&plain_body, &HeaderMap::new(), "model").await,
None
);
}
const MOCK_SSE: &str = concat!(
"data: {\"choices\":[{\"delta\":{\"content\":\"Hello\"},\"finish_reason\":null}]}\n",
"data: {\"choices\":[{\"delta\":{\"content\":\", world\"},\"finish_reason\":\"stop\"}]}\n",
"data: {\"choices\":[],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":3}}\n",
"data: [DONE]\n",
);
async fn run_stream_ollama(streaming: bool) -> (String, String) {
let app = Router::new().route(
"/v1/chat/completions",
post(|| async { ([("content-type", "text/event-stream")], MOCK_SSE) }),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let resp = stream_ollama(
streaming,
Client::new(),
Target::Local(addr.port()),
OAIChatRequest {
messages: vec![OAIMessage::text("user", "hi".to_string())],
stream: true,
..Default::default()
},
ActivityGuard::new(&test_state(), "m"),
Instant::now(),
Duration::from_millis(1),
|delta| OllamaChatChunk {
model: "m".into(),
created_at: now_rfc3339(),
message: OllamaMessage {
role: "assistant".into(),
content: delta.content,
thinking: delta.thinking,
tool_calls: delta.tool_calls,
..Default::default()
},
done: delta.done,
done_reason: delta.done_reason,
metrics: delta.metrics,
},
)
.await
.expect("mock backend answers");
let ct = resp.headers()["content-type"].to_str().unwrap().to_string();
let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
.await
.unwrap();
(ct, String::from_utf8(body.to_vec()).unwrap())
}
#[tokio::test]
async fn stream_ollama_false_returns_one_json_object() {
let (content_type, body) = run_stream_ollama(false).await;
assert_eq!(content_type, "application/json");
let one: serde_json::Value =
serde_json::from_str(&body).expect("the whole body is one JSON object");
assert_eq!(one["message"]["content"], "Hello, world");
assert_eq!(one["done"], true);
assert_eq!(one["done_reason"], "stop");
assert_eq!(one["prompt_eval_count"], 4);
assert_eq!(one["eval_count"], 3);
assert_eq!(one["load_duration"], 1_000_000);
assert!(one["total_duration"].is_u64());
assert!(one.get("eval_duration").is_none());
}
#[tokio::test]
async fn stream_ollama_true_returns_ndjson_chunks() {
let (content_type, body) = run_stream_ollama(true).await;
assert_eq!(content_type, "application/x-ndjson");
let chunks: Vec<serde_json::Value> = body
.lines()
.filter(|l| !l.trim().is_empty())
.map(|l| serde_json::from_str(l).expect("each line is its own object"))
.collect();
assert!(chunks.len() > 1, "got {} chunks", chunks.len());
let joined: String = chunks
.iter()
.map(|c| c["message"]["content"].as_str().unwrap_or_default())
.collect();
assert_eq!(joined, "Hello, world");
let done: Vec<_> = chunks.iter().filter(|c| c["done"] == true).collect();
assert_eq!(done.len(), 1);
assert_eq!(done[0]["eval_count"], 3);
assert!(chunks[0].get("eval_count").is_none());
}
fn rendered_registry() -> String {
metrics::render(&metrics::Snapshot {
version: "test".into(),
start_time_seconds: 0,
scheduling_requests_in_flight: 0,
scheduling_capacity: 1,
models_loaded: 0,
models_loading: 0,
models: Vec::new(),
})
}
fn unload_count(model: &str, reason: &str) -> u64 {
let rendered = rendered_registry();
let needle =
format!("llmman_model_unloads_total{{model=\"{model}\",reason=\"{reason}\"}} ");
rendered
.lines()
.find_map(|l| l.strip_prefix(needle.as_str()))
.and_then(|v| v.parse().ok())
.unwrap_or(0)
}
#[tokio::test]
async fn evicting_for_an_oom_retry_counts_every_model_it_freed() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"loading-now".into(),
running_model_fixture(None, Duration::from_secs(0), 0),
);
mgr.running.insert(
"evictable-a".into(),
running_model_fixture(None, Duration::from_secs(0), 0),
);
mgr.running.insert(
"evictable-b".into(),
running_model_fixture(None, Duration::from_secs(0), 0),
);
mgr.running.insert(
"busy".into(),
running_model_fixture(None, Duration::from_secs(0), 1),
);
}
let evicted = evict_other_models(&state, "loading-now").await;
assert!(evicted, "two idle models were evictable");
assert_eq!(unload_count("evictable-a", "oom"), 1);
assert_eq!(unload_count("evictable-b", "oom"), 1);
assert_eq!(
unload_count("busy", "oom"),
0,
"the in-flight model was left alone, so it must not be counted"
);
let mgr = state.0.manager.lock().await;
assert!(mgr.running.contains_key("loading-now"));
assert!(mgr.running.contains_key("busy"));
assert!(!mgr.running.contains_key("evictable-a"));
assert!(!mgr.running.contains_key("evictable-b"));
}
#[tokio::test]
async fn the_reaper_labels_an_already_dead_backend_crashed_not_idle() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
let mut dead =
running_model_fixture(Some(Duration::from_secs(1)), Duration::from_secs(10), 0);
match &mut dead.process {
ModelProcess::Local(_, child, _) | ModelProcess::Container(_, _, child) => {
child.kill().await.expect("kill the placeholder process")
}
}
mgr.running.insert("died-then-expired".into(), dead);
}
reap_idle_models_once(&state).await;
assert_eq!(
unload_count("died-then-expired", "crashed"),
1,
"a backend already dead when the reaper reached it must count as crashed"
);
assert_eq!(
unload_count("died-then-expired", "idle"),
0,
"and must not also be counted idle"
);
assert!(
!state
.0
.manager
.lock()
.await
.running
.contains_key("died-then-expired"),
"it must still be unloaded"
);
}
#[tokio::test]
async fn the_metrics_route_label_is_the_template_not_the_request_path() {
let app = Router::new()
.route(
"/test-matched-path/:id",
get(|| async { StatusCode::NO_CONTENT }),
)
.layer(middleware::from_fn(track_metrics));
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
Client::new()
.get(format!(
"http://127.0.0.1:{}/test-matched-path/abc123",
addr.port()
))
.send()
.await
.expect("request reaches the test router");
let rendered = rendered_registry();
assert!(
rendered.contains(
"llmman_http_requests_total{route=\"/test-matched-path/:id\",status=\"204\"} 1\n"
),
"{rendered}"
);
assert!(!rendered.contains("abc123"), "{rendered}");
}
#[tokio::test]
async fn the_scrape_endpoint_is_outside_the_cors_layer() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let app = build_router(test_state(), true);
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let get = |path: &'static str| {
let url = format!("http://127.0.0.1:{}{path}", addr.port());
async move {
Client::new()
.get(url)
.header("origin", "http://localhost:3000")
.send()
.await
.expect("request reaches the test router")
}
};
let version = get("/api/version").await;
assert_eq!(
version
.headers()
.get("access-control-allow-origin")
.and_then(|v| v.to_str().ok()),
Some("http://localhost:3000"),
"a page served from localhost is meant to be an allowed origin"
);
let scrape = get("/metrics").await;
assert_eq!(scrape.status(), StatusCode::OK, "the scrape route answers");
assert!(
!scrape.headers().contains_key("access-control-allow-origin"),
"a browser page can read the scrape endpoint"
);
}
#[tokio::test]
async fn the_scrape_endpoint_is_absent_unless_the_operator_enabled_it() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let app = build_router(test_state(), false);
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let url = format!("http://127.0.0.1:{}", addr.port());
let scrape = Client::new()
.get(format!("{url}/metrics"))
.send()
.await
.expect("request reaches the test router");
assert_eq!(scrape.status(), StatusCode::NOT_FOUND);
let version = Client::new()
.get(format!("{url}/api/version"))
.send()
.await
.expect("request reaches the test router");
assert_eq!(version.status(), StatusCode::OK);
Client::new()
.get(format!("{url}/ui/app.css"))
.send()
.await
.expect("request reaches the test router");
assert!(
!rendered_registry().contains("route=\"/ui/*path\""),
"a router built with metrics disabled must not write to the registry"
);
}
async fn serve_router(state: AppState) -> String {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let app = build_router(state, false);
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
format!("http://127.0.0.1:{}", addr.port())
}
#[tokio::test]
async fn the_root_is_the_web_ui_for_browsers_and_a_liveness_line_for_the_rest() {
let url = serve_router(test_state()).await;
let curl = Client::new().get(&url).send().await.unwrap();
assert_eq!(curl.status(), StatusCode::OK);
assert_eq!(curl.text().await.unwrap(), "llmman is running");
let browser = Client::new()
.get(&url)
.header("accept", "text/html,application/xhtml+xml")
.send()
.await
.unwrap();
assert_eq!(browser.status(), StatusCode::OK);
assert_eq!(
browser.headers()["content-type"],
"text/html; charset=utf-8"
);
assert_eq!(browser.headers()["content-encoding"], "gzip");
let mut html = String::new();
std::io::Read::read_to_string(
&mut flate2::read::GzDecoder::new(&browser.bytes().await.unwrap()[..]),
&mut html,
)
.unwrap();
assert!(html.contains("<!doctype html>"), "{html}");
assert!(html.contains("ui/app.js"), "{html}");
assert!(
!html.contains("\"/ui/"),
"asset paths must be relative: {html}"
);
}
#[tokio::test]
async fn web_ui_assets_are_gzipped_and_revalidate_by_etag() {
let url = serve_router(test_state()).await;
let client = Client::builder().no_gzip().build().unwrap();
let first = client
.get(format!("{url}/ui/app.css"))
.send()
.await
.unwrap();
assert_eq!(first.status(), StatusCode::OK);
assert_eq!(first.headers()["content-encoding"], "gzip");
assert_eq!(first.headers()["content-type"], "text/css; charset=utf-8");
assert_eq!(first.headers()["cache-control"], "no-cache");
let etag = first.headers()["etag"].to_str().unwrap().to_string();
assert!(etag.starts_with('"') && etag.ends_with('"'), "{etag}");
let body = first.bytes().await.unwrap();
assert_eq!(&body[..2], &[0x1f, 0x8b], "not gzip");
for tag in [etag.clone(), format!("W/{etag}"), "*".to_string()] {
let again = client
.get(format!("{url}/ui/app.css"))
.header("if-none-match", &tag)
.send()
.await
.unwrap();
assert_eq!(again.status(), StatusCode::NOT_MODIFIED, "{tag}");
}
for path in ["/", "/ui/index.html"] {
let page = client
.get(format!("{url}{path}"))
.header("accept", "text/html")
.send()
.await
.unwrap();
assert_eq!(page.headers()["x-frame-options"], "DENY", "{path}");
assert_eq!(
page.headers()["content-security-policy"],
"frame-ancestors 'none'",
"{path}"
);
}
let css = client
.get(format!("{url}/ui/app.css"))
.send()
.await
.unwrap();
assert!(!css.headers().contains_key("x-frame-options"));
for missing in ["/ui/nope.js", "/ui/../Cargo.toml", "/ui/vendor"] {
let r = client.get(format!("{url}{missing}")).send().await.unwrap();
assert_eq!(r.status(), StatusCode::NOT_FOUND, "{missing}");
}
}
fn shell_state(policy: shell::Policy) -> AppState {
let mut inner = test_inner(std::env::temp_dir());
inner.shell = policy;
AppState(Arc::new(inner))
}
fn ws_upgrade(client: &Client, url: &str, origin: Option<&str>) -> reqwest::RequestBuilder {
let mut req = client
.get(url)
.header("connection", "upgrade")
.header("upgrade", "websocket")
.header("sec-websocket-version", "13")
.header("sec-websocket-key", "dGhlIHNhbXBsZSBub25jZQ==");
if let Some(origin) = origin {
req = req.header("origin", origin);
}
req
}
#[tokio::test]
async fn a_disabled_shell_reports_why_and_refuses_the_upgrade() {
let url = serve_router(shell_state(shell::Policy {
disabled: Some("LLMMAN_SHELL is off".into()),
origins: default_allowed_origins(),
command: Vec::new(),
}))
.await;
let client = Client::new();
let status: shell::Status = client
.get(format!("{url}/llmman/shell"))
.send()
.await
.unwrap()
.json()
.await
.unwrap();
assert_eq!(
status,
shell::Status {
enabled: false,
reason: Some("LLMMAN_SHELL is off".into()),
}
);
let upgrade = ws_upgrade(&client, &format!("{url}/llmman/shell"), None)
.send()
.await
.unwrap();
assert_eq!(upgrade.status(), StatusCode::FORBIDDEN);
assert_eq!(upgrade.text().await.unwrap(), "LLMMAN_SHELL is off");
}
#[tokio::test]
async fn only_a_page_the_daemon_would_answer_cors_for_may_open_a_shell() {
let url = serve_router(shell_state(shell::Policy {
disabled: None,
origins: default_allowed_origins(),
command: Vec::new(),
}))
.await;
let client = Client::new();
let shell_url = format!("{url}/llmman/shell");
let evil = ws_upgrade(&client, &shell_url, Some("https://evil.example"))
.send()
.await
.unwrap();
assert_eq!(evil.status(), StatusCode::FORBIDDEN);
assert!(evil.text().await.unwrap().contains("evil.example"));
let status: shell::Status = client
.get(&shell_url)
.header("origin", "https://evil.example")
.send()
.await
.unwrap()
.json()
.await
.unwrap();
assert!(
!status.enabled,
"the status reply must agree with the refusal"
);
for origin in [
None,
Some("http://localhost:3000"),
Some("http://127.0.0.1"),
] {
let status: shell::Status = {
let mut req = client.get(&shell_url);
if let Some(o) = origin {
req = req.header("origin", o);
}
req.send().await.unwrap().json().await.unwrap()
};
assert!(status.enabled, "{origin:?} should be admitted");
}
}
#[tokio::test]
async fn a_shell_session_round_trips_bytes_and_reports_the_exit_status() {
use futures::SinkExt as _;
use tokio_tungstenite::tungstenite::Message as WsMessage;
#[cfg(windows)]
let command = ["cmd.exe", "/c", "echo got:marker& exit 7"];
#[cfg(not(windows))]
let command = ["sh", "-c", "read line; echo \"got:$line\"; exit 7"];
let url = serve_router(shell_state(shell::Policy {
disabled: None,
origins: default_allowed_origins(),
command: command.map(str::to_string).to_vec(),
}))
.await;
let ws_url = format!("{}/llmman/shell", url.replace("http://", "ws://"));
let (mut ws, _) = tokio_tungstenite::connect_async(&ws_url).await.unwrap();
ws.send(WsMessage::Text(
r#"{"resize":{"cols":120,"rows":40}}"#.into(),
))
.await
.unwrap();
#[cfg(not(windows))]
ws.send(WsMessage::Binary(b"marker\r".to_vec()))
.await
.unwrap();
let mut output = Vec::new();
let mut exit = None;
let deadline = tokio::time::sleep(Duration::from_secs(30));
tokio::pin!(deadline);
loop {
tokio::select! {
_ = &mut deadline => panic!("no exit frame; output so far: {:?}", String::from_utf8_lossy(&output)),
frame = ws.next() => match frame {
Some(Ok(WsMessage::Binary(bytes))) => {
if bytes.windows(4).any(|w| w == b"\x1b[6n") {
ws.send(WsMessage::Binary(b"\x1b[1;1R".to_vec())).await.unwrap();
}
output.extend_from_slice(&bytes);
}
Some(Ok(WsMessage::Text(text))) => {
exit = Some(serde_json::from_str::<serde_json::Value>(&text).unwrap());
}
Some(Ok(WsMessage::Close(_))) | None => break,
Some(Ok(_)) => {}
Some(Err(e)) => panic!("socket error: {e}"),
}
}
}
let text = String::from_utf8_lossy(&output);
assert!(text.contains("got:marker"), "{text:?}");
assert_eq!(exit, Some(serde_json::json!({ "exit": 7 })), "{text:?}");
}
#[tokio::test]
async fn timing_a_response_does_not_cost_it_its_content_length() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let app = build_router(test_state(), true);
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let url = format!("http://127.0.0.1:{}", addr.port());
for route in ["/api/version", "/metrics"] {
let response = Client::new()
.get(format!("{url}{route}"))
.send()
.await
.expect("request reaches the test router");
assert_eq!(response.status(), StatusCode::OK, "{route}");
assert!(
response.headers().contains_key("content-length"),
"{route} lost its content-length: {:?}",
response.headers()
);
assert!(
!response.headers().contains_key("transfer-encoding"),
"{route} was made chunked: {:?}",
response.headers()
);
}
}
#[tokio::test]
async fn the_total_duration_clock_stops_at_the_end_of_the_body_not_the_headers() {
let app = Router::new()
.route(
"/test-slow-body",
get(|| async {
Body::from_stream(futures::stream::once(async {
sleep(Duration::from_millis(200)).await;
Ok::<_, std::convert::Infallible>(Bytes::from_static(b"done"))
}))
}),
)
.layer(middleware::from_fn(track_metrics));
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let body = Client::new()
.get(format!("http://127.0.0.1:{}/test-slow-body", addr.port()))
.send()
.await
.expect("request reaches the test router")
.text()
.await
.expect("the body arrives in full");
assert_eq!(body, "done");
let rendered = rendered_registry();
assert!(
rendered.contains(
"llmman_http_request_ttfb_seconds_bucket{route=\"/test-slow-body\",le=\"0.05\"} 1\n"
),
"headers were ready immediately:\n{rendered}"
);
assert!(
rendered.contains(
"llmman_http_request_duration_seconds_bucket{route=\"/test-slow-body\",le=\"0.05\"} 0\n"
),
"the body took 200ms, so total duration cannot be under 50ms:\n{rendered}"
);
assert!(
rendered.contains(
"llmman_http_request_duration_seconds_count{route=\"/test-slow-body\"} 1\n"
),
"the body ended, so it was recorded exactly once:\n{rendered}"
);
}
#[tokio::test]
async fn record_prompt_logs_generation_requests_and_hands_the_body_on() {
let log =
std::env::temp_dir().join(format!("llmman-record-prompt-{}.jsonl", std::process::id()));
let _ = std::fs::remove_file(&log);
let mut inner = test_inner(std::env::temp_dir());
inner.prompt_log = Some(log.clone());
let state = AppState(Arc::new(inner));
let echo = || post(|body: Bytes| async move { body });
let app = Router::new()
.route("/api/chat", echo())
.route("/api/embed", echo())
.route("/v1/chat/completions", echo())
.layer(middleware::from_fn_with_state(state.clone(), record_prompt))
.with_state(state);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let send = |route: &str, body: &'static str, headers: &'static [(&str, &str)]| {
let url = format!("http://127.0.0.1:{}{route}", addr.port());
async move {
let mut map = reqwest::header::HeaderMap::new();
for (k, v) in [
("user-agent", "test-agent/1"),
("content-type", "application/json"),
]
.iter()
.chain(headers)
{
map.insert(*k, v.parse().unwrap());
}
Client::new()
.post(url)
.headers(map)
.body(body)
.send()
.await
.expect("request reaches the test router")
.text()
.await
.unwrap()
}
};
let chat = r#"{"model":"m","messages":[{"role":"user","content":"hello there"}]}"#;
assert_eq!(
send("/api/chat", chat, &[]).await,
chat,
"the handler got the body"
);
send("/api/embed", r#"{"model":"m","input":"vector me"}"#, &[]).await;
send("/api/chat", r#"{"model":"m","messages":[]}"#, &[]).await;
send(
"/api/chat",
chat,
&[
("content-type", "text/plain"),
("origin", "https://evil.example"),
],
)
.await;
Client::new()
.get(format!("http://127.0.0.1:{}/api/chat", addr.port()))
.body(chat)
.send()
.await
.expect("request reaches the test router");
send(
"/v1/chat/completions",
chat,
&[
("content-type", "text/plain"),
("origin", "https://evil.example"),
],
)
.await;
let entries = crate::promptlog::read(&log).unwrap();
let _ = std::fs::remove_file(&log);
let routes: Vec<&str> = entries.iter().map(|e| e.route.as_str()).collect();
assert_eq!(routes, ["/api/chat", "/v1/chat/completions"], "{entries:?}");
assert_eq!(entries[0].model, "m");
assert_eq!(entries[0].prompt, "hello there");
assert_eq!(entries[0].client.as_deref(), Some("test-agent/1"));
}
#[tokio::test]
async fn record_prompt_keeps_the_body_limit() {
let mut inner = test_inner(std::env::temp_dir());
inner.prompt_log = Some(std::env::temp_dir().join("llmman-unwritten.jsonl"));
let state = AppState(Arc::new(inner));
let app = Router::new()
.route("/api/chat", post(|body: Bytes| async move { body }))
.layer(middleware::from_fn_with_state(state.clone(), record_prompt))
.layer(DefaultBodyLimit::max(64))
.with_state(state);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let send = |len: usize| {
let url = format!("http://127.0.0.1:{}/api/chat", addr.port());
async move {
Client::new()
.post(url)
.body(vec![b' '; len])
.send()
.await
.expect("request reaches the test router")
.status()
}
};
assert_eq!(send(256).await, StatusCode::PAYLOAD_TOO_LARGE);
assert_eq!(send(32).await, StatusCode::OK);
}
}