use std::collections::{HashMap, HashSet};
use std::path::{Path, PathBuf};
use std::sync::{Arc, LazyLock, Mutex as StdMutex};
use anyhow::{anyhow, Context};
use axum::body::{Body, Bytes};
use axum::extract::{Path as UrlPath, State};
use axum::http::{HeaderMap, StatusCode};
use axum::response::{IntoResponse, Response};
use axum::Json;
use futures::StreamExt;
use reqwest::Client;
use serde::Deserialize;
use tokio::time::{sleep, Duration, Instant};
use super::backend::*;
use super::hybrid::send_with_hybrid_fallback;
use super::openai::{effort_budget_for, effort_budget_tokens, thinking_for, EFFORT_BUDGET_MESSAGE};
use super::refusal::wire_refusal;
use super::sched::*;
use super::stream::*;
use super::types::*;
use super::{
aggregation, backend_wire_model, ensure_model_for, forward_ollama, model_lock, now_rfc3339,
pull_serialized, release_model_lock, remote_status, unload_everywhere, AppError, AppState,
LoadOptions, Target, Task,
};
use crate::fsutil::HashingWriter;
use crate::metrics::{self, UnloadReason};
use crate::storage::OciStore;
pub(super) async fn handle_version(State(state): State<AppState>) -> impl IntoResponse {
Json(serde_json::json!({
"version": env!("LLMMAN_VERSION"),
"exe": state.0.exe.as_ref().map(|p| p.to_string_lossy()),
"pid": std::process::id(),
}))
}
fn served_format(store: &OciStore, manifest: &crate::storage::oci::Manifest) -> String {
crate::modelpack::stored_manifest_format(store, manifest)
.map(crate::modelpack::ModelFormat::as_str)
.unwrap_or_default()
.to_string()
}
#[derive(Debug, Clone, Default)]
struct HeaderDetails {
family: String,
parameter_size: String,
quantization_level: String,
}
impl HeaderDetails {
fn read(gguf: Option<&crate::gguf::Info>, manifest: &crate::storage::oci::Manifest) -> Self {
Self {
family: gguf
.and_then(|i| i.architecture())
.unwrap_or_default()
.to_string(),
parameter_size: gguf
.and_then(|i| {
i.u64("general.parameter_count").or_else(|| {
(crate::modelpack::gguf_shard_count(manifest) == 1)
.then_some(i.parameter_count)
})
})
.filter(|n| *n > 0)
.map(crate::fmt::human_count)
.unwrap_or_default(),
quantization_level: gguf
.and_then(|i| {
i.u64("general.file_type")
.and_then(crate::gguf::file_type_name)
.map(str::to_string)
.or_else(|| i.quantization.clone())
})
.unwrap_or_default(),
}
}
fn details(self, format: String) -> OllamaModelDetails {
OllamaModelDetails {
parent_model: String::new(),
format,
families: if self.family.is_empty() {
Vec::new()
} else {
vec![self.family.clone()]
},
family: self.family,
parameter_size: self.parameter_size,
quantization_level: self.quantization_level,
}
}
}
#[derive(Debug, Clone, Default)]
pub(super) struct ModelFacts {
format: String,
header: HeaderDetails,
context_length: Option<u64>,
capabilities: Vec<String>,
architecture: Option<String>,
projector: Option<crate::gguf::ProjectorCaps>,
has_chat_template: bool,
}
impl ModelFacts {
fn details(&self) -> OllamaModelDetails {
self.header.clone().details(self.format.clone())
}
pub(super) fn context_length(&self) -> Option<u64> {
self.context_length
}
pub(super) fn starting_context_window(
&self,
state: &AppState,
gguf_engine: GgufEngine,
) -> Option<u32> {
let trained = super::config::trained_ctx(self.context_length)?;
let has = |c: &str| self.capabilities.iter().any(|have| have == c);
let embedding = has(crate::modelpack::CAPABILITY_EMBEDDING);
let decision = has(crate::modelpack::CAPABILITY_DECISION);
if !(has(crate::modelpack::CAPABILITY_COMPLETION) || embedding || decision) {
return None;
}
if ds4_serves_gguf(
gguf_engine,
self.architecture.as_deref(),
embedding,
decision,
ds4_server_on_path,
) {
return None;
}
let (configured, explicit) = state.0.ctx_setting(embedding);
let ctx_size = super::initial_ctx_size(configured, explicit, Some(trained), embedding);
let num_parallel = super::effective_num_parallel(ctx_size, state.0.num_parallel, embedding);
let window = super::loaded_context_window(
super::Engine::LlamaServer,
ctx_size,
Some(trained),
num_parallel,
explicit,
)?;
Some(window.min(trained))
}
pub(super) fn input_modalities(&self) -> Option<Vec<&'static str>> {
let has = |c: &str| self.capabilities.iter().any(|have| have == c);
has(crate::modelpack::CAPABILITY_COMPLETION).then(|| {
let mut modalities = vec!["text"];
if has(crate::modelpack::CAPABILITY_VISION) {
modalities.push("image");
}
if self.projector.is_some_and(|p| p.audio) {
modalities.push("audio");
}
modalities
})
}
pub(super) fn output_modalities(&self) -> Vec<&'static str> {
let gen_audio = self.projector.is_some_and(|p| p.gen_audio);
let mut modalities = Vec::new();
if !gen_audio || self.has_chat_template {
modalities.push("text");
}
if gen_audio {
modalities.push("audio");
}
modalities
}
}
static MODEL_FACTS: LazyLock<StdMutex<ModelFactsCache>> =
LazyLock::new(|| StdMutex::new(HashMap::new()));
type ModelFactsCache = HashMap<(PathBuf, String), Arc<ModelFacts>>;
pub(super) fn listed_model_facts(
store: &OciStore,
store_path: &Path,
cache_path: &Path,
list: &[crate::storage::oci::ImageSummary],
) -> Vec<Arc<ModelFacts>> {
let key = |digest: &str| (store_path.to_path_buf(), digest.to_string());
let facts = list
.iter()
.map(|img| {
if let Some(facts) = MODEL_FACTS.lock().unwrap().get(&key(&img.digest)) {
return facts.clone();
}
let Ok(manifest) = store.read_manifest(&img.digest) else {
return Arc::default();
};
let gguf = crate::modelpack::gguf_info(store_path, cache_path, &manifest);
let projector = crate::modelpack::projector_info(store_path, cache_path, &manifest);
let facts = Arc::new(ModelFacts {
format: served_format(store, &manifest),
header: HeaderDetails::read(gguf.as_ref(), &manifest),
context_length: gguf.as_ref().and_then(crate::gguf::Info::context_length),
capabilities: crate::modelpack::model_capabilities(
crate::modelpack::with_projector_caps(
crate::modelpack::capabilities(store, &manifest),
projector.as_ref(),
),
gguf.as_ref(),
None,
),
architecture: gguf
.as_ref()
.and_then(|info| info.architecture())
.map(str::to_string),
projector: projector.as_ref().map(crate::gguf::Info::projector_caps),
has_chat_template: gguf
.as_ref()
.is_some_and(|info| info.str("tokenizer.chat_template").is_some()),
});
let projector_pending =
projector.is_none() && crate::modelpack::has_projector_layer(&manifest);
if gguf.is_some() && !projector_pending {
MODEL_FACTS
.lock()
.unwrap()
.insert(key(&img.digest), facts.clone());
}
facts
})
.collect();
let listed: HashSet<&str> = list.iter().map(|img| img.digest.as_str()).collect();
MODEL_FACTS
.lock()
.unwrap()
.retain(|(store, digest), _| store != store_path || listed.contains(digest.as_str()));
facts
}
#[cfg(test)]
pub(super) fn cached_model_facts(store_path: &Path) -> usize {
let cache = MODEL_FACTS.lock().unwrap();
cache
.keys()
.filter(|(store, _)| store == store_path)
.count()
}
pub(super) async fn handle_tags(
State(state): State<AppState>,
headers: HeaderMap,
) -> Result<impl IntoResponse, AppError> {
let (store_path, cache_path) = (state.0.store_path.clone(), state.0.cache_path.clone());
let mut models = tokio::task::spawn_blocking(move || -> anyhow::Result<_> {
let store = OciStore::open(&store_path)?;
let list = store.list()?;
let facts = listed_model_facts(&store, &store_path, &cache_path, &list);
Ok(list
.into_iter()
.zip(facts)
.map(|(img, facts)| OllamaModelInfo {
name: img.reference.clone(),
model: img.reference,
size: img.size,
digest: img.digest,
modified_at: img
.modified_at
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
.and_then(|d| chrono::DateTime::from_timestamp(d.as_secs() as i64, 0))
.map(|dt| dt.to_rfc3339())
.unwrap_or_else(now_rfc3339),
details: facts.details(),
})
.collect::<Vec<_>>())
})
.await
.context("list models task panicked")??;
if aggregation::aggregates(&state, &headers) {
for (_, peer) in aggregation::poll::<OllamaTagsResponse>(&state, "/api/tags").await {
for m in peer.models {
if !models.iter().any(|have| have.name == m.name) {
models.push(m);
}
}
}
}
Ok(Json(OllamaTagsResponse { models }))
}
struct PsEntry {
name: String,
digest: String,
size: u64,
port: u16,
pid: Option<u32>,
processor: String,
started_at: String,
expires_at: Option<chrono::DateTime<chrono::Utc>>,
context_window: Option<u32>,
}
pub(super) async fn handle_ps(
State(state): State<AppState>,
headers: HeaderMap,
) -> impl IntoResponse {
let entries: Vec<PsEntry> = {
let mgr = state.0.manager.lock().await;
mgr.running
.iter()
.map(|(name, m)| PsEntry {
name: name.clone(),
digest: m.digest.clone(),
size: m.size,
port: m.port,
pid: m.pid(),
processor: m.processor(),
started_at: m.started_at.clone(),
expires_at: m
.keep_alive
.and_then(|d| chrono::Duration::from_std(d).ok())
.map(|d| m.last_active_wall + d),
context_window: m.context_window,
})
.collect()
};
let mut models = Vec::with_capacity(entries.len());
for entry in entries {
let context_length = query_context_length(&state.0.client, entry.port)
.await
.or_else(|| entry.context_window.map(u64::from));
models.push(OllamaRunningModelInfo {
name: entry.name.clone(),
model: entry.name,
digest: entry.digest,
size: entry.size,
size_vram: 0, pid: entry.pid,
port: entry.port,
processor: entry.processor,
context_length,
started_at: entry.started_at,
expires_at: entry
.expires_at
.map(|t| t.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)),
node: None,
});
}
if aggregation::aggregates(&state, &headers) {
for (origin, peer) in aggregation::poll::<OllamaPsResponse>(&state, "/api/ps").await {
models.extend(peer.models.into_iter().map(|m| OllamaRunningModelInfo {
node: Some(origin.clone()),
..m
}));
}
}
Json(OllamaPsResponse { models })
}
async fn query_context_length(client: &Client, port: u16) -> Option<u64> {
let body = backend_json(client, port, "/props", Duration::from_millis(500))
.await
.ok()?;
body.get("default_generation_settings")?
.get("n_ctx")?
.as_u64()
}
pub(super) async fn handle_show(
State(state): State<AppState>,
Json(req): Json<OllamaShowRequest>,
) -> Result<impl IntoResponse, AppError> {
let model_ref = req
.name
.as_deref()
.filter(|s| !s.is_empty())
.unwrap_or(&req.model);
if crate::providers::is_remote_ref(model_ref) {
eprintln!("[llmman] /api/show model={model_ref:?} (provider-routed)");
return Ok(Json(OllamaShowResponse {
model_info: serde_json::json!({}),
details: OllamaModelDetails {
parent_model: String::new(),
format: String::new(),
family: String::new(),
families: Vec::new(),
parameter_size: String::new(),
quantization_level: String::new(),
},
capabilities: Vec::new(),
template: None,
thinking: None,
}));
}
let model_ref =
crate::shortnames::resolve_ollama_api(model_ref).map_err(AppError::bad_request)?;
let model_ref = model_ref.as_str();
eprintln!("[llmman] /api/show model={model_ref:?}");
let store = OciStore::open(&state.0.store_path)?;
let desc = store.find(model_ref).map_err(|e| {
if e.downcast_ref::<crate::storage::oci::NotFound>().is_some() {
AppError::status(
StatusCode::NOT_FOUND,
format!("model not found: {model_ref}"),
)
} else {
AppError::from(e)
}
})?;
let manifest = store.read_manifest(&desc.digest)?;
let capabilities = crate::modelpack::capabilities(&store, &manifest);
let (gguf, projector) = {
let (store_path, cache_path, manifest) = (
state.0.store_path.clone(),
state.0.cache_path.clone(),
manifest.clone(),
);
tokio::task::spawn_blocking(move || {
(
crate::modelpack::gguf_info(&store_path, &cache_path, &manifest),
crate::modelpack::projector_info(&store_path, &cache_path, &manifest),
)
})
.await
.map_err(|e| AppError::from(anyhow::anyhow!("reading GGUF metadata: {e}")))?
};
let capabilities = crate::modelpack::with_projector_caps(capabilities, projector.as_ref());
let template = match &gguf {
Some(info) => info.str("tokenizer.chat_template").map(str::to_string),
None => crate::modelpack::chat_template(
&store,
&state.0.store_path,
&state.0.cache_path,
&manifest,
),
};
let capabilities =
crate::modelpack::model_capabilities(capabilities, gguf.as_ref(), template.as_deref());
let thinking = template
.as_deref()
.zip(gguf.as_ref())
.filter(|(_, info)| {
capabilities
.iter()
.any(|c| c == crate::modelpack::CAPABILITY_THINKING)
&& !ds4_serves_gguf(
gguf_engine_from_env(),
info.architecture(),
false,
false,
ds4_server_on_path,
)
})
.and_then(|(template, _)| {
let controls = crate::chat_template::thinking_controls(template);
llama_thinking(&controls, &extra_args_from_env(LLAMA_ARGS_VAR), true).0
});
Ok(Json(OllamaShowResponse {
model_info: gguf
.as_ref()
.map_or_else(|| serde_json::json!({}), model_info_json),
details: HeaderDetails::read(gguf.as_ref(), &manifest)
.details(served_format(&store, &manifest)),
capabilities,
template,
thinking,
}))
}
const MODEL_INFO_OMITTED: [&str; 2] = ["general.name", "tokenizer.chat_template"];
fn model_info_json(info: &crate::gguf::Info) -> serde_json::Value {
info.metadata
.iter()
.filter(|(k, _)| !MODEL_INFO_OMITTED.contains(&k.as_str()))
.map(|(k, v)| (k.clone(), metadata_cell(v)))
.collect::<serde_json::Map<String, serde_json::Value>>()
.into()
}
fn metadata_cell(v: &crate::gguf::Value) -> serde_json::Value {
use crate::gguf::Value;
match v {
Value::U8(n) => serde_json::Value::from(*n),
Value::I8(n) => serde_json::Value::from(*n),
Value::U16(n) => serde_json::Value::from(*n),
Value::I16(n) => serde_json::Value::from(*n),
Value::U32(n) => serde_json::Value::from(*n),
Value::I32(n) => serde_json::Value::from(*n),
Value::U64(n) => serde_json::Value::from(*n),
Value::I64(n) => serde_json::Value::from(*n),
Value::F32(f) => serde_json::Value::from(*f),
Value::F64(f) => serde_json::Value::from(*f),
Value::Bool(b) => serde_json::Value::from(*b),
Value::String(s) => serde_json::Value::from(s.as_str()),
Value::LargeArray(_) => serde_json::Value::Array(Vec::new()),
Value::Array(a) if a.len() > crate::gguf::MAX_ARRAY_ELEMENTS => {
serde_json::Value::Array(Vec::new())
}
Value::Array(a) if !a.is_empty() && a.iter().all(|v| matches!(v, Value::U8(_))) => {
use base64::Engine as _;
let bytes: Vec<u8> = a
.iter()
.filter_map(|v| match v {
Value::U8(n) => Some(*n),
_ => None,
})
.collect();
serde_json::Value::from(base64::engine::general_purpose::STANDARD.encode(bytes))
}
Value::Array(a) => serde_json::Value::Array(a.iter().map(metadata_cell).collect()),
}
}
#[derive(Debug, Deserialize)]
pub(super) struct OllamaPullRequest {
#[serde(default)]
pub(super) model: String,
#[serde(default)]
pub(super) name: String,
}
pub(super) async fn handle_pull(
State(state): State<AppState>,
Json(req): Json<OllamaPullRequest>,
) -> Result<Response, AppError> {
let model_ref = if req.model.is_empty() {
req.name.as_str()
} else {
req.model.as_str()
};
if model_ref.is_empty() {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
"model is required",
));
}
let model = crate::shortnames::resolve_ollama_api(model_ref).map_err(AppError::bad_request)?;
eprintln!("[llmman] /api/pull model={model:?}");
let store_path = state.0.store_path.clone();
let model_for_task = model.clone();
let pull_task =
tokio::task::spawn_blocking(move || pull_serialized(&store_path, &model_for_task));
Ok(stream_ffi_progress(
model,
"pull",
"pulling manifest",
pull_task,
))
}
#[derive(Debug, Deserialize)]
pub(super) struct OllamaPushRequest {
#[serde(default)]
pub(super) model: String,
#[serde(default)]
pub(super) name: String,
}
pub(super) async fn handle_push(
State(state): State<AppState>,
Json(req): Json<OllamaPushRequest>,
) -> Result<Response, AppError> {
let model_ref = if req.model.is_empty() {
req.name.as_str()
} else {
req.model.as_str()
};
push_impl(state, model_ref, "/api/push").await
}
async fn push_impl(
state: AppState,
model_ref: &str,
route: &'static str,
) -> Result<Response, AppError> {
if model_ref.is_empty() {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
"model is required",
));
}
let model = crate::shortnames::resolve_ollama_api(model_ref).map_err(AppError::bad_request)?;
eprintln!("[llmman] {route} model={model:?}");
let store_path = state.0.store_path.clone();
if OciStore::open(&store_path)
.and_then(|s| s.find(&model))
.is_err()
{
return Err(AppError::status(
StatusCode::NOT_FOUND,
format!("model not found: {model}"),
));
}
let model_for_task = model.clone();
let push_task = tokio::task::spawn_blocking(move || {
let lock = model_lock(&model_for_task);
let result = (|| {
let _guard = lock.blocking_lock();
let layout_dir = store_path
.to_str()
.ok_or_else(|| anyhow!("store path is not valid UTF-8"))?;
crate::ffi::push(layout_dir, &model_for_task)?;
let desc = OciStore::open(&store_path)?.find(&model_for_task)?;
Ok(PushOutcome {
digest: desc.digest,
})
})();
drop(lock);
release_model_lock(&model_for_task);
result
});
Ok(stream_ffi_progress(
model,
"push",
"retrieving manifest",
push_task,
))
}
struct PushOutcome {
digest: String,
}
trait StreamedOutcome {
fn into_lines(self) -> Vec<serde_json::Value>;
}
impl StreamedOutcome for Vec<String> {
fn into_lines(self) -> Vec<serde_json::Value> {
self.into_iter()
.map(|notice| serde_json::json!({"notice": notice}))
.collect()
}
}
impl StreamedOutcome for PushOutcome {
fn into_lines(self) -> Vec<serde_json::Value> {
vec![serde_json::json!({"digest": self.digest})]
}
}
fn progress_line(
verb: &str,
model: &str,
snap: (String, i64, i64),
saw_bytes: &mut bool,
) -> Option<serde_json::Value> {
let (status, total, completed) = snap;
if total > 0 {
*saw_bytes = true;
return Some(serde_json::json!({
"status": if status.is_empty() { format!("{verb}ing {model}") } else { status },
"total": total,
"completed": completed.clamp(0, total),
}));
}
if !status.is_empty() {
return Some(serde_json::json!({"status": status}));
}
if *saw_bytes {
return None;
}
Some(serde_json::json!({"status": format!("{verb}ing {model}")}))
}
fn stream_ffi_progress<T: StreamedOutcome + Send + 'static>(
model: String,
verb: &'static str,
first_status: &'static str,
task: tokio::task::JoinHandle<anyhow::Result<T>>,
) -> Response {
let first_line = serde_json::json!({"status": first_status}).to_string() + "\n";
let stream = futures::stream::once(futures::future::ready(Bytes::from(first_line)))
.chain(futures::stream::unfold((Some(task), false), move |state| {
let (task, mut saw_bytes) = state;
let model = model.clone();
async move {
let mut task = task?;
tokio::select! {
result = &mut task => {
let mut out = String::new();
let line = match result {
Ok(Ok(outcome)) => {
for field in outcome.into_lines() {
out.push_str(&field.to_string());
out.push('\n');
}
serde_json::json!({"status": "success"}).to_string()
}
Ok(Err(e)) => serde_json::json!({"error": format!("{e:#}")}).to_string(),
Err(e) => serde_json::json!({"error": format!("{verb} task panicked: {e}")}).to_string(),
};
out.push_str(&line);
out.push('\n');
Some((Bytes::from(out), (None, saw_bytes)))
}
_ = sleep(Duration::from_millis(200)) => {
let rust_snap = crate::hf::progress::poll(&model);
let go_snap = (rust_snap.total == 0).then(|| crate::ffi::progress(&model).ok()).flatten();
let (status, total, completed) = if rust_snap.total > 0 {
(rust_snap.status, rust_snap.total, rust_snap.completed)
} else if !rust_snap.status.is_empty() {
(rust_snap.status, 0, 0)
} else if let Some(p) = &go_snap {
(p.status.clone(), p.total, p.completed)
} else {
(String::new(), 0, 0)
};
let line = progress_line(verb, &model, (status, total, completed), &mut saw_bytes);
let out = line.map(|l| l.to_string() + "\n").unwrap_or_default();
Some((Bytes::from(out), (Some(task), saw_bytes)))
}
}
}
}))
.map(Ok::<_, std::convert::Infallible>);
Response::builder()
.status(StatusCode::OK)
.header("content-type", "application/x-ndjson")
.body(Body::from_stream(stream))
.unwrap()
}
pub(super) async fn handle_delete(
State(state): State<AppState>,
Json(req): Json<OllamaDeleteRequest>,
) -> Result<impl IntoResponse, AppError> {
let model_ref = req
.name
.as_deref()
.filter(|s| !s.is_empty())
.unwrap_or(&req.model);
let model_ref =
crate::shortnames::resolve_ollama_api(model_ref).map_err(AppError::bad_request)?;
let store = OciStore::open(&state.0.store_path)?;
store.remove(&model_ref)?;
Ok(StatusCode::OK)
}
pub(super) async fn handle_copy(
State(state): State<AppState>,
Json(req): Json<OllamaCopyRequest>,
) -> Result<impl IntoResponse, AppError> {
if req.source.is_empty() || req.destination.is_empty() {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
"source and destination are required",
));
}
let source =
crate::shortnames::resolve_ollama_api(&req.source).map_err(AppError::bad_request)?;
let destination =
crate::shortnames::resolve_ollama_api(&req.destination).map_err(AppError::bad_request)?;
eprintln!("[llmman] /api/copy {source:?} -> {destination:?}");
let store = OciStore::open(&state.0.store_path)?;
if store.find(&source).is_err() {
return Err(AppError::status(
StatusCode::NOT_FOUND,
format!("model '{}' not found", req.source),
));
}
let digest = copy_blocking(&state, source, destination.clone()).await?;
evict_if_retagged(&state, &destination, &digest).await;
Ok(StatusCode::OK)
}
async fn copy_blocking(
state: &AppState,
source: String,
destination: String,
) -> Result<String, AppError> {
let (store_path, cache_path) = (state.0.store_path.clone(), state.0.cache_path.clone());
tokio::task::spawn_blocking(move || {
let store = OciStore::open(store_path)?;
crate::cmd::cp::copy(&store, &cache_path, &source, &destination)
})
.await
.context("copy task panicked")?
.map_err(AppError::from)
}
async fn evict_if_retagged(state: &AppState, reference: &str, digest: &str) {
let want = crate::storage::default_tag(reference);
let mut mgr = state.0.manager.lock().await;
let stale: Vec<String> = mgr
.running
.iter()
.filter(|(k, m)| m.digest != digest && crate::storage::default_tag(k) == want)
.map(|(k, _)| k.clone())
.collect();
for key in stale {
mgr.running.remove(&key);
metrics::record_model_unload(&key, UnloadReason::Requested);
}
}
fn blob_staging_dir(state: &AppState) -> PathBuf {
state.0.cache_path.join("blobs")
}
static STAGING_COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
fn staging_temp_name(prefix: &str) -> String {
let n = STAGING_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
format!("{prefix}-{}-{n}", std::process::id())
}
fn staged_blob_path(state: &AppState, digest: &str) -> Result<PathBuf, AppError> {
let hex = digest.strip_prefix("sha256:").unwrap_or_default();
if hex.len() != 64 || !hex.bytes().all(|b| b.is_ascii_hexdigit()) {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
format!("invalid digest {digest:?}: expected sha256:<64 hex digits>"),
));
}
Ok(blob_staging_dir(state).join(hex.to_ascii_lowercase()))
}
pub(super) async fn handle_blob_head(
State(state): State<AppState>,
UrlPath(digest): UrlPath<String>,
) -> Result<StatusCode, AppError> {
let path = staged_blob_path(&state, &digest)?;
Ok(if path.is_file() {
StatusCode::OK
} else {
StatusCode::NOT_FOUND
})
}
pub(super) async fn handle_blob_upload(
State(state): State<AppState>,
UrlPath(digest): UrlPath<String>,
body: Body,
) -> Result<StatusCode, AppError> {
use tokio::io::AsyncWriteExt as _;
let dest = staged_blob_path(&state, &digest)?;
if dest.is_file() {
return Ok(StatusCode::OK);
}
let dir = blob_staging_dir(&state);
tokio::fs::create_dir_all(&dir).await?;
let tmp = dir.join(staging_temp_name("tmp"));
let mut file = HashingWriter::new(tokio::fs::File::create(&tmp).await?);
let mut stream = body.into_data_stream();
let written = async {
while let Some(chunk) = stream.next().await {
let chunk = chunk.context("read upload body")?;
file.write_all(&chunk).await?;
}
file.flush().await?;
anyhow::Ok(())
}
.await;
if let Err(e) = written {
let _ = tokio::fs::remove_file(&tmp).await;
return Err(e.into());
}
let actual = format!("sha256:{}", file.finish().1);
if !actual.eq_ignore_ascii_case(&digest) {
let _ = tokio::fs::remove_file(&tmp).await;
return Err(AppError::status(
StatusCode::BAD_REQUEST,
format!("digest mismatch, expected {digest:?}, got {actual:?}"),
));
}
tokio::fs::rename(&tmp, &dest).await?;
eprintln!("[llmman] /api/blobs stored {digest} ({})", dest.display());
Ok(StatusCode::CREATED)
}
pub(super) async fn handle_create(
State(state): State<AppState>,
Json(req): Json<OllamaCreateRequest>,
) -> Result<Response, AppError> {
let model = if req.model.is_empty() {
req.name.as_str()
} else {
req.model.as_str()
};
if model.is_empty() {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
"model is required",
));
}
let refused: Vec<&str> = req
.unsupported
.iter()
.filter(|(_, v)| !is_empty_json(v))
.map(|(k, _)| k.as_str())
.collect();
if !refused.is_empty() {
let mut refused = refused;
refused.sort_unstable();
return Err(AppError::status(
StatusCode::BAD_REQUEST,
format!(
"/api/create: {} not supported by llmman — models are plain OCI artifacts \
whose GGUF carries its own chat template; only `from` (alias an existing \
model) and `files` (package uploaded blobs) are honoured",
refused.join(", ")
),
));
}
let model = &crate::shortnames::resolve_ollama_api(model).map_err(AppError::bad_request)?;
if crate::storage::split_ref_digest(model).1.is_some() {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
"/api/create: the model name can't be a digest reference",
));
}
let files = req.files.unwrap_or_default();
let from = req.from.filter(|f| !f.is_empty());
let statuses: Vec<String> = match (from, files.is_empty()) {
(Some(_), false) => {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
"/api/create: `from` and `files` are mutually exclusive here",
))
}
(None, true) => {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
"/api/create: one of `from` or `files` is required",
))
}
(Some(from), true) => {
let source =
crate::shortnames::resolve_ollama_api(&from).map_err(AppError::bad_request)?;
eprintln!("[llmman] /api/create {model:?} from {source:?}");
let store = OciStore::open(&state.0.store_path)?;
if store.find(&source).is_err() {
return Err(AppError::status(
StatusCode::NOT_FOUND,
format!("model '{from}' not found"),
));
}
let digest = copy_blocking(&state, source.clone(), model.to_string()).await?;
evict_if_retagged(&state, model, &digest).await;
vec![format!("using existing layer {source}")]
}
(None, false) => {
eprintln!(
"[llmman] /api/create {model:?} from {} uploaded file(s)",
files.len()
);
let staged = files
.iter()
.map(|(name, digest)| Ok((name.clone(), staged_file(&state, name, digest)?)))
.collect::<Result<Vec<_>, AppError>>()?;
let store_path = state.0.store_path.clone();
let cache_path = state.0.cache_path.clone();
let staging = blob_staging_dir(&state);
let model_owned = model.to_string();
let (digest, statuses) = tokio::task::spawn_blocking(move || {
create_from_staged_blobs(&store_path, &cache_path, &staging, &model_owned, &staged)
})
.await
.context("create task panicked")??;
evict_if_retagged(&state, model, &digest).await;
statuses
}
};
let mut lines: Vec<serde_json::Value> = statuses
.into_iter()
.map(|s| serde_json::json!({"status": s}))
.collect();
lines.push(serde_json::json!({"status": "success"}));
Ok(if req.stream {
let body: String = lines.iter().map(|l| l.to_string() + "\n").collect();
([("content-type", "application/x-ndjson")], body).into_response()
} else {
Json(lines.pop().unwrap_or_default()).into_response()
})
}
fn is_empty_json(v: &serde_json::Value) -> bool {
match v {
serde_json::Value::Null => true,
serde_json::Value::String(s) => s.is_empty(),
serde_json::Value::Object(m) => m.is_empty(),
serde_json::Value::Array(a) => a.is_empty(),
_ => false,
}
}
fn staged_file(state: &AppState, name: &str, digest: &str) -> Result<PathBuf, AppError> {
let bare = Path::new(name)
.file_name()
.and_then(|n| n.to_str())
.is_some_and(|n| n == name);
if !bare {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
format!("files: {name:?} must be a bare file name"),
));
}
let src = staged_blob_path(state, digest)?;
if !src.is_file() {
return Err(AppError::status(
StatusCode::NOT_FOUND,
format!("files: {digest} for {name:?} was never uploaded to /api/blobs"),
));
}
Ok(src)
}
fn create_from_staged_blobs(
store_path: &Path,
cache_path: &Path,
staging: &Path,
model: &str,
files: &[(String, PathBuf)],
) -> anyhow::Result<(String, Vec<String>)> {
let tmp = (0..3)
.map(|_| staging.join(staging_temp_name("create")))
.find_map(|dir| std::fs::create_dir(&dir).ok().map(|()| dir))
.ok_or_else(|| {
anyhow!(
"couldn't create a fresh build directory under {}",
staging.display()
)
})?;
let result = (|| {
let mut statuses = Vec::new();
for (name, src) in files {
let dst = tmp.join(name);
if std::fs::hard_link(src, &dst).is_err() {
std::fs::copy(src, &dst)
.with_context(|| format!("stage {name} from {}", src.display()))?;
}
let digest = src.file_name().unwrap_or_default().to_string_lossy();
statuses.push(format!("using sha256:{digest} as {name}"));
}
let store = OciStore::open(store_path)?;
let desc = crate::storage::gc::after_retag(&store, cache_path, model, || {
store.build(&tmp, model, &HashMap::new(), false)
})?;
statuses.push(format!("writing manifest {}", desc.digest));
Ok((desc.digest, statuses))
})();
let _ = std::fs::remove_dir_all(&tmp);
result
}
fn empty_chat_chunk(model: String, done_reason: &str) -> OllamaChatChunk {
OllamaChatChunk {
model,
created_at: now_rfc3339(),
message: OllamaMessage {
role: "assistant".into(),
..Default::default()
},
done: true,
done_reason: Some(done_reason.into()),
metrics: OllamaMetrics::default(),
}
}
fn check_think(think: &Option<serde_json::Value>) -> Result<(), AppError> {
match think {
None
| Some(
serde_json::Value::Null | serde_json::Value::Bool(_) | serde_json::Value::String(_),
) => Ok(()),
Some(_) => Err(AppError::status(
StatusCode::BAD_REQUEST,
"think must be a boolean or string",
)),
}
}
pub(super) async fn handle_ollama_chat(
State(state): State<AppState>,
headers: HeaderMap,
Json(req): Json<OllamaChatRequest>,
) -> Result<Response, AppError> {
eprintln!(
"[llmman] /api/chat model={:?} messages={}",
req.model,
req.messages.len()
);
check_think(&req.think)?;
if req.messages.is_empty() && is_explicit_unload(&req.keep_alive) {
unload_everywhere(&state, crate::hybrid::local_half(&req.model), &headers).await?;
return Ok(Json(empty_chat_chunk(req.model, "unload")).into_response());
}
let model_ref = req.model.clone();
let started = Instant::now();
send_with_hybrid_fallback(
&state,
&model_ref,
Some(&headers),
opt_load_options(&req.options),
|model, target, guard| {
ollama_chat_to(&state, &headers, &req, model, target, guard, started)
},
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn ollama_chat_to(
state: &AppState,
headers: &HeaderMap,
req: &OllamaChatRequest,
model: String,
target: Target,
guard: ActivityGuard,
started: Instant,
) -> Result<Response, AppError> {
let load_duration = if target.is_remote() {
Duration::ZERO
} else {
started.elapsed()
};
if matches!(target, Target::Peer(_)) {
return forward_ollama(state, &target, "/api/chat", headers, req, &model, guard).await;
}
if req.messages.is_empty() {
refresh_activity(guard, resolve_keep_alive(&req.keep_alive)).await;
return Ok(Json(empty_chat_chunk(req.model.clone(), "load")).into_response());
}
let keep_alive = resolve_keep_alive(&req.keep_alive);
let activity = begin_activity(guard, Some(keep_alive)).await;
let wire_model = backend_wire_model(state, &target, &model).await;
let think = resolve_think(&req.think, thinking_for(state, &target).await.as_ref());
let mut oai = OAIChatRequest {
model: wire_model,
messages: req.messages.iter().map(ollama_message_to_oai).collect(),
stream: true,
chat_template_kwargs: think_to_chat_template_kwargs(&think),
tools: req.tools.clone(),
response_format: format_to_response_format(&req.format),
..options_to_oai(&req.options)
};
apply_think_budget(state, &target, &think, &mut oai).await;
stream_ollama(
req.stream,
state.0.client.clone(),
target,
oai,
activity,
started,
load_duration,
move |delta| OllamaChatChunk {
model: model.clone(),
created_at: now_rfc3339(),
message: OllamaMessage {
role: "assistant".into(),
content: delta.content,
thinking: delta.thinking,
tool_calls: delta.tool_calls,
..Default::default()
},
done: delta.done,
done_reason: delta.done_reason,
metrics: delta.metrics,
},
)
.await
}
async fn apply_think_budget(
state: &AppState,
target: &Target,
think: &Option<serde_json::Value>,
oai: &mut OAIChatRequest,
) {
let Some(effort) = think.as_ref().and_then(|t| t.as_str()) else {
return;
};
let Some(message) = effort_budget_for(state, target).await else {
return;
};
oai.reasoning_budget_tokens = effort_budget_tokens(effort, oai.max_tokens.map(u64::from));
oai.reasoning_budget_message =
(message && oai.reasoning_budget_tokens.is_some()).then_some(EFFORT_BUDGET_MESSAGE);
}
fn options_to_oai(options: &Option<serde_json::Value>) -> OAIChatRequest {
OAIChatRequest {
temperature: opt_f64(options, "temperature"),
top_p: opt_f64(options, "top_p"),
max_tokens: opt_u32(options, "num_predict"),
repeat_penalty: opt_f64(options, "repeat_penalty"),
seed: options.as_ref().and_then(|o| o.get("seed")?.as_u64()),
stop: opt_stop(options),
presence_penalty: opt_f64(options, "presence_penalty"),
frequency_penalty: opt_f64(options, "frequency_penalty"),
top_k: opt_u32(options, "top_k"),
min_p: opt_f64(options, "min_p"),
..Default::default()
}
}
fn opt_stop(options: &Option<serde_json::Value>) -> Option<Vec<String>> {
match options.as_ref()?.get("stop")? {
serde_json::Value::String(s) => Some(vec![s.clone()]),
serde_json::Value::Array(a) => Some(
a.iter()
.filter_map(|v| v.as_str().map(str::to_string))
.collect(),
),
_ => None,
}
}
pub(super) async fn handle_ollama_generate(
State(state): State<AppState>,
headers: HeaderMap,
Json(req): Json<OllamaGenerateRequest>,
) -> Result<Response, AppError> {
eprintln!(
"[llmman] /api/generate model={:?} prompt_len={}",
req.model,
req.prompt.len()
);
check_think(&req.think)?;
let is_unload = req.prompt.is_empty() && is_explicit_unload(&req.keep_alive);
if is_unload {
unload_everywhere(&state, crate::hybrid::local_half(&req.model), &headers).await?;
return Ok(Json(OllamaGenerateChunk {
model: req.model,
created_at: now_rfc3339(),
response: String::new(),
thinking: None,
done: true,
done_reason: Some("unload".into()),
metrics: OllamaMetrics::default(),
})
.into_response());
}
let unsupported = if req.raw {
Some("raw")
} else if req.suffix.as_deref().is_some_and(|s| !s.is_empty()) {
Some("suffix")
} else if req.template.as_deref().is_some_and(|t| !t.is_empty()) {
Some("template")
} else {
None
};
if let Some(field) = unsupported {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
format!("llmman does not support the {field:?} field of /api/generate"),
));
}
let model_ref = req.model.clone();
let started = Instant::now();
send_with_hybrid_fallback(
&state,
&model_ref,
Some(&headers),
opt_load_options(&req.options),
|model, target, guard| {
ollama_generate_to(&state, &headers, &req, model, target, guard, started)
},
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn ollama_generate_to(
state: &AppState,
headers: &HeaderMap,
req: &OllamaGenerateRequest,
model: String,
target: Target,
guard: ActivityGuard,
started: Instant,
) -> Result<Response, AppError> {
let load_duration = if target.is_remote() {
Duration::ZERO
} else {
started.elapsed()
};
if matches!(target, Target::Peer(_)) {
return forward_ollama(state, &target, "/api/generate", headers, req, &model, guard).await;
}
if req.prompt.is_empty() {
refresh_activity(guard, resolve_keep_alive(&req.keep_alive)).await;
return Ok(Json(OllamaGenerateChunk {
model: req.model.clone(),
created_at: now_rfc3339(),
response: String::new(),
thinking: None,
done: true,
done_reason: Some("load".into()),
metrics: OllamaMetrics::default(),
})
.into_response());
}
let keep_alive = resolve_keep_alive(&req.keep_alive);
let activity = begin_activity(guard, Some(keep_alive)).await;
let wire_model = backend_wire_model(state, &target, &model).await;
let user = OllamaMessage {
role: "user".into(),
content: req.prompt.clone(),
images: req.images.clone(),
..Default::default()
};
let messages = req
.system
.as_deref()
.filter(|s| !s.is_empty())
.map(|s| OAIMessage::text("system", s))
.into_iter()
.chain(std::iter::once(ollama_message_to_oai(&user)))
.collect();
let think = resolve_think(&req.think, thinking_for(state, &target).await.as_ref());
let mut oai = OAIChatRequest {
model: wire_model,
messages,
stream: true,
chat_template_kwargs: think_to_chat_template_kwargs(&think),
tools: None,
response_format: format_to_response_format(&req.format),
..options_to_oai(&req.options)
};
apply_think_budget(state, &target, &think, &mut oai).await;
stream_ollama(
req.stream,
state.0.client.clone(),
target,
oai,
activity,
started,
load_duration,
move |delta| OllamaGenerateChunk {
model: model.clone(),
created_at: now_rfc3339(),
response: delta.content,
thinking: delta.thinking,
done: delta.done,
done_reason: delta.done_reason,
metrics: delta.metrics,
},
)
.await
}
fn embed_inputs(input: &serde_json::Value) -> Result<Vec<String>, AppError> {
let invalid = || AppError::status(StatusCode::BAD_REQUEST, "invalid input type");
match input {
serde_json::Value::Null => Ok(Vec::new()),
serde_json::Value::String(s) if s.is_empty() => Ok(Vec::new()),
serde_json::Value::String(s) => Ok(vec![s.clone()]),
serde_json::Value::Array(items) => items
.iter()
.map(|v| v.as_str().map(str::to_owned).ok_or_else(invalid))
.collect(),
_ => Err(invalid()),
}
}
async fn embed_via_backend(
state: &AppState,
headers: &HeaderMap,
model_ref: &str,
inputs: &[String],
truncate: bool,
keep_alive: &Option<serde_json::Value>,
) -> Result<(Vec<Vec<f32>>, u64, Duration), AppError> {
let mlx_unsupported = |model: &str| {
AppError::status(
StatusCode::NOT_IMPLEMENTED,
format!(
"{model} is served by mlx_lm.server, which llmman never starts with \
--embedding-model; use a GGUF or vllm-served model for embeddings"
),
)
};
if !crate::providers::is_remote_ref(model_ref) {
if let Some(canonical) = would_use_mlx(state, model_ref).await {
return Err(mlx_unsupported(&canonical));
}
}
let started = Instant::now();
let (model, target, guard) = ensure_model_for(
state,
model_ref,
Some(headers),
LoadOptions::default(),
Task::Embed,
)
.await?;
let loaded = started.elapsed();
if !target.is_remote() && would_use_mlx(state, &model).await.is_some() {
return Err(mlx_unsupported(&model));
}
if let Some(refusal) = wire_refusal(&target, "/v1/embeddings") {
return Err(AppError::status(StatusCode::NOT_IMPLEMENTED, refusal));
}
let keep_alive = resolve_keep_alive(keep_alive);
if inputs.is_empty() {
refresh_activity(guard, keep_alive).await;
return Ok((Vec::new(), 0, loaded));
}
let _activity = begin_activity(guard, Some(keep_alive)).await;
let wire_model = backend_wire_model(state, &target, &model).await;
use futures::TryStreamExt as _;
let calls: Vec<_> = inputs
.iter()
.map(|text| embed_one(state, &target, &wire_model, text, truncate))
.collect();
let results: Vec<(Vec<f32>, u64)> = futures::stream::iter(calls)
.buffered(EMBED_CONCURRENCY)
.try_collect()
.await?;
let total_tokens = results.iter().map(|(_, n)| n).sum();
let embeddings = results
.into_iter()
.map(|(mut v, _)| {
normalize_in_place(&mut v)?;
Ok(v)
})
.collect::<Result<_, AppError>>()?;
Ok((embeddings, total_tokens, loaded))
}
pub(super) fn is_embedding_overflow(error: &str) -> bool {
error.contains("too large")
}
const EMBED_CONCURRENCY: usize = 8;
async fn embed_one(
state: &AppState,
target: &Target,
wire_model: &str,
text: &str,
truncate: bool,
) -> Result<(Vec<f32>, u64), AppError> {
match post_embeddings(&state.0.client, target, wire_model, text).await {
Ok(ok) => Ok(ok),
Err((status, body)) if truncate && !target.is_remote() && status.is_server_error() => {
let Target::Local(port) = target else {
unreachable!("guarded by !is_remote")
};
let limit = query_context_length(&state.0.client, *port)
.await
.ok_or_else(|| anyhow!("{} {status}: {body}", target.describe()))?;
let limit = usize::try_from(limit)
.unwrap_or(usize::MAX)
.saturating_sub(2);
let truncated = truncate_to_tokens(&state.0.client, *port, text, limit).await?;
if truncated.len() >= text.len() {
return Err(AppError(
anyhow!("{} {status}: {body}", target.describe()),
StatusCode::INTERNAL_SERVER_ERROR,
));
}
post_embeddings(&state.0.client, target, wire_model, &truncated)
.await
.map_err(|(status, body)| {
AppError(
anyhow!("{} {status}: {body}", target.describe()),
remote_status(target, status),
)
})
}
Err((status, body))
if !truncate
&& !target.is_remote()
&& status.is_server_error()
&& is_embedding_overflow(&body) =>
{
Err(AppError(
anyhow!("the input length exceeds the context length: {body}"),
StatusCode::BAD_REQUEST,
))
}
Err((status, body)) => Err(AppError(
anyhow!("{} {status}: {body}", target.describe()),
remote_status(target, status),
)),
}
}
async fn post_embeddings(
client: &Client,
target: &Target,
wire_model: &str,
text: &str,
) -> Result<(Vec<f32>, u64), (StatusCode, String)> {
let resp = target
.authorize(
client
.post(target.url("/v1/embeddings"))
.timeout(EMBEDDINGS_TIMEOUT)
.json(&serde_json::json!({ "model": wire_model, "input": text })),
)
.send()
.await
.map_err(|e| {
(
StatusCode::BAD_GATEWAY,
format!("send to {}: {e}", target.describe()),
)
})?;
let status = resp.status();
let body = resp.bytes().await.unwrap_or_default();
if !status.is_success() {
return Err((status, String::from_utf8_lossy(&body).into_owned()));
}
let mut parsed: OAIEmbeddingsResponse = serde_json::from_slice(&body).map_err(|e| {
(
StatusCode::BAD_GATEWAY,
format!(
"{} returned an unexpected embeddings body: {e}",
target.describe()
),
)
})?;
parsed.data.sort_by_key(|d| d.index);
let vector = parsed
.data
.into_iter()
.next()
.map(|d| d.embedding)
.ok_or_else(|| {
(
StatusCode::BAD_GATEWAY,
format!("{} returned no embedding", target.describe()),
)
})?;
Ok((vector, parsed.usage.prompt_tokens))
}
const TOKENIZE_TIMEOUT: Duration = Duration::from_secs(30);
const EMBEDDINGS_TIMEOUT: Duration = Duration::from_secs(600);
async fn truncate_to_tokens(
client: &Client,
port: u16,
text: &str,
limit: usize,
) -> anyhow::Result<String> {
#[derive(Deserialize)]
struct Tokens {
tokens: Vec<i64>,
}
#[derive(Deserialize)]
struct Content {
content: String,
}
let base = format!("http://127.0.0.1:{port}");
let Tokens { mut tokens } = client
.post(format!("{base}/tokenize"))
.timeout(TOKENIZE_TIMEOUT)
.json(&serde_json::json!({ "content": text }))
.send()
.await
.context("tokenize for truncation")?
.error_for_status()?
.json()
.await?;
if limit == 0 {
anyhow::bail!("input after truncation exceeds maximum context length");
}
if tokens.len() <= limit {
return Ok(text.to_string());
}
tokens.truncate(limit);
let Content { content } = client
.post(format!("{base}/detokenize"))
.timeout(TOKENIZE_TIMEOUT)
.json(&serde_json::json!({ "tokens": tokens }))
.send()
.await
.context("detokenize for truncation")?
.error_for_status()?
.json()
.await?;
Ok(content)
}
fn normalize_in_place(v: &mut [f32]) -> Result<(), AppError> {
if v.iter().any(|x| !x.is_finite()) {
return Err(AppError::status(
StatusCode::BAD_GATEWAY,
"embedding contains NaN or Inf values",
));
}
let norm = v.iter().map(|&x| f64::from(x).powi(2)).sum::<f64>().sqrt();
if norm > 0.0 {
for x in v.iter_mut() {
*x = (f64::from(*x) / norm) as f32;
}
}
Ok(())
}
pub(super) fn truncate_embedding(v: &mut Vec<f32>, dims: usize) -> Result<(), AppError> {
if dims < v.len() {
v.truncate(dims);
normalize_in_place(v)?;
}
Ok(())
}
pub(super) async fn handle_embed(
State(state): State<AppState>,
headers: HeaderMap,
Json(req): Json<OllamaEmbedRequest>,
) -> Result<Response, AppError> {
let started = Instant::now();
let inputs = embed_inputs(&req.input)?;
eprintln!(
"[llmman] /api/embed model={:?} inputs={}",
req.model,
inputs.len()
);
let (mut embeddings, tokens, loaded) = embed_via_backend(
&state,
&headers,
&req.model,
&inputs,
req.truncate != Some(false),
&req.keep_alive,
)
.await?;
if let Some(dims) = req.dimensions.filter(|d| *d > 0) {
for v in &mut embeddings {
truncate_embedding(v, dims)?;
}
}
Ok(Json(OllamaEmbedResponse {
model: req.model,
embeddings,
total_duration: started.elapsed().as_nanos() as u64,
load_duration: loaded.as_nanos() as u64,
prompt_eval_count: tokens,
})
.into_response())
}
pub(super) async fn handle_embeddings(
State(state): State<AppState>,
headers: HeaderMap,
Json(req): Json<OllamaEmbeddingsRequest>,
) -> Result<Response, AppError> {
eprintln!(
"[llmman] /api/embeddings model={:?} prompt_len={}",
req.model,
req.prompt.len()
);
let inputs = if req.prompt.is_empty() {
Vec::new()
} else {
vec![req.prompt.clone()]
};
let (embeddings, _tokens, _loaded) =
embed_via_backend(&state, &headers, &req.model, &inputs, true, &req.keep_alive).await?;
let embedding = embeddings
.into_iter()
.next()
.unwrap_or_default()
.into_iter()
.map(f64::from)
.collect();
Ok(Json(OllamaEmbeddingsResponse { embedding }).into_response())
}
fn opt_f64(opts: &Option<serde_json::Value>, key: &str) -> Option<f32> {
opts.as_ref()?.get(key)?.as_f64().map(|f| f as f32)
}
fn opt_u32(opts: &Option<serde_json::Value>, key: &str) -> Option<u32> {
opts.as_ref()?.get(key)?.as_u64().map(|n| n as u32)
}
fn opt_num_thread(opts: &Option<serde_json::Value>) -> Option<u32> {
opt_positive_u32(opts, "num_thread")
}
fn opt_num_ctx(opts: &Option<serde_json::Value>) -> Option<u32> {
opt_positive_u32(opts, "num_ctx")
}
fn opt_load_options(opts: &Option<serde_json::Value>) -> LoadOptions {
LoadOptions {
threads: opt_num_thread(opts),
ctx: opt_num_ctx(opts),
}
}
fn opt_positive_u32(opts: &Option<serde_json::Value>, key: &str) -> Option<u32> {
let n = opts.as_ref()?.get(key)?.as_u64()?;
u32::try_from(n).ok().filter(|&n| n != 0)
}
#[cfg(test)]
mod tests {
use super::super::test_support::{running_model_fixture, test_state};
use super::*;
fn facts_with(capabilities: &[&str]) -> ModelFacts {
ModelFacts {
capabilities: capabilities.iter().map(|c| c.to_string()).collect(),
..ModelFacts::default()
}
}
fn speaking(projector: crate::gguf::ProjectorCaps, has_chat_template: bool) -> ModelFacts {
ModelFacts {
projector: Some(projector),
has_chat_template,
..facts_with(&["completion"])
}
}
#[test]
fn modalities_follow_the_projector_and_the_chat_template() {
use crate::gguf::ProjectorCaps;
let omni = ModelFacts {
capabilities: vec!["completion".into(), "vision".into()],
..speaking(
ProjectorCaps {
vision: true,
audio: true,
gen_audio: true,
},
true,
)
};
assert_eq!(omni.input_modalities().unwrap(), ["text", "image", "audio"]);
assert_eq!(omni.output_modalities(), ["text", "audio"]);
let tts = speaking(
ProjectorCaps {
gen_audio: true,
..ProjectorCaps::default()
},
false,
);
assert_eq!(tts.input_modalities().unwrap(), ["text"]);
assert_eq!(tts.output_modalities(), ["audio"]);
assert_eq!(facts_with(&["completion"]).output_modalities(), ["text"]);
assert_eq!(
speaking(ProjectorCaps::default(), true).output_modalities(),
["text"]
);
assert!(facts_with(&["embedding"]).input_modalities().is_none());
}
#[test]
fn a_gguf_sent_to_ds4_has_no_starting_context_window() {
let state = test_state();
let ds4 = |capability: &str| ModelFacts {
context_length: Some(4096),
architecture: Some("deepseek41".to_string()),
..facts_with(&[capability])
};
let text = ds4(crate::modelpack::CAPABILITY_COMPLETION);
assert_eq!(text.starting_context_window(&state, GgufEngine::Ds4), None);
assert_eq!(
text.starting_context_window(&state, GgufEngine::LlamaServer),
Some(4096)
);
let embedding = ds4(crate::modelpack::CAPABILITY_EMBEDDING);
assert_eq!(
embedding.starting_context_window(&state, GgufEngine::Ds4),
Some(4096)
);
}
#[test]
fn input_modalities_follow_the_vision_capability_of_a_text_model() {
let modalities = |caps: &[&str]| facts_with(caps).input_modalities();
assert_eq!(modalities(&["completion"]), Some(vec!["text"]));
assert_eq!(
modalities(&["completion", "vision"]),
Some(vec!["text", "image"])
);
for not_text in [&["embedding"][..], &["decision"], &["image"], &[]] {
assert_eq!(modalities(not_text), None, "{not_text:?}");
}
}
#[test]
fn header_details_list_the_family_only_when_there_is_one() {
let details = |family: &str| {
HeaderDetails {
family: family.into(),
parameter_size: "361.8M".into(),
quantization_level: "Q4_K_M".into(),
}
.details("gguf".into())
};
let llama = details("llama");
assert_eq!(
(llama.family.as_str(), llama.families.as_slice()),
("llama", ["llama".to_string()].as_slice())
);
assert_eq!(llama.parameter_size, "361.8M");
assert_eq!(llama.format, "gguf");
let none = details("");
assert_eq!((none.family.as_str(), none.families.len()), ("", 0));
}
#[test]
fn progress_line_stops_heartbeating_once_byte_counts_have_been_seen() {
let mut saw_bytes = false;
assert_eq!(
progress_line("pull", "m", (String::new(), 0, 0), &mut saw_bytes),
Some(serde_json::json!({"status": "pulling m"}))
);
assert!(!saw_bytes);
assert_eq!(
progress_line("pull", "m", ("pulling".into(), 100, 40), &mut saw_bytes),
Some(serde_json::json!({"status": "pulling", "total": 100, "completed": 40}))
);
assert!(saw_bytes);
assert_eq!(
progress_line("pull", "m", (String::new(), 0, 0), &mut saw_bytes),
None
);
}
#[test]
fn progress_line_reports_status_only_snapshots_and_clamps_completed() {
let mut saw_bytes = false;
assert_eq!(
progress_line("pull", "m", ("verifying".into(), 0, 0), &mut saw_bytes),
Some(serde_json::json!({"status": "verifying"}))
);
assert!(!saw_bytes);
assert_eq!(
progress_line("push", "m", (String::new(), 100, 999), &mut saw_bytes),
Some(serde_json::json!({"status": "pushing m", "total": 100, "completed": 100}))
);
}
#[tokio::test]
async fn handle_push_rejects_an_invalid_ref_with_400() {
let state = test_state();
let req = OllamaPushRequest {
model: "hf.co/../x".to_string(),
name: String::new(),
};
let resp = handle_push(State(state), Json(req)).await.into_response();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn handle_push_returns_404_for_a_model_not_in_the_store() {
let state = test_state();
let req = OllamaPushRequest {
model: "hf.co/does-not-exist/nowhere".to_string(),
name: String::new(),
};
let resp = handle_push(State(state), Json(req)).await.into_response();
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
}
#[test]
fn pull_notices_go_out_as_notice_lines() {
let lines = vec!["warning: unsigned".to_string()].into_lines();
assert_eq!(
lines,
vec![serde_json::json!({"notice": "warning: unsigned"})]
);
}
#[test]
fn a_push_outcome_reports_its_digest_on_the_stream() {
let lines = PushOutcome {
digest: "sha256:abc".into(),
}
.into_lines();
assert_eq!(lines, vec![serde_json::json!({"digest": "sha256:abc"})]);
}
#[test]
fn an_ollama_push_body_still_deserializes() {
let req: OllamaPushRequest =
serde_json::from_str(r#"{"model":"docker.io/org/model:v1"}"#).unwrap();
assert_eq!(req.model, "docker.io/org/model:v1");
let req: OllamaPushRequest = serde_json::from_str(r#"{"name":"x"}"#).unwrap();
assert_eq!(req.name, "x");
}
#[test]
fn ollama_pull_request_accepts_a_name_only_body() {
let req: OllamaPullRequest =
serde_json::from_value(serde_json::json!({"name": "docker.io/ai/gemma4:E2B"}))
.expect("a name-only body must still deserialize");
assert_eq!(req.model, "");
assert_eq!(req.name, "docker.io/ai/gemma4:E2B");
}
#[test]
fn ollama_pull_request_accepts_a_model_only_body() {
let req: OllamaPullRequest =
serde_json::from_value(serde_json::json!({"model": "docker.io/ai/gemma4:E2B"}))
.expect("a model-only body must still deserialize");
assert_eq!(req.model, "docker.io/ai/gemma4:E2B");
assert_eq!(req.name, "");
}
#[test]
fn ollama_push_request_accepts_a_name_only_body() {
let req: OllamaPushRequest =
serde_json::from_value(serde_json::json!({"name": "docker.io/ai/gemma4:E2B"}))
.expect("a name-only body must still deserialize");
assert_eq!(req.model, "");
assert_eq!(req.name, "docker.io/ai/gemma4:E2B");
}
#[test]
fn options_to_oai_maps_the_documented_sampling_options() {
let oai = options_to_oai(&Some(serde_json::json!({
"temperature": 0.5, "top_p": 0.9, "top_k": 40, "min_p": 0.05,
"seed": 42, "num_predict": -1, "repeat_penalty": 1.1,
"presence_penalty": 0.1, "frequency_penalty": 0.2,
"stop": ["a", "b"], "num_ctx": 8192
})));
assert_eq!(oai.temperature, Some(0.5));
assert_eq!(oai.top_k, Some(40));
assert_eq!(oai.min_p, Some(0.05));
assert_eq!(oai.seed, Some(42));
assert_eq!(oai.max_tokens, None);
assert_eq!(oai.presence_penalty, Some(0.1));
assert_eq!(oai.frequency_penalty, Some(0.2));
assert_eq!(
oai.stop.as_deref(),
Some(&["a".to_string(), "b".to_string()][..])
);
let one = options_to_oai(&Some(
serde_json::json!({ "stop": "END", "num_predict": 8 }),
));
assert_eq!(one.stop.as_deref(), Some(&["END".to_string()][..]));
assert_eq!(one.max_tokens, Some(8));
assert_eq!(options_to_oai(&None).seed, None);
}
#[test]
fn empty_chat_chunk_matches_ollamas_message_less_reply() {
let value = serde_json::to_value(empty_chat_chunk("m:latest".into(), "load")).unwrap();
assert_eq!(value["model"], "m:latest");
assert_eq!(value["done"], true);
assert_eq!(value["done_reason"], "load");
assert_eq!(value["message"]["role"], "assistant");
assert_eq!(value["message"]["content"], "");
assert_eq!(
value["message"].as_object().unwrap().len(),
2,
"ollama's message-less reply carries only role and content"
);
assert!(value.get("created_at").is_some());
}
#[test]
fn embed_inputs_accepts_a_string_or_string_array_only() {
assert_eq!(embed_inputs(&serde_json::json!("hi")).unwrap(), vec!["hi"]);
assert_eq!(
embed_inputs(&serde_json::json!(["a", "b"])).unwrap(),
vec!["a", "b"]
);
assert!(embed_inputs(&serde_json::Value::Null).unwrap().is_empty());
assert!(embed_inputs(&serde_json::json!("")).unwrap().is_empty());
for bad in [
serde_json::json!(1),
serde_json::json!(["a", 1]),
serde_json::json!({}),
] {
let err = embed_inputs(&bad).unwrap_err();
assert_eq!(err.1, StatusCode::BAD_REQUEST);
}
}
#[test]
fn normalize_in_place_yields_a_unit_vector_and_rejects_non_finite() {
let mut v = vec![3.0f32, 4.0];
normalize_in_place(&mut v).unwrap();
assert!((v[0] - 0.6).abs() < 1e-6 && (v[1] - 0.8).abs() < 1e-6);
let mut zero = vec![0.0f32, 0.0];
normalize_in_place(&mut zero).unwrap();
assert_eq!(zero, vec![0.0, 0.0]);
assert!(normalize_in_place(&mut [1.0, f32::NAN]).is_err());
assert!(normalize_in_place(&mut [f32::INFINITY]).is_err());
let mut huge = vec![f32::MAX, f32::MAX];
normalize_in_place(&mut huge).unwrap();
assert!((huge[0] - std::f32::consts::FRAC_1_SQRT_2).abs() < 1e-6);
let mut tiny = vec![1e-20f32, 0.0];
normalize_in_place(&mut tiny).unwrap();
assert_eq!(tiny, vec![1.0, 0.0]);
}
#[test]
fn option_extractors_ported_ollama_options_blob_cases() {
let opts = Some(serde_json::json!({
"temperature": 0.5,
"top_p": 0.9,
"num_predict": 128,
"stop": ["### User:"]
}));
assert_eq!(opt_f64(&opts, "temperature"), Some(0.5));
assert_eq!(opt_f64(&opts, "top_p"), Some(0.9));
assert_eq!(opt_u32(&opts, "num_predict"), Some(128));
assert_eq!(opt_f64(&opts, "repeat_penalty"), None);
assert_eq!(opt_u32(&opts, "stop"), None);
assert_eq!(opt_f64(&None, "temperature"), None);
assert_eq!(opt_u32(&None, "num_predict"), None);
}
#[test]
fn opt_num_thread_accepts_a_positive_integer() {
assert_eq!(
opt_num_thread(&Some(serde_json::json!({"num_thread": 4}))),
Some(4)
);
assert_eq!(
opt_num_thread(&Some(serde_json::json!({"num_thread": 1}))),
Some(1)
);
}
#[test]
fn opt_num_thread_rejects_zero_negative_and_non_integer_values() {
let cases = [
(
serde_json::json!(0),
"zero: llama-server rejects --threads 0",
),
(serde_json::json!(-1), "negative"),
(serde_json::json!(2.5), "fractional"),
(serde_json::json!("4"), "string, not a JSON number"),
(
serde_json::json!(u64::from(u32::MAX) + 1),
"above u32: must not truncate to --threads 0",
),
];
for (value, why) in cases {
let opts = Some(serde_json::json!({ "num_thread": value }));
assert_eq!(opt_num_thread(&opts), None, "{why}");
}
assert_eq!(opt_num_thread(&Some(serde_json::json!({}))), None);
assert_eq!(opt_num_thread(&None), None);
}
#[test]
fn opt_load_options_reads_threads_and_context_apart() {
let load = |options: serde_json::Value| opt_load_options(&Some(options));
let both = serde_json::json!({"num_thread": 4, "num_ctx": 8192, "temperature": 0.5});
let expect = |threads, ctx| LoadOptions { threads, ctx };
assert_eq!(load(both), expect(Some(4), Some(8192)));
assert_eq!(
load(serde_json::json!({"num_ctx": 8192})),
expect(None, Some(8192))
);
assert_eq!(
load(serde_json::json!({"num_ctx": 0})),
LoadOptions::default()
);
assert_eq!(opt_load_options(&None), LoadOptions::default());
}
#[tokio::test]
async fn ps_reports_the_configured_window_when_the_backend_exposes_none() {
let state = test_state();
let mut loaded = running_model_fixture(None, Duration::ZERO, 0);
loaded.port = 0;
loaded.context_window = Some(32_768);
state
.0
.manager
.lock()
.await
.running
.insert("m:latest".into(), loaded);
let resp = handle_ps(State(state.clone()), HeaderMap::new())
.await
.into_response();
let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX)
.await
.unwrap();
let ps: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
assert_eq!(ps["models"][0]["context_length"], 32_768);
}
#[tokio::test]
async fn evict_if_retagged_drops_only_a_runner_serving_stale_content() {
let state = test_state();
let key = "hf.co/o/m:latest";
let mut fixture = running_model_fixture(None, Duration::ZERO, 0);
fixture.digest = "sha256:old".into();
state
.0
.manager
.lock()
.await
.running
.insert(key.into(), fixture);
evict_if_retagged(&state, key, "sha256:old").await;
assert!(state.0.manager.lock().await.running.contains_key(key));
evict_if_retagged(&state, "hf.co/o/m", "sha256:new").await;
assert!(!state.0.manager.lock().await.running.contains_key(key));
}
#[test]
fn staged_file_refuses_a_name_that_is_not_a_bare_file_name() {
let state = test_state();
let digest = format!("sha256:{}", "a".repeat(64));
for name in ["../escape.gguf", "sub/dir.gguf", ".", ""] {
let err = staged_file(&state, name, &digest).unwrap_err();
assert_eq!(err.1, StatusCode::BAD_REQUEST, "{name:?}");
}
}
#[test]
fn staged_blob_path_requires_a_well_formed_sha256_digest() {
let state = test_state();
let ok = staged_blob_path(&state, &format!("sha256:{}", "a".repeat(64))).unwrap();
assert_eq!(ok, state.0.cache_path.join("blobs").join("a".repeat(64)));
for bad in [
"sha256:abc",
"md5:0000",
&format!("sha256:{}", "g".repeat(64)),
"../x",
] {
assert_eq!(
staged_blob_path(&state, bad).unwrap_err().1,
StatusCode::BAD_REQUEST
);
}
}
#[test]
fn model_info_json_sends_scalars_and_short_arrays_verbatim() {
use crate::gguf::Value;
let mut info = crate::gguf::Info::default();
for (k, v) in [
("general.architecture", Value::String("llama".into())),
("llama.context_length", Value::U32(4096)),
("tokenizer.ggml.add_eos_token", Value::Bool(false)),
(
"tokenizer.ggml.token_type",
Value::Array(vec![Value::I32(1), Value::I32(3)]),
),
(
"tokenizer.ggml.precompiled_charsmap",
Value::Array(vec![Value::U8(1), Value::U8(2), Value::U8(3)]),
),
("llama.vision.indexes", Value::Array(Vec::new())),
("general.name", Value::String("Qwen3.5 0.8B".into())),
(
"tokenizer.chat_template",
Value::String("{{ bulk }}".into()),
),
] {
info.metadata.insert(k.to_string(), v);
}
let json = model_info_json(&info);
assert_eq!(json["general.architecture"], serde_json::json!("llama"));
assert_eq!(json["llama.context_length"], serde_json::json!(4096));
assert_eq!(
json["tokenizer.ggml.add_eos_token"],
serde_json::json!(false)
);
assert_eq!(
json["tokenizer.ggml.precompiled_charsmap"],
serde_json::json!("AQID")
);
assert_eq!(json["tokenizer.ggml.token_type"], serde_json::json!([1, 3]));
assert_eq!(json["llama.vision.indexes"], serde_json::json!([]));
assert_eq!(json.get("general.name"), None);
assert_eq!(json.get("tokenizer.chat_template"), None);
}
#[test]
fn model_info_json_elides_only_arrays_past_the_ceiling() {
use crate::gguf::Value;
let array = |n: usize| Value::Array(vec![Value::U32(7); n]);
let mut info = crate::gguf::Info::default();
info.metadata.insert("at.ceiling".into(), array(1024));
info.metadata.insert("past.ceiling".into(), array(1025));
let json = model_info_json(&info);
assert_eq!(json["at.ceiling"].as_array().map(Vec::len), Some(1024));
assert_eq!(json["past.ceiling"], serde_json::json!([]));
}
}