use std::collections::HashMap;
use std::path::{Path, PathBuf};
use anyhow::{anyhow, Context};
use axum::body::{Body, Bytes};
use axum::extract::{Path as UrlPath, State};
use axum::http::{HeaderMap, StatusCode};
use axum::response::{IntoResponse, Response};
use axum::Json;
use futures::StreamExt;
use reqwest::Client;
use serde::Deserialize;
use tokio::time::{sleep, Duration, Instant};
use super::backend::*;
use super::sched::*;
use super::stream::*;
use super::types::*;
use super::{
aggregation, backend_wire_model, ensure_model, forward_ollama, model_lock, now_rfc3339,
opt_f64, opt_num_thread, opt_u32, pull_serialized, release_model_lock, remote_status,
send_with_hybrid_fallback, unload_everywhere, wire_refusal, AppError, AppState, Target,
};
use crate::metrics::{self, UnloadReason};
use crate::storage::OciStore;
pub(super) async fn handle_version(State(state): State<AppState>) -> impl IntoResponse {
Json(serde_json::json!({
"version": env!("LLMMAN_VERSION"),
"exe": state.0.exe.as_ref().map(|p| p.to_string_lossy()),
"pid": std::process::id(),
}))
}
pub(super) async fn handle_tags(
State(state): State<AppState>,
headers: HeaderMap,
) -> Result<impl IntoResponse, AppError> {
let store = OciStore::open(&state.0.store_path)?;
let list = store.list()?;
let mut models: Vec<OllamaModelInfo> = list
.into_iter()
.map(|img| OllamaModelInfo {
name: img.reference.clone(),
model: img.reference,
size: img.size,
digest: img.digest,
modified_at: img
.modified_at
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
.and_then(|d| chrono::DateTime::from_timestamp(d.as_secs() as i64, 0))
.map(|dt| dt.to_rfc3339())
.unwrap_or_else(now_rfc3339),
details: OllamaModelDetails {
format: "gguf".into(),
family: String::new(),
parameter_size: String::new(),
quantization_level: String::new(),
},
})
.collect();
if aggregation::aggregates(&state, &headers) {
for (_, peer) in aggregation::poll::<OllamaTagsResponse>(&state, "/api/tags").await {
for m in peer.models {
if !models.iter().any(|have| have.name == m.name) {
models.push(m);
}
}
}
}
Ok(Json(OllamaTagsResponse { models }))
}
struct PsEntry {
name: String,
digest: String,
size: u64,
port: u16,
pid: Option<u32>,
processor: String,
started_at: String,
expires_at: Option<chrono::DateTime<chrono::Utc>>,
}
pub(super) async fn handle_ps(
State(state): State<AppState>,
headers: HeaderMap,
) -> impl IntoResponse {
let entries: Vec<PsEntry> = {
let mgr = state.0.manager.lock().await;
mgr.running
.iter()
.map(|(name, m)| PsEntry {
name: name.clone(),
digest: m.digest.clone(),
size: m.size,
port: m.port,
pid: m.pid(),
processor: m.processor(),
started_at: m.started_at.clone(),
expires_at: m
.keep_alive
.and_then(|d| chrono::Duration::from_std(d).ok())
.map(|d| m.last_active_wall + d),
})
.collect()
};
let mut models = Vec::with_capacity(entries.len());
for entry in entries {
let context_length = query_context_length(&state.0.client, entry.port).await;
models.push(OllamaRunningModelInfo {
name: entry.name.clone(),
model: entry.name,
digest: entry.digest,
size: entry.size,
size_vram: 0, pid: entry.pid,
port: entry.port,
processor: entry.processor,
context_length,
started_at: entry.started_at,
expires_at: entry
.expires_at
.map(|t| t.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)),
node: None,
});
}
if aggregation::aggregates(&state, &headers) {
for (origin, peer) in aggregation::poll::<OllamaPsResponse>(&state, "/api/ps").await {
models.extend(peer.models.into_iter().map(|m| OllamaRunningModelInfo {
node: Some(origin.clone()),
..m
}));
}
}
Json(OllamaPsResponse { models })
}
async fn query_context_length(client: &Client, port: u16) -> Option<u64> {
let url = format!("http://127.0.0.1:{port}/props");
let resp = client
.get(&url)
.timeout(Duration::from_millis(500))
.send()
.await
.ok()?;
let body: serde_json::Value = resp.json().await.ok()?;
body.get("default_generation_settings")?
.get("n_ctx")?
.as_u64()
}
pub(super) async fn handle_show(
State(state): State<AppState>,
Json(req): Json<OllamaShowRequest>,
) -> Result<impl IntoResponse, AppError> {
let model_ref = req
.name
.as_deref()
.filter(|s| !s.is_empty())
.unwrap_or(&req.model);
if crate::providers::is_remote_ref(model_ref) {
eprintln!("[llmman] /api/show model={model_ref:?} (provider-routed)");
return Ok(Json(OllamaShowResponse {
model_info: serde_json::json!({ "digest": "", "size": 0 }),
details: OllamaModelDetails {
format: String::new(),
family: String::new(),
parameter_size: String::new(),
quantization_level: String::new(),
},
capabilities: Vec::new(),
template: None,
}));
}
let model_ref =
crate::shortnames::resolve_ollama_api(model_ref).map_err(AppError::bad_request)?;
let model_ref = model_ref.as_str();
eprintln!("[llmman] /api/show model={model_ref:?}");
let store = OciStore::open(&state.0.store_path)?;
let desc = store.find(model_ref).map_err(|e| {
if e.downcast_ref::<crate::storage::oci::NotFound>().is_some() {
AppError::status(
StatusCode::NOT_FOUND,
format!("model not found: {model_ref}"),
)
} else {
AppError::from(e)
}
})?;
let manifest = store.read_manifest(&desc.digest)?;
let capabilities = crate::modelpack::capabilities(&store, &manifest);
let template = crate::modelpack::chat_template(
&store,
&state.0.store_path,
&state.0.cache_path,
&manifest,
);
Ok(Json(OllamaShowResponse {
model_info: serde_json::json!({ "digest": desc.digest, "size": desc.size }),
details: OllamaModelDetails {
format: "gguf".into(),
family: String::new(),
parameter_size: String::new(),
quantization_level: String::new(),
},
capabilities,
template,
}))
}
#[derive(Debug, Deserialize)]
pub(super) struct OllamaPullRequest {
#[serde(default)]
pub(super) model: String,
#[serde(default)]
pub(super) name: String,
}
pub(super) async fn handle_pull(
State(state): State<AppState>,
Json(req): Json<OllamaPullRequest>,
) -> Result<Response, AppError> {
let model_ref = if req.model.is_empty() {
req.name.as_str()
} else {
req.model.as_str()
};
if model_ref.is_empty() {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
"model is required",
));
}
let model = crate::shortnames::resolve_ollama_api(model_ref).map_err(AppError::bad_request)?;
eprintln!("[llmman] /api/pull model={model:?}");
let store_path = state.0.store_path.clone();
let model_for_task = model.clone();
let pull_task =
tokio::task::spawn_blocking(move || pull_serialized(&store_path, &model_for_task));
Ok(stream_ffi_progress(
model,
"pull",
"pulling manifest",
pull_task,
))
}
#[derive(Debug, Deserialize)]
pub(super) struct OllamaPushRequest {
#[serde(default)]
pub(super) model: String,
#[serde(default)]
pub(super) name: String,
}
pub(super) async fn handle_push(
State(state): State<AppState>,
Json(req): Json<OllamaPushRequest>,
) -> Result<Response, AppError> {
let model_ref = if req.model.is_empty() {
req.name.as_str()
} else {
req.model.as_str()
};
push_impl(state, model_ref, "/api/push").await
}
async fn push_impl(
state: AppState,
model_ref: &str,
route: &'static str,
) -> Result<Response, AppError> {
if model_ref.is_empty() {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
"model is required",
));
}
let model = crate::shortnames::resolve_ollama_api(model_ref).map_err(AppError::bad_request)?;
eprintln!("[llmman] {route} model={model:?}");
let store_path = state.0.store_path.clone();
if OciStore::open(&store_path)
.and_then(|s| s.find(&model))
.is_err()
{
return Err(AppError::status(
StatusCode::NOT_FOUND,
format!("model not found: {model}"),
));
}
let model_for_task = model.clone();
let push_task = tokio::task::spawn_blocking(move || {
let lock = model_lock(&model_for_task);
let result = (|| {
let _guard = lock.blocking_lock();
let layout_dir = store_path
.to_str()
.ok_or_else(|| anyhow!("store path is not valid UTF-8"))?;
crate::ffi::push(layout_dir, &model_for_task)?;
let desc = OciStore::open(&store_path)?.find(&model_for_task)?;
Ok(PushOutcome {
digest: desc.digest,
})
})();
drop(lock);
release_model_lock(&model_for_task);
result
});
Ok(stream_ffi_progress(
model,
"push",
"retrieving manifest",
push_task,
))
}
pub(super) struct PushOutcome {
pub(super) digest: String,
}
pub(super) trait StreamedOutcome {
fn into_lines(self) -> Vec<serde_json::Value>;
}
impl StreamedOutcome for Vec<String> {
fn into_lines(self) -> Vec<serde_json::Value> {
self.into_iter()
.map(|notice| serde_json::json!({"notice": notice}))
.collect()
}
}
impl StreamedOutcome for PushOutcome {
fn into_lines(self) -> Vec<serde_json::Value> {
vec![serde_json::json!({"digest": self.digest})]
}
}
pub(super) fn progress_line(
verb: &str,
model: &str,
snap: (String, i64, i64),
saw_bytes: &mut bool,
) -> Option<serde_json::Value> {
let (status, total, completed) = snap;
if total > 0 {
*saw_bytes = true;
return Some(serde_json::json!({
"status": if status.is_empty() { format!("{verb}ing {model}") } else { status },
"total": total,
"completed": completed.clamp(0, total),
}));
}
if !status.is_empty() {
return Some(serde_json::json!({"status": status}));
}
if *saw_bytes {
return None;
}
Some(serde_json::json!({"status": format!("{verb}ing {model}")}))
}
fn stream_ffi_progress<T: StreamedOutcome + Send + 'static>(
model: String,
verb: &'static str,
first_status: &'static str,
task: tokio::task::JoinHandle<anyhow::Result<T>>,
) -> Response {
let first_line = serde_json::json!({"status": first_status}).to_string() + "\n";
let stream = futures::stream::once(futures::future::ready(Bytes::from(first_line)))
.chain(futures::stream::unfold((Some(task), false), move |state| {
let (task, mut saw_bytes) = state;
let model = model.clone();
async move {
let mut task = task?;
tokio::select! {
result = &mut task => {
let mut out = String::new();
let line = match result {
Ok(Ok(outcome)) => {
for field in outcome.into_lines() {
out.push_str(&field.to_string());
out.push('\n');
}
serde_json::json!({"status": "success"}).to_string()
}
Ok(Err(e)) => serde_json::json!({"error": format!("{e:#}")}).to_string(),
Err(e) => serde_json::json!({"error": format!("{verb} task panicked: {e}")}).to_string(),
};
out.push_str(&line);
out.push('\n');
Some((Bytes::from(out), (None, saw_bytes)))
}
_ = sleep(Duration::from_millis(200)) => {
let rust_snap = crate::hf::progress::poll(&model);
let go_snap = (rust_snap.total == 0).then(|| crate::ffi::progress(&model).ok()).flatten();
let (status, total, completed) = if rust_snap.total > 0 {
(rust_snap.status, rust_snap.total, rust_snap.completed)
} else if !rust_snap.status.is_empty() {
(rust_snap.status, 0, 0)
} else if let Some(p) = &go_snap {
(p.status.clone(), p.total, p.completed)
} else {
(String::new(), 0, 0)
};
let line = progress_line(verb, &model, (status, total, completed), &mut saw_bytes);
let out = line.map(|l| l.to_string() + "\n").unwrap_or_default();
Some((Bytes::from(out), (Some(task), saw_bytes)))
}
}
}
}))
.map(Ok::<_, std::convert::Infallible>);
Response::builder()
.status(StatusCode::OK)
.header("content-type", "application/x-ndjson")
.body(Body::from_stream(stream))
.unwrap()
}
pub(super) async fn handle_delete(
State(state): State<AppState>,
Json(req): Json<OllamaDeleteRequest>,
) -> Result<impl IntoResponse, AppError> {
let model_ref = req
.name
.as_deref()
.filter(|s| !s.is_empty())
.unwrap_or(&req.model);
let model_ref =
crate::shortnames::resolve_ollama_api(model_ref).map_err(AppError::bad_request)?;
let store = OciStore::open(&state.0.store_path)?;
store.remove(&model_ref)?;
Ok(StatusCode::OK)
}
pub(super) async fn handle_copy(
State(state): State<AppState>,
Json(req): Json<OllamaCopyRequest>,
) -> Result<impl IntoResponse, AppError> {
if req.source.is_empty() || req.destination.is_empty() {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
"source and destination are required",
));
}
let source =
crate::shortnames::resolve_ollama_api(&req.source).map_err(AppError::bad_request)?;
let destination =
crate::shortnames::resolve_ollama_api(&req.destination).map_err(AppError::bad_request)?;
eprintln!("[llmman] /api/copy {source:?} -> {destination:?}");
let store = OciStore::open(&state.0.store_path)?;
if store.find(&source).is_err() {
return Err(AppError::status(
StatusCode::NOT_FOUND,
format!("model '{}' not found", req.source),
));
}
let digest = crate::cmd::cp::copy(&store, &source, &destination)?;
evict_if_retagged(&state, &destination, &digest).await;
Ok(StatusCode::OK)
}
pub(super) async fn evict_if_retagged(state: &AppState, reference: &str, digest: &str) {
let want = crate::storage::default_tag(reference);
let mut mgr = state.0.manager.lock().await;
let stale: Vec<String> = mgr
.running
.iter()
.filter(|(k, m)| m.digest != digest && crate::storage::default_tag(k) == want)
.map(|(k, _)| k.clone())
.collect();
for key in stale {
mgr.running.remove(&key);
metrics::record_model_unload(&key, UnloadReason::Requested);
}
}
fn blob_staging_dir(state: &AppState) -> PathBuf {
state.0.cache_path.join("blobs")
}
static STAGING_COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
fn staging_temp_name(prefix: &str) -> String {
let n = STAGING_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
format!("{prefix}-{}-{n}", std::process::id())
}
pub(super) fn staged_blob_path(state: &AppState, digest: &str) -> Result<PathBuf, AppError> {
let hex = digest.strip_prefix("sha256:").unwrap_or_default();
if hex.len() != 64 || !hex.bytes().all(|b| b.is_ascii_hexdigit()) {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
format!("invalid digest {digest:?}: expected sha256:<64 hex digits>"),
));
}
Ok(blob_staging_dir(state).join(hex.to_ascii_lowercase()))
}
pub(super) async fn handle_blob_head(
State(state): State<AppState>,
UrlPath(digest): UrlPath<String>,
) -> Result<StatusCode, AppError> {
let path = staged_blob_path(&state, &digest)?;
Ok(if path.is_file() {
StatusCode::OK
} else {
StatusCode::NOT_FOUND
})
}
pub(super) async fn handle_blob_upload(
State(state): State<AppState>,
UrlPath(digest): UrlPath<String>,
body: Body,
) -> Result<StatusCode, AppError> {
use sha2::Digest as _;
use tokio::io::AsyncWriteExt as _;
let dest = staged_blob_path(&state, &digest)?;
if dest.is_file() {
return Ok(StatusCode::OK);
}
let dir = blob_staging_dir(&state);
tokio::fs::create_dir_all(&dir).await?;
let tmp = dir.join(staging_temp_name("tmp"));
let mut file = tokio::fs::File::create(&tmp).await?;
let mut hasher = sha2::Sha256::new();
let mut stream = body.into_data_stream();
let written = async {
while let Some(chunk) = stream.next().await {
let chunk = chunk.context("read upload body")?;
hasher.update(&chunk);
file.write_all(&chunk).await?;
}
file.flush().await?;
anyhow::Ok(())
}
.await;
if let Err(e) = written {
let _ = tokio::fs::remove_file(&tmp).await;
return Err(e.into());
}
let actual = format!("sha256:{}", hex::encode(hasher.finalize()));
if !actual.eq_ignore_ascii_case(&digest) {
let _ = tokio::fs::remove_file(&tmp).await;
return Err(AppError::status(
StatusCode::BAD_REQUEST,
format!("digest mismatch, expected {digest:?}, got {actual:?}"),
));
}
tokio::fs::rename(&tmp, &dest).await?;
eprintln!("[llmman] /api/blobs stored {digest} ({})", dest.display());
Ok(StatusCode::CREATED)
}
pub(super) async fn handle_create(
State(state): State<AppState>,
Json(req): Json<OllamaCreateRequest>,
) -> Result<Response, AppError> {
let model = if req.model.is_empty() {
req.name.as_str()
} else {
req.model.as_str()
};
if model.is_empty() {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
"model is required",
));
}
let refused: Vec<&str> = req
.unsupported
.iter()
.filter(|(_, v)| !is_empty_json(v))
.map(|(k, _)| k.as_str())
.collect();
if !refused.is_empty() {
let mut refused = refused;
refused.sort_unstable();
return Err(AppError::status(
StatusCode::BAD_REQUEST,
format!(
"/api/create: {} not supported by llmman — models are plain OCI artifacts \
whose GGUF carries its own chat template; only `from` (alias an existing \
model) and `files` (package uploaded blobs) are honoured",
refused.join(", ")
),
));
}
let model = &crate::shortnames::resolve_ollama_api(model).map_err(AppError::bad_request)?;
if crate::storage::split_ref_digest(model).1.is_some() {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
"/api/create: the model name can't be a digest reference",
));
}
let files = req.files.unwrap_or_default();
let from = req.from.filter(|f| !f.is_empty());
let statuses: Vec<String> = match (from, files.is_empty()) {
(Some(_), false) => {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
"/api/create: `from` and `files` are mutually exclusive here",
))
}
(None, true) => {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
"/api/create: one of `from` or `files` is required",
))
}
(Some(from), true) => {
let source =
crate::shortnames::resolve_ollama_api(&from).map_err(AppError::bad_request)?;
eprintln!("[llmman] /api/create {model:?} from {source:?}");
let store = OciStore::open(&state.0.store_path)?;
if store.find(&source).is_err() {
return Err(AppError::status(
StatusCode::NOT_FOUND,
format!("model '{from}' not found"),
));
}
let digest = crate::cmd::cp::copy(&store, &source, model)?;
evict_if_retagged(&state, model, &digest).await;
vec![format!("using existing layer {source}")]
}
(None, false) => {
eprintln!(
"[llmman] /api/create {model:?} from {} uploaded file(s)",
files.len()
);
let staged = files
.iter()
.map(|(name, digest)| Ok((name.clone(), staged_file(&state, name, digest)?)))
.collect::<Result<Vec<_>, AppError>>()?;
let store_path = state.0.store_path.clone();
let staging = blob_staging_dir(&state);
let model_owned = model.to_string();
let (digest, statuses) = tokio::task::spawn_blocking(move || {
create_from_staged_blobs(&store_path, &staging, &model_owned, &staged)
})
.await
.context("create task panicked")??;
evict_if_retagged(&state, model, &digest).await;
statuses
}
};
let mut lines: Vec<serde_json::Value> = statuses
.into_iter()
.map(|s| serde_json::json!({"status": s}))
.collect();
lines.push(serde_json::json!({"status": "success"}));
Ok(if req.stream {
let body: String = lines.iter().map(|l| l.to_string() + "\n").collect();
([("content-type", "application/x-ndjson")], body).into_response()
} else {
Json(lines.pop().unwrap_or_default()).into_response()
})
}
fn is_empty_json(v: &serde_json::Value) -> bool {
match v {
serde_json::Value::Null => true,
serde_json::Value::String(s) => s.is_empty(),
serde_json::Value::Object(m) => m.is_empty(),
serde_json::Value::Array(a) => a.is_empty(),
_ => false,
}
}
pub(super) fn staged_file(state: &AppState, name: &str, digest: &str) -> Result<PathBuf, AppError> {
let bare = Path::new(name)
.file_name()
.and_then(|n| n.to_str())
.is_some_and(|n| n == name);
if !bare {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
format!("files: {name:?} must be a bare file name"),
));
}
let src = staged_blob_path(state, digest)?;
if !src.is_file() {
return Err(AppError::status(
StatusCode::NOT_FOUND,
format!("files: {digest} for {name:?} was never uploaded to /api/blobs"),
));
}
Ok(src)
}
fn create_from_staged_blobs(
store_path: &Path,
staging: &Path,
model: &str,
files: &[(String, PathBuf)],
) -> anyhow::Result<(String, Vec<String>)> {
let tmp = (0..3)
.map(|_| staging.join(staging_temp_name("create")))
.find_map(|dir| std::fs::create_dir(&dir).ok().map(|()| dir))
.ok_or_else(|| {
anyhow!(
"couldn't create a fresh build directory under {}",
staging.display()
)
})?;
let result = (|| {
let mut statuses = Vec::new();
for (name, src) in files {
let dst = tmp.join(name);
if std::fs::hard_link(src, &dst).is_err() {
std::fs::copy(src, &dst)
.with_context(|| format!("stage {name} from {}", src.display()))?;
}
let digest = src.file_name().unwrap_or_default().to_string_lossy();
statuses.push(format!("using sha256:{digest} as {name}"));
}
let store = OciStore::open(store_path)?;
let desc = store.build(&tmp, model, &HashMap::new())?;
statuses.push(format!("writing manifest {}", desc.digest));
Ok((desc.digest, statuses))
})();
let _ = std::fs::remove_dir_all(&tmp);
result
}
pub(super) fn empty_chat_chunk(model: String, done_reason: &str) -> OllamaChatChunk {
OllamaChatChunk {
model,
created_at: now_rfc3339(),
message: OllamaMessage {
role: "assistant".into(),
..Default::default()
},
done: true,
done_reason: Some(done_reason.into()),
metrics: OllamaMetrics::default(),
}
}
pub(super) async fn handle_ollama_chat(
State(state): State<AppState>,
headers: HeaderMap,
Json(req): Json<OllamaChatRequest>,
) -> Result<Response, AppError> {
eprintln!(
"[llmman] /api/chat model={:?} messages={}",
req.model,
req.messages.len()
);
if req.messages.is_empty() && is_explicit_unload(&req.keep_alive) {
unload_everywhere(&state, crate::hybrid::local_half(&req.model), &headers).await?;
return Ok(Json(empty_chat_chunk(req.model, "unload")).into_response());
}
let model_ref = req.model.clone();
let request_threads = opt_num_thread(&req.options);
let started = Instant::now();
send_with_hybrid_fallback(
&state,
&model_ref,
Some(&headers),
request_threads,
|model, target, guard| {
ollama_chat_to(&state, &headers, &req, model, target, guard, started)
},
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn ollama_chat_to(
state: &AppState,
headers: &HeaderMap,
req: &OllamaChatRequest,
model: String,
target: Target,
guard: ActivityGuard,
started: Instant,
) -> Result<Response, AppError> {
let load_duration = if target.is_remote() {
Duration::ZERO
} else {
started.elapsed()
};
if matches!(target, Target::Peer(_)) {
return forward_ollama(state, &target, "/api/chat", headers, req, &model, guard).await;
}
if req.messages.is_empty() {
refresh_activity(guard, resolve_keep_alive(&req.keep_alive)).await;
return Ok(Json(empty_chat_chunk(req.model.clone(), "load")).into_response());
}
let keep_alive = resolve_keep_alive(&req.keep_alive);
let activity = begin_activity(guard, Some(keep_alive)).await;
let wire_model = backend_wire_model(state, &target, &model).await;
let oai = OAIChatRequest {
model: wire_model,
messages: req.messages.iter().map(ollama_message_to_oai).collect(),
stream: true,
chat_template_kwargs: think_to_chat_template_kwargs(&req.think),
tools: req.tools.clone(),
response_format: format_to_response_format(&req.format),
..options_to_oai(&req.options)
};
stream_ollama(
req.stream,
state.0.client.clone(),
target,
oai,
activity,
started,
load_duration,
move |delta| OllamaChatChunk {
model: model.clone(),
created_at: now_rfc3339(),
message: OllamaMessage {
role: "assistant".into(),
content: delta.content,
thinking: delta.thinking,
tool_calls: delta.tool_calls,
..Default::default()
},
done: delta.done,
done_reason: delta.done_reason,
metrics: delta.metrics,
},
)
.await
}
pub(super) fn options_to_oai(options: &Option<serde_json::Value>) -> OAIChatRequest {
OAIChatRequest {
temperature: opt_f64(options, "temperature"),
top_p: opt_f64(options, "top_p"),
max_tokens: opt_u32(options, "num_predict"),
repeat_penalty: opt_f64(options, "repeat_penalty"),
seed: options.as_ref().and_then(|o| o.get("seed")?.as_u64()),
stop: opt_stop(options),
presence_penalty: opt_f64(options, "presence_penalty"),
frequency_penalty: opt_f64(options, "frequency_penalty"),
top_k: opt_u32(options, "top_k"),
min_p: opt_f64(options, "min_p"),
..Default::default()
}
}
fn opt_stop(options: &Option<serde_json::Value>) -> Option<Vec<String>> {
match options.as_ref()?.get("stop")? {
serde_json::Value::String(s) => Some(vec![s.clone()]),
serde_json::Value::Array(a) => Some(
a.iter()
.filter_map(|v| v.as_str().map(str::to_string))
.collect(),
),
_ => None,
}
}
pub(super) async fn handle_ollama_generate(
State(state): State<AppState>,
headers: HeaderMap,
Json(req): Json<OllamaGenerateRequest>,
) -> Result<Response, AppError> {
eprintln!(
"[llmman] /api/generate model={:?} prompt_len={}",
req.model,
req.prompt.len()
);
let is_unload = req.prompt.is_empty() && is_explicit_unload(&req.keep_alive);
if is_unload {
unload_everywhere(&state, crate::hybrid::local_half(&req.model), &headers).await?;
return Ok(Json(OllamaGenerateChunk {
model: req.model,
created_at: now_rfc3339(),
response: String::new(),
thinking: None,
done: true,
done_reason: Some("unload".into()),
metrics: OllamaMetrics::default(),
})
.into_response());
}
let unsupported = if req.raw {
Some("raw")
} else if req.suffix.as_deref().is_some_and(|s| !s.is_empty()) {
Some("suffix")
} else if req.template.as_deref().is_some_and(|t| !t.is_empty()) {
Some("template")
} else {
None
};
if let Some(field) = unsupported {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
format!("llmman does not support the {field:?} field of /api/generate"),
));
}
let model_ref = req.model.clone();
let request_threads = opt_num_thread(&req.options);
let started = Instant::now();
send_with_hybrid_fallback(
&state,
&model_ref,
Some(&headers),
request_threads,
|model, target, guard| {
ollama_generate_to(&state, &headers, &req, model, target, guard, started)
},
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn ollama_generate_to(
state: &AppState,
headers: &HeaderMap,
req: &OllamaGenerateRequest,
model: String,
target: Target,
guard: ActivityGuard,
started: Instant,
) -> Result<Response, AppError> {
let load_duration = if target.is_remote() {
Duration::ZERO
} else {
started.elapsed()
};
if matches!(target, Target::Peer(_)) {
return forward_ollama(state, &target, "/api/generate", headers, req, &model, guard).await;
}
if req.prompt.is_empty() {
refresh_activity(guard, resolve_keep_alive(&req.keep_alive)).await;
return Ok(Json(OllamaGenerateChunk {
model: req.model.clone(),
created_at: now_rfc3339(),
response: String::new(),
thinking: None,
done: true,
done_reason: Some("load".into()),
metrics: OllamaMetrics::default(),
})
.into_response());
}
let keep_alive = resolve_keep_alive(&req.keep_alive);
let activity = begin_activity(guard, Some(keep_alive)).await;
let wire_model = backend_wire_model(state, &target, &model).await;
let user = OllamaMessage {
role: "user".into(),
content: req.prompt.clone(),
images: req.images.clone(),
..Default::default()
};
let messages = req
.system
.as_deref()
.filter(|s| !s.is_empty())
.map(|s| OAIMessage::text("system", s))
.into_iter()
.chain(std::iter::once(ollama_message_to_oai(&user)))
.collect();
let oai = OAIChatRequest {
model: wire_model,
messages,
stream: true,
chat_template_kwargs: think_to_chat_template_kwargs(&req.think),
tools: None,
response_format: format_to_response_format(&req.format),
..options_to_oai(&req.options)
};
stream_ollama(
req.stream,
state.0.client.clone(),
target,
oai,
activity,
started,
load_duration,
move |delta| OllamaGenerateChunk {
model: model.clone(),
created_at: now_rfc3339(),
response: delta.content,
thinking: delta.thinking,
done: delta.done,
done_reason: delta.done_reason,
metrics: delta.metrics,
},
)
.await
}
pub(super) fn embed_inputs(input: &serde_json::Value) -> Result<Vec<String>, AppError> {
let invalid = || AppError::status(StatusCode::BAD_REQUEST, "invalid input type");
match input {
serde_json::Value::Null => Ok(Vec::new()),
serde_json::Value::String(s) if s.is_empty() => Ok(Vec::new()),
serde_json::Value::String(s) => Ok(vec![s.clone()]),
serde_json::Value::Array(items) => items
.iter()
.map(|v| v.as_str().map(str::to_owned).ok_or_else(invalid))
.collect(),
_ => Err(invalid()),
}
}
async fn embed_via_backend(
state: &AppState,
headers: &HeaderMap,
model_ref: &str,
inputs: &[String],
truncate: bool,
keep_alive: &Option<serde_json::Value>,
) -> Result<(Vec<Vec<f32>>, u64, Duration), AppError> {
let mlx_unsupported = |model: &str| {
AppError::status(
StatusCode::NOT_IMPLEMENTED,
format!(
"{model} is served by mlx_lm.server, which llmman never starts with \
--embedding-model; use a GGUF or vllm-served model for embeddings"
),
)
};
if !crate::providers::is_remote_ref(model_ref) {
if let Some(canonical) = would_use_mlx(state, model_ref).await {
return Err(mlx_unsupported(&canonical));
}
}
let started = Instant::now();
let (model, target, guard) = ensure_model(state, model_ref, Some(headers), None).await?;
let loaded = started.elapsed();
if !target.is_remote() && would_use_mlx(state, &model).await.is_some() {
return Err(mlx_unsupported(&model));
}
if let Some(refusal) = wire_refusal(&target, "/v1/embeddings") {
return Err(AppError::status(StatusCode::NOT_IMPLEMENTED, refusal));
}
let keep_alive = resolve_keep_alive(keep_alive);
if inputs.is_empty() {
refresh_activity(guard, keep_alive).await;
return Ok((Vec::new(), 0, loaded));
}
let _activity = begin_activity(guard, Some(keep_alive)).await;
let wire_model = backend_wire_model(state, &target, &model).await;
use futures::TryStreamExt as _;
let calls: Vec<_> = inputs
.iter()
.map(|text| embed_one(state, &target, &wire_model, text, truncate))
.collect();
let results: Vec<(Vec<f32>, u64)> = futures::stream::iter(calls)
.buffered(EMBED_CONCURRENCY)
.try_collect()
.await?;
let total_tokens = results.iter().map(|(_, n)| n).sum();
let embeddings = results
.into_iter()
.map(|(mut v, _)| {
normalize_in_place(&mut v)?;
Ok(v)
})
.collect::<Result<_, AppError>>()?;
Ok((embeddings, total_tokens, loaded))
}
const EMBED_CONCURRENCY: usize = 8;
async fn embed_one(
state: &AppState,
target: &Target,
wire_model: &str,
text: &str,
truncate: bool,
) -> Result<(Vec<f32>, u64), AppError> {
match post_embeddings(&state.0.client, target, wire_model, text).await {
Ok(ok) => Ok(ok),
Err((status, body)) if truncate && !target.is_remote() && status.is_server_error() => {
let Target::Local(port) = target else {
unreachable!("guarded by !is_remote")
};
let limit = query_context_length(&state.0.client, *port)
.await
.ok_or_else(|| anyhow!("{} {status}: {body}", target.describe()))?;
let limit = usize::try_from(limit)
.unwrap_or(usize::MAX)
.saturating_sub(2);
let truncated = truncate_to_tokens(&state.0.client, *port, text, limit).await?;
if truncated.len() >= text.len() {
return Err(AppError(
anyhow!("{} {status}: {body}", target.describe()),
StatusCode::INTERNAL_SERVER_ERROR,
));
}
post_embeddings(&state.0.client, target, wire_model, &truncated)
.await
.map_err(|(status, body)| {
AppError(
anyhow!("{} {status}: {body}", target.describe()),
remote_status(target, status),
)
})
}
Err((status, body))
if !truncate
&& !target.is_remote()
&& status.is_server_error()
&& body.contains("too large") =>
{
Err(AppError(
anyhow!("the input length exceeds the context length: {body}"),
StatusCode::BAD_REQUEST,
))
}
Err((status, body)) => Err(AppError(
anyhow!("{} {status}: {body}", target.describe()),
remote_status(target, status),
)),
}
}
async fn post_embeddings(
client: &Client,
target: &Target,
wire_model: &str,
text: &str,
) -> Result<(Vec<f32>, u64), (StatusCode, String)> {
let resp = target
.authorize(
client
.post(target.url("/v1/embeddings"))
.timeout(EMBEDDINGS_TIMEOUT)
.json(&serde_json::json!({ "model": wire_model, "input": text })),
)
.send()
.await
.map_err(|e| {
(
StatusCode::BAD_GATEWAY,
format!("send to {}: {e}", target.describe()),
)
})?;
let status = resp.status();
let body = resp.bytes().await.unwrap_or_default();
if !status.is_success() {
return Err((status, String::from_utf8_lossy(&body).into_owned()));
}
let mut parsed: OAIEmbeddingsResponse = serde_json::from_slice(&body).map_err(|e| {
(
StatusCode::BAD_GATEWAY,
format!(
"{} returned an unexpected embeddings body: {e}",
target.describe()
),
)
})?;
parsed.data.sort_by_key(|d| d.index);
let vector = parsed
.data
.into_iter()
.next()
.map(|d| d.embedding)
.ok_or_else(|| {
(
StatusCode::BAD_GATEWAY,
format!("{} returned no embedding", target.describe()),
)
})?;
Ok((vector, parsed.usage.prompt_tokens))
}
const TOKENIZE_TIMEOUT: Duration = Duration::from_secs(30);
const EMBEDDINGS_TIMEOUT: Duration = Duration::from_secs(600);
async fn truncate_to_tokens(
client: &Client,
port: u16,
text: &str,
limit: usize,
) -> anyhow::Result<String> {
#[derive(Deserialize)]
struct Tokens {
tokens: Vec<i64>,
}
#[derive(Deserialize)]
struct Content {
content: String,
}
let base = format!("http://127.0.0.1:{port}");
let Tokens { mut tokens } = client
.post(format!("{base}/tokenize"))
.timeout(TOKENIZE_TIMEOUT)
.json(&serde_json::json!({ "content": text }))
.send()
.await
.context("tokenize for truncation")?
.error_for_status()?
.json()
.await?;
if limit == 0 {
anyhow::bail!("input after truncation exceeds maximum context length");
}
if tokens.len() <= limit {
return Ok(text.to_string());
}
tokens.truncate(limit);
let Content { content } = client
.post(format!("{base}/detokenize"))
.timeout(TOKENIZE_TIMEOUT)
.json(&serde_json::json!({ "tokens": tokens }))
.send()
.await
.context("detokenize for truncation")?
.error_for_status()?
.json()
.await?;
Ok(content)
}
pub(super) fn normalize_in_place(v: &mut [f32]) -> Result<(), AppError> {
if v.iter().any(|x| !x.is_finite()) {
return Err(AppError::status(
StatusCode::BAD_GATEWAY,
"embedding contains NaN or Inf values",
));
}
let norm = v.iter().map(|&x| f64::from(x).powi(2)).sum::<f64>().sqrt();
if norm > 0.0 {
for x in v.iter_mut() {
*x = (f64::from(*x) / norm) as f32;
}
}
Ok(())
}
pub(super) async fn handle_embed(
State(state): State<AppState>,
headers: HeaderMap,
Json(req): Json<OllamaEmbedRequest>,
) -> Result<Response, AppError> {
let started = Instant::now();
let inputs = embed_inputs(&req.input)?;
eprintln!(
"[llmman] /api/embed model={:?} inputs={}",
req.model,
inputs.len()
);
let (mut embeddings, tokens, loaded) = embed_via_backend(
&state,
&headers,
&req.model,
&inputs,
req.truncate != Some(false),
&req.keep_alive,
)
.await?;
if let Some(dims) = req.dimensions.filter(|d| *d > 0) {
for v in &mut embeddings {
if dims < v.len() {
v.truncate(dims);
normalize_in_place(v)?;
}
}
}
Ok(Json(OllamaEmbedResponse {
model: req.model,
embeddings,
total_duration: started.elapsed().as_nanos() as u64,
load_duration: loaded.as_nanos() as u64,
prompt_eval_count: tokens,
})
.into_response())
}
pub(super) async fn handle_embeddings(
State(state): State<AppState>,
headers: HeaderMap,
Json(req): Json<OllamaEmbeddingsRequest>,
) -> Result<Response, AppError> {
eprintln!(
"[llmman] /api/embeddings model={:?} prompt_len={}",
req.model,
req.prompt.len()
);
let inputs = if req.prompt.is_empty() {
Vec::new()
} else {
vec![req.prompt.clone()]
};
let (embeddings, _tokens, _loaded) =
embed_via_backend(&state, &headers, &req.model, &inputs, true, &req.keep_alive).await?;
let embedding = embeddings
.into_iter()
.next()
.unwrap_or_default()
.into_iter()
.map(f64::from)
.collect();
Ok(Json(OllamaEmbeddingsResponse { embedding }).into_response())
}