use anyhow::Context;
use axum::body::{Body, Bytes};
use axum::extract::State;
use axum::http::{HeaderMap, StatusCode};
use axum::response::{IntoResponse, Response};
use axum::Json;
use futures::StreamExt;
use tokio::time::{sleep, Duration, Instant};
use super::backend::{
backend_json, extra_args_from_env, gguf_engine_from_env, llama_sets_slot_context,
would_use_mlx, LLAMA_ARGS_VAR,
};
use super::hybrid::{request_pin, send_with_hybrid_fallback};
use super::messages::content_text;
use super::ollama::{is_embedding_overflow, listed_model_facts, truncate_embedding};
use super::refusal::{explain_missing_route, unsupported_on_wire};
use super::relay::{
proxy, proxy_buffered, proxy_rewriting_model, relay, relay_chat_upstream,
stream_rewriting_model,
};
use super::responses::{
is_responses_route, remote_responses, sanitize_responses_request, RESPONSES_ROUTE,
};
use super::sched::{begin_activity, resolve_keep_alive, ActivityGuard};
use super::types::*;
use super::{
aggregation, backend_wire_model, ensure_model, ensure_model_for, provider_compat,
running_by_digest, send_chat_completion, strip_llama_fields, AppError, AppState, Engine,
LoadOptions, Target, Task, CHAT_COMPLETIONS_ROUTE,
};
use crate::ordered::Ordered;
use crate::storage::OciStore;
pub(super) async fn handle_openai_models(
State(state): State<AppState>,
headers: HeaderMap,
) -> Result<impl IntoResponse, AppError> {
let (store_path, cache_path) = (state.0.store_path.clone(), state.0.cache_path.clone());
let rows = tokio::task::spawn_blocking(move || -> anyhow::Result<_> {
let store = OciStore::open(&store_path)?;
let list = store.list()?;
let facts = listed_model_facts(&store, &store_path, &cache_path, &list);
Ok(list.into_iter().zip(facts).collect::<Vec<_>>())
})
.await
.context("list models task panicked")??;
let serving: Vec<Option<Option<u16>>> = {
let mgr = state.0.manager.lock().await;
rows.iter()
.map(|(img, _)| {
let m = mgr.running.get(&img.reference).or_else(|| {
running_by_digest(&mgr.running, &img.reference, &img.digest).map(|(_, m)| m)
})?;
Some((m.process.engine() == Engine::LlamaServer).then_some(m.port))
})
.collect()
};
let live = futures::future::join_all(serving.iter().map(|port| async {
match port {
Some(Some(port)) => backend_model_facts(&state.0.client, *port).await,
_ => None,
}
}))
.await;
let user_sizes_slots = llama_sets_slot_context(&extra_args_from_env(LLAMA_ARGS_VAR), true);
let gguf_engine = gguf_engine_from_env();
let mut data: Vec<serde_json::Value> = rows
.into_iter()
.zip(serving.iter().zip(live))
.map(|((img, facts), (serving, live))| {
let mut entry = serde_json::json!({
"id": img.reference,
"object": "model",
"created": 0,
"owned_by": "llmman",
"status": { "value": if serving.is_some() { "loaded" } else { "unloaded" } },
});
if let Some(trained) = facts.context_length() {
entry["meta"] = serde_json::json!({ "n_ctx_train": trained });
if serving.is_none() && !user_sizes_slots {
if let Some(n_ctx) = facts.starting_context_window(&state, gguf_engine) {
entry["meta"]["n_ctx"] = n_ctx.into();
}
}
}
if let Some(input) = facts.input_modalities() {
entry["architecture"] = serde_json::json!({
"input_modalities": input,
"output_modalities": facts.output_modalities(),
});
}
if let Some(live) = live {
entry.as_object_mut().unwrap().extend(live);
}
entry
})
.collect();
if aggregation::aggregates(&state, &headers) {
for (_, peer) in aggregation::poll::<serde_json::Value>(&state, "/v1/models").await {
for m in peer["data"].as_array().into_iter().flatten() {
match data.iter_mut().find(|have| have["id"] == m["id"]) {
Some(have)
if m["status"]["value"] == "loaded"
&& have["status"]["value"] != "loaded" =>
{
if let Some(meta) = have["meta"].as_object_mut() {
meta.remove("n_ctx");
}
for key in ["status", "meta", "architecture"] {
if let Some(value) = m.get(key) {
have[key] = value.clone();
}
}
}
Some(_) => {}
None => data.push(m.clone()),
}
}
}
}
Ok(Json(serde_json::json!({ "object": "list", "data": data })))
}
async fn backend_model_facts(
client: &reqwest::Client,
port: u16,
) -> Option<serde_json::Map<String, serde_json::Value>> {
let body = backend_json(client, port, "/v1/models", Duration::from_millis(500))
.await
.ok()?;
let entry = body["data"].as_array()?.first()?;
let facts: serde_json::Map<String, serde_json::Value> = ["meta", "architecture"]
.into_iter()
.filter_map(|key| {
let value = entry.get(key).filter(|v| v.is_object())?;
Some((key.to_string(), value.clone()))
})
.collect();
(!facts.is_empty()).then_some(facts)
}
pub(super) fn apply_default_repeat_penalty(req: &mut serde_json::Value) {
if req.get("repeat_penalty").is_none() {
req["repeat_penalty"] = serde_json::json!(DEFAULT_REPEAT_PENALTY);
}
}
pub(super) fn consolidate_chat_system_messages(req: &mut serde_json::Value) {
fn role(message: &serde_json::Value) -> &str {
message.get("role").and_then(|v| v.as_str()).unwrap_or("")
}
fn leads(message: &serde_json::Value) -> bool {
matches!(role(message), "system" | "developer")
}
let Some(messages) = req.get_mut("messages").and_then(|v| v.as_array_mut()) else {
return;
};
let conforming = messages
.iter()
.enumerate()
.all(|(i, m)| !leads(m) || i == 0)
&& messages.first().map(role) != Some("developer");
if conforming {
return;
}
let mut merged = MergedInstructions::default();
messages.retain(|message| {
if !leads(message) {
return true;
}
merged.push(message.get("content").unwrap_or(&serde_json::Value::Null));
false
});
let Some(content) = merged.finish() else {
return;
};
messages.insert(
0,
serde_json::json!({ "role": "system", "content": content }),
);
}
pub(super) fn merged_after_refusal(
target: &Target,
status: StatusCode,
reply: &[u8],
body: &[u8],
) -> Option<Bytes> {
let refused = status == StatusCode::BAD_REQUEST
&& String::from_utf8_lossy(reply)
.to_ascii_lowercase()
.contains("roles must alternate");
let mut req: serde_json::Value = serde_json::from_slice(body).ok().filter(|_| refused)?;
if !merge_adjacent_roles(&mut req) {
return None;
}
eprintln!(
"[llmman] {} refused consecutive same-role messages; retrying with them merged",
target.describe()
);
serde_json::to_vec(&req).ok().map(Bytes::from)
}
fn merge_adjacent_roles(req: &mut serde_json::Value) -> bool {
fn mergeable_role(message: &serde_json::Value) -> Option<&str> {
let message = message.as_object()?;
let role = message.get("role")?.as_str()?;
(matches!(role, "user" | "assistant")
&& message.keys().all(|key| key == "role" || key == "content"))
.then_some(role)
}
let Some(messages) = req.get_mut("messages").and_then(|v| v.as_array_mut()) else {
return false;
};
let mut merged_any = false;
let mut merged = Vec::with_capacity(messages.len());
let mut pending = std::mem::take(messages).into_iter().peekable();
while let Some(first) = pending.next() {
let Some(role) = mergeable_role(&first).map(str::to_owned) else {
merged.push(first);
continue;
};
let mut run = vec![first];
while let Some(next) = pending.next_if(|m| mergeable_role(m) == Some(role.as_str())) {
run.push(next);
}
if run.len() == 1 {
merged.extend(run);
continue;
}
merged_any = true;
let mut content = MergedInstructions::default();
for message in &run {
content.push(message.get("content").unwrap_or(&serde_json::Value::Null));
}
merged.push(serde_json::json!({
"role": role,
"content": content.finish().unwrap_or_else(|| serde_json::json!("")),
}));
}
*messages = merged;
merged_any
}
#[derive(Default)]
struct MergedInstructions {
blocks: Vec<serde_json::Value>,
text: Vec<String>,
}
impl MergedInstructions {
fn push(&mut self, content: &serde_json::Value) {
match content {
serde_json::Value::Array(parts) => {
let mut run: Vec<&str> = Vec::new();
for part in parts {
if part.get("type").and_then(|v| v.as_str()) == Some("text") {
let text = part.get("text").and_then(|v| v.as_str()).unwrap_or("");
if !text.is_empty() {
run.push(text);
}
} else {
self.push_run(&mut run);
self.flush_text();
self.blocks.push(part.clone());
}
}
self.push_run(&mut run);
}
other => self.push_text(&content_text(other)),
}
}
fn push_run(&mut self, run: &mut Vec<&str>) {
if run.is_empty() {
return;
}
self.push_text(&std::mem::take(run).join("\n"));
}
fn push_text(&mut self, text: &str) {
if !text.is_empty() {
self.text.push(text.to_string());
}
}
fn flush_text(&mut self) {
if !self.text.is_empty() {
let text = std::mem::take(&mut self.text).join("\n\n");
self.blocks
.push(serde_json::json!({"type": "text", "text": text}));
}
}
fn finish(mut self) -> Option<serde_json::Value> {
self.flush_text();
match self.blocks.len() {
0 => None,
1 if self.blocks[0].get("type").and_then(|v| v.as_str()) == Some("text") => {
self.blocks[0].get("text").cloned()
}
_ => Some(serde_json::Value::Array(self.blocks)),
}
}
}
fn apply_reasoning_effort(req: &mut serde_json::Value) {
let Some(effort) = req.get("reasoning_effort").and_then(|v| v.as_str()) else {
return;
};
let think = if effort == "none" {
serde_json::Value::Bool(false)
} else {
serde_json::Value::String(effort.to_string())
};
let Some(serde_json::Value::Object(kwargs)) = think_to_chat_template_kwargs(&Some(think))
else {
return;
};
let Some(o) = req.as_object_mut() else {
return;
};
let slot = o
.entry("chat_template_kwargs")
.or_insert_with(|| serde_json::json!({}));
let Some(existing) = slot.as_object_mut() else {
return;
};
for (key, value) in kwargs {
existing.entry(key).or_insert(value);
}
}
const EFFORT_BUDGET_SHARES: [(&str, u64); 2] = [("minimal", 8), ("low", 4)];
pub(super) const EFFORT_BUDGET_MESSAGE: &str = "Okay, I have to answer now.";
pub(super) fn effort_budget_tokens(effort: &str, limit: Option<u64>) -> Option<u64> {
let effort = effort.trim();
let &(_, share) = EFFORT_BUDGET_SHARES.iter().find(|(l, _)| *l == effort)?;
let &(_, cap) = super::anthropic::THINKING_BUDGETS
.iter()
.find(|(l, _)| *l == effort)?;
let cap = u64::from(cap);
Some(limit.map_or(cap, |limit| (limit / share).clamp(1, cap)))
}
pub(super) async fn effort_budget_for(state: &AppState, target: &Target) -> Option<bool> {
local_model(state, target, |m| {
m.effort_budget.then_some(m.effort_budget_message)
})
.await
.flatten()
}
pub(super) async fn thinking_for(
state: &AppState,
target: &Target,
) -> Option<crate::chat_template::Thinking> {
local_model(state, target, |m| m.thinking.clone())
.await
.flatten()
}
fn apply_effort_budget(req: &mut serde_json::Value, message: bool) {
let Some(o) = req.as_object_mut() else {
return;
};
let Some(effort) = o.get("reasoning_effort").and_then(|v| v.as_str()) else {
return;
};
let thinking_off = o
.get("chat_template_kwargs")
.and_then(|k| k.get("enable_thinking"))
== Some(&serde_json::Value::Bool(false));
let sent = |key: &str| o.get(key).is_some_and(|v| !v.is_null());
if thinking_off || sent("reasoning_budget_tokens") || sent("thinking_budget_tokens") {
return;
}
let send_message = message && !sent("reasoning_budget_message");
let limit = ["n_predict", "max_completion_tokens", "max_tokens"]
.iter()
.find_map(|key| o.get(*key).filter(|v| !v.is_null()))
.and_then(|v| v.as_f64())
.filter(|n| *n >= 1.0)
.map(|n| n as u64);
let Some(budget) = effort_budget_tokens(effort, limit) else {
return;
};
o.insert("reasoning_budget_tokens".into(), budget.into());
if send_message {
o.insert(
"reasoning_budget_message".into(),
EFFORT_BUDGET_MESSAGE.into(),
);
}
}
pub(super) async fn resolve_openai_request(
state: &AppState,
headers: &HeaderMap,
body: Bytes,
task: Task,
) -> Result<(serde_json::Value, Target, ActivityGuard, Option<String>), AppError> {
let req = parse_openai_request(&body)?;
let model = req["model"].as_str().unwrap_or("").to_string();
let (model, target, guard) =
ensure_model_for(state, &model, Some(headers), LoadOptions::default(), task).await?;
prepare_openai_request(state, req, model, target, guard).await
}
fn parse_openai_request(body: &Bytes) -> Result<serde_json::Value, AppError> {
serde_json::from_slice(body).map_err(|e| {
AppError::status(
StatusCode::BAD_REQUEST,
format!("parse OpenAI request body: {e}"),
)
})
}
async fn prepare_openai_request(
state: &AppState,
mut req: serde_json::Value,
model: String,
target: Target,
guard: ActivityGuard,
) -> Result<(serde_json::Value, Target, ActivityGuard, Option<String>), AppError> {
let activity = begin_activity(guard, None).await;
let wire_model = backend_wire_model(state, &target, &model).await;
let response_model_override = (wire_model != model).then_some(model);
req["model"] = serde_json::Value::String(wire_model);
Ok((req, target, activity, response_model_override))
}
pub(super) async fn proxy_openai_generation(
state: &AppState,
headers: &HeaderMap,
body: Bytes,
llama_path: &str,
) -> Result<Response, AppError> {
let req = parse_openai_request(&body)?;
let model_ref = req["model"].as_str().unwrap_or("").to_string();
send_with_hybrid_fallback(
state,
&model_ref,
Some(headers),
LoadOptions::default(),
|model, target, guard| {
proxy_openai_generation_to(
state,
headers,
req.clone(),
llama_path,
&model_ref,
model,
target,
guard,
)
},
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn proxy_openai_generation_to(
state: &AppState,
headers: &HeaderMap,
req: serde_json::Value,
llama_path: &str,
model_ref: &str,
model: String,
target: Target,
guard: ActivityGuard,
) -> Result<Response, AppError> {
let reply_model = model_ref.to_string();
let (mut req, target, activity, _) =
prepare_openai_request(state, req, model, target, guard).await?;
if let Some(refusal) = unsupported_on_wire(&target, llama_path) {
drop(activity);
return Ok(refusal);
}
if llama_path == RESPONSES_ROUTE && target.is_remote() {
strip_llama_fields(&mut req);
return remote_responses(
&state.0.client,
&target,
headers,
req,
activity,
reply_model,
)
.await;
}
if llama_path == CHAT_COMPLETIONS_ROUTE && target.is_anthropic() {
let upstream = send_chat_completion(&state.0.client, &target, &req, &reply_model).await?;
return Ok(relay_chat_upstream(upstream, activity));
}
if is_responses_route(llama_path) && !target.is_remote() {
sanitize_responses_request(&mut req);
}
if (llama_path == CHAT_COMPLETIONS_ROUTE || llama_path == "/v1/completions")
&& asks_for_usage(state, &target).await
{
super::usage::ask_for_stream_usage(&mut req);
}
match &target {
Target::Remote(remote) if llama_path == CHAT_COMPLETIONS_ROUTE => {
provider_compat(remote, &mut req)
}
Target::Remote(_) => strip_llama_fields(&mut req),
_ => {
apply_default_repeat_penalty(&mut req);
if llama_path == CHAT_COMPLETIONS_ROUTE {
apply_reasoning_effort(&mut req);
if req.get("reasoning_effort").is_some() {
if let Some(message) = effort_budget_for(state, &target).await {
apply_effort_budget(&mut req, message);
}
}
consolidate_chat_system_messages(&mut req);
}
}
}
let streaming = req.get("stream").and_then(|v| v.as_bool()).unwrap_or(false);
let body = Bytes::from(serde_json::to_vec(&req).context("re-serialize OpenAI request body")?);
let resp = if streaming {
stream_rewriting_model(
&state.0.client,
&target,
llama_path,
headers,
body,
activity,
reply_model,
)
.await
} else {
proxy_rewriting_model(
&state.0.client,
&target,
llama_path,
headers,
body,
activity,
&reply_model,
)
.await
}?;
Ok(explain_missing_route(&target, llama_path, resp))
}
async fn asks_for_usage(state: &AppState, target: &Target) -> bool {
match target {
Target::Remote(remote) => !remote.refuses_stream_options(),
Target::Local(_) => local_engine(state, target).await != Some(Engine::LlamaServer),
Target::Peer(_) => false,
}
}
pub(super) async fn proxy_openai_passthrough(
state: &AppState,
headers: &HeaderMap,
body: Bytes,
llama_path: &str,
) -> Result<Response, AppError> {
let embeddings = llama_path == "/v1/embeddings";
if embeddings {
if let Ok(peek) = serde_json::from_slice::<serde_json::Value>(&body) {
if let Some(model_ref) = peek["model"].as_str() {
if !crate::providers::is_remote_ref(model_ref) {
if let Some(canonical) = would_use_mlx(state, model_ref).await {
return Ok(mlx_embeddings_unsupported_response(&canonical));
}
}
}
}
}
let task = if embeddings {
Task::Embed
} else {
Task::Forward
};
let (req, target, activity, response_model_override) =
resolve_openai_request(state, headers, body, task).await?;
forward_openai_request(
state,
headers,
req,
target,
activity,
response_model_override,
llama_path,
)
.await
}
async fn forward_openai_request(
state: &AppState,
headers: &HeaderMap,
mut req: serde_json::Value,
target: Target,
activity: ActivityGuard,
response_model_override: Option<String>,
llama_path: &str,
) -> Result<Response, AppError> {
let embeddings = llama_path == "/v1/embeddings";
if let Some(refusal) = unsupported_on_wire(&target, llama_path) {
drop(activity);
return Ok(refusal);
}
if is_responses_route(llama_path) && !target.is_remote() {
sanitize_responses_request(&mut req);
}
if embeddings && !target.is_remote() {
if let Some(canonical) = &response_model_override {
drop(activity);
return Ok(mlx_embeddings_unsupported_response(canonical));
}
}
let dimensions = (embeddings && !target.is_remote())
.then(|| requested_dimensions(&req))
.flatten();
let body = Bytes::from(serde_json::to_vec(&req).context("re-serialize OpenAI request body")?);
let resp = match (response_model_override, dimensions) {
(_, Some(dims)) => {
proxy_buffered(
&state.0.client,
&target,
llama_path,
headers,
body,
activity,
|raw| apply_embedding_dimensions(raw, dims),
)
.await
}
(Some(canonical), None) => {
proxy_rewriting_model(
&state.0.client,
&target,
llama_path,
headers,
body,
activity,
&canonical,
)
.await
}
(None, None) => {
proxy(
&state.0.client,
&target,
llama_path,
headers,
body,
activity,
)
.await
}
}?;
let resp = if embeddings && !target.is_remote() {
explain_embedding_overflow(resp).await?
} else {
resp
};
Ok(explain_missing_route(&target, llama_path, resp))
}
pub(super) async fn explain_embedding_overflow(resp: Response) -> Result<Response, AppError> {
batch_overflow_as_400(resp, |text| {
is_embedding_overflow(text).then(|| {
let message = format!(
"the input is longer than {} this model reads at once; shorten it",
batch_size_limit(text)
);
let error = serde_json::json!({
"error": { "code": 400, "message": message, "type": "invalid_request_error" }
});
(StatusCode::BAD_REQUEST, Json(error)).into_response()
})
})
.await
}
async fn explain_prompt_too_long(resp: Response) -> Result<Response, AppError> {
batch_overflow_as_400(resp, |text| {
text.contains("physical batch size").then(|| {
let message = format!(
"the text and questions are longer than {} this model reads at once; \
shorten them, or raise LLMMAN_CONTEXT_LENGTH",
batch_size_limit(text)
);
AppError::status(StatusCode::BAD_REQUEST, message).into_response()
})
})
.await
}
async fn batch_overflow_as_400(
resp: Response,
rewrite: impl FnOnce(&str) -> Option<Response>,
) -> Result<Response, AppError> {
if resp.status() != StatusCode::INTERNAL_SERVER_ERROR {
return Ok(resp);
}
let (parts, body) = resp.into_parts();
let body = axum::body::to_bytes(body, usize::MAX)
.await
.context("read inference backend response")?;
Ok(rewrite(&String::from_utf8_lossy(&body))
.unwrap_or_else(|| Response::from_parts(parts, Body::from(body))))
}
fn batch_size_limit(text: &str) -> String {
let digits: String = text
.split("current batch size: ")
.nth(1)
.unwrap_or_default()
.chars()
.take_while(char::is_ascii_digit)
.collect();
if digits.is_empty() {
"what".into()
} else {
format!("the {digits} tokens")
}
}
fn requested_dimensions(req: &serde_json::Value) -> Option<usize> {
let dims = req.get("dimensions")?.as_u64().filter(|d| *d > 0)?;
usize::try_from(dims).ok()
}
#[derive(serde::Deserialize, serde::Serialize)]
struct EmbeddingsBody {
data: Vec<EmbeddingItem>,
#[serde(flatten)]
rest: serde_json::Map<String, serde_json::Value>,
}
#[derive(serde::Deserialize, serde::Serialize)]
struct EmbeddingItem {
embedding: EmbeddingVector,
#[serde(flatten)]
rest: serde_json::Map<String, serde_json::Value>,
}
#[derive(serde::Deserialize, serde::Serialize)]
#[serde(untagged)]
enum EmbeddingVector {
Floats(Vec<f32>),
Base64(String),
}
impl EmbeddingVector {
fn truncate(&mut self, dims: usize) -> Result<bool, AppError> {
match self {
EmbeddingVector::Floats(v) if v.len() > dims => {
truncate_embedding(v, dims)?;
Ok(true)
}
EmbeddingVector::Floats(_) => Ok(false),
EmbeddingVector::Base64(encoded) => {
let Some(mut v) = decode_f32s(encoded).filter(|v| v.len() > dims) else {
return Ok(false);
};
truncate_embedding(&mut v, dims)?;
*encoded = encode_f32s(&v);
Ok(true)
}
}
}
}
pub(super) fn encode_f32s(values: &[f32]) -> String {
use base64::Engine as _;
let bytes: Vec<u8> = values.iter().flat_map(|v| v.to_le_bytes()).collect();
base64::engine::general_purpose::STANDARD.encode(bytes)
}
pub(super) fn decode_f32s(encoded: &str) -> Option<Vec<f32>> {
use base64::Engine as _;
let bytes = base64::engine::general_purpose::STANDARD
.decode(encoded)
.ok()?;
let (chunks, rest) = bytes.as_chunks::<4>();
rest.is_empty()
.then(|| chunks.iter().map(|c| f32::from_le_bytes(*c)).collect())
}
fn apply_embedding_dimensions(raw: &Bytes, dims: usize) -> Result<Bytes, AppError> {
let Ok(mut body) = serde_json::from_slice::<EmbeddingsBody>(raw) else {
return Ok(raw.clone());
};
let mut changed = false;
for item in &mut body.data {
changed |= item.embedding.truncate(dims)?;
}
if !changed {
return Ok(raw.clone());
}
Ok(Bytes::from(
serde_json::to_vec(&body).context("re-serialize embeddings response")?,
))
}
fn mlx_embeddings_unsupported_response(canonical_model: &str) -> Response {
let body = serde_json::json!({
"error": {
"message": format!(
"{canonical_model} is served by mlx_lm.server, which llmman never starts with \
--embedding-model — /v1/embeddings isn't supported for it; use a GGUF or \
vllm-served model for embeddings instead"
),
"type": "invalid_request_error",
}
});
(StatusCode::NOT_IMPLEMENTED, Json(body)).into_response()
}
pub(super) async fn handle_openai_chat(
State(state): State<AppState>,
headers: HeaderMap,
body: Bytes,
) -> Result<Response, AppError> {
proxy_openai_generation(&state, &headers, body, "/v1/chat/completions").await
}
pub(super) async fn handle_openai_completions(
State(state): State<AppState>,
headers: HeaderMap,
body: Bytes,
) -> Result<Response, AppError> {
proxy_openai_generation(&state, &headers, body, "/v1/completions").await
}
pub(super) async fn handle_openai_embeddings(
State(state): State<AppState>,
headers: HeaderMap,
body: Bytes,
) -> Result<Response, AppError> {
proxy_openai_passthrough(&state, &headers, body, "/v1/embeddings").await
}
const SYSTEMONE_ROUTE: &str = "/v1/systemone";
type Members = Ordered<Box<serde_json::value::RawValue>>;
fn member<T: serde::de::DeserializeOwned>(members: &Members, key: &str) -> Option<T> {
serde_json::from_str(members.get(key)?.get()).ok()
}
fn raw_member(value: &impl serde::Serialize) -> Result<Box<serde_json::value::RawValue>, AppError> {
Ok(serde_json::value::to_raw_value(value).context("encode a request member")?)
}
pub(super) async fn handle_openai_systemone(
State(state): State<AppState>,
headers: HeaderMap,
body: Bytes,
) -> Result<Response, AppError> {
let mut req: Members = serde_json::from_slice(&body).map_err(|e| {
AppError::status(
StatusCode::BAD_REQUEST,
format!("parse OpenAI request body: {e}"),
)
})?;
let model_ref: String = member(&req, "model").unwrap_or_default();
let keep_alive = resolve_keep_alive(&member(&req, "keep_alive"));
if let Some(mut images) = member::<Vec<serde_json::Value>>(&req, "images") {
for image in &mut images {
if let Some(base64) = image.as_str() {
*image = serde_json::Value::String(image_data_uri(base64));
}
}
req.set("images", raw_member(&images)?);
}
let (model, target, guard) = ensure_model_for(
&state,
&model_ref,
Some(&headers),
LoadOptions::default(),
Task::Forward,
)
.await?;
let activity = begin_activity(guard, Some(keep_alive)).await;
if !matches!(target, Target::Peer(_)) {
req.remove("keep_alive");
}
req.set(
"model",
raw_member(&backend_wire_model(&state, &target, &model).await)?,
);
if let Some(refusal) = unsupported_on_wire(&target, SYSTEMONE_ROUTE) {
drop(activity);
return Ok(refusal);
}
let body = Bytes::from(serde_json::to_vec(&req).context("re-serialize the request")?);
let resp = proxy_buffered(
&state.0.client,
&target,
SYSTEMONE_ROUTE,
&headers,
body,
activity,
|raw| Ok(with_model(raw, &model_ref)),
)
.await?;
explain_prompt_too_long(resp).await
}
fn with_model(raw: &Bytes, model: &str) -> Bytes {
let renamed = serde_json::from_slice::<Members>(raw)
.ok()
.filter(|reply| reply.get("model").is_some())
.and_then(|mut reply| {
reply.set("model", serde_json::value::to_raw_value(model).ok()?);
serde_json::to_vec(&reply).ok()
});
renamed.map_or_else(|| raw.clone(), Bytes::from)
}
pub(super) async fn handle_openai_images(
State(state): State<AppState>,
headers: HeaderMap,
body: Bytes,
) -> Result<Response, AppError> {
handle_openai_media(&state, &headers, body, "/v1/images/generations").await
}
pub(super) async fn handle_openai_videos(
State(state): State<AppState>,
headers: HeaderMap,
body: Bytes,
) -> Result<Response, AppError> {
handle_openai_media(&state, &headers, body, "/v1/videos").await
}
async fn handle_openai_media(
state: &AppState,
headers: &HeaderMap,
body: Bytes,
route: &str,
) -> Result<Response, AppError> {
let (req, target, activity, override_) =
resolve_openai_request(state, headers, body, Task::Forward).await?;
if local_engine(state, &target).await == Some(Engine::VllmOmni) {
return if route == "/v1/videos" {
omni_videos(state, &target, req, activity).await
} else {
omni_images(state, &target, req, activity).await
};
}
forward_openai_request(state, headers, req, target, activity, override_, route).await
}
pub(super) async fn local_engine(state: &AppState, target: &Target) -> Option<Engine> {
local_model(state, target, |m| m.process.engine()).await
}
async fn local_model<T>(
state: &AppState,
target: &Target,
read: impl FnOnce(&super::RunningModel) -> T,
) -> Option<T> {
let Target::Local(port) = target else {
return None;
};
let mgr = state.0.manager.lock().await;
mgr.running.values().find(|m| m.port == *port).map(read)
}
fn omni_image_request(mut req: serde_json::Value) -> Result<(serde_json::Value, bool), String> {
let stream = req["stream"].as_bool().unwrap_or(false);
let Some(obj) = req.as_object_mut() else {
return Ok((req, stream));
};
obj.remove("stream");
let dim = |v: Option<serde_json::Value>| v.and_then(|v| v.as_u64()).filter(|n| *n > 0);
let (width, height) = (dim(obj.remove("width")), dim(obj.remove("height")));
match (width, height) {
(Some(w), Some(h)) if !obj.contains_key("size") => {
obj.insert("size".into(), serde_json::json!(format!("{w}x{h}")));
}
(Some(_), None) | (None, Some(_)) => {
return Err("vLLM-Omni needs both width and height, or neither".into());
}
_ => {}
}
for (llama, omni) in [
("steps", "num_inference_steps"),
("cfg_scale", "guidance_scale"),
] {
if let Some(v) = obj.remove(llama) {
if !obj.contains_key(omni) && v.as_f64().is_some_and(|n| n > 0.0) {
obj.insert(omni.into(), v);
}
}
}
if !obj.contains_key("response_format") {
obj.insert("response_format".into(), "b64_json".into());
}
Ok((req, stream))
}
async fn omni_images(
state: &AppState,
target: &Target,
req: serde_json::Value,
activity: ActivityGuard,
) -> Result<Response, AppError> {
let (req, stream) = match omni_image_request(req) {
Ok(ok) => ok,
Err(m) => {
drop(activity);
return Err(AppError::status(StatusCode::BAD_REQUEST, m));
}
};
let resp = state
.0
.client
.post(target.url("/v1/images/generations"))
.json(&req)
.send()
.await
.with_context(|| format!("proxy request to {}", target.describe()))?;
if !stream || !resp.status().is_success() {
return Ok(relay(resp, activity));
}
let body: serde_json::Value = resp
.json()
.await
.context("decode vllm-omni image response")?;
let created = body["created"]
.as_i64()
.unwrap_or_else(|| chrono::Utc::now().timestamp());
let mut events = String::new();
for image in body["data"].as_array().into_iter().flatten() {
let ev = serde_json::json!({
"type": "image_generation.completed",
"b64_json": image["b64_json"],
"revised_prompt": image["revised_prompt"],
"created_at": created,
});
events.push_str(&format!("data: {ev}\n\n"));
}
drop(activity);
Ok(Response::builder()
.status(StatusCode::OK)
.header("content-type", "text/event-stream")
.header("cache-control", "no-cache")
.body(Body::from(events))
.context("build image event stream")?)
}
fn omni_video_fields(req: &serde_json::Value) -> Vec<(String, String)> {
let Some(obj) = req.as_object() else {
return Vec::new();
};
let renamed = |name: &str| match name {
"steps" => Some("num_inference_steps"),
"cfg_scale" => Some("guidance_scale"),
"frames" => Some("num_frames"),
"audio" => Some("generate_sound"),
_ => None,
};
let text = |value: &serde_json::Value| match value {
serde_json::Value::String(s) => Some(s.clone()),
serde_json::Value::Null => None,
other => Some(other.to_string()),
};
let mut fields: Vec<(String, String)> = Vec::new();
for (name, value) in obj {
if matches!(name.as_str(), "stream" | "response_format") || renamed(name).is_some() {
continue;
}
let value = if name == "seconds" {
match value.as_f64().or_else(|| value.as_str()?.parse().ok()) {
Some(s) if s > 0.0 => {
serde_json::Value::String((s.round().max(1.0) as u64).to_string())
}
_ => continue,
}
} else {
value.clone()
};
if let Some(text) = text(&value) {
fields.push((name.clone(), text));
}
}
for (name, value) in obj {
let Some(native) = renamed(name) else {
continue;
};
if fields.iter().any(|(existing, _)| existing == native) {
continue;
}
if let Some(text) = text(value) {
fields.push((native.to_string(), text));
}
}
fields
}
fn multipart_form(fields: &[(String, String)]) -> (Vec<u8>, String) {
let fields: Vec<&(String, String)> = fields
.iter()
.filter(|(name, _)| {
!name.is_empty() && name.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'_')
})
.collect();
let stamp = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0);
let boundary = (0u32..)
.map(|salt| format!("----llmman{}{stamp:x}{salt:x}", std::process::id()))
.find(|b| !fields.iter().any(|(_, v)| v.contains(b.as_str())))
.unwrap_or_default();
let mut body = Vec::new();
for (name, value) in fields {
body.extend_from_slice(
format!("--{boundary}\r\nContent-Disposition: form-data; name=\"{name}\"\r\n\r\n")
.as_bytes(),
);
body.extend_from_slice(value.as_bytes());
body.extend_from_slice(b"\r\n");
}
body.extend_from_slice(format!("--{boundary}--\r\n").as_bytes());
(body, format!("multipart/form-data; boundary={boundary}"))
}
const OMNI_VIDEO_POLL: Duration = Duration::from_secs(1);
const OMNI_VIDEO_MAX_WAIT: Duration = Duration::from_secs(60 * 60);
async fn omni_videos(
state: &AppState,
target: &Target,
req: serde_json::Value,
activity: ActivityGuard,
) -> Result<Response, AppError> {
let (body, content_type) = multipart_form(&omni_video_fields(&req));
let resp = state
.0
.client
.post(target.url("/v1/videos"))
.header("content-type", content_type)
.body(body)
.send()
.await
.with_context(|| format!("proxy request to {}", target.describe()))?;
if !resp.status().is_success() {
return Ok(relay(resp, activity));
}
let mut job: serde_json::Value = resp.json().await.context("decode vllm-omni video job")?;
let id = job["id"]
.as_str()
.context("vllm-omni video job has no id")?
.to_string();
let poll_url = target.url(&format!("/v1/videos/{id}"));
let deadline = Instant::now() + OMNI_VIDEO_MAX_WAIT;
while !matches!(job["status"].as_str(), Some("completed" | "failed")) {
if Instant::now() >= deadline {
drop(activity);
return Err(AppError::status(
StatusCode::GATEWAY_TIMEOUT,
format!("video job {id} did not finish within {OMNI_VIDEO_MAX_WAIT:?}"),
));
}
sleep(OMNI_VIDEO_POLL).await;
let resp = state
.0
.client
.get(&poll_url)
.timeout(Duration::from_secs(30))
.send()
.await
.with_context(|| format!("poll video job {id} on {}", target.describe()))?;
if !resp.status().is_success() {
return Ok(relay(resp, activity));
}
job = resp.json().await.context("decode vllm-omni video job")?;
}
drop(activity);
if job["status"] == "failed" {
let message = job["error"]["message"]
.as_str()
.unwrap_or("video generation failed")
.to_string();
let body = serde_json::json!({
"error": { "message": message, "type": "server_error" }
});
return Ok((StatusCode::INTERNAL_SERVER_ERROR, Json(body)).into_response());
}
job["content_url"] = serde_json::json!(format!("/v1/videos/{id}/content"));
Ok(Json(job).into_response())
}
pub(super) async fn handle_openai_video_get(
State(state): State<AppState>,
axum::extract::OriginalUri(uri): axum::extract::OriginalUri,
) -> Result<Response, AppError> {
let ports: Vec<u16> = {
let mgr = state.0.manager.lock().await;
mgr.running.values().map(|m| m.port).collect()
};
let path = uri.path();
let probes = ports.into_iter().map(|port| {
state
.0
.client
.get(format!("http://127.0.0.1:{port}{path}"))
.timeout(Duration::from_secs(30))
.send()
});
let found = futures::future::join_all(probes)
.await
.into_iter()
.flatten()
.find(|r| r.status() != reqwest::StatusCode::NOT_FOUND);
if let Some(resp) = found {
let status = resp.status();
let mut builder = Response::builder().status(status);
for name in ["content-type", "content-disposition"] {
if let Some(v) = resp.headers().get(name) {
builder = builder.header(name, v);
}
}
let stream = resp
.bytes_stream()
.map(|item| item.map_err(|e| Box::new(e) as Box<dyn std::error::Error + Send + Sync>));
return Ok(builder
.body(Body::from_stream(stream))
.context("build video response")?);
}
let body = serde_json::json!({
"error": { "message": "video not found", "type": "not_found_error" }
});
Ok((StatusCode::NOT_FOUND, Json(body)).into_response())
}
pub(super) async fn handle_openai_speech(
State(state): State<AppState>,
headers: HeaderMap,
body: Bytes,
) -> Result<Response, AppError> {
proxy_openai_passthrough(&state, &headers, body, "/v1/audio/speech").await
}
pub(super) const TRANSCRIPTION_BODY_LIMIT_BYTES: usize = 200 * 1024 * 1024;
pub(super) fn multipart_parser(
body: &Bytes,
headers: &HeaderMap,
) -> Option<(String, multer::Multipart<'static>)> {
let content_type = headers.get("content-type")?.to_str().ok()?;
let boundary = multer::parse_boundary(content_type).ok()?;
let chunk = body.clone();
let stream = futures::stream::once(async { Ok::<_, std::io::Error>(chunk) });
Some((boundary.clone(), multer::Multipart::new(stream, boundary)))
}
async fn multipart_text_field(
body: &Bytes,
headers: &HeaderMap,
field_name: &str,
) -> Option<String> {
let (_, mut multipart) = multipart_parser(body, headers)?;
while let Ok(Some(field)) = multipart.next_field().await {
if field.name() == Some(field_name) {
return field.text().await.ok();
}
}
None
}
pub(super) const DEFAULT_TRANSCRIPTION_PROMPT: &str = "Transcribe the speech in this audio. Reply with only the words spoken, with no introduction, heading or quotation marks.";
async fn has_prompt(body: &Bytes, headers: &HeaderMap) -> bool {
multipart_text_field(body, headers, "prompt")
.await
.is_some_and(|p| !p.trim().is_empty())
}
async fn multipart_with_prompt(body: &Bytes, headers: &HeaderMap, prompt: &str) -> Option<Bytes> {
let (boundary, mut multipart) = multipart_parser(body, headers)?;
let mut out = Vec::with_capacity(body.len() + prompt.len() + 256);
while let Some(mut field) = multipart.next_field().await.ok()? {
if field.name() == Some("prompt") {
continue;
}
if (field.headers().keys()).any(|k| k != "content-disposition" && k != "content-type") {
return None;
}
let disposition = field.headers().get("content-disposition")?.clone();
let part_type = field.headers().get("content-type").cloned();
out.extend_from_slice(format!("--{boundary}\r\nContent-Disposition: ").as_bytes());
out.extend_from_slice(disposition.as_bytes());
if let Some(part_type) = part_type {
out.extend_from_slice(b"\r\nContent-Type: ");
out.extend_from_slice(part_type.as_bytes());
}
out.extend_from_slice(b"\r\n\r\n");
while let Some(chunk) = field.chunk().await.ok()? {
out.extend_from_slice(&chunk);
}
out.extend_from_slice(b"\r\n");
}
out.extend_from_slice(
format!(
"--{boundary}\r\nContent-Disposition: form-data; name=\"prompt\"\r\n\r\n{prompt}\r\n--{boundary}--\r\n"
)
.as_bytes(),
);
Some(Bytes::from(out))
}
fn is_lfm2_template(template: &str) -> bool {
template.contains("<|tool_list_start|>") && template.contains("<|tool_list_end|>")
}
async fn with_default_transcription_prompt(
state: &AppState,
target: &Target,
headers: &HeaderMap,
body: Bytes,
) -> Bytes {
let Target::Local(port) = target else {
return body;
};
if local_engine(state, target).await != Some(Engine::LlamaServer)
|| has_prompt(&body, headers).await
{
return body;
}
let props = backend_json(&state.0.client, *port, "/props", Duration::from_secs(1)).await;
let Ok(props) = props else {
return body;
};
if props["chat_template"]
.as_str()
.is_some_and(is_lfm2_template)
{
return body;
}
multipart_with_prompt(&body, headers, DEFAULT_TRANSCRIPTION_PROMPT)
.await
.unwrap_or(body)
}
pub(super) async fn handle_openai_transcriptions(
State(state): State<AppState>,
headers: HeaderMap,
body: Bytes,
) -> Result<Response, AppError> {
proxy_multipart(
&state,
&headers,
body,
&LocalRoute {
path: "/v1/audio/transcriptions",
what: "transcription",
hint: "use a locally served model with audio support",
default_prompt: true,
},
)
.await
}
pub(super) struct LocalRoute {
pub(super) path: &'static str,
pub(super) what: &'static str,
pub(super) hint: &'static str,
pub(super) default_prompt: bool,
}
pub(super) async fn handle_tts(
State(state): State<AppState>,
headers: HeaderMap,
body: Bytes,
) -> Result<Response, AppError> {
const TTS: LocalRoute = LocalRoute {
path: "/tts",
what: "text-to-speech",
hint: "use a locally served text-to-speech model",
default_prompt: false,
};
let multipart = headers
.get("content-type")
.and_then(|v| v.to_str().ok())
.is_some_and(|v| v.to_ascii_lowercase().starts_with("multipart/form-data"));
if multipart {
return proxy_multipart(&state, &headers, body, &TTS).await;
}
let req = parse_openai_request(&body)?;
let model = route_model(&headers, req["model"].as_str().unwrap_or(""), &TTS)?;
let (model, target, guard) = ensure_model_for(
&state,
&model,
Some(&headers),
LoadOptions::default(),
Task::Forward,
)
.await?;
let (req, target, activity, response_model_override) =
prepare_openai_request(&state, req, model, target, guard).await?;
forward_openai_request(
&state,
&headers,
req,
target,
activity,
response_model_override,
TTS.path,
)
.await
}
pub(super) fn provider_refusal(path: &str, hint: &str) -> AppError {
AppError::status(
StatusCode::BAD_REQUEST,
format!("llmman does not route {path} to a provider: {hint}"),
)
}
pub(super) fn route_model(
headers: &HeaderMap,
model: &str,
route: &LocalRoute,
) -> Result<String, AppError> {
let model = match crate::hybrid::split_ref(model) {
Some(pair) => {
let side = match request_pin(Some(headers))? {
Some(crate::hybrid::Side::Cloud) => crate::hybrid::Side::Cloud,
_ => crate::hybrid::Side::Local,
};
eprintln!(
"[llmman] hybrid {:?} + {:?} -> {} ({})",
pair.local,
pair.remote_ref(),
side.as_str(),
route.what
);
pair.side_ref(side)
}
None => model.to_string(),
};
if crate::providers::is_remote_ref(&model) {
return Err(provider_refusal(route.path, route.hint));
}
Ok(model)
}
async fn proxy_multipart(
state: &AppState,
headers: &HeaderMap,
body: Bytes,
route: &LocalRoute,
) -> Result<Response, AppError> {
let Some(model) = multipart_text_field(&body, headers, "model")
.await
.filter(|m| !m.is_empty())
else {
return Err(AppError::status(
StatusCode::BAD_REQUEST,
format!(
"{} request is missing a required \"model\" form field",
route.what
),
));
};
let model = route_model(headers, &model, route)?;
let (_, target, guard) = ensure_model(state, &model, Some(headers), None).await?;
let activity = begin_activity(guard, None).await;
let body = if route.default_prompt {
with_default_transcription_prompt(state, &target, headers, body).await
} else {
body
};
proxy(
&state.0.client,
&target,
route.path,
headers,
body,
activity,
)
.await
}
#[cfg(test)]
mod tests {
use super::super::test_support::{multipart_parts, multipart_upload, BOUNDARY};
use super::*;
fn embeddings_response(embedding: serde_json::Value) -> Bytes {
Bytes::from(
serde_json::to_vec(&serde_json::json!({
"object": "list",
"model": "/models/abc",
"data": [{ "object": "embedding", "index": 0, "embedding": embedding }],
"usage": { "prompt_tokens": 2, "total_tokens": 2 },
}))
.unwrap(),
)
}
#[test]
fn dimensions_cut_each_vector_and_renormalise_it() {
let raw = embeddings_response(serde_json::json!([3.0, 4.0, 12.0]));
let out = apply_embedding_dimensions(&raw, 2).unwrap();
let text = std::str::from_utf8(&out).unwrap();
assert!(text.contains("[0.6,0.8]"), "{text}");
let body: serde_json::Value = serde_json::from_slice(&out).unwrap();
assert_eq!(body["data"][0]["embedding"], serde_json::json!([0.6, 0.8]));
assert_eq!(body["data"][0]["index"], 0);
assert_eq!(body["data"][0]["object"], "embedding");
assert_eq!(body["usage"]["prompt_tokens"], 2);
assert_eq!(body["object"], "list");
assert_eq!(body["model"], "/models/abc");
}
#[test]
fn dimensions_cut_a_base64_vector_too() {
let raw = embeddings_response(serde_json::json!(encode_f32s(&[3.0, 4.0, 12.0])));
let out = apply_embedding_dimensions(&raw, 2).unwrap();
let body: serde_json::Value = serde_json::from_slice(&out).unwrap();
let encoded = body["data"][0]["embedding"].as_str().unwrap();
assert_eq!(decode_f32s(encoded).unwrap(), [0.6, 0.8]);
}
#[test]
fn dimensions_leave_everything_else_untouched() {
let raw = embeddings_response(serde_json::json!([3.0, 4.0, 12.0]));
for dims in [3, 4, 768] {
assert_eq!(apply_embedding_dimensions(&raw, dims).unwrap(), raw);
}
let short = embeddings_response(serde_json::json!(encode_f32s(&[3.0, 4.0])));
assert_eq!(apply_embedding_dimensions(&short, 2).unwrap(), short);
let not_f32s = embeddings_response(serde_json::json!("AAAAAAA="));
assert_eq!(apply_embedding_dimensions(¬_f32s, 1).unwrap(), not_f32s);
let not_base64 = embeddings_response(serde_json::json!("%%%"));
assert_eq!(
apply_embedding_dimensions(¬_base64, 1).unwrap(),
not_base64
);
for body in [
Bytes::from_static(br#"{"error":{"message":"input too large"}}"#),
Bytes::from_static(b"not json"),
Bytes::new(),
] {
assert_eq!(apply_embedding_dimensions(&body, 2).unwrap(), body);
}
}
#[test]
fn dimensions_refuse_a_vector_that_is_not_finite() {
let raw = embeddings_response(serde_json::json!(encode_f32s(&[f32::NAN, 4.0, 12.0])));
let err = apply_embedding_dimensions(&raw, 2).unwrap_err();
assert_eq!(err.1, StatusCode::BAD_GATEWAY);
}
#[test]
fn dimensions_are_a_positive_whole_number() {
let asked = |dimensions: serde_json::Value| {
requested_dimensions(&serde_json::json!({ "model": "m", "dimensions": dimensions }))
};
assert_eq!(asked(serde_json::json!(256)), Some(256));
for not_a_request in [
serde_json::json!(0),
serde_json::json!(-1),
serde_json::json!(256.5),
serde_json::json!("256"),
serde_json::Value::Null,
] {
assert_eq!(asked(not_a_request.clone()), None, "{not_a_request}");
}
assert_eq!(
requested_dimensions(&serde_json::json!({ "model": "m" })),
None
);
}
fn multipart_body(fields: &[(&str, &str)]) -> (Bytes, HeaderMap) {
let parts: Vec<_> = fields
.iter()
.map(|(name, value)| (*name, None, value.as_bytes()))
.collect();
multipart_upload(&parts)
}
#[tokio::test]
async fn multipart_text_field_finds_a_named_field_among_several() {
let (body, headers) = multipart_body(&[
("language", "en"),
("model", "docker.io/ai/whisper:latest"),
("response_format", "json"),
]);
assert_eq!(
multipart_text_field(&body, &headers, "model").await,
Some("docker.io/ai/whisper:latest".to_string())
);
assert_eq!(
multipart_text_field(&body, &headers, "language").await,
Some("en".to_string())
);
}
#[tokio::test]
async fn multipart_text_field_leaves_the_original_body_untouched() {
let (body, headers) = multipart_body(&[("model", "m"), ("prompt", "hello")]);
let before = body.clone();
let _ = multipart_text_field(&body, &headers, "model").await;
assert_eq!(body, before);
}
#[tokio::test]
async fn multipart_text_field_is_none_for_a_missing_field_or_non_multipart_body() {
let (body, headers) = multipart_body(&[("language", "en")]);
assert_eq!(multipart_text_field(&body, &headers, "model").await, None);
let plain_body = Bytes::from_static(b"{\"model\":\"m\"}");
let mut json_headers = HeaderMap::new();
json_headers.insert("content-type", "application/json".parse().unwrap());
assert_eq!(
multipart_text_field(&plain_body, &json_headers, "model").await,
None
);
assert_eq!(
multipart_text_field(&plain_body, &HeaderMap::new(), "model").await,
None
);
}
#[tokio::test]
async fn mlx_embeddings_unsupported_response_explains_why_and_names_the_model() {
let resp = mlx_embeddings_unsupported_response("gemma4:latest");
assert_eq!(resp.status(), StatusCode::NOT_IMPLEMENTED);
let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
.await
.unwrap();
let value: serde_json::Value = serde_json::from_slice(&body).unwrap();
let message = value["error"]["message"].as_str().unwrap();
assert!(message.contains("gemma4:latest"));
assert!(message.contains("mlx_lm.server"));
assert!(message.contains("/v1/embeddings"));
}
#[test]
fn omni_image_request_translates_llama_server_fields() {
let req = serde_json::json!({
"model": "nvidia/Cosmos3-Edge",
"prompt": "a manatee",
"stream": true,
"response_format": "b64_json",
"width": 640,
"height": 480,
"steps": 8,
"seed": 42,
"cfg_scale": 5.0,
"negative_prompt": "blurry"
});
let (out, stream) = omni_image_request(req).unwrap();
assert!(stream);
assert_eq!(
out,
serde_json::json!({
"model": "nvidia/Cosmos3-Edge",
"prompt": "a manatee",
"response_format": "b64_json",
"size": "640x480",
"num_inference_steps": 8,
"seed": 42,
"guidance_scale": 5.0,
"negative_prompt": "blurry"
})
);
}
#[test]
fn omni_image_request_leaves_a_native_request_alone_and_invents_nothing() {
let req = serde_json::json!({
"model": "m", "prompt": "p", "size": "1024x1024",
"num_inference_steps": 50, "guidance_scale": 7.0
});
let (out, stream) = omni_image_request(req.clone()).unwrap();
assert!(!stream);
let mut expected = req;
expected["response_format"] = "b64_json".into();
assert_eq!(out, expected);
let (out, _) =
omni_image_request(serde_json::json!({"model": "m", "prompt": "p"})).unwrap();
assert!(out.get("size").is_none());
assert!(out.get("num_inference_steps").is_none());
let (out, _) = omni_image_request(
serde_json::json!({"prompt": "p", "width": 0, "height": 0, "steps": 0}),
)
.unwrap();
assert!(out.get("size").is_none());
assert!(out.get("num_inference_steps").is_none());
let (out, _) = omni_image_request(
serde_json::json!({"prompt": "p", "steps": 8, "num_inference_steps": 30}),
)
.unwrap();
assert_eq!(out["num_inference_steps"], 30);
let err = omni_image_request(serde_json::json!({"prompt": "p", "width": 768})).unwrap_err();
assert!(err.contains("both width and height"), "{err}");
}
#[test]
fn omni_video_fields_translate_and_round_seconds() {
let req = serde_json::json!({
"model": "nvidia/Cosmos3-Edge",
"prompt": "waves",
"stream": false,
"seconds": 2.4,
"fps": 24,
"steps": 20,
"cfg_scale": 5.0,
"audio": false,
"seed": 7,
"extra_params": {"guardrails": false}
});
let fields: std::collections::HashMap<String, String> =
omni_video_fields(&req).into_iter().collect();
assert_eq!(fields["model"], "nvidia/Cosmos3-Edge");
assert_eq!(fields["prompt"], "waves");
assert_eq!(fields["seconds"], "2", "SecondStr is a whole number");
assert_eq!(fields["fps"], "24");
assert_eq!(fields["num_inference_steps"], "20");
assert_eq!(fields["guidance_scale"], "5.0");
assert_eq!(fields["generate_sound"], "false");
assert_eq!(fields["seed"], "7");
assert_eq!(fields["extra_params"], r#"{"guardrails":false}"#);
assert!(!fields.contains_key("stream"));
assert!(!fields.contains_key("steps"));
let fields: std::collections::HashMap<String, String> =
omni_video_fields(&serde_json::json!({"prompt": "p", "seconds": 0.3}))
.into_iter()
.collect();
assert_eq!(fields["seconds"], "1");
let fields: std::collections::HashMap<String, String> =
omni_video_fields(&serde_json::json!({"prompt": "p", "frames": 33, "num_frames": 49}))
.into_iter()
.collect();
assert_eq!(fields["num_frames"], "49");
}
#[test]
fn multipart_form_is_well_formed_and_readable_back() {
let fields = vec![
("model".to_string(), "m".to_string()),
("prompt".to_string(), "a b\r\nc".to_string()),
];
let (body, content_type) = multipart_form(&fields);
let boundary = content_type
.strip_prefix("multipart/form-data; boundary=")
.unwrap();
let text = String::from_utf8(body.clone()).unwrap();
assert!(text.starts_with(&format!("--{boundary}\r\n")));
assert!(text.ends_with(&format!("--{boundary}--\r\n")));
let (filtered, _) = multipart_form(&[
("ok_1".to_string(), "v".to_string()),
("bad\"\r\nX: y".to_string(), "v".to_string()),
]);
let filtered = String::from_utf8(filtered).unwrap();
assert!(filtered.contains("name=\"ok_1\""));
assert!(!filtered.contains("name=\"bad\""), "{filtered}");
assert!(!filtered.contains("X: y"), "{filtered}");
let (_, ct2) = multipart_form(&[("prompt".to_string(), format!("x{boundary}y"))]);
assert_ne!(ct2, content_type);
let mut headers = HeaderMap::new();
headers.insert("content-type", content_type.parse().unwrap());
let body = Bytes::from(body);
let rt = tokio::runtime::Runtime::new().unwrap();
assert_eq!(
rt.block_on(multipart_text_field(&body, &headers, "prompt")),
Some("a b\r\nc".to_string())
);
assert_eq!(
rt.block_on(multipart_text_field(&body, &headers, "model")),
Some("m".to_string())
);
}
#[test]
fn consolidate_chat_system_messages_merges_them_into_one_leading_message() {
let mut req = serde_json::json!({
"model": "docker.io/ai/qwen3.5:0.8b",
"messages": [
{"role": "system", "content": "you are a helpful assistant"},
{"role": "user", "content": "hi"},
{"role": "system", "content": "## Shell Tools"},
{"role": "developer", "content": [{"type": "text", "text": "## Filesystem Tools"}]}
]
});
consolidate_chat_system_messages(&mut req);
assert_eq!(
req["messages"],
serde_json::json!([
{"role": "system", "content":
"you are a helpful assistant\n\n## Shell Tools\n\n## Filesystem Tools"},
{"role": "user", "content": "hi"}
])
);
}
#[test]
fn consolidate_chat_system_messages_leaves_a_single_leading_one_alone() {
let mut req = serde_json::json!({
"messages": [
{"role": "system", "content": "you are a helpful assistant"},
{"role": "user", "content": "hi"}
]
});
let before = req.clone();
consolidate_chat_system_messages(&mut req);
assert_eq!(req, before);
}
#[test]
fn consolidate_chat_system_messages_keeps_block_content_when_conforming() {
let mut req = serde_json::json!({
"messages": [
{"role": "system", "content": [
{"type": "text", "text": "be terse"},
{"type": "input_image", "image_url": "data:image/png;base64,AAAA"}
]},
{"role": "user", "content": "hi"}
]
});
let before = req.clone();
consolidate_chat_system_messages(&mut req);
assert_eq!(req, before);
}
#[test]
fn consolidate_chat_system_messages_keeps_non_text_blocks_while_merging() {
let mut req = serde_json::json!({
"messages": [
{"role": "system", "content": [
{"type": "text", "text": "be terse"},
{"type": "input_image", "image_url": "data:image/png;base64,AAAA"}
]},
{"role": "user", "content": "hi"},
{"role": "developer", "content": "and cite sources"}
]
});
consolidate_chat_system_messages(&mut req);
assert_eq!(
req["messages"],
serde_json::json!([
{"role": "system", "content": [
{"type": "text", "text": "be terse"},
{"type": "input_image", "image_url": "data:image/png;base64,AAAA"},
{"type": "text", "text": "and cite sources"}
]},
{"role": "user", "content": "hi"}
])
);
}
#[test]
fn consolidate_chat_system_messages_keep_a_messages_own_parts_off_the_blank_line() {
let mut req = serde_json::json!({
"messages": [
{"role": "system", "content": [
{"type": "text", "text": "part one"},
{"type": "text", "text": "part two"}
]},
{"role": "user", "content": "hi"},
{"role": "developer", "content": "developer instruction"}
]
});
consolidate_chat_system_messages(&mut req);
assert_eq!(
req["messages"][0]["content"],
"part one\npart two\n\ndeveloper instruction"
);
}
#[test]
fn consolidate_chat_system_messages_let_a_non_text_block_end_the_run() {
let image = serde_json::json!({
"type": "image_url",
"image_url": {"url": "data:image/png;base64,AAAA", "detail": "high"}
});
let mut req = serde_json::json!({
"messages": [
{"role": "system", "content": [
{"type": "text", "text": "before one"},
{"type": "text", "text": "before two"},
image,
{"type": "text", "text": "after"}
]},
{"role": "user", "content": "hi"},
{"role": "developer", "content": "and cite sources"}
]
});
consolidate_chat_system_messages(&mut req);
assert_eq!(
req["messages"][0]["content"],
serde_json::json!([
{"type": "text", "text": "before one\nbefore two"},
{"type": "image_url",
"image_url": {"url": "data:image/png;base64,AAAA", "detail": "high"}},
{"type": "text", "text": "after\n\nand cite sources"}
])
);
assert_eq!(
req["messages"][1],
serde_json::json!({"role": "user", "content": "hi"})
);
assert_eq!(req["messages"].as_array().map(Vec::len), Some(2));
}
#[test]
fn consolidate_chat_system_messages_drop_empty_parts_from_the_run() {
let cases = [
(
serde_json::json!([{"type": "text", "text": "a"},
{"type": "text", "text": ""}]),
"a\n\ndev",
),
(
serde_json::json!([{"type": "text", "text": ""},
{"type": "text", "text": "b"}]),
"b\n\ndev",
),
(
serde_json::json!([{"type": "text", "text": ""},
{"type": "text", "text": ""}]),
"dev",
),
(
serde_json::json!([{"type": "text", "text": "a"},
{"type": "text", "text": ""},
{"type": "text", "text": "b"}]),
"a\nb\n\ndev",
),
];
for (content, want) in cases {
let mut req = serde_json::json!({
"messages": [
{"role": "system", "content": content},
{"role": "user", "content": "hi"},
{"role": "developer", "content": "dev"}
]
});
consolidate_chat_system_messages(&mut req);
assert_eq!(req["messages"][0]["content"], want);
}
}
#[test]
fn consolidate_chat_system_messages_moves_a_late_lone_system_turn_to_the_front() {
let mut req = serde_json::json!({
"messages": [
{"role": "user", "content": "hi"},
{"role": "system", "content": "a mid-conversation reminder"}
]
});
consolidate_chat_system_messages(&mut req);
assert_eq!(
req["messages"],
serde_json::json!([
{"role": "system", "content": "a mid-conversation reminder"},
{"role": "user", "content": "hi"}
])
);
}
#[test]
fn consolidate_chat_system_messages_is_a_no_op_without_any() {
let mut req = serde_json::json!({
"messages": [{"role": "user", "content": "hi"}]
});
let before = req.clone();
consolidate_chat_system_messages(&mut req);
assert_eq!(req, before);
let mut input_only = serde_json::json!({"input": "hi"});
let before = input_only.clone();
consolidate_chat_system_messages(&mut input_only);
assert_eq!(input_only, before);
}
#[test]
fn consolidate_chat_system_messages_drops_empty_ones() {
let mut req = serde_json::json!({
"messages": [
{"role": "system", "content": ""},
{"role": "user", "content": "hi"},
{"role": "system", "content": "the only real instruction"}
]
});
consolidate_chat_system_messages(&mut req);
assert_eq!(
req["messages"],
serde_json::json!([
{"role": "system", "content": "the only real instruction"},
{"role": "user", "content": "hi"}
])
);
}
#[test]
fn merge_adjacent_roles_joins_a_run_into_one_turn() {
let mut req = serde_json::json!({
"model": "gemma3:1b",
"messages": [
{"role": "system", "content": "be terse"},
{"role": "user", "content": "a"},
{"role": "user", "content": [{"type": "text", "text": "b"}]},
]
});
assert!(merge_adjacent_roles(&mut req));
assert_eq!(
req["messages"],
serde_json::json!([
{"role": "system", "content": "be terse"},
{"role": "user", "content": "a\n\nb"},
])
);
assert_eq!(req["model"], "gemma3:1b", "only the messages change");
let mut req = serde_json::json!({
"messages": [
{"role": "user", "content": "q1"},
{"role": "assistant", "content": "a1"},
{"role": "assistant", "content": "a2"},
{"role": "user", "content": "q2"},
{"role": "user", "content": "q3"},
{"role": "user", "content": "q4"},
]
});
assert!(merge_adjacent_roles(&mut req));
assert_eq!(
req["messages"],
serde_json::json!([
{"role": "user", "content": "q1"},
{"role": "assistant", "content": "a1\n\na2"},
{"role": "user", "content": "q2\n\nq3\n\nq4"},
])
);
}
#[test]
fn merge_adjacent_roles_leaves_an_alternating_request_alone() {
let mut req = serde_json::json!({
"messages": [
{"role": "system", "content": "s"},
{"role": "user", "content": [{"type": "text", "text": "a"}]},
{"role": "assistant", "content": "b"},
{"role": "user", "content": "c"},
]
});
let before = req.clone();
assert!(!merge_adjacent_roles(&mut req));
assert_eq!(req, before);
let mut no_messages = serde_json::json!({"input": "hi"});
assert!(!merge_adjacent_roles(&mut no_messages));
}
#[test]
fn merge_adjacent_roles_keeps_tool_turns_apart() {
let call = serde_json::json!([{"id": "c1", "type": "function",
"function": {"name": "f", "arguments": "{}"}}]);
let mut req = serde_json::json!({
"messages": [
{"role": "user", "content": "go"},
{"role": "assistant", "content": null, "tool_calls": call},
{"role": "assistant", "content": "calling"},
{"role": "assistant", "content": "again"},
{"role": "tool", "tool_call_id": "c1", "content": "1"},
{"role": "tool", "tool_call_id": "c2", "content": "2"},
{"role": "user", "content": "x", "name": "alice"},
{"role": "user", "content": "y"},
]
});
assert!(merge_adjacent_roles(&mut req));
assert_eq!(
req["messages"],
serde_json::json!([
{"role": "user", "content": "go"},
{"role": "assistant", "content": null, "tool_calls": call},
{"role": "assistant", "content": "calling\n\nagain"},
{"role": "tool", "tool_call_id": "c1", "content": "1"},
{"role": "tool", "tool_call_id": "c2", "content": "2"},
{"role": "user", "content": "x", "name": "alice"},
{"role": "user", "content": "y"},
])
);
}
#[test]
fn merge_adjacent_roles_keeps_non_text_blocks() {
let image = serde_json::json!({"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}});
let mut req = serde_json::json!({
"messages": [
{"role": "user", "content": "look"},
{"role": "user", "content": [image, {"type": "text", "text": "what is it"}]},
]
});
assert!(merge_adjacent_roles(&mut req));
assert_eq!(
req["messages"],
serde_json::json!([{"role": "user", "content": [
{"type": "text", "text": "look"},
image,
{"type": "text", "text": "what is it"},
]}])
);
let mut req = serde_json::json!({
"messages": [{"role": "user", "content": ""}, {"role": "user", "content": null}]
});
assert!(merge_adjacent_roles(&mut req));
assert_eq!(
req["messages"],
serde_json::json!([{"role": "user", "content": ""}])
);
}
#[test]
fn merged_after_refusal_reads_the_templates_own_rule() {
let gemma = r#"{"error":{"code":400,"message":"Unable to generate parser for this template. Automatic parser generation failed: Conversation roles must alternate user/assistant/user/assistant/...","type":"invalid_request_error"}}"#;
let mistral = "After the optional system message, conversation roles must alternate user and assistant roles except for tool calls and results.";
let sent = br#"{"messages":[{"role":"user","content":"a"},{"role":"user","content":"b"}]}"#;
let target = Target::Local(1);
let retry = |status, reply: &str, body: &[u8]| {
merged_after_refusal(&target, status, reply.as_bytes(), body)
};
for reply in [gemma, mistral] {
let merged = retry(StatusCode::BAD_REQUEST, reply, sent).expect(reply);
let merged: serde_json::Value = serde_json::from_slice(&merged).unwrap();
assert_eq!(
merged["messages"],
serde_json::json!([{"role": "user", "content": "a\n\nb"}])
);
assert_eq!(retry(StatusCode::INTERNAL_SERVER_ERROR, reply, sent), None);
}
let other = "the request exceeds the available context size";
assert_eq!(retry(StatusCode::BAD_REQUEST, other, sent), None);
let alternating = br#"{"messages":[{"role":"user","content":"a"}]}"#;
assert_eq!(retry(StatusCode::BAD_REQUEST, gemma, alternating), None);
assert_eq!(retry(StatusCode::BAD_REQUEST, gemma, b"not json"), None);
}
#[test]
fn apply_reasoning_effort_mirrors_it_into_template_kwargs() {
let with = |req: serde_json::Value| {
let mut req = req;
apply_reasoning_effort(&mut req);
req
};
assert_eq!(
with(serde_json::json!({ "model": "m", "reasoning_effort": "none" })),
serde_json::json!({
"model": "m", "reasoning_effort": "none",
"chat_template_kwargs": { "enable_thinking": false }
})
);
for level in crate::chat_template::EFFORT_LEVELS {
assert_eq!(
with(serde_json::json!({ "model": "m", "reasoning_effort": level })),
serde_json::json!({
"model": "m", "reasoning_effort": level,
"chat_template_kwargs": { "enable_thinking": true, "reasoning_effort": level }
}),
"{level}"
);
}
assert_eq!(
with(serde_json::json!({
"model": "m", "reasoning_effort": "high",
"chat_template_kwargs": { "enable_thinking": false }
})),
serde_json::json!({
"model": "m", "reasoning_effort": "high",
"chat_template_kwargs": { "enable_thinking": false, "reasoning_effort": "high" }
})
);
let unknown = serde_json::json!({ "model": "m", "reasoning_effort": "verbose" });
assert_eq!(with(unknown.clone()), unknown);
let absent = serde_json::json!({ "model": "m", "chat_template_kwargs": { "a": 1 } });
assert_eq!(with(absent.clone()), absent);
let not_a_string = serde_json::json!({ "model": "m", "reasoning_effort": 3 });
assert_eq!(with(not_a_string.clone()), not_a_string);
}
#[test]
fn apply_effort_budget_bounds_minimal_and_low() {
let budget = |req: serde_json::Value| {
let mut req = req;
apply_effort_budget(&mut req, true);
req
};
let low = |extra: serde_json::Value| {
let mut req = serde_json::json!({ "model": "m", "reasoning_effort": "low" });
req.as_object_mut()
.unwrap()
.extend(extra.as_object().unwrap().clone());
budget(req)["reasoning_budget_tokens"].clone()
};
assert_eq!(
budget(serde_json::json!({
"model": "m", "reasoning_effort": "minimal", "max_tokens": 120
})),
serde_json::json!({
"model": "m", "reasoning_effort": "minimal", "max_tokens": 120,
"reasoning_budget_tokens": 15,
"reasoning_budget_message": EFFORT_BUDGET_MESSAGE
})
);
assert_eq!(low(serde_json::json!({ "max_tokens": 120 })), 30);
assert_eq!(low(serde_json::json!({ "max_tokens": 100_000 })), 2048);
assert_eq!(low(serde_json::json!({})), 2048);
assert_eq!(low(serde_json::json!({ "max_tokens": 1 })), 1);
assert_eq!(low(serde_json::json!({ "max_tokens": 120.0 })), 30);
assert_eq!(low(serde_json::json!({ "max_tokens": 0.5 })), 2048);
assert_eq!(low(serde_json::json!({ "max_tokens": 0 })), 2048);
assert_eq!(
low(serde_json::json!({ "n_predict": 400, "max_tokens": 120 })),
100
);
assert_eq!(
low(serde_json::json!({ "max_completion_tokens": 400, "max_tokens": 120 })),
100
);
assert_eq!(
low(serde_json::json!({ "n_predict": -1, "max_tokens": 120 })),
2048
);
assert_eq!(
budget(serde_json::json!({
"model": "m", "reasoning_effort": "low", "max_tokens": 120,
"reasoning_budget_message": "mine"
}))["reasoning_budget_message"],
"mine"
);
assert_eq!(
budget(serde_json::json!({
"model": "m", "reasoning_effort": "low", "max_tokens": 120,
"reasoning_budget_tokens": null, "thinking_budget_tokens": null,
"reasoning_budget_message": null
})),
serde_json::json!({
"model": "m", "reasoning_effort": "low", "max_tokens": 120,
"reasoning_budget_tokens": 30, "thinking_budget_tokens": null,
"reasoning_budget_message": EFFORT_BUDGET_MESSAGE
})
);
let mut own_message = serde_json::json!({ "model": "m", "reasoning_effort": "low" });
apply_effort_budget(&mut own_message, false);
assert_eq!(
own_message,
serde_json::json!({
"model": "m", "reasoning_effort": "low", "reasoning_budget_tokens": 2048
})
);
for unchanged in [
serde_json::json!({ "model": "m", "max_tokens": 120 }),
serde_json::json!({ "model": "m", "reasoning_effort": "none" }),
serde_json::json!({ "model": "m", "reasoning_effort": "medium" }),
serde_json::json!({ "model": "m", "reasoning_effort": "high" }),
serde_json::json!({ "model": "m", "reasoning_effort": "LOW" }),
serde_json::json!({
"model": "m", "reasoning_effort": "low",
"chat_template_kwargs": { "enable_thinking": false }
}),
serde_json::json!({
"model": "m", "reasoning_effort": "low", "reasoning_budget_tokens": -1
}),
serde_json::json!({
"model": "m", "reasoning_effort": "low", "thinking_budget_tokens": 64
}),
] {
assert_eq!(budget(unchanged.clone()), unchanged);
}
}
const AUDIO: &[u8] = b"RIFF\x00\xff\r\n--not-the-boundary\r\n\x00\x01WAVE";
#[tokio::test]
async fn a_prompt_is_added_and_every_other_part_carried_over() {
let wav = Some(("a b.wav", "audio/wav"));
let (body, headers) = multipart_upload(&[
("model", None, b"voxtral"),
("file", wav, AUDIO),
("language", None, b"en"),
]);
let out = multipart_with_prompt(&body, &headers, "Transcribe it.")
.await
.unwrap();
let owned = |s: &str| Some(s.to_string());
assert_eq!(
multipart_parts(&out, &headers).await,
vec![
("model".to_string(), None, None, b"voxtral".to_vec()),
(
"file".to_string(),
owned("a b.wav"),
owned("audio/wav"),
AUDIO.to_vec()
),
("language".to_string(), None, None, b"en".to_vec()),
("prompt".to_string(), None, None, b"Transcribe it.".to_vec()),
]
);
}
#[tokio::test]
async fn a_large_part_is_carried_over_byte_for_byte() {
let mut audio: Vec<u8> = (0..300_000u32).map(|i| (i * 31 % 251) as u8).collect();
let lookalike = format!("\r\n--{}\r\n", &BOUNDARY[..10]);
audio.splice(65_529..65_529 + lookalike.len(), lookalike.bytes());
let wav = Some(("long.wav", "audio/wav"));
let (body, headers) = multipart_upload(&[("file", wav, &audio), ("model", None, b"m")]);
let out = multipart_with_prompt(&body, &headers, "p").await.unwrap();
let parts = multipart_parts(&out, &headers).await;
assert_eq!(parts.len(), 3);
assert!(parts[0].3 == audio, "the audio changed in transit");
assert_eq!(parts[1].3, b"m");
assert_eq!(parts[2].3, b"p");
}
#[tokio::test]
async fn an_existing_prompt_part_is_replaced_not_duplicated() {
for old in ["", " \r\n ", "x"] {
let (body, headers) = multipart_upload(&[
("prompt", None, old.as_bytes()),
("file", Some(("a.wav", "audio/wav")), AUDIO),
]);
let out = multipart_with_prompt(&body, &headers, "Transcribe it.")
.await
.unwrap();
let parts = multipart_parts(&out, &headers).await;
let names: Vec<_> = parts.iter().map(|p| p.0.as_str()).collect();
assert_eq!(names, ["file", "prompt"], "{old:?}");
assert_eq!(parts[0].3, AUDIO);
assert_eq!(parts[1].3, b"Transcribe it.");
}
}
#[tokio::test]
async fn only_a_non_blank_prompt_counts_as_one() {
let cases = [
(Some("Transcribe in French."), true),
(Some(" \r\n"), false),
(Some(""), false),
(None, false),
];
for (prompt, expected) in cases {
let mut fields = vec![("model", "m")];
fields.extend(prompt.map(|p| ("prompt", p)));
let (body, headers) = multipart_body(&fields);
assert_eq!(has_prompt(&body, &headers).await, expected, "{prompt:?}");
}
assert!(!has_prompt(&Bytes::from_static(b"{}"), &HeaderMap::new()).await);
}
#[tokio::test]
async fn what_is_not_a_plain_multipart_upload_is_left_alone() {
let (body, headers) = multipart_body(&[("model", "m")]);
assert!(multipart_with_prompt(&body, &HeaderMap::new(), "x")
.await
.is_none());
let odd = format!(
"--{BOUNDARY}\r\nContent-Disposition: form-data; name=\"file\"\r\nContent-Transfer-Encoding: base64\r\n\r\nAAAA\r\n--{BOUNDARY}--\r\n"
);
assert!(multipart_with_prompt(&Bytes::from(odd), &headers, "x")
.await
.is_none());
}
#[test]
fn lfm2_is_told_by_its_tool_list_tokens() {
assert!(is_lfm2_template(
"a <|tool_list_start|>[]<|tool_list_end|> b"
));
assert!(!is_lfm2_template("<|tool_list_start|> never closed"));
assert!(!is_lfm2_template("{{ messages }}"));
}
}