use super::anthropic::relay_anthropic_messages;
use super::backend::would_use_mlx;
use super::hybrid::{request_pin, resolve_hybrid_side, with_hybrid_fallback};
use super::openai::{
apply_default_repeat_penalty, decode_f32s, encode_f32s, handle_tts,
DEFAULT_TRANSCRIPTION_PROMPT, EFFORT_BUDGET_MESSAGE,
};
use super::refusal::{explain_missing_route, unsupported_on_wire};
use super::responses::{
consolidate_responses_instructions, filter_non_function_tools, remote_responses,
responses_input_item_text, RESPONSES_ROUTE,
};
use super::sched::{reap_idle_models_once, DEFAULT_KEEP_ALIVE};
use super::stream::{fold_ollama_lines, stream_ollama};
use super::test_support::{
headers_with, mock_peer, multipart_parts, multipart_upload, node, remote_target,
remote_target_on, rendered_registry, running_model_fixture, running_model_fixture_with_engine,
serve_router, serve_router_with, shell_token_protocol, test_inner, test_state, test_state_at,
test_state_with_budget, ws_upgrade, HOSTED, PAIR, SHELL_TOKEN,
};
use super::*;
#[test]
fn mediagen_worker_argv_carries_the_pinned_reference_verbatim() {
let pinned = "example.com/ai/model:q4@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef";
assert_eq!(
mediagen_backend_args(pinned, 8_123),
["serve", pinned, "--port", "8123"]
);
}
#[test]
fn detached_serve_forwards_configuration_without_detaching_the_child() {
let args = ServeArgs {
model: Some("docker.io/ai/model:latest".into()),
detach: true,
runtime: Runtime::Path,
no_hardened: false,
llama_cpp_version: Some("b1234".into()),
vllm_version: None,
vllm_image: Some("example/vllm:1".into()),
sglang_version: Some("v0.5".into()),
pull_only: false,
port: None,
host: "127.0.0.1".parse().unwrap(),
};
assert_eq!(
args.detached_child_args(),
[
"--runtime",
"path",
"--llama-cpp-version",
"b1234",
"--vllm-image",
"example/vllm:1",
"--sglang-version",
"v0.5",
]
);
}
#[test]
fn hardened_is_on_by_default_and_the_opt_out_reaches_the_detached_child() {
#[derive(clap::Parser)]
struct Cli {
#[command(flatten)]
args: ServeArgs,
}
let parse = |argv: &[&str]| {
<Cli as clap::Parser>::try_parse_from(["serve"].iter().chain(argv))
.map(|c| c.args)
.expect("serve args parse")
};
let on = parse(&[]);
assert!(!on.no_hardened);
assert!(test_state().0.hardened);
assert_eq!(on.detached_child_args(), ["--runtime", "auto"]);
let off = parse(&["--no-hardened"]);
assert!(off.no_hardened);
assert_eq!(
off.detached_child_args(),
["--runtime", "auto", "--no-hardened"]
);
}
#[test]
fn hardened_from_turns_off_on_the_flag_or_the_variable() {
for (no_hardened, env, expected) in [
(false, None, true),
(false, Some(""), true),
(false, Some("0"), true),
(false, Some("Off"), true),
(false, Some("1"), false),
(false, Some("yes"), false),
(true, None, false),
(true, Some("0"), false),
] {
assert_eq!(
hardened_from(no_hardened, env),
expected,
"--no-hardened={no_hardened} LLMMAN_NO_HARDENED={env:?}"
);
}
}
#[test]
fn ffi_pull_ref_is_the_polled_key_not_the_tag_normalized_ref() {
assert_eq!(
ffi_pull_ref("docker.io/ai/qwen3.8", "docker.io/ai/qwen3.8:latest"),
"docker.io/ai/qwen3.8"
);
assert_eq!(
ffi_pull_ref("docker.io/ai/qwen3.8:0.8b", "docker.io/ai/qwen3.8:0.8b"),
"docker.io/ai/qwen3.8:0.8b"
);
}
#[test]
fn a_local_target_addresses_loopback_unchanged() {
let target = Target::Local(17434);
assert_eq!(
target.url("/v1/chat/completions"),
"http://127.0.0.1:17434/v1/chat/completions"
);
assert_eq!(
target.url("/v1/audio/transcriptions"),
"http://127.0.0.1:17434/v1/audio/transcriptions"
);
assert_eq!(
target.url("/v1/responses/input_tokens"),
"http://127.0.0.1:17434/v1/responses/input_tokens"
);
assert!(!target.is_remote());
}
#[test]
fn a_remote_target_rebases_routes_onto_the_provider_url() {
assert_eq!(
remote_target("https://openrouter.ai/api/v1").url("/v1/chat/completions"),
"https://openrouter.ai/api/v1/chat/completions"
);
assert_eq!(
remote_target("https://generativelanguage.googleapis.com/v1beta/openai")
.url("/v1/chat/completions"),
"https://generativelanguage.googleapis.com/v1beta/openai/chat/completions"
);
assert_eq!(
remote_target("https://api.perplexity.ai").url("/v1/chat/completions"),
"https://api.perplexity.ai/chat/completions"
);
assert!(remote_target("https://example.invalid/v1").is_remote());
}
#[test]
fn client_api_key_reads_bearer_and_x_api_key() {
let mut headers = HeaderMap::new();
headers.insert("authorization", "Bearer sk-openai-style".parse().unwrap());
assert_eq!(
client_api_key(Some(&headers)),
Some("sk-openai-style".to_string())
);
let mut headers = HeaderMap::new();
headers.insert("x-api-key", "sk-anthropic-style".parse().unwrap());
assert_eq!(
client_api_key(Some(&headers)),
Some("sk-anthropic-style".to_string())
);
let mut headers = HeaderMap::new();
headers.insert("authorization", "Bearer sk-bearer".parse().unwrap());
headers.insert("x-api-key", "sk-other".parse().unwrap());
assert_eq!(
client_api_key(Some(&headers)),
Some("sk-bearer".to_string())
);
}
#[test]
fn client_api_key_ignores_the_local_placeholder_and_empty_values() {
for header in ["Bearer llmman", "Bearer ", "Bearer ", ""] {
let mut headers = HeaderMap::new();
headers.insert("authorization", header.parse().unwrap());
assert_eq!(
client_api_key(Some(&headers)),
None,
"{header:?} was treated as a credential"
);
}
let mut headers = HeaderMap::new();
headers.insert("x-api-key", PLACEHOLDER_API_KEY.parse().unwrap());
assert_eq!(client_api_key(Some(&headers)), None);
assert_eq!(client_api_key(None), None);
assert_eq!(client_api_key(Some(&HeaderMap::new())), None);
}
#[test]
fn client_api_key_ignores_non_bearer_authorization() {
for header in ["Basic dXNlcjpwYXNz", "sk-no-scheme", "Bearer"] {
let mut headers = HeaderMap::new();
headers.insert("authorization", header.parse().unwrap());
assert_eq!(
client_api_key(Some(&headers)),
None,
"{header:?} was treated as a bearer token"
);
}
}
#[test]
fn client_api_key_accepts_any_spelling_of_bearer() {
for header in ["Bearer sk-real", "bearer sk-real", "BEARER sk-real"] {
let mut headers = HeaderMap::new();
headers.insert("authorization", header.parse().unwrap());
assert_eq!(
client_api_key(Some(&headers)),
Some("sk-real".to_string()),
"{header:?}"
);
}
}
#[test]
fn client_api_key_falls_through_an_unusable_authorization() {
for header in ["Bearer llmman", "Bearer ", "Basic dXNlcjpwYXNz"] {
let mut headers = HeaderMap::new();
headers.insert("authorization", header.parse().unwrap());
headers.insert("x-api-key", "sk-real".parse().unwrap());
assert_eq!(
client_api_key(Some(&headers)),
Some("sk-real".to_string()),
"{header:?} shadowed a real x-api-key"
);
}
}
#[tokio::test]
async fn a_providers_missing_responses_route_is_explained() {
let remote = remote_target("https://example.invalid/v1");
let not_found = (StatusCode::NOT_FOUND, "{}").into_response();
let explained = explain_missing_route(&remote, "/v1/responses", not_found);
assert_eq!(explained.status(), StatusCode::NOT_IMPLEMENTED);
let body = axum::body::to_bytes(explained.into_body(), usize::MAX)
.await
.unwrap();
let body = String::from_utf8_lossy(&body);
assert!(body.contains("mockprov"), "{body}");
assert!(body.contains("Responses API"), "{body}");
let counted = explain_missing_route(
&remote,
"/v1/responses/input_tokens",
(StatusCode::NOT_FOUND, "{}").into_response(),
);
assert_eq!(counted.status(), StatusCode::NOT_IMPLEMENTED);
for (target, route, status) in [
(&remote, "/v1/chat/completions", StatusCode::NOT_FOUND),
(&remote, "/v1/embeddings", StatusCode::NOT_FOUND),
(
&Target::Local(17434),
"/v1/responses",
StatusCode::NOT_FOUND,
),
(&remote, "/v1/responses", StatusCode::OK),
(&remote, "/v1/responses", StatusCode::UNAUTHORIZED),
] {
let resp = explain_missing_route(target, route, (status, "{}").into_response());
assert_eq!(resp.status(), status, "{route} {status}");
}
}
async fn fake_provider(native: StatusCode) -> String {
use axum::routing::post;
let app = Router::new()
.route(
"/v1/responses",
post(move || async move {
(
native,
[("x-provider", "native")],
r#"{"error":{"message":"native"}}"#,
)
}),
)
.route(
"/v1/chat/completions",
post(|| async {
(
[("content-type", "text/event-stream")],
"data: {\"choices\":[{\"index\":0,\"delta\":{\"content\":\"OK\"},\"finish_reason\":null}]}\n\n\
data: {\"choices\":[{\"index\":0,\"delta\":{},\"finish_reason\":\"stop\"}]}\n\n\
data: {\"choices\":[],\"usage\":{\"prompt_tokens\":3,\"completion_tokens\":1}}\n\n\
data: [DONE]\n\n",
)
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
format!("http://{addr}/v1")
}
async fn call_remote_responses_target(
target: Target,
stream: bool,
) -> (StatusCode, HeaderMap, String) {
let state = test_state();
let req = serde_json::json!({ "model": "mock-model", "input": "hi", "stream": stream });
let resp = remote_responses(
&state.0.client,
&target,
&HeaderMap::new(),
req,
ActivityGuard::new(&state, "m"),
"llmman.provider/mockprov/mock-model".to_string(),
)
.await
.unwrap();
let status = resp.status();
let headers = resp.headers().clone();
let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
.await
.unwrap();
(status, headers, String::from_utf8_lossy(&body).into_owned())
}
async fn call_remote_responses(base_url: &str, stream: bool) -> (StatusCode, HeaderMap, String) {
call_remote_responses_target(remote_target(base_url), stream).await
}
#[tokio::test]
async fn remote_responses_falls_back_only_on_a_missing_or_broken_route() {
for native in [StatusCode::NOT_FOUND] {
let base = fake_provider(native).await;
let (status, headers, body) = call_remote_responses(&base, true).await;
assert_eq!(status, StatusCode::OK, "{native}");
assert_eq!(headers["content-type"], "text/event-stream");
assert!(body.contains("event: response.created"), "{body}");
assert!(body.contains("\"delta\":\"OK\""), "{body}");
assert!(body.contains("event: response.completed"), "{body}");
assert!(
body.contains("\"model\":\"llmman.provider/mockprov/mock-model\""),
"{body}"
);
let (status, headers, body) = call_remote_responses(&base, false).await;
assert_eq!(status, StatusCode::OK, "{native}");
assert_eq!(headers["content-type"], "application/json");
let response: serde_json::Value = serde_json::from_str(&body).unwrap();
assert_eq!(response["status"], "completed");
assert_eq!(response["output"][0]["content"][0]["text"], "OK");
assert_eq!(response["usage"]["total_tokens"], 4);
}
for native in [
StatusCode::INTERNAL_SERVER_ERROR,
StatusCode::BAD_GATEWAY,
StatusCode::SERVICE_UNAVAILABLE,
] {
let base = fake_provider(native).await;
let target = Target::Remote(Arc::new(RemoteTarget {
provider: "opencode".into(),
base_url: base,
wire: Wire::OpenAi,
model: "mock-model".into(),
max_output: None,
cost: None,
api_key: Some("sk-test".into()),
session: None,
}));
let (status, headers, body) = call_remote_responses_target(target, true).await;
assert_eq!(status, StatusCode::OK, "{native}");
assert_eq!(headers["content-type"], "text/event-stream");
assert!(body.contains("event: response.completed"), "{body}");
}
for native in [
StatusCode::OK,
StatusCode::UNAUTHORIZED,
StatusCode::TOO_MANY_REQUESTS,
StatusCode::INTERNAL_SERVER_ERROR,
StatusCode::BAD_GATEWAY,
StatusCode::SERVICE_UNAVAILABLE,
] {
let base = fake_provider(native).await;
let (status, headers, body) = call_remote_responses(&base, true).await;
assert_eq!(status, native);
assert!(body.contains("\"native\""), "{native}: {body}");
assert!(headers.contains_key("x-provider"), "{native}");
}
}
#[test]
fn only_a_browser_acting_for_another_site_is_refused_the_daemons_key() {
for site in ["cross-site", "same-site", "Cross-Site", " cross-site "] {
let mut headers = HeaderMap::new();
headers.insert("sec-fetch-site", site.parse().unwrap());
assert!(is_cross_site(Some(&headers)), "{site:?}");
}
for site in ["same-origin", "none"] {
let mut headers = HeaderMap::new();
headers.insert("sec-fetch-site", site.parse().unwrap());
assert!(!is_cross_site(Some(&headers)), "{site:?}");
}
assert!(!is_cross_site(Some(&HeaderMap::new())));
assert!(!is_cross_site(None));
}
#[test]
fn a_remote_targets_debug_output_omits_the_key() {
let rendered = format!("{:?}", remote_target("https://example.invalid/v1"));
assert!(!rendered.contains("sk-test"), "{rendered}");
assert!(rendered.contains("mockprov"), "{rendered}");
}
#[test]
fn repeat_penalty_is_local_only() {
assert!(repeat_penalty_applies(&Target::Local(17434)));
assert!(!repeat_penalty_applies(&remote_target(
"https://example.invalid/v1"
)));
}
#[test]
fn a_providers_own_status_is_not_buried_in_a_500() {
let remote = remote_target("https://example.invalid/v1");
assert_eq!(
remote_status(&remote, StatusCode::UNAUTHORIZED),
StatusCode::UNAUTHORIZED
);
assert_eq!(
remote_status(&remote, StatusCode::TOO_MANY_REQUESTS),
StatusCode::TOO_MANY_REQUESTS
);
assert_eq!(
remote_status(&remote, StatusCode::BAD_GATEWAY),
StatusCode::BAD_GATEWAY
);
assert_eq!(
remote_status(&remote, StatusCode::INTERNAL_SERVER_ERROR),
StatusCode::BAD_GATEWAY
);
let local = Target::Local(17434);
for upstream in [StatusCode::UNAUTHORIZED, StatusCode::INTERNAL_SERVER_ERROR] {
assert_eq!(
remote_status(&local, upstream),
StatusCode::INTERNAL_SERVER_ERROR,
"a local backend's status changed"
);
}
assert_eq!(
remote_status(&local, StatusCode::BAD_REQUEST),
StatusCode::BAD_REQUEST
);
assert_eq!(local.describe(), "inference backend");
assert_eq!(remote.describe(), "provider mockprov");
}
#[tokio::test]
async fn a_remote_request_reaches_the_provider_as_plain_openai() {
let seen = Arc::new(tokio::sync::Mutex::new(None));
let captured = seen.clone();
let app = Router::new().route(
"/v1/chat/completions",
post(|headers: HeaderMap, body: Bytes| async move {
let body: serde_json::Value = serde_json::from_slice(&body).unwrap();
let auth = headers["authorization"].to_str().unwrap().to_string();
*captured.lock().await = Some((auth, body));
([("content-type", "application/json")], "{}")
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let target = remote_target(&format!("http://127.0.0.1:{}/v1", addr.port()));
let mut req = OAIChatRequest {
model: "mock-model".into(),
stream: false,
repeat_penalty: Some(1.1),
top_k: Some(40),
min_p: Some(0.05),
chat_template_kwargs: Some(serde_json::json!({ "enable_thinking": true })),
..Default::default()
};
let _body = post_chat(&Client::new(), &target, &mut req).await.unwrap();
let (auth, body) = seen.lock().await.take().expect("provider was not called");
assert_eq!(auth, "Bearer sk-test");
assert_eq!(body["model"], "mock-model");
for field in ["repeat_penalty", "top_k", "min_p", "chat_template_kwargs"] {
assert!(
body.get(field).is_none(),
"llama.cpp-only field {field} sent to a provider: {body}"
);
}
assert!(body.get("reasoning_effort").is_none(), "{body}");
}
async fn mock_anthropic(
reply: &'static str,
) -> (
String,
Arc<tokio::sync::Mutex<Vec<(String, HeaderMap, serde_json::Value)>>>,
) {
let seen = Arc::new(tokio::sync::Mutex::new(Vec::new()));
let captured = seen.clone();
let app = Router::new()
.route(
"/v1/messages",
post(move |headers: HeaderMap, body: Bytes| async move {
let body: serde_json::Value = serde_json::from_slice(&body).unwrap();
captured
.lock()
.await
.push(("/v1/messages".to_string(), headers, body));
(
[
("content-type", "text/event-stream"),
("request-id", "req_mock"),
],
reply,
)
}),
)
.fallback(|req: Request| async move {
(
StatusCode::NOT_FOUND,
format!("no such route: {}", req.uri().path()),
)
});
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let base = format!(
"http://127.0.0.1:{}/v1",
listener.local_addr().unwrap().port()
);
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
(base, seen)
}
const MOCK_MESSAGES_STREAM: &str = "event: message_start\n\
data: {\"type\":\"message_start\",\"message\":{\"id\":\"msg_1\",\"model\":\"claude-x\",\"usage\":{\"input_tokens\":3,\"output_tokens\":1}}}\n\n\
event: content_block_start\n\
data: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n\
event: content_block_delta\n\
data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"hi there\"}}\n\n\
event: message_delta\n\
data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"},\"usage\":{\"output_tokens\":2}}\n\n\
event: message_stop\n\
data: {\"type\":\"message_stop\"}\n\n";
#[tokio::test]
async fn a_typed_request_to_an_anthropic_provider_speaks_messages() {
let (base, seen) = mock_anthropic(MOCK_MESSAGES_STREAM).await;
let target = remote_target_on(&base, Wire::Anthropic);
let mut req = OAIChatRequest {
model: "mock-model".into(),
messages: vec![
OAIMessage::text("system", "be brief"),
OAIMessage::text("user", "hello"),
],
stream: true,
repeat_penalty: Some(1.1),
..Default::default()
};
let body = post_chat(&Client::new(), &target, &mut req).await.unwrap();
let lines: Vec<String> = bytes_to_lines(body).collect().await;
let calls = seen.lock().await;
let (path, headers, sent) = &calls[0];
assert_eq!(path, "/v1/messages");
assert_eq!(headers["x-api-key"], "sk-test");
assert_eq!(headers["anthropic-version"], anthropic::VERSION);
assert!(
headers.get("authorization").is_none(),
"bearer sent to Anthropic"
);
assert_eq!(sent["model"], "mock-model");
assert_eq!(sent["system"][0]["text"], "be brief");
assert_eq!(sent["messages"][0]["role"], "user");
assert_eq!(sent["messages"][0]["content"][0]["text"], "hello");
assert_eq!(sent["system"][0]["cache_control"]["type"], "ephemeral");
assert!(headers.get("anthropic-beta").is_none());
assert_eq!(sent["max_tokens"], anthropic::DEFAULT_MAX_TOKENS);
assert_eq!(sent["stream"], true);
assert!(sent.get("repeat_penalty").is_none(), "{sent}");
let fold = fold_ollama_lines(lines);
assert_eq!(fold.content, "hi there");
assert!(fold.done);
}
#[tokio::test]
async fn a_non_streaming_request_to_an_anthropic_provider_is_folded() {
let (base, seen) = mock_anthropic(MOCK_MESSAGES_STREAM).await;
let target = remote_target_on(&base, Wire::Anthropic);
let req = serde_json::json!({
"model": "mock-model",
"messages": [{ "role": "user", "content": "hello" }],
"stream": false
});
let upstream = send_chat_completion(
&Client::new(),
&target,
&req,
"llmman.provider/mockprov/mock-model",
)
.await
.unwrap();
assert_eq!(upstream.status, StatusCode::OK);
assert_eq!(upstream.headers["content-type"], "application/json");
assert!(upstream.headers.get("content-length").is_none());
assert_eq!(upstream.headers["request-id"], "req_mock");
let body: serde_json::Value = serde_json::from_str(&upstream.text().await).unwrap();
assert_eq!(body["object"], "chat.completion");
assert_eq!(body["model"], "llmman.provider/mockprov/mock-model");
assert_eq!(body["choices"][0]["message"]["content"], "hi there");
assert_eq!(body["choices"][0]["finish_reason"], "stop");
assert_eq!(body["usage"]["total_tokens"], 5);
assert_eq!(seen.lock().await[0].2["stream"], true);
}
#[tokio::test]
async fn max_tokens_is_clamped_to_the_catalogs_ceiling_for_an_anthropic_provider() {
let (base, seen) = mock_anthropic(MOCK_MESSAGES_STREAM).await;
let capped = Target::Remote(Arc::new(RemoteTarget {
provider: "mockprov".into(),
base_url: base.clone(),
wire: Wire::Anthropic,
model: "mock-model".into(),
max_output: Some(64_000),
cost: None,
api_key: Some("sk-test".into()),
session: None,
}));
let send = |target: &Target, req: serde_json::Value| {
let target = target.clone();
async move {
send_chat_completion(&Client::new(), &target, &req, "m")
.await
.unwrap()
}
};
let ask = |max_tokens: u64| {
serde_json::json!({
"model": "mock-model",
"messages": [{ "role": "user", "content": "hello" }],
"max_tokens": max_tokens
})
};
send(&capped, ask(65_535)).await;
send(&capped, ask(1_000)).await;
send(&remote_target_on(&base, Wire::Anthropic), ask(65_535)).await;
let calls = seen.lock().await;
assert_eq!(calls[0].2["max_tokens"], 64_000);
assert_eq!(calls[1].2["max_tokens"], 1_000);
assert_eq!(calls[2].2["max_tokens"], 65_535);
}
#[tokio::test]
async fn a_thinking_level_reaches_each_claude_in_its_own_form() {
let (base, seen) = mock_anthropic(MOCK_MESSAGES_STREAM).await;
for (model, effort) in [
("claude-sonnet-5", "high"),
("claude-sonnet-5", "none"),
("claude-sonnet-4-5", "high"),
] {
let target = Target::Remote(Arc::new(RemoteTarget {
provider: "anthropic".into(),
base_url: base.clone(),
wire: Wire::Anthropic,
model: model.into(),
max_output: Some(64_000),
cost: None,
api_key: Some("sk-test".into()),
session: None,
}));
let req = serde_json::json!({
"model": model,
"messages": [{ "role": "user", "content": "hello" }],
"reasoning_effort": effort, "max_tokens": 64_000, "temperature": 0.2
});
let upstream = send_chat_completion(&Client::new(), &target, &req, "m")
.await
.unwrap();
assert_eq!(upstream.status, StatusCode::OK, "{model} {effort}");
}
let calls = seen.lock().await;
let (_, headers, sent) = &calls[0];
assert_eq!(
sent["thinking"],
serde_json::json!({ "type": "adaptive", "display": "summarized" })
);
assert_eq!(
sent["output_config"],
serde_json::json!({ "effort": "high" })
);
assert!(sent.get("temperature").is_none(), "{sent}");
assert!(headers.get("anthropic-beta").is_none(), "{headers:?}");
let (_, headers, sent) = &calls[1];
assert_eq!(sent["thinking"], serde_json::json!({ "type": "disabled" }));
assert!(sent.get("output_config").is_none(), "{sent}");
assert!(headers.get("anthropic-beta").is_none());
let (_, headers, sent) = &calls[2];
assert_eq!(
sent["thinking"],
serde_json::json!({ "type": "enabled", "budget_tokens": 16_384 })
);
assert!(sent.get("output_config").is_none(), "{sent}");
assert_eq!(
headers["anthropic-beta"],
anthropic::INTERLEAVED_THINKING_BETA
);
}
#[tokio::test]
async fn an_inbound_messages_request_is_relayed_to_an_anthropic_provider_intact() {
let (base, seen) = mock_anthropic(MOCK_MESSAGES_STREAM).await;
let target = remote_target_on(&base, Wire::Anthropic);
let state = test_state();
let raw = serde_json::json!({
"model": "llmman.provider/mockprov/mock-model",
"max_tokens": 32,
"stream": true,
"system": [{ "type": "text", "text": "sys", "cache_control": { "type": "ephemeral" } }],
"messages": [{ "role": "user", "content": "hi" }],
"thinking": { "type": "enabled", "budget_tokens": 1024 },
"tools": [{ "name": "f", "input_schema": { "type": "object" } }]
});
let mut headers = HeaderMap::new();
headers.insert(
"anthropic-beta",
"interleaved-thinking-2025-05-14".parse().unwrap(),
);
headers.insert("anthropic-version", "2024-01-01".parse().unwrap());
headers.insert("x-api-key", "the-clients-own-key".parse().unwrap());
let activity = begin_activity(ActivityGuard::new(&state, "m"), None).await;
let resp = relay_anthropic_messages(
&Client::new(),
&target,
&headers,
&raw,
"mock-model",
activity,
)
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
assert_eq!(resp.headers()["request-id"], "req_mock");
let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
.await
.unwrap();
let body = String::from_utf8_lossy(&body);
assert!(
body.contains("\"model\":\"llmman.provider/mockprov/mock-model\""),
"{body}"
);
assert!(!body.contains("claude-x"), "{body}");
let calls = seen.lock().await;
let (_, sent_headers, sent) = &calls[0];
assert_eq!(
sent_headers["anthropic-beta"],
"interleaved-thinking-2025-05-14"
);
let versions: Vec<_> = sent_headers.get_all("anthropic-version").iter().collect();
assert_eq!(versions, ["2024-01-01"]);
assert_eq!(sent_headers["x-api-key"], "sk-test");
let mut expected = raw.clone();
expected["model"] = serde_json::json!("mock-model");
assert_eq!(*sent, expected);
}
#[test]
fn routes_without_a_messages_equivalent_are_refused_up_front() {
let anthropic = remote_target_on("https://api.anthropic.com/v1", Wire::Anthropic);
for route in [
"/v1/completions",
"/v1/embeddings",
"/v1/responses/input_tokens",
] {
let refusal = unsupported_on_wire(&anthropic, route)
.unwrap_or_else(|| panic!("{route} was not refused"));
assert_eq!(refusal.status(), StatusCode::NOT_IMPLEMENTED);
}
for route in [CHAT_COMPLETIONS_ROUTE, RESPONSES_ROUTE] {
assert!(unsupported_on_wire(&anthropic, route).is_none(), "{route}");
}
let openai = remote_target("https://api.openai.com/v1");
assert!(unsupported_on_wire(&openai, "/v1/embeddings").is_none());
assert!(unsupported_on_wire(&Target::Local(1), "/v1/embeddings").is_none());
}
fn remote(provider: &str, model: &str, wire: Wire) -> RemoteTarget {
RemoteTarget {
provider: provider.into(),
base_url: "https://example.invalid/v1".into(),
wire,
model: model.into(),
max_output: None,
cost: None,
api_key: Some("k".into()),
session: None,
}
}
#[test]
fn provider_compat_translates_think_per_wire() {
let with = |remote: &RemoteTarget, kwargs: serde_json::Value| {
let mut req = serde_json::json!({ "model": "m", "chat_template_kwargs": kwargs });
provider_compat(remote, &mut req);
assert!(req.get("chat_template_kwargs").is_none());
req.get("reasoning_effort").cloned()
};
let openrouter = remote("openrouter", "some/model", Wire::OpenAi);
let anthropic = remote("anthropic", "claude", Wire::Anthropic);
let level = serde_json::json!({ "enable_thinking": true, "reasoning_effort": "high" });
assert_eq!(with(&openrouter, level.clone()), Some("high".into()));
assert_eq!(with(&anthropic, level), Some("high".into()));
let on = serde_json::json!({ "enable_thinking": true });
assert_eq!(with(&openrouter, on.clone()), None);
assert_eq!(with(&anthropic, on), Some("medium".into()));
let off = serde_json::json!({ "enable_thinking": false });
assert_eq!(with(&anthropic, off), Some("none".into()));
}
#[test]
fn provider_compat_applies_openais_reasoning_model_rules() {
let mut req = serde_json::json!({
"model": "o3", "max_tokens": 100, "temperature": 0.2, "top_p": 0.9,
"presence_penalty": 0.1, "stop": ["x"], "reasoning_effort": "high",
"n_probs": 5, "cache_prompt": true, "json_schema": {}, "repeat_last_n": 64,
"n_predict": 8, "grammar_lazy": true, "grammar_triggers": [], "preserved_tokens": []
});
provider_compat(&remote("openai", "o3", Wire::OpenAi), &mut req);
assert_eq!(
req,
serde_json::json!({
"model": "o3", "max_completion_tokens": 100, "stop": ["x"], "reasoning_effort": "high"
})
);
let mut gpt5 = serde_json::json!({ "model": "gpt-5", "temperature": 1, "stop": ["x"], "logit_bias": {}, "max_completion_tokens": 7, "max_tokens": 9 });
provider_compat(&remote("openai", "gpt-5", Wire::OpenAi), &mut gpt5);
assert_eq!(
gpt5,
serde_json::json!({ "model": "gpt-5", "temperature": 1, "max_completion_tokens": 7 })
);
let mut plain = serde_json::json!({ "model": "gpt-4o", "temperature": 0.2, "reasoning_effort": "high", "max_tokens": 5 });
provider_compat(&remote("openai", "gpt-4o", Wire::OpenAi), &mut plain);
assert_eq!(
plain,
serde_json::json!({ "model": "gpt-4o", "temperature": 0.2, "max_tokens": 5 })
);
let mut other = serde_json::json!({ "model": "o3-mini", "temperature": 0.2, "max_tokens": 5, "top_k": 3, "stream_options": { "include_usage": true } });
provider_compat(&remote("groq", "o3-mini", Wire::OpenAi), &mut other);
assert_eq!(
other,
serde_json::json!({ "model": "o3-mini", "temperature": 0.2, "max_tokens": 5, "stream_options": { "include_usage": true } })
);
let mut cohere =
serde_json::json!({ "model": "c", "stream_options": { "include_usage": true } });
provider_compat(&remote("cohere", "c", Wire::OpenAi), &mut cohere);
assert_eq!(cohere, serde_json::json!({ "model": "c" }));
for (model, reasoning) in [
("o1", true),
("o4-mini", true),
("openai/o3", true),
("gpt-5-mini", true),
("gpt-5-chat-latest", false),
("gpt-4o", false),
("omni", false),
] {
assert_eq!(openai_reasoning_model(model), reasoning, "{model}");
}
}
#[test]
fn provider_compat_drops_claudes_rejected_sampling_parameters() {
let both =
serde_json::json!({ "model": "m", "temperature": 0.2, "top_p": 0.9, "max_tokens": 5 });
let with = |provider: &str, model: &str, wire: Wire, req: &serde_json::Value| {
let mut req = req.clone();
provider_compat(&remote(provider, model, wire), &mut req);
req
};
assert_eq!(
with("anthropic", "claude-sonnet-5", Wire::Anthropic, &both),
serde_json::json!({ "model": "m", "max_tokens": 5 })
);
let one = serde_json::json!({ "model": "m", "temperature": 1.0, "top_p": 0.9 });
assert_eq!(
with("anthropic", "claude-opus-4-7", Wire::Anthropic, &one),
serde_json::json!({ "model": "m", "temperature": 1.0 })
);
assert_eq!(
with(
"anthropic",
"claude-sonnet-4-5-20250929",
Wire::Anthropic,
&both
),
serde_json::json!({ "model": "m", "temperature": 0.2, "max_tokens": 5 })
);
let nucleus = serde_json::json!({ "model": "m", "top_p": 0.9, "temperature": null });
assert_eq!(
with("anthropic", "claude-opus-4-6", Wire::Anthropic, &nucleus),
nucleus
);
assert_eq!(
with(
"openrouter",
"anthropic/claude-sonnet-5",
Wire::OpenAi,
&both
),
both
);
assert_eq!(with("minimax", "MiniMax-M2", Wire::Anthropic, &both), both);
}
#[test]
fn a_peer_target_is_a_local_backend_in_all_but_address() {
let peer = aggregation::target(&test_state(), "http://spark:17434".into());
assert_eq!(
peer.url("/v1/chat/completions"),
"http://spark:17434/v1/chat/completions"
);
assert!(!peer.is_remote(), "a peer accepts llama.cpp extensions");
assert!(repeat_penalty_applies(&peer));
assert_eq!(peer.describe(), "peer http://spark:17434");
for upstream in [
StatusCode::NOT_FOUND,
StatusCode::SERVICE_UNAVAILABLE,
StatusCode::INTERNAL_SERVER_ERROR,
] {
assert_eq!(remote_status(&peer, upstream), upstream);
}
}
fn state_with_peers(peers: Vec<String>, memory: u64) -> AppState {
AppState(Arc::new(inner_with_peers(peers, memory)))
}
fn inner_with_peers(peers: Vec<String>, memory: u64) -> Inner {
let dir = std::env::temp_dir().join(format!(
"llmman-aggregation-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let desc = crate::storage::oci::Descriptor {
media_type: "application/vnd.oci.image.manifest.v1+json".into(),
digest: "sha256:a1b2".into(),
size: 0,
annotations: None,
};
OciStore::open(&dir)
.unwrap()
.tag(desc, "docker.io/ai/m:latest")
.unwrap();
let mut inner = test_inner(dir);
inner.peers = peers;
inner.memory = memory;
inner
}
#[tokio::test]
async fn a_peer_request_carries_the_hop_marker_and_nothing_else() {
let (origin, seen) = mock_peer(node(0, &[], &[])).await;
let mut req = OAIChatRequest {
model: "docker.io/ai/m:latest".into(),
..Default::default()
};
let _body = post_chat(
&Client::new(),
&aggregation::target(&test_state(), origin),
&mut req,
)
.await
.unwrap();
let calls = seen.lock().await;
let (path, headers, body) = &calls[0];
assert_eq!(path, "/v1/chat/completions");
assert_eq!(headers[aggregation::HOP], "1");
assert!(headers.get("authorization").is_none());
let body: serde_json::Value = serde_json::from_slice(body).unwrap();
assert_eq!(body["repeat_penalty"], DEFAULT_REPEAT_PENALTY);
}
#[tokio::test]
async fn ensure_model_forwards_to_the_peer_that_has_the_model_loaded() {
let (origin, _) = mock_peer(node(8 << 30, &["docker.io/ai/m:latest"], &[])).await;
let state = state_with_peers(vec![origin.clone()], 0);
let (name, target, _) = ensure_model(&state, "docker.io/ai/m", Some(&HeaderMap::new()), None)
.await
.unwrap();
assert_eq!(name, "docker.io/ai/m:latest");
assert!(
matches!(&target, Target::Peer(p) if p.origin == origin),
"{target:?}"
);
let mut hopped = HeaderMap::new();
hopped.insert(aggregation::HOP, "1".parse().unwrap());
let err = ensure_model(&state, "docker.io/ai/m", Some(&hopped), None)
.await
.err()
.expect("the fixture has no blobs to load");
assert!(
format!("{:#}", err.0).contains("resolve model"),
"{:#}",
err.0
);
let err = ensure_model(&state, "docker.io/ai/m", None, None)
.await
.err()
.unwrap();
assert!(
format!("{:#}", err.0).contains("resolve model"),
"{:#}",
err.0
);
}
#[tokio::test]
async fn ensure_model_places_a_cold_model_on_the_roomiest_reachable_node() {
let (roomy, _) = mock_peer(node(128 << 30, &[], &["docker.io/ai/m:latest"])).await;
let state = state_with_peers(vec![roomy.clone()], 8 << 30);
let (_, target, _) = ensure_model(&state, "docker.io/ai/m", Some(&HeaderMap::new()), None)
.await
.unwrap();
assert!(
matches!(&target, Target::Peer(p) if p.origin == roomy),
"{target:?}"
);
let (bare, _) = mock_peer(node(128 << 30, &[], &[])).await;
let dead = "http://127.0.0.1:9".to_string();
let state = state_with_peers(vec![dead, bare], 8 << 30);
let err = ensure_model(&state, "docker.io/ai/m", Some(&HeaderMap::new()), None)
.await
.err()
.expect("loads (and fails on the fixture) locally");
assert!(
format!("{:#}", err.0).contains("resolve model"),
"{:#}",
err.0
);
}
#[tokio::test]
async fn listings_cover_the_aggregation_except_for_a_peer_asking() {
let ps = serde_json::json!({ "models": [{
"name": "docker.io/ai/m:latest", "model": "docker.io/ai/m:latest",
"expires_at": null, "digest": "sha256:aa", "size": 1, "size_vram": 0,
"pid": null, "port": 1, "processor": "GPU", "context_length": null,
"started_at": "2026-01-01T00:00:00Z"
}]});
let tags = serde_json::json!({ "models": [{
"name": "docker.io/ai/m:latest", "model": "docker.io/ai/m:latest",
"size": 1, "digest": "sha256:aa", "modified_at": "2026-01-01T00:00:00Z",
"details": { "format": "gguf", "family": "", "parameter_size": "", "quantization_level": "" }
}]});
let models = serde_json::json!({ "object": "list", "data": [
{ "id": "docker.io/ai/m:latest", "object": "model", "created": 0,
"owned_by": "llmman", "status": { "value": "loaded" } }
]});
let app = Router::new()
.route("/api/ps", get(move || async move { Json(ps) }))
.route("/api/tags", get(move || async move { Json(tags) }))
.route("/v1/models", get(move || async move { Json(models) }));
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let origin = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port());
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let dir = std::env::temp_dir().join(format!(
"llmman-aggregation-listings-{}",
std::process::id()
));
let _ = std::fs::remove_dir_all(&dir);
OciStore::open(&dir).unwrap();
let mut inner = test_inner(dir);
inner.peers = vec![origin.clone()];
let state = AppState(Arc::new(inner));
async fn body(resp: Response) -> serde_json::Value {
let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX)
.await
.unwrap();
serde_json::from_slice(&bytes).unwrap()
}
let mut hopped = HeaderMap::new();
hopped.insert(aggregation::HOP, "1".parse().unwrap());
let ps = body(
handle_ps(State(state.clone()), HeaderMap::new())
.await
.into_response(),
)
.await;
assert_eq!(ps["models"][0]["name"], "docker.io/ai/m:latest");
assert_eq!(ps["models"][0]["node"], origin);
let own = body(
handle_ps(State(state.clone()), hopped.clone())
.await
.into_response(),
)
.await;
assert_eq!(own["models"].as_array().unwrap().len(), 0);
let tags = body(
handle_tags(State(state.clone()), HeaderMap::new())
.await
.unwrap()
.into_response(),
)
.await;
assert_eq!(tags["models"][0]["name"], "docker.io/ai/m:latest");
assert!(!tags.to_string().contains("\"node\""));
let own = body(
handle_tags(State(state.clone()), hopped.clone())
.await
.unwrap()
.into_response(),
)
.await;
assert_eq!(own["models"].as_array().unwrap().len(), 0);
let models = body(
handle_openai_models(State(state.clone()), HeaderMap::new())
.await
.unwrap()
.into_response(),
)
.await;
assert_eq!(models["data"][0]["id"], "docker.io/ai/m:latest");
assert_eq!(models["data"][0]["status"]["value"], "loaded");
let own = body(
handle_openai_models(State(state), hopped)
.await
.unwrap()
.into_response(),
)
.await;
assert_eq!(own["data"].as_array().unwrap().len(), 0);
}
#[tokio::test]
async fn an_ollama_request_reaches_a_peer_natively() {
let (origin, seen) = mock_peer(node(8 << 30, &["docker.io/ai/m:latest"], &[])).await;
let state = state_with_peers(vec![origin], 0);
let req: OllamaGenerateRequest = serde_json::from_value(
serde_json::json!({ "model": "docker.io/ai/m", "keep_alive": "1h" }),
)
.unwrap();
let resp = handle_ollama_generate(State(state), HeaderMap::new(), Json(req))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let calls = seen.lock().await;
let (path, headers, body) = &calls[0];
assert_eq!(path, "/api/generate");
assert_eq!(headers[aggregation::HOP], "1");
let body: serde_json::Value = serde_json::from_slice(body).unwrap();
assert_eq!(body["model"], "docker.io/ai/m");
assert_eq!(body["keep_alive"], "1h");
assert_eq!(body["prompt"], "");
assert!(body.get("think").is_none(), "{body}");
}
#[tokio::test]
async fn a_pair_reaches_a_peer_as_its_chosen_half() {
let (origin, seen) = mock_peer(node(8 << 30, &["docker.io/ai/m:latest"], &[])).await;
let state = state_with_peers(vec![origin], 0);
let req: OllamaGenerateRequest = serde_json::from_value(serde_json::json!({
"model": "llmman.hybrid/docker.io/ai/m,anthropic/claude-sonnet-5",
}))
.unwrap();
let headers = headers_with(&[("x-llmman-route", "local")]);
let resp = handle_ollama_generate(State(state), headers, Json(req))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let calls = seen.lock().await;
let body: serde_json::Value = serde_json::from_slice(&calls[0].2).unwrap();
assert_eq!(body["model"], "docker.io/ai/m:latest");
}
#[tokio::test]
async fn an_unload_is_forwarded_to_every_peer() {
let (origin, seen) = mock_peer(node(0, &[], &[])).await;
let state = state_with_peers(vec![origin], 0);
unload_everywhere(&state, "docker.io/ai/m", &HeaderMap::new())
.await
.unwrap();
let calls = seen.lock().await;
let (path, headers, body) = &calls[0];
assert_eq!(path, "/api/generate");
assert_eq!(headers[aggregation::HOP], "1");
let body: serde_json::Value = serde_json::from_slice(body).unwrap();
assert_eq!(body["model"], "docker.io/ai/m");
assert_eq!(body["keep_alive"], 0);
drop(calls);
let mut hopped = HeaderMap::new();
hopped.insert(aggregation::HOP, "1".parse().unwrap());
unload_everywhere(&state, "docker.io/ai/m", &hopped)
.await
.unwrap();
assert_eq!(seen.lock().await.len(), 1, "forwarded a forwarded unload");
}
fn pair(reference: &'static str) -> crate::hybrid::Pair<'static> {
crate::hybrid::split_ref(reference).expect("test reference must be a pair")
}
#[tokio::test]
async fn a_pair_resolves_to_an_ordinary_reference_for_the_half_it_picks() {
let state = test_state();
let local = resolve_hybrid_side(&state, &pair(PAIR), Some(&headers_with(&[])))
.await
.unwrap();
assert_eq!(local, "gemma4");
let cloud = resolve_hybrid_side(
&state,
&pair(PAIR),
Some(&headers_with(&[("x-llmman-route", "cloud")])),
)
.await
.unwrap();
assert_eq!(cloud, HOSTED);
assert!(crate::providers::is_remote_ref(&cloud));
}
#[tokio::test]
async fn a_pair_without_headers_stays_local() {
let state = test_state();
assert_eq!(
resolve_hybrid_side(&state, &pair(PAIR), None)
.await
.unwrap(),
"gemma4"
);
}
#[tokio::test]
async fn a_request_too_large_for_this_host_goes_to_the_hosted_half() {
let state = test_state_with_budget(262_144);
let fits = headers_with(&[("content-length", "262144")]);
assert_eq!(
resolve_hybrid_side(&state, &pair(PAIR), Some(&fits))
.await
.unwrap(),
"gemma4"
);
let does_not = headers_with(&[("content-length", "262145")]);
assert_eq!(
resolve_hybrid_side(&state, &pair(PAIR), Some(&does_not))
.await
.unwrap(),
HOSTED
);
}
#[tokio::test]
async fn without_a_budget_size_never_routes_a_pair_away() {
let state = test_state();
let huge = headers_with(&[("content-length", "999999999")]);
assert_eq!(
resolve_hybrid_side(&state, &pair(PAIR), Some(&huge))
.await
.unwrap(),
"gemma4"
);
}
#[tokio::test]
async fn a_transcription_pair_takes_its_local_half_whatever_its_size() {
let state = test_state_with_budget(1);
let huge = headers_with(&[("content-length", "99999999")]);
assert_eq!(
resolve_hybrid_side(&state, &pair(PAIR), Some(&huge))
.await
.unwrap(),
HOSTED,
"the generic path still routes on size"
);
assert_eq!(request_pin(Some(&huge)).unwrap(), None);
let pinned = headers_with(&[("x-llmman-route", "cloud")]);
assert_eq!(
request_pin(Some(&pinned)).unwrap(),
Some(crate::hybrid::Side::Cloud),
"an explicit cloud pin is still refused by that handler"
);
}
#[tokio::test]
async fn a_multipart_tts_request_needs_a_local_model() {
let form = |model: Option<&str>| {
let field = |name: &str, value: &str| {
format!("--B\r\nContent-Disposition: form-data; name=\"{name}\"\r\n\r\n{value}\r\n")
};
let mut body = String::new();
if let Some(model) = model {
body.push_str(&field("model", model));
}
body.push_str(&field("input", "hello"));
body.push_str("--B--\r\n");
Bytes::from(body)
};
let headers = headers_with(&[("content-type", "multipart/form-data; boundary=B")]);
for (model, expected) in [
(None, "missing a required \"model\" form field"),
(Some(HOSTED), "does not route /tts to a provider"),
] {
let response = handle_tts(State(test_state()), headers.clone(), form(model))
.await
.unwrap_or_else(IntoResponse::into_response);
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
let body = axum::body::to_bytes(response.into_body(), 1 << 20)
.await
.unwrap();
let body: serde_json::Value = serde_json::from_slice(&body).unwrap();
let error = body["error"].as_str().unwrap_or_default();
assert!(error.contains(expected), "{body}");
}
}
#[tokio::test]
async fn a_json_tts_request_for_a_provider_model_is_a_400() {
let headers = headers_with(&[("content-type", "application/json")]);
let body = Bytes::from(format!(r#"{{"model":"{HOSTED}","input":"hello"}}"#));
let response = handle_tts(State(test_state()), headers, body)
.await
.unwrap_or_else(IntoResponse::into_response);
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn a_cross_site_upload_is_refused() {
let url = serve_router(test_state()).await;
for path in ["/tts", "/v1/audio/transcriptions", "/audio/transcriptions"] {
for content_type in ["multipart/form-data; boundary=B", "text/plain"] {
let resp = Client::new()
.post(format!("{url}{path}"))
.header("content-type", content_type)
.header("origin", "https://evil.example")
.body("{}")
.send()
.await
.unwrap();
assert_eq!(
resp.status(),
StatusCode::FORBIDDEN,
"{path} {content_type}"
);
}
}
}
#[tokio::test]
async fn a_malformed_tts_body_is_a_400() {
let headers = headers_with(&[("content-type", "application/json")]);
let response = handle_tts(State(test_state()), headers, Bytes::from("{"))
.await
.unwrap_or_else(IntoResponse::into_response);
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn an_unreadable_route_header_is_rejected_rather_than_guessed() {
let state = test_state();
let headers = headers_with(&[("x-llmman-route", "on-device")]);
let err = resolve_hybrid_side(&state, &pair(PAIR), Some(&headers))
.await
.expect_err("an unknown side must not be guessed at");
assert_eq!(err.into_response().status(), StatusCode::BAD_REQUEST);
}
#[test]
fn a_zero_context_length_records_the_trained_context_it_stands_for() {
assert_eq!(
loaded_context_window(Engine::LlamaServer, Some(0), Some(32_768), None, true),
Some(32_768)
);
assert_eq!(
loaded_context_window(Engine::LlamaServer, Some(0), Some(32_768), Some(4), true),
Some(8_192)
);
assert_eq!(
loaded_context_window(Engine::LlamaServer, Some(0), None, None, true),
None
);
}
#[test]
fn a_loads_window_is_the_one_its_own_engine_was_given() {
assert_eq!(
loaded_context_window(Engine::LlamaServer, Some(32_768), Some(32_768), None, false),
Some(32_768)
);
assert_eq!(
loaded_context_window(Engine::Vllm, Some(8_192), None, None, true),
Some(8_192)
);
assert_eq!(
loaded_context_window(Engine::Vllm, Some(8_192), None, None, false),
None
);
assert_eq!(
loaded_context_window(Engine::Mlx, Some(8_192), Some(8_192), None, true),
None
);
assert_eq!(
loaded_context_window(Engine::Ds4, Some(65_536), Some(1_048_576), None, true),
Some(65_536)
);
assert_eq!(
loaded_context_window(Engine::Ds4, Some(262_144), Some(1_048_576), None, false),
None
);
}
#[test]
fn only_ds4_is_probed_somewhere_other_than_health() {
assert_eq!(Engine::Ds4.ready_path(), "/v1/models");
for engine in [
Engine::LlamaServer,
Engine::Vllm,
Engine::VllmOmni,
Engine::Sglang,
Engine::Mlx,
Engine::Speech,
] {
assert_eq!(engine.ready_path(), "/health");
}
}
#[tokio::test]
async fn a_pair_routes_against_the_window_its_local_half_actually_loaded_with() {
let state = test_state_with_budget(262_144 * 4);
let big = headers_with(&[("content-length", "200000")]);
assert_eq!(
resolve_hybrid_side(&state, &pair(PAIR), Some(&big))
.await
.unwrap(),
"gemma4",
"nothing is loaded yet, so the daemon-wide budget stands"
);
let mut loaded = running_model_fixture(None, Duration::ZERO, 0);
loaded.context_window = Some(32_768);
let key = running_key(&state, "gemma4");
state.0.manager.lock().await.running.insert(key, loaded);
assert_eq!(
resolve_hybrid_side(&state, &pair(PAIR), Some(&big))
.await
.unwrap(),
HOSTED,
"past the 32k this half loaded with, whatever the daemon's default"
);
let fits = headers_with(&[("content-length", "131072")]);
assert_eq!(
resolve_hybrid_side(&state, &pair(PAIR), Some(&fits))
.await
.unwrap(),
"gemma4"
);
assert_eq!(
resolve_hybrid_side(&state, &pair(PAIR), Some(&headers_with(&[])))
.await
.unwrap(),
"gemma4"
);
}
#[tokio::test]
async fn a_dead_local_half_is_budgeted_as_if_it_were_not_loaded() {
let state = test_state_with_budget(262_144 * 4);
let big = headers_with(&[("content-length", "200000")]);
let mut dead = running_model_fixture(None, Duration::ZERO, 0);
dead.context_window = Some(32_768);
match &mut dead.process {
ModelProcess::Local(_, child, _) | ModelProcess::Container(_, _, child) => {
child.kill().await.expect("kill the placeholder process")
}
}
let key = running_key(&state, "gemma4");
state.0.manager.lock().await.running.insert(key, dead);
assert_eq!(
resolve_hybrid_side(&state, &pair(PAIR), Some(&big))
.await
.unwrap(),
"gemma4",
"the daemon-wide budget stands, so ensure_model still reloads it"
);
let past_the_daemons_own = headers_with(&[("content-length", "2000000")]);
assert_eq!(
resolve_hybrid_side(&state, &pair(PAIR), Some(&past_the_daemons_own))
.await
.unwrap(),
HOSTED
);
}
#[tokio::test]
async fn a_stated_budget_is_not_overridden_by_the_loaded_window() {
let mut inner = test_inner(std::env::temp_dir());
inner.hybrid_local_bytes = Some(1_000_000);
inner.hybrid_local_bytes_explicit = true;
let state = AppState(Arc::new(inner));
let mut loaded = running_model_fixture(None, Duration::ZERO, 0);
loaded.context_window = Some(4_096);
let key = running_key(&state, "gemma4");
state.0.manager.lock().await.running.insert(key, loaded);
let big = headers_with(&[("content-length", "999999")]);
assert_eq!(
resolve_hybrid_side(&state, &pair(PAIR), Some(&big))
.await
.unwrap(),
"gemma4"
);
}
#[tokio::test]
async fn ensure_model_validates_the_local_half_of_a_pair() {
let state = test_state();
let headers = headers_with(&[("x-llmman-route", "local")]);
let err = ensure_model(
&state,
"llmman.hybrid/hf.co/../x,anthropic/claude-sonnet-4-5",
Some(&headers),
None,
)
.await
.err()
.expect("an invalid local half must be rejected");
assert_eq!(err.into_response().status(), StatusCode::BAD_REQUEST);
}
#[test]
fn a_local_context_refusal_is_recognised_in_every_shape_it_arrives_in() {
let llama = r#"{"error":{"code":400,"message":"request (5213 tokens) exceeds the available context size (2048 tokens), try increasing it","type":"exceed_context_size_error","n_prompt_tokens":5213,"n_ctx":2048}}"#;
let expected =
"request (5213 tokens) exceeds the available context size (2048 tokens), try increasing it";
assert_eq!(
context_overflow_message(StatusCode::BAD_REQUEST, llama).as_deref(),
Some(expected)
);
assert_eq!(
context_overflow_message(
StatusCode::BAD_REQUEST,
&format!("inference backend 400 Bad Request: {llama}")
)
.as_deref(),
Some(expected)
);
let vllm = r#"{"object":"error","message":"This model's maximum context length is 2048 tokens. However, you requested 5213 tokens.","type":"BadRequestError","code":400}"#;
assert!(context_overflow_message(StatusCode::BAD_REQUEST, vllm).is_some());
for (status, body) in [
(
StatusCode::BAD_REQUEST,
r#"{"error":{"code":400,"message":"invalid grammar","type":"invalid_request_error"}}"#,
),
(StatusCode::INTERNAL_SERVER_ERROR, llama),
(StatusCode::BAD_REQUEST, "not json"),
] {
assert_eq!(
context_overflow_message(status, body),
None,
"{status} {body}"
);
}
}
#[test]
fn a_repeated_route_header_is_rejected() {
let mut headers = headers_with(&[("x-llmman-route", "local")]);
headers.append("x-llmman-route", "cloud".parse().unwrap());
let err = request_pin(Some(&headers)).expect_err("two pins must not resolve");
assert_eq!(err.into_response().status(), StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn a_local_refusal_is_retried_on_the_hosted_half_unless_pinned() {
use std::sync::atomic::{AtomicUsize, Ordering};
let state = test_state();
let refusal = r#"{"error":{"code":400,"message":"request (9 tokens) exceeds the available context size (8 tokens), try increasing it","type":"exceed_context_size_error"}}"#;
let resolve = |m: String| {
let state = state.clone();
async move {
let m = crate::hybrid::local_half(&m).to_string();
let target = if crate::providers::is_remote_ref(&m) {
Target::Remote(Arc::new(RemoteTarget {
provider: "mockprov".into(),
base_url: "http://provider".into(),
wire: Wire::OpenAi,
model: "claude".into(),
max_output: None,
cost: None,
api_key: Some("k".into()),
session: None,
}))
} else {
Target::Local(1)
};
Ok((m.clone(), target, ActivityGuard::new(&state, &m)))
}
};
enum Local {
Error,
Relayed,
Fine,
OtherError,
}
let run = |local: Local, headers: HeaderMap| async move {
let calls = AtomicUsize::new(0);
let remote_seen = AtomicUsize::new(0);
let (calls, remote_seen, local) = (&calls, &remote_seen, &local);
let result = with_hybrid_fallback(PAIR, Some(&headers), resolve, |model, target, _| {
calls.fetch_add(1, Ordering::SeqCst);
async move {
if target.is_remote() {
remote_seen.fetch_add(1, Ordering::SeqCst);
assert_eq!(model, HOSTED);
return Ok(StatusCode::OK.into_response());
}
assert_eq!(model, "gemma4");
match local {
Local::Error => Err(AppError(
anyhow::Error::new(ContextOverflow {
message: refusal.into(),
refusal: refusal.into(),
}),
StatusCode::INTERNAL_SERVER_ERROR,
)),
Local::Relayed => Ok(Response::builder()
.status(StatusCode::BAD_REQUEST)
.body(Body::from(refusal))
.unwrap()),
Local::Fine => Ok(StatusCode::OK.into_response()),
Local::OtherError => Err(AppError::status(
StatusCode::INTERNAL_SERVER_ERROR,
"backend died",
)),
}
}
})
.await;
(
result.map(|r| r.status()).map_err(|e| e.1),
calls.load(Ordering::SeqCst),
remote_seen.load(Ordering::SeqCst),
)
};
let none = HeaderMap::new();
assert_eq!(
run(Local::Error, none.clone()).await,
(Ok(StatusCode::OK), 2, 1)
);
assert_eq!(
run(Local::Relayed, none.clone()).await,
(Ok(StatusCode::OK), 2, 1)
);
assert_eq!(
run(Local::Fine, none.clone()).await,
(Ok(StatusCode::OK), 1, 0)
);
assert_eq!(
run(Local::OtherError, none).await,
(Err(StatusCode::INTERNAL_SERVER_ERROR), 1, 0)
);
let pinned = headers_with(&[("x-llmman-route", "local")]);
assert_eq!(
run(Local::Error, pinned.clone()).await,
(Err(StatusCode::INTERNAL_SERVER_ERROR), 1, 0)
);
assert_eq!(
run(Local::Relayed, pinned).await,
(Ok(StatusCode::BAD_REQUEST), 1, 0)
);
}
#[test]
fn apply_default_repeat_penalty_sets_default_when_absent() {
let mut req = serde_json::json!({"model": "qwen3.5:0.8b", "messages": []});
apply_default_repeat_penalty(&mut req);
assert_eq!(
req["repeat_penalty"],
serde_json::json!(DEFAULT_REPEAT_PENALTY)
);
}
#[test]
fn apply_default_repeat_penalty_preserves_an_explicit_value() {
let mut req = serde_json::json!({"model": "qwen3.5:0.8b", "repeat_penalty": 1.3});
apply_default_repeat_penalty(&mut req);
assert_eq!(req["repeat_penalty"], serde_json::json!(1.3));
}
fn oai_chat_request_with_repeat_penalty(repeat_penalty: Option<f32>) -> OAIChatRequest {
OAIChatRequest {
model: "qwen3.5:0.8b".into(),
stream: true,
repeat_penalty,
..Default::default()
}
}
#[test]
fn apply_default_repeat_penalty_typed_sets_default_when_absent() {
let mut oai = oai_chat_request_with_repeat_penalty(None);
apply_default_repeat_penalty_typed(&mut oai);
assert_eq!(oai.repeat_penalty, Some(DEFAULT_REPEAT_PENALTY));
}
#[test]
fn apply_default_repeat_penalty_typed_preserves_an_explicit_value() {
let mut oai = oai_chat_request_with_repeat_penalty(Some(1.3));
apply_default_repeat_penalty_typed(&mut oai);
assert_eq!(oai.repeat_penalty, Some(1.3));
}
#[test]
fn ollama_message_to_oai_gives_each_tool_call_a_distinct_id_even_with_identical_names() {
let m = OllamaMessage {
role: "assistant".into(),
tool_calls: Some(vec![
OllamaToolCall {
id: None,
function: OllamaToolCallFunction {
index: 0,
name: "get_weather".into(),
arguments: serde_json::json!({ "city": "nyc" }),
},
},
OllamaToolCall {
id: None,
function: OllamaToolCallFunction {
index: 0,
name: "get_weather".into(),
arguments: serde_json::json!({ "city": "sf" }),
},
},
]),
..Default::default()
};
let oai = ollama_message_to_oai(&m);
let calls = oai.tool_calls.expect("tool_calls must be carried over");
assert_eq!(calls.len(), 2);
assert_ne!(
calls[0].id, calls[1].id,
"two tool calls in one message must never share an id"
);
}
#[tokio::test]
async fn generate_refuses_what_it_cannot_render() {
for (field, value) in [
("raw", serde_json::json!(true)),
("suffix", serde_json::json!("}")),
("template", serde_json::json!("{{ .Prompt }}")),
] {
let mut body = serde_json::json!({ "model": "docker.io/ai/m", "prompt": "x" });
body[field] = value;
let req: OllamaGenerateRequest = serde_json::from_value(body).unwrap();
let err = handle_ollama_generate(State(test_state()), HeaderMap::new(), Json(req))
.await
.err()
.unwrap_or_else(|| panic!("{field} accepted"));
assert_eq!(err.1, StatusCode::BAD_REQUEST, "{field}");
assert!(err.0.to_string().contains(field), "{field}: {}", err.0);
}
let req: OllamaGenerateRequest = serde_json::from_value(
serde_json::json!({ "model": "docker.io/ai/m", "prompt": "", "raw": false, "suffix": "" }),
)
.unwrap();
assert!(!req.raw);
}
#[test]
fn ollama_message_to_oai_sends_wav_as_input_audio() {
use base64::Engine as _;
let mut wav = b"RIFF\x58\x02\x00\x00WAVEfmt ".to_vec();
wav.resize(64, 0);
let b64 = base64::engine::general_purpose::STANDARD.encode(&wav);
let m = OllamaMessage {
role: "user".into(),
content: "transcribe".into(),
images: Some(vec![b64.clone()]),
..Default::default()
};
let oai = ollama_message_to_oai(&m);
assert_eq!(
oai.content,
serde_json::json!([
{ "type": "text", "text": "transcribe" },
{ "type": "input_audio", "input_audio": { "data": b64, "format": "wav" } }
])
);
assert!(!is_wav_base64("Zm9v"));
assert!(!is_wav_base64("data:image/png;base64,Zm9v"));
}
fn fixture_catalog() -> crate::providers::Catalog {
crate::providers::Catalog::from_json(
br#"{
"openrouter": {
"id": "openrouter", "name": "OpenRouter",
"api": "https://openrouter.ai/api/v1",
"npm": "@openrouter/ai-sdk-provider",
"env": ["LLMMAN_TEST_PROVIDER_KEY_UNSET"],
"models": {
"z-model": { "cost": { "input": 2.5, "output": 10 } },
"a-model": {}
}
}
}"#,
)
.expect("fixture parses")
}
#[test]
fn the_provider_listing_reports_model_counts_not_model_ids() {
let catalog = fixture_catalog();
let summaries: Vec<ProviderSummary> = catalog
.iter()
.map(|p| ProviderSummary::new(&test_state(), p))
.collect();
let json = serde_json::to_value(&summaries).unwrap();
assert_eq!(json[0]["id"], "openrouter");
assert_eq!(json[0]["name"], "OpenRouter");
assert_eq!(json[0]["key_env"], "LLMMAN_TEST_PROVIDER_KEY_UNSET");
assert_eq!(json[0]["models"], 2);
assert_eq!(json[0]["key_set"], false);
assert_eq!(json[0]["key_usable"], false);
}
#[test]
fn a_single_provider_carries_its_models_and_their_prices() {
let catalog = fixture_catalog();
let provider = catalog.get("openrouter").unwrap();
let json = serde_json::to_value(ProviderResponse::new(&test_state(), provider)).unwrap();
assert_eq!(json["base_url"], "https://openrouter.ai/api/v1");
assert_eq!(
json["models"],
serde_json::json!([
{ "id": "a-model" },
{ "id": "z-model", "cost": { "input": 2.5, "output": 10.0 } },
])
);
}
#[test]
fn provider_responses_carry_no_api_key() {
let catalog = fixture_catalog();
let provider = catalog.get("openrouter").unwrap();
for json in [
serde_json::to_value(ProviderSummary::new(&test_state(), provider)).unwrap(),
serde_json::to_value(ProviderResponse::new(&test_state(), provider)).unwrap(),
] {
let fields: Vec<&String> = json.as_object().unwrap().keys().collect();
assert!(
!fields.iter().any(|f| f.contains("api_key")),
"{fields:?} carries a key"
);
}
}
#[test]
fn a_configured_provider_reports_its_key_as_optional() {
let catalog = fixture_catalog().with_configured(&[crate::config::ConfiguredProvider {
id: "inferencebox".into(),
name: "Inference box".into(),
base_url: "http://inferencebox:8000/v1".into(),
wire: Wire::OpenAi,
key_env: None,
}]);
let provider = catalog.get("inferencebox").unwrap();
let json = serde_json::to_value(ProviderSummary::new(&test_state(), provider)).unwrap();
assert_eq!(json["key_optional"], true);
assert_eq!(json["key_set"], false);
assert!(json.get("key_env").is_none(), "{json}");
assert_eq!(json["base_url"], "http://inferencebox:8000/v1");
let json = serde_json::to_value(ProviderSummary::new(
&test_state(),
catalog.get("openrouter").unwrap(),
))
.unwrap();
assert_eq!(json["key_optional"], false);
}
#[test]
fn a_keyless_remote_target_sends_no_credential_header() {
let client = Client::new();
let keyless = |wire: Wire| {
Target::Remote(Arc::new(RemoteTarget {
provider: "inferencebox".into(),
base_url: "http://inferencebox:8000/v1".into(),
wire,
model: "m".into(),
max_output: None,
cost: None,
api_key: None,
session: None,
}))
};
let headers = |target: &Target| {
target
.authorize(client.post("http://inferencebox:8000/v1/x"))
.build()
.unwrap()
.headers()
.clone()
};
let openai = headers(&keyless(Wire::OpenAi));
assert!(
openai.get(reqwest::header::AUTHORIZATION).is_none(),
"{openai:?}"
);
let anthropic = headers(&keyless(Wire::Anthropic));
assert!(anthropic.get("x-api-key").is_none(), "{anthropic:?}");
assert_eq!(
anthropic.get("anthropic-version").unwrap(),
anthropic::VERSION
);
let keyed = headers(&remote_target_on(
"http://inferencebox:8000/v1",
Wire::OpenAi,
));
assert_eq!(
keyed.get(reqwest::header::AUTHORIZATION).unwrap(),
"Bearer sk-test"
);
}
#[test]
fn an_opencode_go_target_sends_a_session_and_user_agent() {
let client = Client::new();
let build = |target: RemoteTarget| {
Target::Remote(Arc::new(target))
.authorize(client.post("http://x/v1/chat/completions"))
.build()
.unwrap()
};
let req1 = build(remote("opencode-go", "m", Wire::OpenAi));
let session1 = req1
.headers()
.get("x-opencode-session")
.expect("x-opencode-session missing")
.to_str()
.unwrap()
.to_owned();
assert!(!session1.is_empty());
let ua1 = req1
.headers()
.get(reqwest::header::USER_AGENT)
.expect("User-Agent missing")
.to_str()
.unwrap();
assert!(
ua1.starts_with("llmman/"),
"expected llmman/<ver>, got {ua1}"
);
let req2 = build(remote("opencode-go", "m", Wire::OpenAi));
let session2 = req2
.headers()
.get("x-opencode-session")
.unwrap()
.to_str()
.unwrap();
assert_eq!(session1, session2, "daemon session must be stable");
let with_session = RemoteTarget {
session: Some("conv-1".into()),
..remote("opencode-go", "m", Wire::OpenAi)
};
let req3 = build(with_session);
assert_eq!(
req3.headers()
.get("x-opencode-session")
.unwrap()
.to_str()
.unwrap(),
"conv-1"
);
for provider in ["openrouter", "opencode", "anthropic"] {
let req = build(remote(provider, "m", Wire::OpenAi));
assert!(
req.headers().get("x-opencode-session").is_none(),
"{provider} should not send x-opencode-session"
);
}
}
#[test]
fn client_opencode_session_reads_the_callers_header() {
use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
let mut h = HeaderMap::new();
h.insert(
HeaderName::from_static("x-opencode-session"),
HeaderValue::from_static(" conv-1 "),
);
assert_eq!(client_opencode_session(Some(&h)), Some("conv-1".to_owned()));
let mut h2 = HeaderMap::new();
h2.insert(
HeaderName::from_static("x-opencode-session"),
HeaderValue::from_static(" "),
);
assert_eq!(client_opencode_session(Some(&h2)), None);
assert_eq!(client_opencode_session(Some(&HeaderMap::new())), None);
assert_eq!(client_opencode_session(None), None);
}
#[test]
fn only_an_opencode_go_target_adopts_the_callers_session() {
use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
let mut h = HeaderMap::new();
h.insert(
HeaderName::from_static(OPENCODE_SESSION_HEADER),
HeaderValue::from_static("conv-1"),
);
let mut t = remote("opencode-go", "m", Wire::OpenAi);
t.adopt_session(Some(&h));
assert_eq!(t.session, Some("conv-1".into()));
for provider in ["openrouter", "opencode", "anthropic"] {
let mut t = remote(provider, "m", Wire::OpenAi);
t.adopt_session(Some(&h));
assert!(t.session.is_none(), "{provider} should not adopt a session");
}
let mut t = remote("opencode-go", "m", Wire::OpenAi);
t.adopt_session(None);
assert!(t.session.is_none());
}
#[tokio::test]
async fn resolve_remote_target_adopts_the_session_for_opencode_go_only() {
let _fx = ProviderKeyFixture::new("resolve-opencode-session");
crate::providers::hold_catalog_for_tests(
r#"{
"opencode-go": {"id": "opencode-go", "name": "OpenCode Go",
"api": "https://opencode.ai/zen/go/v1",
"npm": "@ai-sdk/openai-compatible",
"env": ["LLMMAN_TEST_PROVIDER_KEY_UNSET"],
"models": {"m": {}}},
"openrouter": {"id": "openrouter", "name": "OpenRouter",
"api": "https://openrouter.ai/api/v1",
"npm": "@openrouter/ai-sdk-provider",
"env": ["LLMMAN_TEST_PROVIDER_KEY_UNSET"],
"models": {"m": {}}}
}"#,
);
let state = test_state();
let headers = headers_with(&[
("authorization", "Bearer sk-test"),
(OPENCODE_SESSION_HEADER, "conv-1"),
]);
let Some(Target::Remote(go)) =
resolve_remote_target(&state, "llmman.provider/opencode-go/m", Some(&headers))
.await
.unwrap()
else {
panic!("an opencode-go reference resolves to a remote target");
};
assert_eq!(go.session.as_deref(), Some("conv-1"));
let Some(Target::Remote(other)) =
resolve_remote_target(&state, "llmman.provider/openrouter/m", Some(&headers))
.await
.unwrap()
else {
panic!("an openrouter reference resolves to a remote target");
};
assert!(
other.session.is_none(),
"openrouter must not adopt the caller's session"
);
}
#[tokio::test]
async fn opencode_go_requests_carry_a_session_on_both_routes() {
use axum::routing::post;
async fn gateway(headers: HeaderMap) -> Response {
if headers.get("x-opencode-session").is_none() {
return (
StatusCode::BAD_REQUEST,
r#"{"error":{"type":"MissingSessionID"}}"#,
)
.into_response();
}
(
[("content-type", "application/json")],
r#"{"id":"r1","object":"response","model":"m","output":[]}"#,
)
.into_response()
}
let app = Router::new()
.route("/v1/responses", post(gateway))
.route("/v1/chat/completions", post(gateway));
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let target = Target::Remote(Arc::new(RemoteTarget {
base_url: format!("http://{addr}/v1"),
..remote("opencode-go", "m", Wire::OpenAi)
}));
let chat = serde_json::json!({
"model": "m",
"messages": [{ "role": "user", "content": "hello" }]
});
let upstream = send_chat_completion(&Client::new(), &target, &chat, "m")
.await
.unwrap();
assert_eq!(upstream.status, StatusCode::OK);
let state = test_state();
let responses = serde_json::json!({ "model": "m", "input": "hi", "stream": false });
let resp = remote_responses(
&state.0.client,
&target,
&HeaderMap::new(),
responses,
ActivityGuard::new(&state, "m"),
"llmman.provider/opencode-go/m".to_string(),
)
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
}
#[tokio::test]
async fn a_configured_providers_models_are_asked_of_its_endpoint() {
let seen: Arc<tokio::sync::Mutex<Vec<HeaderMap>>> = Arc::default();
let captured = seen.clone();
let app = Router::new().route(
"/v1/models",
get(move |headers: HeaderMap| async move {
captured.lock().await.push(headers);
Json(serde_json::json!({ "object": "list", "data": [
{ "id": "qwen3-coder", "object": "model" },
{ "id": "gemma4", "object": "model" },
{ "id": "gemma4", "object": "model" }
]}))
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let base = format!(
"http://127.0.0.1:{}/v1",
listener.local_addr().unwrap().port()
);
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let provider =
crate::providers::Provider::from_configured(&crate::config::ConfiguredProvider {
id: "inferencebox".into(),
name: "inferencebox".into(),
base_url: base.clone(),
wire: Wire::OpenAi,
key_env: None,
});
let state = test_state();
let models = configured_provider_models(&state, &provider).await;
assert_eq!(
models,
vec!["gemma4".to_string(), "qwen3-coder".to_string()]
);
let calls = seen.lock().await;
assert!(
calls[0].get("authorization").is_none(),
"a keyless provider was sent a bearer: {:?}",
calls[0]
);
drop(calls);
let mut down = provider.clone();
down.base_url = "http://127.0.0.1:9/v1".into();
assert!(configured_provider_models(&state, &down).await.is_empty());
let mut anthropic = provider.clone();
anthropic.wire = Wire::Anthropic;
assert!(configured_provider_models(&state, &anthropic)
.await
.is_empty());
assert_eq!(seen.lock().await.len(), 1, "no second request was made");
}
#[tokio::test]
async fn the_engine_label_names_the_engine_not_the_runtime() {
for (engine, expected) in [
(Engine::LlamaServer, "llama-server"),
(Engine::Vllm, "vllm"),
(Engine::VllmOmni, "vllm-omni"),
(Engine::Sglang, "sglang"),
(Engine::Mlx, "mlx"),
(Engine::Ds4, "ds4"),
(Engine::Speech, "speech"),
] {
assert_eq!(
running_model_fixture_with_engine(engine, None).engine_label(),
expected
);
}
}
#[tokio::test(flavor = "multi_thread")]
async fn backend_wire_model_is_the_canonical_name_for_every_engine_except_mlx() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"llama-model".into(),
running_model_fixture_with_engine(Engine::LlamaServer, None),
);
mgr.running.insert(
"vllm-model".into(),
running_model_fixture_with_engine(Engine::Vllm, None),
);
mgr.running.insert(
"sglang-model:latest".into(),
running_model_fixture_with_engine(Engine::Sglang, Some("sglang-model-latest")),
);
mgr.running.insert(
"mlx-model".into(),
running_model_fixture_with_engine(Engine::Mlx, Some("/cache/mlx-model/abcd")),
);
mgr.running.insert(
"ds4-model".into(),
running_model_fixture_with_engine(Engine::Ds4, None),
);
}
assert_eq!(
backend_wire_model(&state, &Target::Local(0), "llama-model").await,
"llama-model"
);
assert_eq!(
backend_wire_model(&state, &Target::Local(0), "vllm-model").await,
"vllm-model"
);
assert_eq!(
backend_wire_model(&state, &Target::Local(0), "ds4-model").await,
"ds4-model"
);
assert_eq!(
backend_wire_model(&state, &Target::Local(0), "sglang-model:latest").await,
"sglang-model-latest"
);
assert_eq!(
backend_wire_model(&state, &Target::Local(0), "mlx-model").await,
"/cache/mlx-model/abcd",
"an Engine::Mlx backend must be addressed by its real directory path, not its human-readable name"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn backend_wire_model_falls_back_to_the_canonical_name_when_not_running() {
let state = test_state();
assert_eq!(
backend_wire_model(&state, &Target::Local(0), "not-running").await,
"not-running"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn backend_wire_model_strips_the_routing_prefix_for_a_remote_target() {
let state = test_state();
let target = remote_target("https://example.invalid/v1");
assert_eq!(
backend_wire_model(&state, &target, "llmman.provider/mockprov/mock-model").await,
"mock-model"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn would_use_mlx_finds_an_already_running_mlx_model_without_touching_disk_or_a_process() {
let state = test_state();
let model_ref = "hf.co/mlx-community/foo:latest";
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
model_ref.to_string(),
running_model_fixture_with_engine(Engine::Mlx, Some("/cache/foo/abcd")),
);
}
assert_eq!(
would_use_mlx(&state, model_ref).await,
Some(model_ref.to_string())
);
}
#[tokio::test(flavor = "multi_thread")]
async fn would_use_mlx_is_none_for_an_already_running_non_mlx_model() {
let state = test_state();
let model_ref = "hf.co/mlx-community/foo:latest";
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
model_ref.to_string(),
running_model_fixture_with_engine(Engine::LlamaServer, None),
);
}
assert_eq!(would_use_mlx(&state, model_ref).await, None);
}
#[tokio::test(flavor = "multi_thread")]
async fn would_use_mlx_is_none_when_not_running_and_this_host_never_uses_mlx() {
let state = test_state();
assert_eq!(
would_use_mlx(&state, "hf.co/mlx-community/not-loaded-yet:latest").await,
None
);
}
#[tokio::test(flavor = "multi_thread")]
async fn reap_idle_models_unloads_only_idle_expired_models_not_in_flight_or_forever() {
let _serialised = metrics::GLOBAL_COUNTER_TEST_LOCK.lock().await;
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"expired-and-idle".into(),
running_model_fixture(Some(Duration::from_secs(1)), Duration::from_secs(10), 0),
);
mgr.running.insert(
"expired-but-in-flight".into(),
running_model_fixture(Some(Duration::from_secs(1)), Duration::from_secs(10), 1),
);
mgr.running.insert(
"expired-but-forever".into(),
running_model_fixture(None, Duration::from_secs(10), 0),
);
mgr.running.insert(
"not-yet-expired".into(),
running_model_fixture(Some(Duration::from_secs(300)), Duration::from_secs(1), 0),
);
let mut speech = running_model_fixture_with_engine(Engine::Speech, None);
speech.keep_alive = Some(Duration::from_secs(1));
speech.last_active = Instant::now() - Duration::from_secs(10);
mgr.running.insert("expired-speech-worker".into(), speech);
}
reap_idle_models_once(&state).await;
let mgr = state.0.manager.lock().await;
assert!(
!mgr.running.contains_key("expired-and-idle"),
"an idle model past its keep_alive deadline must be unloaded"
);
assert!(
!mgr.running.contains_key("expired-speech-worker"),
"a speech worker is unloaded like every other local process"
);
assert!(
mgr.running.contains_key("expired-but-in-flight"),
"a model with an in-flight request must survive regardless of its deadline"
);
assert!(
mgr.running.contains_key("expired-but-forever"),
"keep_alive: None (forever) must never be reaped"
);
assert!(
mgr.running.contains_key("not-yet-expired"),
"a model whose keep_alive deadline hasn't passed yet must survive"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn evict_other_models_evicts_everything_except_the_target_and_in_flight_models() {
let _serialised = metrics::GLOBAL_COUNTER_TEST_LOCK.lock().await;
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"the-model-being-loaded".into(),
running_model_fixture(None, Duration::from_secs(0), 0),
);
mgr.running.insert(
"idle-other-model".into(),
running_model_fixture(None, Duration::from_secs(0), 0),
);
mgr.running.insert(
"busy-other-model".into(),
running_model_fixture(None, Duration::from_secs(0), 1),
);
}
let evicted_anything = evict_other_models(&state, "the-model-being-loaded").await;
assert!(evicted_anything);
let mgr = state.0.manager.lock().await;
assert!(
mgr.running.contains_key("the-model-being-loaded"),
"the model ensure_model is trying to load isn't itself an eviction target"
);
assert!(
!mgr.running.contains_key("idle-other-model"),
"an idle other model should be evicted to free memory"
);
assert!(
mgr.running.contains_key("busy-other-model"),
"a model with an in-flight request must survive eviction, same as reap_idle_models"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn evict_other_models_reports_nothing_evicted_when_nothing_is_evictable() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"the-model-being-loaded".into(),
running_model_fixture(None, Duration::from_secs(0), 0),
);
}
assert!(!evict_other_models(&state, "the-model-being-loaded").await);
}
#[tokio::test(flavor = "multi_thread")]
async fn check_running_claims_in_flight_on_a_live_hit() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running
.insert("m".into(), running_model_fixture(None, Duration::ZERO, 0));
}
let (_, _guard) = check_running(&state, "m").await.unwrap();
assert_eq!(
state.0.manager.lock().await.running["m"].in_flight,
1,
"a cache hit must claim in_flight so it can't be evicted before the caller claims it"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn an_unclaimed_activity_guard_still_releases_on_drop() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running
.insert("m".into(), running_model_fixture(None, Duration::ZERO, 0));
}
let (_, guard) = check_running(&state, "m").await.unwrap();
assert_eq!(state.0.manager.lock().await.running["m"].in_flight, 1);
drop(guard); tokio::time::sleep(Duration::from_millis(100)).await;
assert_eq!(
state.0.manager.lock().await.running["m"].in_flight,
0,
"the claim must still be released even if never handed off"
);
}
#[test]
fn try_admit_rejects_once_the_cap_is_reached_and_releases_on_drop() {
static COUNTER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
let first = try_admit_against(&COUNTER, 1).expect("first admission under the cap must succeed");
assert!(
try_admit_against(&COUNTER, 1).is_err(),
"a second admission at the cap must be rejected"
);
drop(first);
assert!(
try_admit_against(&COUNTER, 1).is_ok(),
"dropping an admitted guard must free its slot for the next caller"
);
}
#[test]
fn try_admit_with_a_zero_cap_admits_one_at_a_time_not_none_or_unbounded() {
static COUNTER: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
let first = try_admit_against(&COUNTER, 0).expect("one admission must still succeed");
assert!(
try_admit_against(&COUNTER, 0).is_err(),
"a second concurrent admission must still be rejected"
);
drop(first);
assert!(
try_admit_against(&COUNTER, 0).is_ok(),
"dropping the first must free the slot for the next caller"
);
}
#[test]
fn default_allowed_origins_covers_every_scheme_and_localhost_spelling() {
let origins = default_allowed_origins();
for expected in [
"http://localhost:*",
"https://localhost:*",
"http://127.0.0.1:*",
"https://127.0.0.1:*",
"http://0.0.0.0:*",
"https://0.0.0.0:*",
"http://[::1]:*",
"https://[::1]:*",
] {
assert!(
origins.iter().any(|o| o == expected),
"missing default origin pattern {expected:?}"
);
}
}
#[test]
fn origin_matches_a_trailing_wildcard_port_pattern() {
assert!(origin_matches(
"http://localhost:3000",
"http://localhost:*"
));
assert!(origin_matches("http://localhost:1", "http://localhost:*"));
assert!(origin_matches("http://localhost:", "http://localhost:*"));
assert!(!origin_matches(
"http://evil.example:3000",
"http://localhost:*"
));
assert!(!origin_matches(
"http://localhost.evil.example",
"http://localhost:*"
));
}
#[test]
fn origin_matches_a_wildcard_anywhere_in_the_pattern() {
assert!(origin_matches(
"https://foo.example.com",
"https://*.example.com"
));
assert!(!origin_matches(
"https://example.com",
"https://*.example.com"
));
assert!(origin_matches("https://anything.at.all", "*"));
assert!(!origin_matches("https://example.com", "https://*.*.com"));
}
#[test]
fn origin_matches_a_plain_pattern_only_byte_for_byte() {
assert!(origin_matches("https://example.com", "https://example.com"));
assert!(!origin_matches(
"https://example.com:8080",
"https://example.com"
));
assert!(!origin_matches("http://example.com", "https://example.com"));
}
#[test]
fn allowed_origins_from_env_always_includes_the_localhost_defaults() {
let origins = allowed_origins_from_env();
assert!(origins.iter().any(|o| o == "http://localhost:*"));
}
#[tokio::test(flavor = "multi_thread")]
async fn enforce_max_loaded_models_evicts_nothing_but_still_reserves_when_unbounded() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"m1".into(),
running_model_fixture(None, Duration::from_secs(0), 0),
);
}
let mut guard = enforce_max_loaded_models(&state, 0)
.await
.expect("unbounded admits every load");
{
let mut mgr = state.0.manager.lock().await;
assert_eq!(mgr.running.len(), 1, "nothing was evicted");
assert_eq!(mgr.pending_loads, 1, "the load is still counted as loading");
guard.release_into(&mut mgr);
assert_eq!(mgr.pending_loads, 0);
}
}
#[tokio::test(flavor = "multi_thread")]
async fn releasing_a_reservation_into_the_lock_leaves_no_double_counted_window() {
let state = test_state();
let mut guard = enforce_max_loaded_models(&state, 0).await.unwrap();
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"just-loaded".into(),
running_model_fixture(None, Duration::from_secs(0), 1),
);
guard.release_into(&mut mgr);
assert_eq!(mgr.running.len(), 1);
assert_eq!(mgr.pending_loads, 0);
drop(mgr);
drop(guard);
tokio::time::sleep(Duration::from_millis(50)).await;
assert_eq!(state.0.manager.lock().await.pending_loads, 0);
}
#[tokio::test(flavor = "multi_thread")]
async fn enforce_max_loaded_models_evicts_the_least_recently_active_idle_model_first() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"oldest-idle".into(),
running_model_fixture(None, Duration::from_secs(300), 0),
);
mgr.running.insert(
"newest-idle".into(),
running_model_fixture(None, Duration::from_secs(1), 0),
);
}
assert!(enforce_max_loaded_models(&state, 2).await.is_ok());
let mgr = state.0.manager.lock().await;
assert_eq!(mgr.running.len(), 1);
assert!(
!mgr.running.contains_key("oldest-idle"),
"the least-recently-active idle model must be evicted first"
);
assert!(mgr.running.contains_key("newest-idle"));
}
#[tokio::test(flavor = "multi_thread")]
async fn enforce_max_loaded_models_reserves_its_own_slot_when_evicting() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"victim".into(),
running_model_fixture(None, Duration::from_secs(300), 0),
);
}
let guard = enforce_max_loaded_models(&state, 1).await.unwrap();
let mgr = state.0.manager.lock().await;
assert_eq!(mgr.running.len(), 0, "the sole idle model must be evicted");
assert_eq!(
mgr.pending_loads, 1,
"the freed slot must already be reserved for this caller"
);
drop(mgr);
drop(guard);
}
#[tokio::test(flavor = "multi_thread")]
async fn enforce_max_loaded_models_rejects_with_503_when_every_loaded_model_is_busy() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"busy-1".into(),
running_model_fixture(None, Duration::from_secs(0), 1),
);
}
let err = enforce_max_loaded_models(&state, 1)
.await
.expect_err("every model at/over the cap is busy, so this must reject");
assert_eq!(err.1, StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(state.0.manager.lock().await.running.len(), 1);
}
#[tokio::test(flavor = "multi_thread")]
async fn enforce_max_loaded_models_reserves_a_pending_slot_for_an_in_flight_load() {
let state = test_state();
let guard1 = enforce_max_loaded_models(&state, 1).await.unwrap();
assert_eq!(state.0.manager.lock().await.pending_loads, 1);
let state2 = state.clone();
let second = tokio::spawn(async move { enforce_max_loaded_models(&state2, 1).await });
tokio::time::sleep(Duration::from_millis(50)).await;
assert!(
!second.is_finished(),
"a second load must wait for the first reservation, not double up on it"
);
drop(guard1);
tokio::time::timeout(Duration::from_secs(2), second)
.await
.expect("second load must proceed once the first reservation is released")
.unwrap()
.expect("second load must succeed once a slot is actually free");
}
#[tokio::test(flavor = "multi_thread")]
async fn begin_activity_marks_in_flight_and_its_drop_releases_it_and_updates_keep_alive() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"m".into(),
running_model_fixture(Some(DEFAULT_KEEP_ALIVE), Duration::ZERO, 1),
);
}
let claim = ActivityGuard::new(&state, "m");
let guard = begin_activity(claim, Some(Some(Duration::from_secs(42)))).await;
{
let mgr = state.0.manager.lock().await;
let m = &mgr.running["m"];
assert_eq!(
m.in_flight, 1,
"begin_activity must not add a second claim on top of ensure_model's own"
);
assert_eq!(m.keep_alive, Some(Duration::from_secs(42)));
}
drop(guard);
tokio::time::sleep(Duration::from_millis(100)).await;
let mgr = state.0.manager.lock().await;
let m = &mgr.running["m"];
assert_eq!(
m.in_flight, 0,
"dropping the guard must release the in-flight count"
);
assert_eq!(m.keep_alive, Some(Duration::from_secs(42)));
}
#[tokio::test(flavor = "multi_thread")]
async fn begin_activity_with_no_override_never_touches_an_existing_keep_alive() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"m".into(),
running_model_fixture(None, Duration::from_secs(600), 1),
);
}
let claim = ActivityGuard::new(&state, "m");
let guard = begin_activity(claim, None).await;
{
let mgr = state.0.manager.lock().await;
let m = &mgr.running["m"];
assert_eq!(m.in_flight, 1);
assert_eq!(
m.keep_alive, None,
"a None override must not touch the model's existing keep_alive"
);
assert!(
m.last_active.elapsed() < Duration::from_secs(600),
"the idle clock must still refresh even without a keep_alive override"
);
}
drop(guard);
tokio::time::sleep(Duration::from_millis(100)).await;
let mgr = state.0.manager.lock().await;
assert_eq!(
mgr.running["m"].keep_alive, None,
"dropping the guard must still leave keep_alive untouched"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn refresh_activity_releases_ensure_models_own_claim() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"m".into(),
running_model_fixture(Some(DEFAULT_KEEP_ALIVE), Duration::ZERO, 1),
);
}
refresh_activity(ActivityGuard::new(&state, "m"), None).await;
tokio::time::sleep(Duration::from_millis(100)).await;
assert_eq!(
state.0.manager.lock().await.running["m"].in_flight,
0,
"refresh_activity must release ensure_model's claim, with no guard to do it later"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn refresh_activity_with_none_pins_a_model_to_never_unload() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"preloaded".into(),
running_model_fixture(Some(DEFAULT_KEEP_ALIVE), Duration::ZERO, 1),
);
}
refresh_activity(ActivityGuard::new(&state, "preloaded"), None).await;
let mgr = state.0.manager.lock().await;
assert_eq!(
mgr.running["preloaded"].keep_alive, None,
"a pre-loaded model must be pinned to never unload, not left at the daemon default"
);
}
fn chat_request(body: serde_json::Value) -> OllamaChatRequest {
serde_json::from_value(body).expect("valid OllamaChatRequest")
}
async fn chat_response_json(resp: Response) -> serde_json::Value {
let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
.await
.unwrap();
serde_json::from_slice(&body).unwrap()
}
#[tokio::test(flavor = "multi_thread")]
async fn ollama_chat_with_no_messages_and_keep_alive_zero_unloads_the_model() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"docker.io/ai/m:latest".into(),
running_model_fixture(Some(DEFAULT_KEEP_ALIVE), Duration::ZERO, 0),
);
}
let resp = handle_ollama_chat(
State(state.clone()),
HeaderMap::new(),
Json(chat_request(serde_json::json!({
"model": "docker.io/ai/m:latest",
"messages": [],
"keep_alive": 0,
}))),
)
.await
.expect("unload must not error");
let value = chat_response_json(resp).await;
assert_eq!(value["done_reason"], "unload");
assert_eq!(value["message"]["content"], "");
assert!(
!state
.0
.manager
.lock()
.await
.running
.contains_key("docker.io/ai/m:latest"),
"keep_alive: 0 with no messages must actually unload the model"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn a_tagless_unload_still_finds_a_model_running_under_latest() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"docker.io/ai/m:latest".into(),
running_model_fixture(Some(DEFAULT_KEEP_ALIVE), Duration::ZERO, 0),
);
}
let resp = handle_ollama_chat(
State(state.clone()),
HeaderMap::new(),
Json(chat_request(serde_json::json!({
"model": "docker.io/ai/m",
"messages": [],
"keep_alive": 0,
}))),
)
.await
.expect("unload must not error");
assert_eq!(chat_response_json(resp).await["done_reason"], "unload");
assert!(
!state
.0
.manager
.lock()
.await
.running
.contains_key("docker.io/ai/m:latest"),
"a tagless unload must reach the model stored under :latest"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn unloading_a_model_llmman_does_not_have_is_a_404() {
let state = test_state();
let err = unload_model(&state, "docker.io/ai/nothing-here")
.await
.expect_err("an unknown model must not report a successful unload");
assert_eq!(err.1, StatusCode::NOT_FOUND);
let body = axum::body::to_bytes(err.into_response().into_body(), usize::MAX)
.await
.unwrap();
assert!(
crate::daemon::is_model_not_found_body(
std::str::from_utf8(&body).unwrap(),
"docker.io/ai/nothing-here"
),
"daemon::is_model_not_found_body must accept the body unload_model sends"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn unloading_a_provider_routed_model_is_not_a_404() {
let state = test_state();
unload_model(&state, "llmman.provider/openrouter/qwen/qwen3-coder")
.await
.expect("a provider-routed unload must succeed");
}
#[tokio::test(flavor = "multi_thread")]
async fn a_model_gone_from_the_store_but_still_loaded_unloads_without_a_404() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"docker.io/ai/orphan:latest".into(),
running_model_fixture(Some(DEFAULT_KEEP_ALIVE), Duration::ZERO, 0),
);
}
unload_model(&state, "docker.io/ai/orphan")
.await
.expect("a loaded model must unload even with nothing in the store");
assert!(
!state
.0
.manager
.lock()
.await
.running
.contains_key("docker.io/ai/orphan:latest"),
"the running entry must be gone"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn an_unload_by_digest_reaches_the_entry_stored_under_another_tag() {
let dir = std::env::temp_dir().join(format!(
"llmman-unload-digest-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let _ = std::fs::remove_dir_all(&dir);
let state = test_state_at(dir.clone());
let desc = crate::storage::oci::Descriptor {
media_type: "application/vnd.oci.image.manifest.v1+json".into(),
digest: "sha256:a1b2".into(),
size: 0,
annotations: None,
};
OciStore::open(&dir)
.unwrap()
.tag(desc, "docker.io/ai/m-tagged:v9")
.unwrap();
state.0.manager.lock().await.running.insert(
"docker.io/ai/m-tagged:v9".into(),
running_model_fixture(Some(DEFAULT_KEEP_ALIVE), Duration::ZERO, 0),
);
unload_model(&state, "docker.io/ai/m-tagged@sha256:a1b2")
.await
.expect("the digest spelling must unload the model loaded under its tag");
assert!(
!state
.0
.manager
.lock()
.await
.running
.contains_key("docker.io/ai/m-tagged:v9"),
"the entry under the tag must be gone"
);
unload_model(&state, "docker.io/ai/m-tagged@sha256:a1b2")
.await
.expect("a stored model must unload without a 404 whether or not it is loaded");
let err = unload_model(&state, "docker.io/ai/m-tagged@sha256:ffff")
.await
.expect_err("a digest the store does not hold must be a 404");
assert_eq!(err.1, StatusCode::NOT_FOUND);
let _ = std::fs::remove_dir_all(&dir);
}
#[tokio::test(flavor = "multi_thread")]
async fn an_unload_by_digest_reaches_a_loaded_model_the_store_has_lost() {
let state = test_state();
let running = |digest: &str| {
let mut m = running_model_fixture(Some(DEFAULT_KEEP_ALIVE), Duration::ZERO, 0);
m.digest = digest.into();
m
};
{
let mut mgr = state.0.manager.lock().await;
mgr.running
.insert("docker.io/ai/m-gone:v8".into(), running("sha256:a1b2"));
mgr.running
.insert("docker.io/ai/m-gone:v9".into(), running("sha256:a1b2"));
mgr.running
.insert("docker.io/ai/other:v9".into(), running("sha256:a1b2"));
}
unload_model(&state, "docker.io/ai/m-gone:v9@sha256:A1B2")
.await
.expect("the digest must reach the model running under its lost tag");
let keys = |mgr: &ModelManager| {
let mut k: Vec<String> = mgr.running.keys().cloned().collect();
k.sort();
k
};
assert_eq!(
keys(&*state.0.manager.lock().await),
["docker.io/ai/m-gone:v8", "docker.io/ai/other:v9"],
"the spelled tag goes first; the other tag and the other repository stay"
);
unload_model(&state, "docker.io/ai/m-gone@sha256:a1b2")
.await
.expect("with no spelled tag, the remaining entry with the digest is it");
assert_eq!(
keys(&*state.0.manager.lock().await),
["docker.io/ai/other:v9"]
);
let err = unload_model(&state, "docker.io/ai/m-gone@sha256:a1b2")
.await
.expect_err("nothing running or stored with it is the 404 case");
assert_eq!(err.1, StatusCode::NOT_FOUND);
}
#[tokio::test(flavor = "multi_thread")]
async fn a_tag_reaches_the_process_a_load_by_digest_registered() {
let dir = std::env::temp_dir().join(format!(
"llmman-digest-first-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let _ = std::fs::remove_dir_all(&dir);
let state = test_state_at(dir.clone());
let desc = |digest: &str| crate::storage::oci::Descriptor {
media_type: "application/vnd.oci.image.manifest.v1+json".into(),
digest: digest.into(),
size: 0,
annotations: None,
};
let store = OciStore::open(&dir).unwrap();
store
.tag(desc("sha256:df01"), "docker.io/ai/m-df@sha256:df01")
.unwrap();
store
.tag(desc("sha256:df01"), "docker.io/ai/m-df:latest")
.unwrap();
{
let mut running = running_model_fixture(Some(DEFAULT_KEEP_ALIVE), Duration::ZERO, 0);
running.digest = "sha256:df01".into();
state
.0
.manager
.lock()
.await
.running
.insert("docker.io/ai/m-df@sha256:df01".into(), running);
}
let (key, target, guard) = ensure_model(&state, "docker.io/ai/m-df", None, None)
.await
.expect("the tag must find the running content rather than load again");
assert_eq!(key, "docker.io/ai/m-df@sha256:df01");
assert!(matches!(target, Target::Local(0)));
drop(guard);
unload_model(&state, "docker.io/ai/m-df:latest")
.await
.expect("the tag must unload the process running under the digest key");
assert!(
state.0.manager.lock().await.running.is_empty(),
"the digest-keyed entry must be gone"
);
unload_model(&state, "docker.io/ai/m-df").await.unwrap();
let _ = std::fs::remove_dir_all(&dir);
}
#[tokio::test(flavor = "multi_thread")]
async fn an_unload_waiting_on_a_load_removes_the_key_that_load_inserted() {
let dir = std::env::temp_dir().join(format!(
"llmman-unload-race-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let _ = std::fs::remove_dir_all(&dir);
let state = test_state_at(dir.clone());
let pre_pull_key = "docker.io/ai/m:latest";
let loading = acquire_load_lock(pre_pull_key).await;
let unloader = state.clone();
let unload = tokio::spawn(async move { unload_model(&unloader, "docker.io/ai/m").await });
tokio::time::sleep(Duration::from_millis(100)).await;
let desc = crate::storage::oci::Descriptor {
media_type: "application/vnd.oci.image.manifest.v1+json".into(),
digest: "sha256:0000".into(),
size: 0,
annotations: None,
};
OciStore::open(&dir)
.unwrap()
.tag(desc, "docker.io/ai/m")
.unwrap();
state.0.manager.lock().await.running.insert(
"docker.io/ai/m".into(),
running_model_fixture(Some(DEFAULT_KEEP_ALIVE), Duration::ZERO, 0),
);
drop(loading);
unload
.await
.unwrap()
.expect("the unload must not error once the load releases the lock");
assert!(
!state
.0
.manager
.lock()
.await
.running
.contains_key("docker.io/ai/m"),
"the unload must remove the key the load inserted, not the one it resolved before waiting"
);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn load_identity_folds_the_digest_spelling_onto_the_tag_spelling() {
let store = std::env::temp_dir();
let latest = load_identity(&store, "docker.io/ai/m:latest").unwrap();
assert_eq!(load_identity(&store, "docker.io/ai/m").unwrap(), latest);
assert_eq!(
load_identity(&store, "docker.io/ai/m@sha256:0000000000000000").unwrap(),
latest
);
assert_eq!(
load_identity(&store, "docker.io/ai/m:v9@sha256:0000000000000000").unwrap(),
"docker.io/ai/m:v9"
);
}
#[test]
fn load_identity_takes_the_tag_the_store_holds_a_digest_under() {
let dir = std::env::temp_dir().join(format!(
"llmman-identity-store-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let store = OciStore::open(&dir).unwrap();
let desc = |digest: &str| crate::storage::oci::Descriptor {
media_type: "application/vnd.oci.image.manifest.v1+json".into(),
digest: digest.into(),
size: 0,
annotations: None,
};
store
.tag(desc("sha256:aaaa"), "docker.io/ai/m-key:v9")
.unwrap();
store
.tag(desc("sha256:bbbb"), "docker.io/ai/m-key@sha256:bbbb")
.unwrap();
assert_eq!(
load_identity(&dir, "docker.io/ai/m-key@sha256:aaaa").unwrap(),
"docker.io/ai/m-key:v9"
);
assert_eq!(
load_identity(&dir, "docker.io/ai/m-key:latest@sha256:aaaa").unwrap(),
"docker.io/ai/m-key:v9",
"the tag the content is under wins over the one spelled"
);
assert_eq!(
load_identity(&dir, "docker.io/ai/m-key@sha256:bbbb").unwrap(),
"docker.io/ai/m-key:latest",
"a digest-named entry is what a pull by digest writes; it folds"
);
assert_eq!(
load_identity(&dir, "docker.io/ai/m-key@sha256:ffff").unwrap(),
"docker.io/ai/m-key:latest"
);
assert_eq!(
load_identity(&dir, "docker.io/ai/m-key:v9").unwrap(),
"docker.io/ai/m-key:v9"
);
let _ = std::fs::remove_dir_all(&dir);
}
#[tokio::test(flavor = "multi_thread")]
async fn a_load_by_digest_waits_on_the_tag_spellings_lock_and_takes_its_process() {
let dir = std::env::temp_dir().join(format!(
"llmman-load-digest-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let _ = std::fs::remove_dir_all(&dir);
let state = test_state_at(dir.clone());
let desc = crate::storage::oci::Descriptor {
media_type: "application/vnd.oci.image.manifest.v1+json".into(),
digest: "sha256:d16e".into(),
size: 0,
annotations: None,
};
OciStore::open(&dir)
.unwrap()
.tag(desc, "docker.io/ai/m-digest:v9")
.unwrap();
let loading =
acquire_load_lock(&load_identity(&dir, "docker.io/ai/m-digest:v9").unwrap()).await;
let loader = state.clone();
let load = tokio::spawn(async move {
ensure_model(&loader, "docker.io/ai/m-digest@sha256:d16e", None, None).await
});
tokio::time::sleep(Duration::from_millis(100)).await;
assert!(
!load.is_finished(),
"a load by digest must wait on the lock the tag spelling holds"
);
state.0.manager.lock().await.running.insert(
"docker.io/ai/m-digest:v9".into(),
running_model_fixture(Some(DEFAULT_KEEP_ALIVE), Duration::ZERO, 0),
);
drop(loading);
let (model_ref, target, _guard) = load
.await
.unwrap()
.expect("the load by digest must succeed once the lock releases");
assert_eq!(model_ref, "docker.io/ai/m-digest:v9");
assert!(
matches!(target, Target::Local(0)),
"the process the tag spelling started must be the one handed back"
);
let _ = std::fs::remove_dir_all(&dir);
}
#[tokio::test(flavor = "multi_thread")]
async fn an_unload_after_the_pull_but_before_the_insert_still_waits_for_the_load() {
let dir = std::env::temp_dir().join(format!(
"llmman-unload-postpull-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let _ = std::fs::remove_dir_all(&dir);
let state = test_state_at(dir.clone());
let desc = crate::storage::oci::Descriptor {
media_type: "application/vnd.oci.image.manifest.v1+json".into(),
digest: "sha256:0000".into(),
size: 0,
annotations: None,
};
OciStore::open(&dir)
.unwrap()
.tag(desc, "docker.io/ai/m")
.unwrap();
let loading = acquire_load_lock(&load_identity(&dir, "docker.io/ai/m").unwrap()).await;
let unloader = state.clone();
let unload = tokio::spawn(async move { unload_model(&unloader, "docker.io/ai/m").await });
tokio::time::sleep(Duration::from_millis(100)).await;
assert!(
!unload.is_finished(),
"the unload must block on the load in flight, not resolve past it"
);
state.0.manager.lock().await.running.insert(
"docker.io/ai/m".into(),
running_model_fixture(Some(DEFAULT_KEEP_ALIVE), Duration::ZERO, 0),
);
drop(loading);
unload
.await
.unwrap()
.expect("the unload must succeed once the load releases");
assert!(
!state
.0
.manager
.lock()
.await
.running
.contains_key("docker.io/ai/m"),
"the entry the load inserted must be gone"
);
let _ = std::fs::remove_dir_all(&dir);
}
#[tokio::test(flavor = "multi_thread")]
async fn ollama_chat_with_no_messages_loads_without_generating() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"docker.io/ai/m:latest".into(),
running_model_fixture(Some(DEFAULT_KEEP_ALIVE), Duration::ZERO, 0),
);
}
let resp = handle_ollama_chat(
State(state.clone()),
HeaderMap::new(),
Json(chat_request(serde_json::json!({
"model": "docker.io/ai/m:latest",
"messages": [],
}))),
)
.await
.expect("pre-load must not error");
let value = chat_response_json(resp).await;
assert_eq!(value["done_reason"], "load");
assert_eq!(value["message"]["content"], "");
assert!(
state
.0
.manager
.lock()
.await
.running
.contains_key("docker.io/ai/m:latest"),
"a pre-load must leave the model loaded"
);
}
#[test]
fn vllm_omni_engine_has_its_own_label() {
assert_eq!(Engine::VllmOmni.label(), "vllm-omni");
}
#[test]
fn is_json_content_type_matches_json_and_json_suffixed_types_only() {
assert!(is_json_content_type("application/json"));
assert!(is_json_content_type("Application/JSON; charset=utf-8"));
assert!(is_json_content_type("application/vnd.api+json"));
assert!(!is_json_content_type("text/plain;charset=UTF-8"));
assert!(!is_json_content_type("application/x-www-form-urlencoded"));
assert!(!is_json_content_type(""));
}
#[tokio::test]
async fn ollama_routes_accept_json_without_a_json_content_type() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let app = build_router(test_state(), false);
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let url = format!("http://127.0.0.1:{}/api/show", addr.port());
let post = |content_type: Option<&str>, origin: Option<&str>| {
let mut req = Client::new()
.post(&url)
.body(r#"{"model":"hf:///bad ref"}"#);
if let Some(ct) = content_type {
req = req.header("content-type", ct);
}
if let Some(origin) = origin {
req = req.header("origin", origin);
}
req.send()
};
for content_type in [None, Some("text/plain;charset=UTF-8")] {
let resp = post(content_type, None).await.unwrap();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST, "{content_type:?}");
}
let resp = post(Some("text/plain"), Some("http://localhost:3000"))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
let resp = post(Some("text/plain"), Some("https://evil.example"))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
let resp = post(None, Some("https://evil.example")).await.unwrap();
assert_eq!(resp.status(), StatusCode::FORBIDDEN);
}
#[tokio::test]
async fn create_refuses_unsupported_modelfile_fields_by_name() {
let state = test_state();
let req: OllamaCreateRequest = serde_json::from_value(serde_json::json!({
"model": "mine", "from": "gemma4",
"system": "be terse", "quantize": "q4_K_M", "template": "", "adapters": null
}))
.unwrap();
let resp = handle_create(State(state), Json(req)).await.into_response();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
.await
.unwrap();
let body = String::from_utf8_lossy(&body);
assert!(
body.contains("/api/create: quantize, system not supported"),
"{body}"
);
}
#[tokio::test]
async fn create_needs_exactly_one_of_from_or_files() {
let state = test_state();
for body in [
serde_json::json!({"model": "mine"}),
serde_json::json!({"model": format!("mine@sha256:{}", "a".repeat(64)), "from": "a"}),
serde_json::json!({"model": "mine", "from": "a", "files": {"m.gguf": "sha256:00"}}),
] {
let req: OllamaCreateRequest = serde_json::from_value(body).unwrap();
let resp = handle_create(State(state.clone()), Json(req))
.await
.into_response();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
}
}
const ABC_DIGEST: &str = "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad";
fn blob_upload_state(cache: &Path) -> AppState {
let mut inner = test_inner(cache.to_path_buf());
inner.cache_path = cache.to_path_buf();
AppState(Arc::new(inner))
}
fn chunked_body(chunks: &[&'static str]) -> Body {
let chunks: Vec<Result<&'static str, std::io::Error>> =
chunks.iter().map(|chunk| Ok(*chunk)).collect();
Body::from_stream(futures::stream::iter(chunks))
}
#[tokio::test]
async fn blob_upload_stores_a_body_that_matches_its_digest() {
let cache = tempfile::tempdir().unwrap();
let resp = handle_blob_upload(
State(blob_upload_state(cache.path())),
UrlPath(ABC_DIGEST.to_string()),
chunked_body(&["a", "bc"]),
)
.await
.into_response();
assert_eq!(resp.status(), StatusCode::CREATED);
let staging = cache.path().join("blobs");
let stored = staging.join(ABC_DIGEST.trim_start_matches("sha256:"));
assert_eq!(std::fs::read(stored).unwrap(), b"abc");
assert_eq!(std::fs::read_dir(&staging).unwrap().count(), 1);
}
#[tokio::test]
async fn blob_upload_refuses_a_body_that_does_not_match_and_keeps_nothing() {
let cache = tempfile::tempdir().unwrap();
let resp = handle_blob_upload(
State(blob_upload_state(cache.path())),
UrlPath(format!("sha256:{}", "0".repeat(64))),
chunked_body(&["a", "bc"]),
)
.await
.into_response();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
.await
.unwrap();
let body = String::from_utf8_lossy(&body);
assert!(body.contains(ABC_DIGEST), "{body}");
let staging = cache.path().join("blobs");
assert_eq!(std::fs::read_dir(&staging).unwrap().count(), 0);
}
#[test]
fn filter_non_function_tools_drops_non_function_entries_only() {
let mut req = serde_json::json!({
"tools": [
{"type": "function", "name": "exec_command"},
{"type": "namespace", "name": "multi_agent_v1", "tools": [{"type": "function", "name": "close_agent"}]},
{"type": "web_search"},
{"type": "function", "name": "update_plan"}
]
});
filter_non_function_tools(&mut req);
assert_eq!(
req["tools"],
serde_json::json!([
{"type": "function", "name": "exec_command"},
{"type": "function", "name": "update_plan"}
])
);
}
#[test]
fn filter_non_function_tools_is_a_no_op_without_a_tools_field() {
let mut req = serde_json::json!({"model": "x"});
filter_non_function_tools(&mut req);
assert_eq!(req, serde_json::json!({"model": "x"}));
}
#[test]
fn consolidate_responses_instructions_folds_developer_and_system_input_items() {
let mut req = serde_json::json!({
"model": "docker.io/ai/qwen3.5:0.8b",
"instructions": "top-level instructions",
"input": [
{"type": "message", "role": "developer", "content": [
{"type": "input_text", "text": "permissions instructions"}
]},
{"type": "message", "role": "user", "content": [
{"type": "input_text", "text": "hi"}
]},
{"type": "message", "role": "system", "content": "a plain-string system item"}
]
});
consolidate_responses_instructions(&mut req);
assert_eq!(
req["instructions"],
"top-level instructions\n\npermissions instructions\n\na plain-string system item"
);
assert_eq!(
req["input"],
serde_json::json!([
{"type": "message", "role": "user", "content": [
{"type": "input_text", "text": "hi"}
]}
])
);
}
#[test]
fn consolidate_responses_instructions_is_a_no_op_without_developer_or_system_items() {
let mut req = serde_json::json!({
"instructions": "top-level instructions",
"input": [{"type": "message", "role": "user", "content": "hi"}]
});
let before = req.clone();
consolidate_responses_instructions(&mut req);
assert_eq!(req, before);
}
#[test]
fn responses_input_item_text_ported_ollama_polymorphic_input_cases() {
assert_eq!(
responses_input_item_text(&serde_json::json!({"role": "user", "content": "plain"})),
Some("plain".into())
);
assert_eq!(
responses_input_item_text(&serde_json::json!({"role": "user", "content": [
{"type": "input_text", "text": "a"},
{"type": "output_text", "text": "b"}
]})),
Some("ab".into())
);
assert_eq!(
responses_input_item_text(&serde_json::json!({"content": [{"type": "input_image"}]})),
Some(String::new())
);
assert_eq!(
responses_input_item_text(&serde_json::json!({"type": "function_call", "name": "f"})),
None
);
assert_eq!(
responses_input_item_text(&serde_json::json!({"content": 42})),
None
);
}
#[test]
fn keyed_lock_is_per_key_and_release_only_drops_unreferenced_entries() {
let registry: StdMutex<HashMap<String, Arc<tokio::sync::Mutex<()>>>> =
StdMutex::new(HashMap::new());
let a1 = keyed_lock(®istry, "model-a");
let a2 = keyed_lock(®istry, "model-a");
assert!(Arc::ptr_eq(&a1, &a2), "same key must return the same lock");
let b = keyed_lock(®istry, "model-b");
assert!(
!Arc::ptr_eq(&a1, &b),
"different keys must not share a lock"
);
drop(a1);
release_keyed_lock(®istry, "model-a");
assert!(registry.lock().unwrap().contains_key("model-a"));
drop(a2);
release_keyed_lock(®istry, "model-a");
assert!(!registry.lock().unwrap().contains_key("model-a"));
drop(b);
}
#[tokio::test(flavor = "multi_thread")]
async fn load_lock_serializes_same_model_but_not_different_models() {
let slow = load_lock("test-load-lock-slow-model");
let guard = slow.lock().await;
let other = load_lock("test-load-lock-other-model");
let _other_guard = tokio::time::timeout(std::time::Duration::from_millis(200), other.lock())
.await
.expect("a different model's load must not block on an unrelated one");
let same = load_lock("test-load-lock-slow-model");
assert!(
tokio::time::timeout(std::time::Duration::from_millis(50), same.lock())
.await
.is_err(),
"a second load of the same model must block while the first is in flight"
);
drop(guard);
let same_guard = tokio::time::timeout(std::time::Duration::from_millis(200), same.lock())
.await
.expect("must acquire promptly once the first load releases");
drop(same_guard);
drop(_other_guard);
drop(same);
drop(other);
drop(slow);
release_load_lock("test-load-lock-slow-model");
release_load_lock("test-load-lock-other-model");
}
#[test]
fn ensure_model_key_pipeline_converges_aliases_before_the_lock() {
let store = std::env::temp_dir();
let tagless = load_identity(&store, "regression-test-model").unwrap();
let tagged = load_identity(&store, "regression-test-model:latest").unwrap();
let digest = load_identity(&store, "regression-test-model@sha256:0000").unwrap();
assert_eq!(
tagless, tagged,
"tagless and :latest aliases must resolve to one key"
);
assert_eq!(
tagless, digest,
"the digest spelling must resolve to the same key"
);
let a = load_lock(&tagless);
let b = load_lock(&tagged);
let c = load_lock(&digest);
assert!(
Arc::ptr_eq(&a, &b) && Arc::ptr_eq(&a, &c),
"all three aliases must take the same load lock"
);
drop(a);
drop(b);
drop(c);
release_load_lock(&tagless);
}
#[tokio::test]
async fn ensure_model_rejects_an_invalid_ref_with_400() {
let state = test_state();
let err = ensure_model(&state, "hf.co/../x", None, None)
.await
.err()
.expect("invalid ref must be rejected");
assert_eq!(err.into_response().status(), StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn handle_delete_rejects_an_invalid_ref_with_400() {
let state = test_state();
let req = OllamaDeleteRequest {
model: "hf.co//foo".to_string(),
name: None,
};
let resp = handle_delete(State(state), Json(req)).await.into_response();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
}
fn store_one_model(store: &OciStore, name: &str, media_type: &str, filepath: &str, blob: &[u8]) {
store_one_model_with_config(store, name, media_type, filepath, blob, b"{}");
}
fn store_one_model_with_config(
store: &OciStore,
name: &str,
media_type: &str,
filepath: &str,
blob: &[u8],
config: &[u8],
) {
let mut layer = store.write_blob(media_type, blob).unwrap();
layer.annotations = Some(HashMap::from([(
"org.cncf.model.filepath".to_string(),
filepath.to_string(),
)]));
let config = store
.write_blob("application/vnd.cncf.model.config.v1+json", config)
.unwrap();
let manifest = crate::storage::oci::Manifest {
schema_version: 2,
media_type: "application/vnd.oci.image.manifest.v1+json".into(),
artifact_type: None,
config,
layers: vec![layer],
annotations: None,
};
let mdesc = store
.write_blob(
"application/vnd.oci.image.manifest.v1+json",
&serde_json::to_vec(&manifest).unwrap(),
)
.unwrap();
store
.tag(mdesc, &format!("docker.io/ai/{name}:latest"))
.unwrap();
}
fn fresh_store_dir(name: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!(
"llmman-show-{name}-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let _ = std::fs::remove_dir_all(&dir);
dir
}
async fn show_one_model(
name: &str,
media_type: &str,
filepath: &str,
blob: &[u8],
) -> serde_json::Value {
let dir = fresh_store_dir(name);
let state = test_state_at(dir.clone());
store_one_model(
&OciStore::open(&dir).unwrap(),
name,
media_type,
filepath,
blob,
);
let req = OllamaShowRequest {
model: format!("docker.io/ai/{name}"),
name: None,
};
let resp = handle_show(State(state), Json(req)).await.into_response();
assert_eq!(resp.status(), StatusCode::OK);
let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
.await
.unwrap();
let _ = std::fs::remove_dir_all(&dir);
serde_json::from_slice(&body).unwrap()
}
async fn show_with_template(name: &str, u32s: &[(&str, u32)], template: &str) -> serde_json::Value {
let path =
crate::gguf::write_test_gguf_with_strings(u32s, &[("tokenizer.chat_template", template)]);
let bytes = std::fs::read(&path).unwrap();
let _ = std::fs::remove_file(&path);
show_one_model(
name,
"application/vnd.docker.ai.gguf.v3",
"model.gguf",
&bytes,
)
.await
}
#[tokio::test]
async fn handle_show_reports_the_stored_ggufs_own_metadata() {
let path = crate::gguf::write_test_gguf_with(&[("general.file_type", 15)]);
let bytes = std::fs::read(&path).unwrap();
let _ = std::fs::remove_file(&path);
let v = show_one_model(
"showmeta",
"application/vnd.docker.ai.gguf.v3",
"model.gguf",
&bytes,
)
.await;
assert_eq!(v["details"]["format"], "gguf");
assert_eq!(v["details"]["family"], "llama");
assert_eq!(v["details"]["families"], serde_json::json!(["llama"]));
assert_eq!(v["details"]["quantization_level"], "Q4_K_M");
assert_eq!(v["model_info"]["general.architecture"], "llama");
assert_eq!(v["model_info"]["llama.context_length"], 4096);
assert_eq!(v["model_info"].get("digest"), None);
assert_eq!(v["model_info"].get("size"), None);
}
#[tokio::test]
async fn handle_show_reports_a_safetensors_model_as_safetensors() {
let v = show_one_model(
"showst",
"application/vnd.cncf.model.weight.v1.tar",
"model.safetensors",
b"weights",
)
.await;
assert_eq!(v["details"]["format"], "safetensors");
assert_eq!(v["details"]["family"], "");
assert_eq!(v["details"]["families"], serde_json::json!([]));
assert_eq!(v["model_info"], serde_json::json!({}));
assert_eq!(v["template"], serde_json::Value::Null);
}
#[tokio::test]
async fn handle_show_reports_what_the_chat_template_handles() {
async fn capabilities(
name: &str,
u32s: &[(&str, u32)],
template: &str,
) -> (serde_json::Value, serde_json::Value) {
let v = show_with_template(name, u32s, template).await;
(v["capabilities"].clone(), v["template"].clone())
}
let template = "{%- if tools %}{{- tools | tojson }}{%- endif %}\
{%- if enable_thinking %}<think>{%- endif %}";
let (caps, shown) = capabilities("showtools", &[], template).await;
assert_eq!(caps, serde_json::json!(["completion", "tools", "thinking"]));
assert_eq!(shown, template);
let (caps, _) = capabilities("showplain", &[], "{{ messages[0].content }}").await;
assert_eq!(caps, serde_json::json!(["completion"]));
let (caps, _) = capabilities("showembed", &[("llama.pooling_type", 1)], template).await;
assert_eq!(caps, serde_json::json!(["embedding"]));
let path = crate::gguf::write_test_gguf_with_strings(
&[],
&[
("tokenizer.chat_template", template),
("llama.decision.type", "clef"),
],
);
let bytes = std::fs::read(&path).unwrap();
let _ = std::fs::remove_file(&path);
let v = show_one_model(
"showdecision",
"application/vnd.docker.ai.gguf.v3",
"model.gguf",
&bytes,
)
.await;
assert_eq!(v["capabilities"], serde_json::json!(["decision"]));
}
#[tokio::test]
async fn handle_show_reports_the_thinking_values_the_template_takes() {
async fn thinking(name: &str, template: &str) -> Option<serde_json::Value> {
show_with_template(name, &[], template)
.await
.get("thinking")
.cloned()
}
let qwen3_5 = "{%- if enable_thinking is defined and enable_thinking is false %}\
<think></think>{%- endif %}";
assert_eq!(
thinking("showqwen35", qwen3_5).await,
Some(serde_json::json!({
"values": [false, "minimal", "low", "medium", "high"],
"default": "medium",
}))
);
let levels = "{%- if reasoning_effort not in ('low', 'medium', 'xhigh') %}\
{{ raise_exception('think') }}{%- endif %}";
assert_eq!(
thinking("showlevels", levels).await,
Some(serde_json::json!({
"values": [false, "low", "medium", "xhigh"],
"default": "medium",
}))
);
assert_eq!(
thinking("showplainthink", "{{ messages[0].content }}").await,
None
);
}
#[tokio::test]
async fn handle_tags_reports_each_models_own_format() {
let dir = fresh_store_dir("tags");
let state = test_state_at(dir.clone());
let store = OciStore::open(&dir).unwrap();
let path = crate::gguf::write_test_gguf_with(&[]);
let gguf = std::fs::read(&path).unwrap();
let _ = std::fs::remove_file(&path);
store_one_model(
&store,
"tagsgguf",
"application/vnd.docker.ai.gguf.v3",
"model.gguf",
&gguf,
);
store_one_model(
&store,
"tagsst",
"application/vnd.cncf.model.weight.v1.tar",
"model.safetensors",
b"weights",
);
store_one_model(&store, "tagsnone", "text/plain", "README.md", b"no weights");
let resp = handle_tags(State(state), HeaderMap::new())
.await
.into_response();
assert_eq!(resp.status(), StatusCode::OK);
let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
.await
.unwrap();
let _ = std::fs::remove_dir_all(&dir);
let v: serde_json::Value = serde_json::from_slice(&body).unwrap();
let format = |name: &str| {
v["models"]
.as_array()
.unwrap()
.iter()
.find(|m| m["name"] == format!("docker.io/ai/{name}:latest"))
.unwrap_or_else(|| panic!("{name} not listed: {v}"))["details"]["format"]
.clone()
};
assert_eq!(format("tagsgguf"), "gguf");
assert_eq!(format("tagsst"), "safetensors");
assert_eq!(format("tagsnone"), "");
}
#[tokio::test]
async fn handle_show_rejects_an_invalid_ref_with_400() {
let state = test_state();
let req = OllamaShowRequest {
model: "hf:///foo".to_string(),
name: None,
};
let resp = handle_show(State(state), Json(req)).await.into_response();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn handle_show_answers_404_for_a_model_not_in_the_store() {
let state = test_state();
let req = OllamaShowRequest {
model: "docker.io/ai/nothing-here".to_string(),
name: None,
};
let resp = handle_show(State(state), Json(req)).await.into_response();
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
}
#[tokio::test]
async fn load_lock_release_actually_removes_the_entry_once_unused() {
let key = "test-load-lock-release-cleanup";
let lock = load_lock(key);
let guard = lock.lock().await;
drop(guard);
drop(lock);
release_load_lock(key);
assert!(
!LOAD_LOCKS.lock().unwrap().contains_key(key),
"release_load_lock must drop the registry entry once nothing else references it"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn load_lock_guard_releases_on_task_cancellation() {
let key = "test-load-lock-guard-cancel";
let started = Arc::new(tokio::sync::Notify::new());
let started_tx = started.clone();
let handle = tokio::spawn(async move {
let _guard = acquire_load_lock("test-load-lock-guard-cancel").await;
started_tx.notify_one();
std::future::pending::<()>().await;
});
started.notified().await;
handle.abort();
let _ = handle.await;
assert!(
!LOAD_LOCKS.lock().unwrap().contains_key(key),
"aborting a task holding LoadLockGuard must still release the registry entry"
);
}
const MOCK_SSE: &str = concat!(
"data: {\"choices\":[{\"delta\":{\"content\":\"Hello\"},\"finish_reason\":null}]}\n",
"data: {\"choices\":[{\"delta\":{\"content\":\", world\"},\"finish_reason\":\"stop\"}]}\n",
"data: {\"choices\":[],\"usage\":{\"prompt_tokens\":4,\"completion_tokens\":3}}\n",
"data: [DONE]\n",
);
async fn run_stream_ollama(streaming: bool) -> (String, String) {
let app = Router::new().route(
"/v1/chat/completions",
post(|| async { ([("content-type", "text/event-stream")], MOCK_SSE) }),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let resp = stream_ollama(
streaming,
Client::new(),
Target::Local(addr.port()),
OAIChatRequest {
messages: vec![OAIMessage::text("user", "hi".to_string())],
stream: true,
..Default::default()
},
ActivityGuard::new(&test_state(), "m"),
Instant::now(),
Duration::from_millis(1),
|delta| OllamaChatChunk {
model: "m".into(),
created_at: now_rfc3339(),
message: OllamaMessage {
role: "assistant".into(),
content: delta.content,
thinking: delta.thinking,
tool_calls: delta.tool_calls,
..Default::default()
},
done: delta.done,
done_reason: delta.done_reason,
metrics: delta.metrics,
},
)
.await
.expect("mock backend answers");
let ct = resp.headers()["content-type"].to_str().unwrap().to_string();
let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
.await
.unwrap();
(ct, String::from_utf8(body.to_vec()).unwrap())
}
#[tokio::test]
async fn stream_ollama_false_returns_one_json_object() {
let (content_type, body) = run_stream_ollama(false).await;
assert_eq!(content_type, "application/json");
let one: serde_json::Value =
serde_json::from_str(&body).expect("the whole body is one JSON object");
assert_eq!(one["message"]["content"], "Hello, world");
assert_eq!(one["done"], true);
assert_eq!(one["done_reason"], "stop");
assert_eq!(one["prompt_eval_count"], 4);
assert_eq!(one["eval_count"], 3);
assert_eq!(one["load_duration"], 1_000_000);
assert!(one["total_duration"].is_u64());
assert!(one.get("eval_duration").is_none());
}
#[tokio::test]
async fn stream_ollama_true_returns_ndjson_chunks() {
let (content_type, body) = run_stream_ollama(true).await;
assert_eq!(content_type, "application/x-ndjson");
let chunks: Vec<serde_json::Value> = body
.lines()
.filter(|l| !l.trim().is_empty())
.map(|l| serde_json::from_str(l).expect("each line is its own object"))
.collect();
assert!(chunks.len() > 1, "got {} chunks", chunks.len());
let joined: String = chunks
.iter()
.map(|c| c["message"]["content"].as_str().unwrap_or_default())
.collect();
assert_eq!(joined, "Hello, world");
let done: Vec<_> = chunks.iter().filter(|c| c["done"] == true).collect();
assert_eq!(done.len(), 1);
assert_eq!(done[0]["eval_count"], 3);
assert!(chunks[0].get("eval_count").is_none());
}
fn unload_count(model: &str, reason: &str) -> u64 {
let rendered = rendered_registry();
let needle = format!("llmman_model_unloads_total{{model=\"{model}\",reason=\"{reason}\"}} ");
rendered
.lines()
.find_map(|l| l.strip_prefix(needle.as_str()))
.and_then(|v| v.parse().ok())
.unwrap_or(0)
}
#[tokio::test]
async fn evicting_for_an_oom_retry_counts_every_model_it_freed() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
mgr.running.insert(
"loading-now".into(),
running_model_fixture(None, Duration::from_secs(0), 0),
);
mgr.running.insert(
"evictable-a".into(),
running_model_fixture(None, Duration::from_secs(0), 0),
);
mgr.running.insert(
"evictable-b".into(),
running_model_fixture(None, Duration::from_secs(0), 0),
);
mgr.running.insert(
"busy".into(),
running_model_fixture(None, Duration::from_secs(0), 1),
);
}
let evicted = evict_other_models(&state, "loading-now").await;
assert!(evicted, "two idle models were evictable");
assert_eq!(unload_count("evictable-a", "oom"), 1);
assert_eq!(unload_count("evictable-b", "oom"), 1);
assert_eq!(
unload_count("busy", "oom"),
0,
"the in-flight model was left alone, so it must not be counted"
);
let mgr = state.0.manager.lock().await;
assert!(mgr.running.contains_key("loading-now"));
assert!(mgr.running.contains_key("busy"));
assert!(!mgr.running.contains_key("evictable-a"));
assert!(!mgr.running.contains_key("evictable-b"));
}
#[tokio::test]
async fn the_reaper_labels_an_already_dead_backend_crashed_not_idle() {
let state = test_state();
{
let mut mgr = state.0.manager.lock().await;
let mut dead =
running_model_fixture(Some(Duration::from_secs(1)), Duration::from_secs(10), 0);
match &mut dead.process {
ModelProcess::Local(_, child, _) | ModelProcess::Container(_, _, child) => {
child.kill().await.expect("kill the placeholder process")
}
}
mgr.running.insert("died-then-expired".into(), dead);
}
reap_idle_models_once(&state).await;
assert_eq!(
unload_count("died-then-expired", "crashed"),
1,
"a backend already dead when the reaper reached it must count as crashed"
);
assert_eq!(
unload_count("died-then-expired", "idle"),
0,
"and must not also be counted idle"
);
assert!(
!state
.0
.manager
.lock()
.await
.running
.contains_key("died-then-expired"),
"it must still be unloaded"
);
}
#[tokio::test]
async fn the_metrics_route_label_is_the_template_not_the_request_path() {
let app = Router::new()
.route(
"/test-matched-path/:id",
get(|| async { StatusCode::NO_CONTENT }),
)
.layer(middleware::from_fn(track_metrics));
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
Client::new()
.get(format!(
"http://127.0.0.1:{}/test-matched-path/abc123",
addr.port()
))
.send()
.await
.expect("request reaches the test router");
let rendered = rendered_registry();
assert!(
rendered.contains(
"llmman_http_requests_total{route=\"/test-matched-path/:id\",status=\"204\"} 1\n"
),
"{rendered}"
);
assert!(!rendered.contains("abc123"), "{rendered}");
}
#[tokio::test]
async fn the_scrape_shows_requests_queued_behind_the_parallel_slots() {
let state = test_state();
let mut model = running_model_fixture(None, Duration::ZERO, 3);
model.parallel_slots = Some(1);
state
.0
.manager
.lock()
.await
.running
.insert("queued-model:latest".into(), model);
let resp = handle_metrics(State(state)).await.into_response();
let body = axum::body::to_bytes(resp.into_body(), usize::MAX)
.await
.unwrap();
let scrape = String::from_utf8(body.to_vec()).unwrap();
for expected in [
"llmman_requests_in_flight{model=\"queued-model:latest\"} 3\n",
"llmman_requests_queued{model=\"queued-model:latest\"} 2\n",
] {
assert!(scrape.contains(expected), "{expected} in:\n{scrape}");
}
}
#[tokio::test]
async fn the_scrape_endpoint_is_outside_the_cors_layer() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let app = build_router(test_state(), true);
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let get = |path: &'static str| {
let url = format!("http://127.0.0.1:{}{path}", addr.port());
async move {
Client::new()
.get(url)
.header("origin", "http://localhost:3000")
.send()
.await
.expect("request reaches the test router")
}
};
let version = get("/api/version").await;
assert_eq!(
version
.headers()
.get("access-control-allow-origin")
.and_then(|v| v.to_str().ok()),
Some("http://localhost:3000"),
"a page served from localhost is meant to be an allowed origin"
);
let scrape = get("/metrics").await;
assert_eq!(scrape.status(), StatusCode::OK, "the scrape route answers");
assert!(
!scrape.headers().contains_key("access-control-allow-origin"),
"a browser page can read the scrape endpoint"
);
}
#[tokio::test]
async fn the_scrape_endpoint_is_absent_unless_the_operator_enabled_it() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let app = build_router(test_state(), false);
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let url = format!("http://127.0.0.1:{}", addr.port());
let scrape = Client::new()
.get(format!("{url}/metrics"))
.send()
.await
.expect("request reaches the test router");
assert_eq!(scrape.status(), StatusCode::NOT_FOUND);
let version = Client::new()
.get(format!("{url}/api/version"))
.send()
.await
.expect("request reaches the test router");
assert_eq!(version.status(), StatusCode::OK);
Client::new()
.get(format!("{url}/ui/index.html"))
.send()
.await
.expect("request reaches the test router");
assert!(
!rendered_registry().contains("route=\"/ui/*path\""),
"a router built with metrics disabled must not write to the registry"
);
}
fn shell_state(policy: shell::Policy) -> AppState {
let mut inner = test_inner(std::env::temp_dir());
inner.shell = policy;
AppState(Arc::new(inner))
}
#[tokio::test]
async fn a_disabled_shell_reports_why_and_refuses_the_upgrade() {
let url = serve_router(shell_state(shell::Policy {
disabled: Some("LLMMAN_SHELL is off".into()),
origins: default_allowed_origins(),
command: Vec::new(),
token_digest: auth::digest(SHELL_TOKEN),
token_unsaved: Default::default(),
}))
.await;
let client = Client::new();
let status: shell::Status = client
.get(format!("{url}/llmman/shell"))
.send()
.await
.unwrap()
.json()
.await
.unwrap();
assert_eq!(
status,
shell::Status {
enabled: false,
reason: Some("LLMMAN_SHELL is off".into()),
token_required: true,
}
);
let upgrade = ws_upgrade(&client, &format!("{url}/llmman/shell"), None)
.send()
.await
.unwrap();
assert_eq!(upgrade.status(), StatusCode::FORBIDDEN);
assert_eq!(upgrade.text().await.unwrap(), "LLMMAN_SHELL is off");
}
#[tokio::test]
async fn only_a_page_the_daemon_would_answer_cors_for_may_open_a_shell() {
let url = serve_router(shell_state(shell::Policy {
disabled: None,
origins: default_allowed_origins(),
command: Vec::new(),
token_digest: auth::digest(SHELL_TOKEN),
token_unsaved: Default::default(),
}))
.await;
let client = Client::new();
let shell_url = format!("{url}/llmman/shell");
let evil = ws_upgrade(&client, &shell_url, Some("https://evil.example"))
.send()
.await
.unwrap();
assert_eq!(evil.status(), StatusCode::FORBIDDEN);
assert!(evil.text().await.unwrap().contains("evil.example"));
let status: shell::Status = client
.get(&shell_url)
.header("origin", "https://evil.example")
.send()
.await
.unwrap()
.json()
.await
.unwrap();
assert!(
!status.enabled,
"the status reply must agree with the refusal"
);
for origin in [
None,
Some("http://localhost:3000"),
Some("http://127.0.0.1"),
] {
let status: shell::Status = {
let mut req = client.get(&shell_url);
if let Some(o) = origin {
req = req.header("origin", o);
}
req.send().await.unwrap().json().await.unwrap()
};
assert!(status.enabled, "{origin:?} should be admitted");
}
}
#[tokio::test]
async fn a_shell_session_round_trips_bytes_and_reports_the_exit_status() {
use futures::SinkExt as _;
use tokio_tungstenite::tungstenite::client::IntoClientRequest as _;
use tokio_tungstenite::tungstenite::Message as WsMessage;
#[cfg(windows)]
let command = ["cmd.exe", "/c", "echo got:marker& exit 7"];
#[cfg(not(windows))]
let command = ["sh", "-c", "read line; echo \"got:$line\"; exit 7"];
let url = serve_router(shell_state(shell::Policy {
disabled: None,
origins: default_allowed_origins(),
command: command.map(str::to_string).to_vec(),
token_digest: auth::digest(SHELL_TOKEN),
token_unsaved: Default::default(),
}))
.await;
let ws_url = format!("{}/llmman/shell", url.replace("http://", "ws://"));
let mut request = ws_url.into_client_request().unwrap();
request.headers_mut().insert(
"sec-websocket-protocol",
shell_token_protocol().parse().unwrap(),
);
let (mut ws, _) = tokio_tungstenite::connect_async(request).await.unwrap();
ws.send(WsMessage::Text(
r#"{"resize":{"cols":120,"rows":40}}"#.into(),
))
.await
.unwrap();
#[cfg(not(windows))]
ws.send(WsMessage::Binary(b"marker\r".to_vec()))
.await
.unwrap();
let mut output = Vec::new();
let mut exit = None;
let deadline = tokio::time::sleep(Duration::from_secs(30));
tokio::pin!(deadline);
loop {
tokio::select! {
_ = &mut deadline => panic!("no exit frame; output so far: {:?}", String::from_utf8_lossy(&output)),
frame = ws.next() => match frame {
Some(Ok(WsMessage::Binary(bytes))) => {
if bytes.windows(4).any(|w| w == b"\x1b[6n") {
ws.send(WsMessage::Binary(b"\x1b[1;1R".to_vec())).await.unwrap();
}
output.extend_from_slice(&bytes);
}
Some(Ok(WsMessage::Text(text))) => {
exit = Some(serde_json::from_str::<serde_json::Value>(&text).unwrap());
}
Some(Ok(WsMessage::Close(_))) | None => break,
Some(Ok(_)) => {}
Some(Err(e)) => panic!("socket error: {e}"),
}
}
}
let text = String::from_utf8_lossy(&output);
assert!(text.contains("got:marker"), "{text:?}");
assert_eq!(exit, Some(serde_json::json!({ "exit": 7 })), "{text:?}");
}
#[tokio::test]
async fn timing_a_response_does_not_cost_it_its_content_length() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let app = build_router(test_state(), true);
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let url = format!("http://127.0.0.1:{}", addr.port());
for route in ["/api/version", "/metrics"] {
let response = Client::new()
.get(format!("{url}{route}"))
.send()
.await
.expect("request reaches the test router");
assert_eq!(response.status(), StatusCode::OK, "{route}");
assert!(
response.headers().contains_key("content-length"),
"{route} lost its content-length: {:?}",
response.headers()
);
assert!(
!response.headers().contains_key("transfer-encoding"),
"{route} was made chunked: {:?}",
response.headers()
);
}
}
#[tokio::test]
async fn the_total_duration_clock_stops_at_the_end_of_the_body_not_the_headers() {
let app = Router::new()
.route(
"/test-slow-body",
get(|| async {
Body::from_stream(futures::stream::once(async {
sleep(Duration::from_millis(200)).await;
Ok::<_, std::convert::Infallible>(Bytes::from_static(b"done"))
}))
}),
)
.layer(middleware::from_fn(track_metrics));
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let body = Client::new()
.get(format!("http://127.0.0.1:{}/test-slow-body", addr.port()))
.send()
.await
.expect("request reaches the test router")
.text()
.await
.expect("the body arrives in full");
assert_eq!(body, "done");
let rendered = rendered_registry();
assert!(
rendered.contains(
"llmman_http_request_ttfb_seconds_bucket{route=\"/test-slow-body\",le=\"0.05\"} 1\n"
),
"headers were ready immediately:\n{rendered}"
);
assert!(
rendered.contains(
"llmman_http_request_duration_seconds_bucket{route=\"/test-slow-body\",le=\"0.05\"} 0\n"
),
"the body took 200ms, so total duration cannot be under 50ms:\n{rendered}"
);
assert!(
rendered
.contains("llmman_http_request_duration_seconds_count{route=\"/test-slow-body\"} 1\n"),
"the body ended, so it was recorded exactly once:\n{rendered}"
);
}
#[tokio::test]
async fn record_prompt_logs_generation_requests_and_hands_the_body_on() {
let log =
std::env::temp_dir().join(format!("llmman-record-prompt-{}.jsonl", std::process::id()));
let _ = std::fs::remove_file(&log);
let mut inner = test_inner(std::env::temp_dir());
inner.prompt_log = Some(log.clone());
let state = AppState(Arc::new(inner));
let echo = || post(|body: Bytes| async move { body });
let app = Router::new()
.route("/api/chat", echo())
.route("/api/embed", echo())
.route("/v1/chat/completions", echo())
.route("/gemini/:model/*gemini_path", echo())
.layer(middleware::from_fn_with_state(state.clone(), record_prompt))
.with_state(state);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let send = |route: &str, body: &'static str, headers: &'static [(&str, &str)]| {
let url = format!("http://127.0.0.1:{}{route}", addr.port());
async move {
let mut map = reqwest::header::HeaderMap::new();
for (k, v) in [
("user-agent", "test-agent/1"),
("content-type", "application/json"),
]
.iter()
.chain(headers)
{
map.insert(*k, v.parse().unwrap());
}
Client::new()
.post(url)
.headers(map)
.body(body)
.send()
.await
.expect("request reaches the test router")
.text()
.await
.unwrap()
}
};
let chat = r#"{"model":"m","messages":[{"role":"user","content":"hello there"}]}"#;
assert_eq!(
send("/api/chat", chat, &[]).await,
chat,
"the handler got the body"
);
send("/api/embed", r#"{"model":"m","input":"vector me"}"#, &[]).await;
send("/api/chat", r#"{"model":"m","messages":[]}"#, &[]).await;
send(
"/api/chat",
chat,
&[
("content-type", "text/plain"),
("origin", "https://evil.example"),
],
)
.await;
Client::new()
.get(format!("http://127.0.0.1:{}/api/chat", addr.port()))
.body(chat)
.send()
.await
.expect("request reaches the test router");
send(
"/v1/chat/completions",
chat,
&[
("content-type", "text/plain"),
("origin", "https://evil.example"),
],
)
.await;
let gemini = r#"{"model":"ignored-body-model","contents":[{"role":"user","parts":[{"text":"describe this"},{"inlineData":{"mimeType":"image/png","data":"aGVsbG8="}}]}]}"#;
let encoded_model = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(PAIR);
let base = format!("/gemini/{encoded_model}/v1beta/models/agy-helper-model");
assert_eq!(
send(&format!("{base}:streamGenerateContent"), gemini, &[]).await,
gemini
);
send(&format!("{base}:countTokens"), gemini, &[]).await;
send(
"/gemini/invalid!/v1beta/models/m:streamGenerateContent",
gemini,
&[],
)
.await;
let entries = crate::promptlog::read(&log).unwrap();
let _ = std::fs::remove_file(&log);
let routes: Vec<&str> = entries.iter().map(|e| e.route.as_str()).collect();
assert_eq!(
routes,
[
"/api/chat",
"/v1/chat/completions",
"/gemini/:model/*gemini_path"
],
"{entries:?}"
);
assert_eq!(entries[0].model, "m");
assert_eq!(entries[0].prompt, "hello there");
assert_eq!(entries[0].client.as_deref(), Some("test-agent/1"));
assert_eq!(entries[2].model, PAIR);
assert_eq!(entries[2].prompt, "describe this");
assert_eq!(entries[2].client.as_deref(), Some("test-agent/1"));
}
#[tokio::test]
async fn generation_routes_accept_large_bodies_and_keep_the_32_mib_cap() {
let dir = tempfile::tempdir().unwrap();
let body = Bytes::from(vec![b' '; GENERATION_BODY_LIMIT_BYTES + 1]);
let client = Client::new();
for logging in [false, true] {
let mut inner = test_inner(dir.path().to_path_buf());
if logging {
inner.prompt_log = Some(dir.path().join("prompts.jsonl"));
}
let url = serve_router(AppState(Arc::new(inner))).await;
for route in [
"/v1/chat/completions",
"/v1/completions",
"/v1/messages",
"/v1/responses",
"/v1/responses/input_tokens",
"/api/chat",
"/api/generate",
"/gemini/m/v1beta/models/m:generateContent",
] {
for (size, too_large) in [
(GENERATION_BODY_LIMIT_BYTES, false),
(GENERATION_BODY_LIMIT_BYTES + 1, true),
] {
let bytes = body.slice(..size);
let response = client
.post(format!("{url}{route}"))
.header("content-type", "application/json")
.body(reqwest::Body::wrap_stream(futures::stream::once(async {
Ok::<_, std::io::Error>(bytes)
})))
.send()
.await
.unwrap();
let status = response.status();
assert_eq!(
status == StatusCode::PAYLOAD_TOO_LARGE,
too_large,
"{route}, logging={logging}, size={size}, status={status}"
);
}
}
let response = client
.post(format!("{url}/api/show"))
.header("content-type", "application/json")
.body(reqwest::Body::wrap_stream(futures::stream::once(async {
Ok::<_, std::io::Error>(Bytes::from(vec![b' '; 3 * 1024 * 1024]))
})))
.send()
.await
.unwrap();
assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE);
}
}
#[tokio::test]
async fn record_prompt_keeps_the_body_limit() {
let mut inner = test_inner(std::env::temp_dir());
inner.prompt_log = Some(std::env::temp_dir().join("llmman-unwritten.jsonl"));
let state = AppState(Arc::new(inner));
let app = Router::new()
.route("/api/chat", post(|body: Bytes| async move { body }))
.layer(middleware::from_fn_with_state(state.clone(), record_prompt))
.layer(DefaultBodyLimit::max(64))
.with_state(state);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let send = |len: usize| {
let url = format!("http://127.0.0.1:{}/api/chat", addr.port());
async move {
Client::new()
.post(url)
.body(vec![b' '; len])
.send()
.await
.expect("request reaches the test router")
.status()
}
};
assert_eq!(send(256).await, StatusCode::PAYLOAD_TOO_LARGE);
assert_eq!(send(32).await, StatusCode::OK);
}
fn keyed_state(keys: &[&str]) -> AppState {
let mut inner = test_inner(std::env::temp_dir());
inner.auth = auth::Policy::with_keys(keys.iter().copied());
AppState(Arc::new(inner))
}
#[tokio::test]
async fn a_keyed_daemon_refuses_everything_but_its_own_page_without_the_key() {
let url = serve_router_with(keyed_state(&["k1", "k2"]), true).await;
let client = Client::new();
for path in [
"/api/version",
"/v1/models",
"/llmman/node",
"/llmman/search?q=qwen",
"/llmman/search/popular",
"/llmman/search/model?name=hf.co/o/r",
"/llmman/search/avatar?name=hf.co/o/r",
"/llmman/shell",
"/metrics",
] {
let r = client.get(format!("{url}{path}")).send().await.unwrap();
assert_eq!(r.status(), StatusCode::UNAUTHORIZED, "{path}");
assert_eq!(r.headers()["www-authenticate"], "Bearer realm=\"llmman\"");
let body: serde_json::Value = r.json().await.unwrap();
assert!(
body["error"].as_str().unwrap().contains("API key"),
"{body}"
);
let wrong = client
.get(format!("{url}{path}"))
.bearer_auth("k3")
.send()
.await
.unwrap();
assert_eq!(wrong.status(), StatusCode::UNAUTHORIZED, "{path}");
}
let r = client
.post(format!("{url}/api/show"))
.body("{}")
.send()
.await
.unwrap();
assert_eq!(r.status(), StatusCode::UNAUTHORIZED);
let bearer = client
.get(format!("{url}/api/version"))
.bearer_auth("k1")
.send()
.await
.unwrap();
assert_eq!(bearer.status(), StatusCode::OK);
let x_api_key = client
.get(format!("{url}/api/version"))
.header("x-api-key", "k2")
.send()
.await
.unwrap();
assert_eq!(x_api_key.status(), StatusCode::OK);
for (key, expected) in [("k2", StatusCode::OK), ("k3", StatusCode::UNAUTHORIZED)] {
let response = client
.get(format!("{url}/api/version"))
.header("x-goog-api-key", key)
.send()
.await
.unwrap();
assert_eq!(response.status(), expected);
}
let scrape = client
.get(format!("{url}/metrics"))
.bearer_auth("k2")
.send()
.await
.unwrap();
assert_eq!(scrape.status(), StatusCode::OK);
let url = serve_router(keyed_state(&["k1"])).await;
for path in ["/", "/ui/index.html"] {
let r = client.get(format!("{url}{path}")).send().await.unwrap();
assert_eq!(r.status(), StatusCode::OK, "{path}");
let r = client.head(format!("{url}{path}")).send().await.unwrap();
assert_eq!(r.status(), StatusCode::OK, "HEAD {path}");
}
for (path, expected) in [
("/models", StatusCode::OK),
("/api/version", StatusCode::UNAUTHORIZED),
("/api/nope", StatusCode::UNAUTHORIZED),
("//evil.example/c/x", StatusCode::UNAUTHORIZED),
] {
let r = client
.get(format!("{url}{path}"))
.header("accept", "text/html")
.send()
.await
.unwrap();
assert_eq!(r.status(), expected, "{path}");
}
}
#[tokio::test]
async fn a_keyed_daemon_still_answers_cors_preflights() {
let url = serve_router(keyed_state(&["k1"])).await;
let r = Client::new()
.request(
reqwest::Method::OPTIONS,
format!("{url}/v1/chat/completions"),
)
.header("origin", "http://localhost:3000")
.header("access-control-request-method", "POST")
.header(
"access-control-request-headers",
"authorization,content-type",
)
.send()
.await
.unwrap();
assert_eq!(r.status(), StatusCode::OK);
assert_eq!(
r.headers()["access-control-allow-origin"],
"http://localhost:3000"
);
}
#[tokio::test]
async fn the_daemon_key_is_stripped_before_the_handler_sees_the_headers() {
let state = keyed_state(&["daemon-key"]);
let seen = Arc::new(tokio::sync::Mutex::new(Vec::<HeaderMap>::new()));
let captured = seen.clone();
let app = Router::new()
.route(
"/api/version",
get(move |headers: HeaderMap| async move {
captured.lock().await.push(headers);
"ok"
}),
)
.layer(middleware::from_fn_with_state(
state.clone(),
auth::require_key,
))
.with_state(state);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!(
"http://127.0.0.1:{}/api/version",
listener.local_addr().unwrap().port()
);
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let client = Client::new();
client
.get(&url)
.bearer_auth("daemon-key")
.send()
.await
.unwrap();
client
.get(&url)
.bearer_auth("sk-provider")
.header("x-api-key", "daemon-key")
.send()
.await
.unwrap();
client
.get(&url)
.bearer_auth("daemon-key")
.header("x-api-key", "sk-provider")
.send()
.await
.unwrap();
client
.get(&url)
.bearer_auth("daemon-key")
.header("x-api-key", "daemon-key")
.header("x-goog-api-key", "daemon-key")
.send()
.await
.unwrap();
client
.get(&url)
.header("x-goog-api-key", "daemon-key")
.send()
.await
.unwrap();
client
.get(&url)
.bearer_auth("daemon-key")
.header("x-goog-api-key", "sk-provider")
.send()
.await
.unwrap();
let seen = seen.lock().await;
assert_eq!(seen.len(), 6);
assert!(seen[0].get("authorization").is_none());
assert_eq!(client_api_key(Some(&seen[0])), None);
assert!(seen[1].get("x-api-key").is_none());
assert_eq!(
client_api_key(Some(&seen[1])).as_deref(),
Some("sk-provider")
);
assert!(seen[2].get("authorization").is_none());
assert_eq!(
client_api_key(Some(&seen[2])).as_deref(),
Some("sk-provider")
);
assert!(seen[3].get("x-goog-api-key").is_none());
assert_eq!(client_api_key(Some(&seen[3])), None);
assert!(seen[4].get("x-goog-api-key").is_none());
assert_eq!(client_api_key(Some(&seen[4])), None);
assert_eq!(seen[5]["x-goog-api-key"], "sk-provider");
}
#[test]
fn an_authenticated_caller_may_spend_the_daemon_provider_key() {
let mut cross_site = HeaderMap::new();
cross_site.insert("sec-fetch-site", "cross-site".parse().unwrap());
let keyed = keyed_state(&["k"]);
assert!(daemon_key_spendable(&keyed, None));
assert!(daemon_key_spendable(&keyed, Some(&cross_site)));
assert!(!daemon_key_spendable(&test_state(), Some(&cross_site)));
}
#[test]
fn provider_keys_are_set_only_by_whoever_may_spend_them() {
let mut cross_site = HeaderMap::new();
cross_site.insert("sec-fetch-site", "cross-site".parse().unwrap());
let keyed = keyed_state(&["k"]);
assert_eq!(provider_key_write_refusal(&keyed, &cross_site), None);
assert_eq!(provider_key_write_refusal(&keyed, &HeaderMap::new()), None);
assert!(provider_key_write_refusal(&test_state(), &cross_site)
.is_some_and(|r| r.contains("another site")));
}
struct ProviderKeyFixture {
dir: std::path::PathBuf,
path: std::path::PathBuf,
_one: std::sync::MutexGuard<'static, ()>,
}
impl ProviderKeyFixture {
fn new(name: &str) -> Self {
let one = crate::config::PROVIDER_KEYS_TEST_LOCK
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let dir = std::env::temp_dir().join(format!("llmman-{name}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
let path = dir.join("llmman.conf");
*crate::config::TEST_PROVIDER_FILES.lock().unwrap() = vec![path.clone()];
crate::providers::hold_catalog_for_tests(
r#"{"anthropic": {"id": "anthropic", "name": "Anthropic",
"npm": "@ai-sdk/anthropic", "env": ["ANTHROPIC_API_KEY"],
"models": {"claude-sonnet-5": {}}}}"#,
);
Self {
dir,
path,
_one: one,
}
}
}
impl Drop for ProviderKeyFixture {
fn drop(&mut self) {
crate::config::TEST_PROVIDER_FILES.lock().unwrap().clear();
let _ = crate::config::reload_provider_keys();
let _ = std::fs::remove_dir_all(&self.dir);
}
}
#[tokio::test]
async fn a_provider_key_put_then_delete_takes_effect_at_once() {
let fx = ProviderKeyFixture::new("provider-key-e2e");
let url = serve_router(keyed_state(&["k"])).await;
let client = Client::new();
let route = format!("{url}/llmman/providers/anthropic/key");
let secret = "sk-e2e-0123456789";
let r = client
.put(&route)
.bearer_auth("k")
.json(&serde_json::json!({ "api_key": secret }))
.send()
.await
.unwrap();
assert_eq!(r.status(), StatusCode::OK);
let body = r.text().await.unwrap();
assert!(!body.contains(secret), "{body}");
let v: serde_json::Value = serde_json::from_str(&body).unwrap();
assert_eq!(v["id"], "anthropic");
assert_eq!(v["key_env"], "ANTHROPIC_API_KEY");
assert_eq!(
v["env_override"],
crate::providers::key_from_env("ANTHROPIC_API_KEY").is_some()
);
assert_eq!(v["key_usable"], true);
assert!(std::fs::read_to_string(&fx.path).unwrap().contains(secret));
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mode = std::fs::metadata(&fx.path).unwrap().permissions().mode() & 0o777;
assert_eq!(mode, 0o600);
}
assert_eq!(
crate::config::provider_api_key("anthropic").as_deref(),
Some(secret),
"spent from the next request on"
);
let r = client.delete(&route).bearer_auth("k").send().await.unwrap();
assert_eq!(r.status(), StatusCode::OK);
assert!(!std::fs::read_to_string(&fx.path).unwrap().contains(secret));
assert_eq!(crate::config::provider_api_key("anthropic"), None);
let r = client.delete(&route).bearer_auth("k").send().await.unwrap();
assert_eq!(r.status(), StatusCode::OK);
}
#[tokio::test]
async fn a_malformed_config_never_reaches_the_reply() {
let fx = ProviderKeyFixture::new("provider-key-redact");
let leaked = "sk-must-not-leak-42";
std::fs::write(
&fx.path,
format!("[providers.anthropic]\napi_key = \"{leaked}\n"),
)
.unwrap();
let url = serve_router(keyed_state(&["k"])).await;
let client = Client::new();
let route = format!("{url}/llmman/providers/anthropic/key");
let put = client
.put(&route)
.bearer_auth("k")
.json(&serde_json::json!({"api_key": "sk-new-one"}))
.send()
.await
.unwrap();
let delete = client.delete(&route).bearer_auth("k").send().await.unwrap();
for r in [put, delete] {
assert_eq!(r.status(), StatusCode::INTERNAL_SERVER_ERROR);
let body = r.text().await.unwrap();
assert!(!body.contains(leaked), "{body}");
assert!(!body.contains("api_key ="), "{body}");
assert!(body.contains("log"), "{body}");
}
}
#[tokio::test]
async fn the_provider_key_route_checks_before_it_writes() {
let mut inner = test_inner(std::env::temp_dir());
inner.auth = auth::Policy::with_keys(["k"]);
let keyed = serve_router(AppState(Arc::new(inner))).await;
let client = Client::new();
let route = format!("{keyed}/llmman/providers/anthropic/key");
let anonymous = client
.put(&route)
.json(&serde_json::json!({"api_key": "sk-secret"}))
.send()
.await
.unwrap();
assert_eq!(anonymous.status(), StatusCode::UNAUTHORIZED);
let anonymous = client.delete(&route).send().await.unwrap();
assert_eq!(anonymous.status(), StatusCode::UNAUTHORIZED);
for blank in ["", " ", "sk-a\nsk-b"] {
let r = client
.put(&route)
.bearer_auth("k")
.json(&serde_json::json!({ "api_key": blank }))
.send()
.await
.unwrap();
assert_eq!(r.status(), StatusCode::BAD_REQUEST, "{blank:?}");
assert!(!r.text().await.unwrap().contains("sk-"), "{blank:?}");
}
let open = serve_router(test_state()).await;
let r = client
.put(format!("{open}/llmman/providers/anthropic/key"))
.header("sec-fetch-site", "cross-site")
.json(&serde_json::json!({"api_key": "sk-secret"}))
.send()
.await
.unwrap();
assert_eq!(r.status(), StatusCode::FORBIDDEN);
assert!(!r.text().await.unwrap().contains("sk-secret"));
}
#[tokio::test]
async fn a_keyless_daemon_opens_a_shell_only_with_its_token() {
let url = serve_router(shell_state(shell::Policy {
disabled: None,
origins: default_allowed_origins(),
command: Vec::new(),
token_digest: auth::digest(SHELL_TOKEN),
token_unsaved: Default::default(),
}))
.await;
let client = Client::new();
let shell_url = format!("{url}/llmman/shell");
let bare = ws_upgrade(&client, &shell_url, None).send().await.unwrap();
assert_eq!(bare.status(), StatusCode::UNAUTHORIZED);
assert!(bare
.text()
.await
.unwrap()
.contains("the shell needs the token"));
let wrong = ws_upgrade(&client, &shell_url, None)
.header("sec-websocket-protocol", "llmman.shell.nope")
.send()
.await
.unwrap();
assert_eq!(wrong.status(), StatusCode::UNAUTHORIZED);
assert!(wrong
.text()
.await
.unwrap()
.contains("the shell needs the token"));
let protocol = shell_token_protocol();
let opened = ws_upgrade(&client, &shell_url, None)
.header("sec-websocket-protocol", protocol.as_str())
.send()
.await
.unwrap();
assert_eq!(opened.status(), StatusCode::SWITCHING_PROTOCOLS);
assert_eq!(
opened.headers()["sec-websocket-protocol"],
protocol.as_str()
);
}
#[tokio::test]
async fn a_shell_upgrade_presents_its_key_as_a_subprotocol() {
let mut inner = test_inner(std::env::temp_dir());
inner.auth = auth::Policy::with_keys(["k"]);
inner.shell = shell::Policy {
disabled: None,
origins: default_allowed_origins(),
command: Vec::new(),
token_digest: auth::digest(SHELL_TOKEN),
token_unsaved: Default::default(),
};
let url = serve_router(AppState(Arc::new(inner))).await;
let client = Client::new();
let shell_url = format!("{url}/llmman/shell");
let bare = ws_upgrade(&client, &shell_url, None).send().await.unwrap();
assert_eq!(bare.status(), StatusCode::UNAUTHORIZED);
let wrong = ws_upgrade(&client, &shell_url, None)
.header("sec-websocket-protocol", crate::auth::ws_protocol("nope"))
.send()
.await
.unwrap();
assert_eq!(wrong.status(), StatusCode::UNAUTHORIZED);
let protocol = crate::auth::ws_protocol("k");
let key_only = ws_upgrade(&client, &shell_url, None)
.header("sec-websocket-protocol", format!("chat, {protocol}"))
.send()
.await
.unwrap();
assert_eq!(key_only.status(), StatusCode::UNAUTHORIZED);
assert!(key_only
.text()
.await
.unwrap()
.contains("the shell needs the token"));
let token = shell_token_protocol();
let keyed = ws_upgrade(&client, &shell_url, None)
.header(
"sec-websocket-protocol",
format!("chat, {protocol}, {token}"),
)
.send()
.await
.unwrap();
assert_eq!(keyed.status(), StatusCode::SWITCHING_PROTOCOLS);
assert_eq!(keyed.headers()["sec-websocket-protocol"], token.as_str());
}
#[tokio::test]
async fn a_peer_request_carries_the_peer_key_when_there_is_one() {
let (origin, seen) = mock_peer(node(0, &[], &[])).await;
let mut inner = test_inner(std::env::temp_dir());
inner.peers = vec![origin.clone()];
inner.peer_key = Some("pool-key".into());
let state = AppState(Arc::new(inner));
let mut req = OAIChatRequest {
model: "docker.io/ai/m:latest".into(),
..Default::default()
};
let _body = post_chat(
&Client::new(),
&aggregation::target(&state, origin),
&mut req,
)
.await
.unwrap();
let mut headers = HeaderMap::new();
aggregation::unload(&state, "docker.io/ai/m:latest", &headers).await;
headers.insert(aggregation::HOP, "1".parse().unwrap());
assert!(
!aggregation::unload(&state, "docker.io/ai/m:latest", &headers).await,
"a hopped request is not forwarded again"
);
let calls = seen.lock().await;
assert_eq!(calls.len(), 2, "{calls:?}");
for (_, headers, _) in calls.iter() {
assert_eq!(headers[aggregation::HOP], "1");
assert_eq!(headers["authorization"], "Bearer pool-key");
}
}
#[tokio::test]
async fn an_unenforced_policy_strips_its_keys_without_refusing_anyone() {
let seen = Arc::new(tokio::sync::Mutex::new(Vec::<HeaderMap>::new()));
let captured = seen.clone();
let mut inner = test_inner(std::env::temp_dir());
inner.auth = auth::Policy::with_keys(["daemon-key"]).optional();
let state = AppState(Arc::new(inner));
let app = Router::new()
.route(
"/api/version",
get(move |headers: HeaderMap| async move {
captured.lock().await.push(headers);
"ok"
}),
)
.layer(middleware::from_fn_with_state(
state.clone(),
auth::require_key,
))
.with_state(state);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!(
"http://127.0.0.1:{}/api/version",
listener.local_addr().unwrap().port()
);
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let client = Client::new();
let bare = client.get(&url).send().await.unwrap();
assert_eq!(bare.status(), StatusCode::OK);
let keyed = client
.get(&url)
.bearer_auth("daemon-key")
.header("x-api-key", "sk-provider")
.header("x-goog-api-key", "daemon-key")
.send()
.await
.unwrap();
assert_eq!(keyed.status(), StatusCode::OK);
let seen = seen.lock().await;
assert!(seen[1].get("authorization").is_none());
assert!(seen[1].get("x-goog-api-key").is_none());
assert_eq!(
client_api_key(Some(&seen[1])).as_deref(),
Some("sk-provider")
);
}
fn temp_log(label: &str) -> std::path::PathBuf {
let path = std::env::temp_dir().join(format!(
"llmman-{label}-{}-{}.jsonl",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap()
.as_nanos()
));
let _ = std::fs::remove_file(&path);
path
}
async fn ledger_of(path: &std::path::Path, n: usize) -> Vec<crate::usage::Entry> {
for _ in 0..100 {
let entries = crate::usage::read(path).unwrap();
if entries.len() >= n {
return entries;
}
sleep(Duration::from_millis(10)).await;
}
crate::usage::read(path).unwrap()
}
async fn usage_peer() -> (String, Arc<tokio::sync::Mutex<Vec<serde_json::Value>>>) {
let seen = Arc::new(tokio::sync::Mutex::new(Vec::new()));
let captured = seen.clone();
let node = node(8 << 30, &["docker.io/ai/m:latest"], &[]);
let app = Router::new()
.route("/llmman/node", get(move || async move { Json(node) }))
.route(
"/v1/chat/completions",
post(move |Json(req): Json<serde_json::Value>| async move {
captured.lock().await.push(req.clone());
let usage = serde_json::json!({
"prompt_tokens": 120, "completion_tokens": 7,
"prompt_tokens_details": { "cached_tokens": 100 }
});
if req["stream"] != true {
let body = serde_json::json!({
"choices": [{ "message": { "content": "hi" } }],
"usage": usage,
});
return ([("content-type", "application/json")], body.to_string());
}
let timings =
serde_json::json!({ "prompt_n": 20, "cache_n": 100, "predicted_n": 7 });
let finish =
serde_json::json!({ "choices": [{ "delta": {}, "finish_reason": "stop" }] });
let mut sse =
String::from("data: {\"choices\":[{\"delta\":{\"content\":\"hi\"}}]}\n\n");
if req["stream_options"]["include_usage"] == true {
let last =
serde_json::json!({ "choices": [], "usage": usage, "timings": timings });
sse.push_str(&format!("data: {finish}\n\ndata: {last}\n\n"));
} else {
let mut finish = finish;
finish["timings"] = timings;
sse.push_str(&format!("data: {finish}\n\n"));
}
sse.push_str("data: [DONE]\n\n");
([("content-type", "text/event-stream")], sse)
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let origin = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port());
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
(origin, seen)
}
#[tokio::test]
async fn the_usage_ledger_records_each_reply_and_leaves_the_stream_as_asked() {
let (origin, seen) = usage_peer().await;
let usage_log = temp_log("usage");
let prompt_log = temp_log("prompts");
let mut inner = inner_with_peers(vec![origin], 0);
inner.usage_log = Some(usage_log.clone());
inner.prompt_log = Some(prompt_log.clone());
let base = serve_router(AppState(Arc::new(inner))).await;
let chat = |extra: serde_json::Value| {
let mut body = serde_json::json!({
"model": "docker.io/ai/m",
"messages": [{ "role": "user", "content": "hello" }],
});
body.as_object_mut()
.unwrap()
.extend(extra.as_object().unwrap().clone());
let url = format!("{base}/v1/chat/completions");
async move {
let resp = Client::new()
.post(url)
.header("user-agent", "test-agent/1")
.json(&body)
.send()
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
resp.text().await.unwrap()
}
};
let unasked = chat(serde_json::json!({ "stream": true })).await;
assert!(unasked.ends_with("data: [DONE]\n\n"), "{unasked}");
assert!(!unasked.contains("usage"), "not asked for: {unasked}");
assert!(unasked.contains("\"timings\""), "{unasked}");
assert!(seen.lock().await[0].get("stream_options").is_none());
let asked = chat(serde_json::json!({
"stream": true, "stream_options": { "include_usage": true }
}))
.await;
assert!(asked.contains("\"prompt_tokens\":120"), "{asked}");
let whole = chat(serde_json::json!({})).await;
assert!(whole.contains("\"usage\""), "{whole}");
assert!(
seen.lock().await[2].get("stream_options").is_none(),
"an unstreamed request is left alone"
);
let entries = ledger_of(&usage_log, 3).await;
let prompts = crate::promptlog::read(&prompt_log).unwrap();
let _ = std::fs::remove_file(&usage_log);
let _ = std::fs::remove_file(&prompt_log);
assert_eq!(entries.len(), 3, "{entries:?}");
assert_eq!(prompts.len(), 3, "{prompts:?}");
for (entry, prompt) in entries.iter().zip(&prompts) {
assert_eq!(entry.id, prompt.id, "the two logs join");
assert_eq!(entry.route, "/v1/chat/completions");
assert_eq!(entry.model, "docker.io/ai/m:latest");
assert_eq!(entry.provider, None, "a peer is not a provider");
assert_eq!(entry.client.as_deref(), Some("test-agent/1"));
assert_eq!(entry.tokens, crate::usage::Tokens::new(120, 100, 0, 7));
assert_eq!((entry.rate, entry.cost), (None, None), "no price, not free");
}
}
#[tokio::test]
async fn the_usage_ledger_prices_a_provider_reply_and_skips_the_rest() {
let usage_log = temp_log("usage-priced");
let mut inner = test_inner(std::env::temp_dir());
inner.usage_log = Some(usage_log.clone());
let state = AppState(Arc::new(inner));
fn routed() {
let cost = crate::providers::Cost {
cache_read: Some(0.3),
cache_write: Some(3.75),
..crate::providers::Cost::flat(3.0, 15.0)
};
let target = Target::Remote(Arc::new(RemoteTarget {
provider: "openrouter".into(),
base_url: "https://openrouter.invalid/api/v1".into(),
wire: Wire::OpenAi,
model: "anthropic/claude-sonnet-4".into(),
max_output: None,
cost: Some(cost),
api_key: None,
session: None,
}));
usage::note_target(
"llmman.provider/openrouter/anthropic/claude-sonnet-4",
&target,
);
}
let messages = r#"{"type":"message","usage":{"input_tokens":130,"cache_read_input_tokens":17010,"cache_creation_input_tokens":1000,"output_tokens":412}}"#;
let app = Router::new()
.route(
"/v1/messages",
post(move || async move {
routed();
([("content-type", "application/json")], messages)
}),
)
.route(
"/v1/chat/completions",
post(|| async {
r#"{"usage":{"prompt_tokens":5,"completion_tokens":5}}"#
}),
)
.route(
"/v1/completions",
post(|| async {
let mut req = serde_json::json!({ "stream": true });
usage::ask_for_stream_usage(&mut req);
assert_eq!(req["stream_options"]["include_usage"], true);
routed();
let sse = "data: {\"choices\":[{\"text\":\"hi\",\"finish_reason\":\"stop\"}]}\n\n\
data: {\"choices\":[],\"usage\":{\"prompt_tokens\":10,\"completion_tokens\":1}}\n\n\
data: [DONE]\n\n";
let body = futures::stream::iter(sse.as_bytes().chunks(7).map(|c| {
Ok::<_, std::convert::Infallible>(Bytes::copy_from_slice(c))
}));
([("content-type", "text/event-stream")], Body::from_stream(body))
}),
)
.route(
"/api/generate",
post(|| async {
usage::note_target("m", &Target::Local(1));
r#"{"done":true,"prompt_eval_count":5,"eval_count":5}"#
}),
)
.route(
"/api/chat",
post(|| async {
routed();
(
StatusCode::BAD_GATEWAY,
r#"{"prompt_eval_count":5,"eval_count":5}"#,
)
}),
)
.layer(middleware::from_fn_with_state(
state.clone(),
usage::record_usage,
))
.with_state(state);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let base = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port());
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
let client = Client::new();
let resp = client
.post(format!("{base}/v1/messages"))
.send()
.await
.unwrap();
assert_eq!(
resp.headers().get("content-length").map(|v| v.as_bytes()),
Some(messages.len().to_string().as_bytes()),
"an in-memory reply stays unchunked"
);
assert_eq!(resp.text().await.unwrap(), messages);
let stream = client
.post(format!("{base}/v1/completions"))
.send()
.await
.unwrap();
assert!(!stream.text().await.unwrap().contains("usage"));
for route in ["/v1/chat/completions", "/api/chat"] {
client.post(format!("{base}{route}")).send().await.unwrap();
}
for route in ["/api/generate", "/v1/messages"] {
client
.post(format!("{base}{route}"))
.header(aggregation::HOP, "1")
.send()
.await
.unwrap();
}
let mut entries = ledger_of(&usage_log, 3).await;
let _ = std::fs::remove_file(&usage_log);
assert_eq!(entries.len(), 3, "{entries:?}");
let streamed = entries
.iter()
.position(|e| e.route == "/v1/completions")
.unwrap();
assert_eq!(
entries.remove(streamed).tokens,
crate::usage::Tokens::new(10, 0, 0, 1)
);
assert_eq!(entries[0].tokens, entries[1].tokens);
let entry = &entries[0];
assert_eq!(entry.id.len(), 40);
assert_eq!(entry.provider.as_deref(), Some("openrouter"));
assert_eq!(
entry.tokens,
crate::usage::Tokens::new(18_140, 17_010, 1_000, 412)
);
let want = (130.0 * 3.0 + 17_010.0 * 0.3 + 1_000.0 * 3.75 + 412.0 * 15.0) / 1e6;
assert!((entry.cost.unwrap() - want).abs() < 1e-12, "{entry:?}");
assert_eq!(entry.rate.unwrap().cache_read, 0.3);
}
#[test]
fn a_blank_model_hint_is_no_hint() {
let hint = |model: Option<&str>| {
super::ProviderQuery {
model: model.map(str::to_string),
}
.model()
.map(str::to_string)
};
assert_eq!(hint(None), None);
assert_eq!(hint(Some("")), None);
assert_eq!(hint(Some(" \t")), None);
assert_eq!(
hint(Some(" claude-sonnet-5-5 ")),
Some("claude-sonnet-5-5".into())
);
}
const BACKEND_MODEL_PATH: &str =
"/models/bd258782e35f7f458f8aced1adc053e6e92e89bc735ba3be89d38a06121dc517";
fn llama_server_standin() -> Router {
let completion = || {
Json(serde_json::json!({
"id": "chatcmpl-1",
"object": "chat.completion",
"model": BACKEND_MODEL_PATH,
"choices": [{
"index": 0,
"message": { "role": "assistant", "content": "Hello" },
"finish_reason": "stop"
}],
}))
};
let sse = |body: String| ([("content-type", "text/event-stream")], body).into_response();
Router::new()
.route(
"/v1/chat/completions",
post(move |Json(req): Json<serde_json::Value>| async move {
if req["stream"] != true {
return completion().into_response();
}
let chunk = |content: &str| {
let chunk = serde_json::json!({
"id": "chatcmpl-1",
"object": "chat.completion.chunk",
"model": BACKEND_MODEL_PATH,
"choices": [{ "index": 0, "delta": { "content": content } }],
});
format!("data: {chunk}\n\n")
};
sse(format!("{}{}data: [DONE]\n\n", chunk("Hel"), chunk("lo")))
}),
)
.route("/v1/completions", post(move || async move { completion() }))
.route(
"/v1/responses",
post(move |Json(req): Json<serde_json::Value>| async move {
let response = serde_json::json!({
"id": "resp_1", "object": "response",
"model": BACKEND_MODEL_PATH, "status": "completed",
});
if req["stream"] != true {
return Json(response).into_response();
}
let created =
serde_json::json!({ "type": "response.created", "response": response });
sse(format!(
"event: response.created\ndata: {created}\n\n\
event: response.output_text.delta\ndata: {{\"delta\":\"Hi\"}}\n\n"
))
}),
)
.route(
"/v1/embeddings",
post(|Json(req): Json<serde_json::Value>| async move {
let embedding = if req["encoding_format"] == "base64" {
serde_json::json!(encode_f32s(&[3.0, 4.0, 12.0]))
} else {
serde_json::json!([3.0, 4.0, 12.0])
};
Json(serde_json::json!({
"object": "list",
"model": BACKEND_MODEL_PATH,
"data": [{ "object": "embedding", "index": 0, "embedding": embedding }],
"usage": { "prompt_tokens": 2, "total_tokens": 2 },
}))
}),
)
}
fn decision_server_standin() -> Router {
Router::new().route(
"/v1/systemone",
post(|body: String| async move {
let received = serde_json::to_string(&body).unwrap();
let usage = r#"{"input_tokens":3,"output_tokens":0}"#;
(
[("content-type", "application/json")],
format!(
r#"{{"model":"{BACKEND_MODEL_PATH}","answers":{{"zebra?":{{"type":"noul","noul":0.9}},"apple?":{{"type":"noul","noul":0.1}}}},"usage":{usage},"received":{received}}}"#
),
)
})
.layer(DefaultBodyLimit::disable()),
)
}
async fn serve_over(model: &str, backend: Router) -> String {
serve_over_state(model, backend, false).await.0
}
async fn serve_over_state(model: &str, backend: Router, decision: bool) -> (String, AppState) {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let mut loaded = running_model_fixture(None, Duration::ZERO, 0);
sleep(Duration::from_millis(250)).await;
assert!(
loaded.process.is_alive(),
"the placeholder process exited as soon as it was spawned"
);
loaded.port = listener.local_addr().unwrap().port();
loaded.decision = decision;
tokio::spawn(async move { axum::serve(listener, backend).await.unwrap() });
let state = test_state();
let key = running_key(&state, model);
state.0.manager.lock().await.running.insert(key, loaded);
(serve_router(state.clone()).await, state)
}
fn stand_in_client() -> Client {
Client::builder()
.timeout(Duration::from_secs(30))
.build()
.unwrap()
}
async fn post_with(
base: &str,
route: &str,
mut body: serde_json::Value,
extra: serde_json::Value,
) -> reqwest::Response {
body.as_object_mut()
.unwrap()
.extend(extra.as_object().unwrap().clone());
let resp = stand_in_client()
.post(format!("{base}{route}"))
.json(&body)
.send()
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK, "{route}");
resp
}
fn sse_payloads(stream: &str) -> Vec<serde_json::Value> {
stream
.lines()
.filter_map(|l| l.strip_prefix("data: "))
.filter(|payload| *payload != "[DONE]")
.map(|payload| serde_json::from_str(payload).unwrap())
.collect()
}
#[tokio::test(flavor = "multi_thread")]
async fn openai_replies_name_the_model_the_client_asked_for() {
let base = serve_over("qwen3.5:0.8b", llama_server_standin()).await;
let ask = |route: &'static str, extra: serde_json::Value| {
let body = serde_json::json!({
"model": "qwen3.5:0.8b",
"messages": [{ "role": "user", "content": "hi" }],
"prompt": "hi",
"input": "hi",
"max_tokens": 8,
});
let base = base.clone();
async move { post_with(&base, route, body, extra).await }
};
let stream = serde_json::json!({ "stream": true });
for route in ["/v1/chat/completions", "/v1/completions", "/v1/responses"] {
let body: serde_json::Value = ask(route, serde_json::json!({}))
.await
.json()
.await
.unwrap();
assert_eq!(body["model"], "qwen3.5:0.8b", "{route}");
}
let text = ask("/v1/chat/completions", stream.clone())
.await
.text()
.await
.unwrap();
assert!(
!text.contains("/models/") && text.ends_with("data: [DONE]\n\n"),
"{text}"
);
let chunks = sse_payloads(&text);
assert_eq!(chunks.len(), 2, "{text}");
for chunk in &chunks {
assert_eq!(chunk["model"], "qwen3.5:0.8b");
assert_eq!(chunk["id"], "chatcmpl-1");
}
assert_eq!(chunks[1]["choices"][0]["delta"]["content"], "lo");
let text = ask("/v1/responses", stream).await.text().await.unwrap();
assert!(!text.contains("/models/"), "{text}");
assert_eq!(sse_payloads(&text)[0]["response"]["model"], "qwen3.5:0.8b");
assert!(
text.contains("event: response.output_text.delta\n"),
"{text}"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn a_backend_dying_mid_stream_aborts_the_clients_stream() {
use futures::StreamExt as _;
let dying = Router::new().route(
"/v1/chat/completions",
post(|| async {
let first = futures::stream::once(async {
Ok::<_, std::io::Error>(Bytes::from("data: {\"model\":\"x\"}\n\n"))
});
let then = futures::stream::once(async {
sleep(Duration::from_millis(50)).await;
Err(std::io::Error::other("backend died"))
});
(
[("content-type", "text/event-stream")],
axum::body::Body::from_stream(first.chain(then)),
)
}),
);
let base = serve_over("qwen3.5:0.8b", dying).await;
let body = serde_json::json!({
"model": "qwen3.5:0.8b", "messages": [{ "role": "user", "content": "hi" }],
});
let resp = post_with(
&base,
"/v1/chat/completions",
body,
serde_json::json!({ "stream": true }),
)
.await;
let err = resp
.text()
.await
.expect_err("a truncated reply must not look complete");
assert!(
!err.is_timeout(),
"the abort must reach the client before its timeout: {err}"
);
}
fn loaded_with(window: Option<u32>, trained: Option<u32>) -> RunningModel {
let mut model = running_model_fixture(None, Duration::ZERO, 0);
model.context_window = window;
model.trained_ctx = trained;
model
}
#[tokio::test]
async fn num_ctx_is_applied_as_ollama_applies_it() {
assert_eq!(effective_ctx(4096, Some(40960)), 4096);
assert_eq!(effective_ctx(1 << 20, Some(40960)), 40960);
assert_eq!(effective_ctx(1 << 20, None), 1 << 20);
assert_eq!(effective_ctx(1, Some(40960)), 4);
let model = loaded_with(Some(8192), Some(40960));
assert!(!model.needs_context(Some(8192)) && !model.needs_context(None));
assert!(model.needs_context(Some(4096)) && model.needs_context(Some(1 << 20)));
assert!(!loaded_with(Some(40960), Some(40960)).needs_context(Some(1 << 20)));
assert!(loaded_with(None, Some(4096)).needs_context(Some(4096)));
for engine in [Engine::Mlx, Engine::Vllm, Engine::Sglang, Engine::VllmOmni] {
let mut other = running_model_fixture_with_engine(engine, None);
other.context_window = Some(8192);
assert!(!other.needs_context(Some(4096)), "{engine:?}");
}
let gguf = ModelPath::Gguf(PathBuf::from("m.gguf"), None);
assert_eq!(ctx_for_load(Some(4096), &gguf, false), Some(4096));
assert_eq!(ctx_for_load(Some(4096), &gguf, true), None, "ds4");
assert_eq!(ctx_for_load(None, &gguf, false), None);
let diffusion = ModelPath::Diffusion(Default::default());
assert_eq!(ctx_for_load(Some(4096), &diffusion, false), Some(4096));
for other in [
ModelPath::SafeTensors(PathBuf::from("m")),
ModelPath::Omni(PathBuf::from("m")),
] {
assert_eq!(ctx_for_load(Some(4096), &other, false), None);
}
}
#[tokio::test]
async fn a_model_at_another_window_is_not_reused_for_a_request_that_names_one() {
let state = test_state();
let key = running_key(&state, "qwen3:0.6b");
let in_flight = |state: &AppState, key: &str| {
let (state, key) = (state.clone(), key.to_string());
async move { state.0.manager.lock().await.running[&key].in_flight }
};
state
.0
.manager
.lock()
.await
.running
.insert(key.clone(), loaded_with(Some(40960), Some(40960)));
assert!(check_running_for(&state, &key, Some(4096)).await.is_none());
assert_eq!(in_flight(&state, &key).await, 0);
for ctx in [None, Some(40960), Some(1 << 20)] {
let (_, guard) = check_running_for(&state, &key, ctx).await.unwrap();
assert_eq!(in_flight(&state, &key).await, 1);
drop(guard);
for _ in 0..100 {
if in_flight(&state, &key).await == 0 {
break;
}
sleep(Duration::from_millis(10)).await;
}
assert_eq!(in_flight(&state, &key).await, 0, "{ctx:?}");
}
}
#[tokio::test]
async fn retiring_a_model_for_its_window_unloads_only_a_mismatch() {
let state = test_state();
let key = running_key(&state, "qwen3:0.6b");
let other = running_key(&state, "gemma3:1b");
{
let mut mgr = state.0.manager.lock().await;
mgr.running
.insert(key.clone(), loaded_with(Some(40960), Some(40960)));
mgr.running
.insert(other.clone(), loaded_with(Some(8192), Some(40960)));
}
for ctx in [None, Some(40960), Some(1 << 20)] {
assert!(
retire_for_context(&state, &key, ctx).await.is_none(),
"{ctx:?}"
);
}
let never = "docker.io/ai/never-loaded:latest";
assert!(retire_for_context(&state, never, Some(4096))
.await
.is_none());
assert_eq!(state.0.manager.lock().await.pending_loads, 0);
let reserved = retire_for_context(&state, &key, Some(4096)).await;
let mgr = state.0.manager.lock().await;
assert!(reserved.is_some() && mgr.pending_loads == 1);
assert!(!mgr.running.contains_key(&key));
assert!(
mgr.running.contains_key(&other),
"another model is untouched"
);
}
#[tokio::test(flavor = "multi_thread")]
async fn the_slot_a_retired_model_frees_is_kept_for_its_replacement() {
let mut inner = test_inner(std::env::temp_dir());
inner.max_loaded_models = 1;
let state = AppState(Arc::new(inner));
let key = running_key(&state, "qwen3:0.6b");
state
.0
.manager
.lock()
.await
.running
.insert(key.clone(), loaded_with(Some(40960), Some(40960)));
let reserved = retire_for_context(&state, &key, Some(4096)).await.unwrap();
let competitor = {
let state = state.clone();
tokio::spawn(async move { enforce_max_loaded_models(&state, 1).await })
};
sleep(Duration::from_millis(300)).await;
assert!(!competitor.is_finished(), "it must not take the room");
drop(reserved);
let slot = tokio::time::timeout(Duration::from_secs(10), competitor)
.await
.expect("takes the room once it is released")
.unwrap();
assert!(slot.is_ok());
}
#[tokio::test(flavor = "multi_thread")]
async fn a_model_waiting_to_be_retired_takes_no_new_requests() {
let state = test_state();
let key = running_key(&state, "qwen3:0.6b");
let mut busy = loaded_with(Some(40960), Some(40960));
busy.in_flight = 1;
state
.0
.manager
.lock()
.await
.running
.insert(key.clone(), busy);
let retire = || {
let (state, key) = (state.clone(), key.clone());
tokio::spawn(async move { retire_for_context(&state, &key, Some(4096)).await })
};
let retiring = || async {
for _ in 0..200 {
if state.0.manager.lock().await.running[&key]
.retiring
.load(std::sync::atomic::Ordering::SeqCst)
{
return true;
}
sleep(Duration::from_millis(10)).await;
}
false
};
let waiting = retire();
assert!(retiring().await, "it marks the model");
assert!(!waiting.is_finished(), "and waits for the request");
for ctx in [None, Some(40960), Some(4096)] {
assert!(
check_running_for(&state, &key, ctx).await.is_none(),
"{ctx:?}"
);
}
assert_eq!(state.0.manager.lock().await.running[&key].in_flight, 1);
waiting.abort();
let _ = waiting.await;
let (_, guard) = check_running_for(&state, &key, None).await.unwrap();
drop(guard);
let waiting = retire();
assert!(retiring().await);
change_loaded(&state, "qwen3:0.6b", |m| m.in_flight = 0).await;
let reserved = tokio::time::timeout(Duration::from_secs(10), waiting)
.await
.expect("retires once idle")
.unwrap();
assert!(reserved.is_some());
assert!(!state.0.manager.lock().await.running.contains_key(&key));
}
#[tokio::test]
async fn retiring_leaves_the_same_content_under_another_name_alone() {
let state = test_state();
let by_digest = "docker.io/ai/m@sha256:d16e".to_string();
let mut model = loaded_with(Some(40960), Some(40960));
model.digest = "sha256:d16e".into();
state
.0
.manager
.lock()
.await
.running
.insert(by_digest.clone(), model);
let retired = retire_for_context(&state, "docker.io/ai/m:v1", Some(4096)).await;
assert!(retired.is_none());
assert!(state
.0
.manager
.lock()
.await
.running
.contains_key(&by_digest));
}
#[cfg(unix)]
#[test]
#[ignore = "entered by FakeLlamaServer, not a test of its own"]
fn fake_llama_server_child() {
use std::io::{Read, Write};
let Ok(args) = std::env::var("LLMMAN_FAKE_LLAMA_ARGS") else {
return;
};
let flag = |name: &str| {
args.lines()
.skip_while(|arg| *arg != name)
.nth(1)
.map(str::to_string)
};
let number = |name: &str| flag(name).and_then(|n| n.parse::<u64>().ok());
let port = number("--port").expect("--port <n>");
let trained = flag("--model")
.and_then(|model| crate::gguf::read_info(std::path::Path::new(&model)).ok())
.and_then(|info| info.context_length())
.expect("a --model with a trained context");
let pad = |n: u64| n.div_ceil(256) * 256;
let n_ctx = pad(number("--ctx-size").filter(|n| *n > 0).unwrap_or(trained));
let slot = number("--parallel")
.map_or(n_ctx, |slots| pad(n_ctx / slots))
.min(trained);
let models = serde_json::json!({
"data": [{ "meta": { "n_ctx": slot, "n_ctx_train": trained } }],
})
.to_string();
let listener = std::net::TcpListener::bind(("127.0.0.1", port as u16)).unwrap();
for stream in listener.incoming().flatten() {
let mut stream = stream;
let mut request = [0u8; 1024];
let read = stream.read(&mut request).unwrap_or(0);
let body = if request[..read].starts_with(b"GET /v1/models ") {
models.as_str()
} else {
"{}"
};
let _ = write!(
stream,
"HTTP/1.1 200 OK\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
body.len()
);
}
}
#[cfg(unix)]
struct FakeLlamaServer {
dir: PathBuf,
bin: PathBuf,
log: PathBuf,
}
#[cfg(unix)]
impl FakeLlamaServer {
fn new(name: &str) -> Self {
use std::os::unix::fs::PermissionsExt;
let dir = fresh_store_dir(name);
std::fs::create_dir_all(&dir).unwrap();
let (bin, log) = (dir.join("llama-server"), dir.join("args.log"));
let test = format!(
"{}::fake_llama_server_child",
module_path!().split_once("::").unwrap().1
);
let script = format!(
"#!/bin/sh\ncase \"$1\" in --selftest) exit 0;; esac\necho \"$*\" >> '{}'\n\
LLMMAN_FAKE_LLAMA_ARGS=\"$(printf '%s\\n' \"$@\")\" exec '{}' --ignored --exact '{test}'\n",
log.display(),
std::env::current_exe().unwrap().display(),
);
std::fs::write(&bin, script).unwrap();
std::fs::set_permissions(&bin, std::fs::Permissions::from_mode(0o755)).unwrap();
for attempt in 0.. {
match std::process::Command::new(&bin).arg("--selftest").status() {
Ok(status) if status.success() => break,
Err(e) if e.raw_os_error() == Some(26) && attempt < 100 => {
std::thread::sleep(Duration::from_millis(20));
}
other => panic!("the fake llama-server will not run: {other:?}"),
}
}
Self { dir, bin, log }
}
fn state(&self) -> AppState {
self.state_with("ctxmodel", &[], &[], |_| {})
}
fn state_with(
&self,
name: &str,
u32s: &[(&str, u32)],
strings: &[(&str, &str)],
configure: impl FnOnce(&mut Inner),
) -> AppState {
let store = self.dir.join("store");
let gguf = crate::gguf::write_test_gguf_with_strings(u32s, strings);
let bytes = std::fs::read(&gguf).unwrap();
let _ = std::fs::remove_file(&gguf);
store_one_model(
&OciStore::open(&store).unwrap(),
name,
"application/vnd.docker.ai.gguf.v3",
"model.gguf",
&bytes,
);
let mut inner = test_inner(store);
inner.runtime = runtime::Lazy::new(
runtime::Runtime::Path,
None,
true,
Some(runtime::Resolved::Local {
source: runtime::Runtime::Path,
bin: self.bin.clone(),
}),
);
configure(&mut inner);
AppState(Arc::new(inner))
}
fn ctx_sizes(&self) -> Vec<String> {
std::fs::read_to_string(&self.log)
.unwrap_or_default()
.lines()
.map(|line| {
line.split_whitespace()
.skip_while(|arg| *arg != "--ctx-size")
.nth(1)
.unwrap_or("(none)")
.to_string()
})
.collect()
}
}
#[cfg(unix)]
impl Drop for FakeLlamaServer {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.dir);
}
}
#[cfg(unix)]
#[tokio::test(flavor = "multi_thread")]
async fn num_ctx_sizes_the_load_and_a_different_one_reloads_it() {
let fake = FakeLlamaServer::new("num-ctx");
let state = fake.state();
let model = "docker.io/ai/ctxmodel";
let mut ports = Vec::new();
for (ctx, spawns, window) in [
(Some(2048), vec!["2048"], 2048),
(Some(2048), vec!["2048"], 2048),
(None, vec!["2048"], 2048),
(Some(3072), vec!["2048", "3072"], 3072),
(Some(1 << 20), vec!["2048", "3072", "4096"], 4096),
(Some(4096), vec!["2048", "3072", "4096"], 4096),
(Some(1 << 20), vec!["2048", "3072", "4096"], 4096),
] {
let opts = LoadOptions {
ctx,
..Default::default()
};
let (name, target, guard) = ensure_model_for(&state, model, None, opts, Task::Other)
.await
.unwrap_or_else(|e| panic!("{ctx:?}: {:#}", e.0));
let Target::Local(port) = target else {
panic!("{ctx:?}: not a local target");
};
ports.push(port);
drop(guard);
assert_eq!(fake.ctx_sizes(), spawns, "{ctx:?}");
for _ in 0..200 {
if state
.0
.manager
.lock()
.await
.running
.values()
.all(|m| m.in_flight == 0)
{
break;
}
sleep(Duration::from_millis(10)).await;
}
let mgr = state.0.manager.lock().await;
assert_eq!(mgr.running.len(), 1, "{ctx:?}: the old load is gone");
assert_eq!(mgr.running[&name].context_window, Some(window), "{ctx:?}");
}
assert_eq!(
ports
.iter()
.map(|p| ports.iter().position(|q| q == p).unwrap())
.collect::<Vec<_>>(),
[0, 0, 0, 3, 4, 4, 4]
);
}
#[cfg(unix)]
#[tokio::test(flavor = "multi_thread")]
async fn an_unloaded_models_n_ctx_is_what_its_load_reports() {
type Header = (
&'static str,
&'static [(&'static str, u32)],
&'static [(&'static str, &'static str)],
);
let kinds: [Header; 3] = [
("text", &[], &[]),
("embedding", &[("llama.pooling_type", 1)], &[]),
("decision", &[], &[("llama.decision.type", "clef")]),
];
let configs = [
(Some(DEFAULT_CTX_SIZE), false, None, None),
(Some(DEFAULT_CTX_SIZE), false, Some(4), None),
(Some(2048), true, None, None),
(Some(65536), true, None, None),
(Some(0), true, None, None),
(Some(0), true, Some(2), None),
(Some(2048), true, Some(4), None),
(Some(DEFAULT_CTX_SIZE), false, None, Some(512)),
(Some(DEFAULT_CTX_SIZE), false, Some(2), Some(0)),
];
let cases = configs
.into_iter()
.flat_map(|config| kinds.map(|kind| (config, kind)))
.enumerate();
let fake = FakeLlamaServer::new("ctx parity");
let tasks: Vec<_> = cases
.map(
|(
i,
((ctx_size, explicit, num_parallel, embedding_ctx_size), (kind, u32s, strings)),
)| {
let case = format!(
"{kind}: ctx_size {ctx_size:?} explicit {explicit} parallel {num_parallel:?} \
embedding {embedding_ctx_size:?}"
);
let name = format!("ctxparity{i}");
let state = fake.state_with(&name, u32s, strings, |inner| {
inner.ctx_size = ctx_size;
inner.ctx_size_explicit = explicit;
inner.num_parallel = num_parallel;
inner.embedding_ctx_size = embedding_ctx_size;
});
tokio::spawn(async move {
let base = serve_router(state.clone()).await;
let n_ctx = |v: &serde_json::Value| {
let row = listed(v, &name);
(row["status"]["value"].clone(), row["meta"]["n_ctx"].clone())
};
let (_, v) = get_json(&base, "/v1/models").await;
let (status, predicted) = n_ctx(&v);
assert_eq!(status, "unloaded", "{case}");
assert!(predicted.is_u64(), "{case}: {predicted}");
let model = format!("docker.io/ai/{name}");
let mut attempts = 0;
let (_, _, guard) = loop {
let opts = LoadOptions::default();
match ensure_model_for(&state, &model, None, opts, Task::Other).await {
Ok(loaded) => break loaded,
Err(e)
if attempts < 3
&& format!("{:#}", e.0).contains("Address already in use") =>
{
attempts += 1;
}
Err(e) => panic!("{case}: {:#}", e.0),
}
};
let (_, v) = get_json(&base, "/v1/models").await;
drop(guard);
assert_eq!(n_ctx(&v), ("loaded".into(), predicted), "{case}");
})
},
)
.collect();
for task in tasks {
if let Err(e) = task.await {
std::panic::resume_unwind(e.into_panic());
}
}
}
struct RoleStandin {
router: Router,
seen: Arc<StdMutex<Vec<serde_json::Value>>>,
}
impl RoleStandin {
fn new(refusal: Option<&'static str>) -> Self {
let seen = Arc::new(StdMutex::new(Vec::new()));
let record = seen.clone();
let router = Router::new().route(
"/v1/chat/completions",
post(move |Json(req): Json<serde_json::Value>| {
let record = record.clone();
async move {
record.lock().unwrap().push(req.clone());
let roles: Vec<&str> = req["messages"]
.as_array()
.unwrap()
.iter()
.map(|m| m["role"].as_str().unwrap_or(""))
.filter(|role| *role != "system")
.collect();
if let Some(message) =
refusal.filter(|_| roles.windows(2).any(|w| w[0] == w[1]))
{
return (
StatusCode::BAD_REQUEST,
Json(serde_json::json!({ "error": {
"code": 400, "message": message, "type": "invalid_request_error",
} })),
)
.into_response();
}
let delta = |content: &str, finish: serde_json::Value| {
let chunk = serde_json::json!({
"id": "chatcmpl-1", "object": "chat.completion.chunk",
"model": BACKEND_MODEL_PATH,
"choices": [{ "index": 0, "delta": { "content": content },
"finish_reason": finish }],
});
format!("data: {chunk}\n\n")
};
if req["stream"] == true {
let body = format!(
"{}{}data: [DONE]\n\n",
delta("Hel", serde_json::Value::Null),
delta("lo", "stop".into())
);
return ([("content-type", "text/event-stream")], body).into_response();
}
Json(serde_json::json!({
"id": "chatcmpl-1", "object": "chat.completion",
"model": BACKEND_MODEL_PATH,
"choices": [{ "index": 0, "finish_reason": "stop",
"message": { "role": "assistant", "content": "Hello" } }],
"usage": { "prompt_tokens": 3, "completion_tokens": 1 },
}))
.into_response()
}
}),
);
Self { router, seen }
}
fn requests(&self) -> Vec<Vec<(String, String)>> {
self.seen
.lock()
.unwrap()
.iter()
.map(|req| {
req["messages"]
.as_array()
.unwrap()
.iter()
.map(|m| {
(
m["role"].as_str().unwrap().to_string(),
m["content"].as_str().unwrap_or_default().to_string(),
)
})
.collect()
})
.collect()
}
}
#[tokio::test(flavor = "multi_thread")]
async fn low_effort_gets_a_thinking_budget_only_on_a_model_flagged_for_one() {
let standin = RoleStandin::new(None);
let (base, state) = serve_over_state("qwen3:0.6b", standin.router.clone(), false).await;
let send = || {
post_with(
&base,
"/v1/chat/completions",
serde_json::json!({
"model": "qwen3:0.6b",
"messages": [{ "role": "user", "content": "hi" }],
"max_tokens": 120,
"reasoning_effort": "low",
}),
serde_json::json!({}),
)
};
let last = || standin.seen.lock().unwrap().last().unwrap().clone();
send().await;
let sent = last();
assert!(sent.get("reasoning_budget_tokens").is_none(), "{sent}");
assert!(sent.get("reasoning_budget_message").is_none(), "{sent}");
change_loaded(&state, "qwen3:0.6b", |m| m.effort_budget = true).await;
send().await;
let sent = last();
assert_eq!(sent["reasoning_budget_tokens"], 30, "{sent}");
assert_eq!(
sent["reasoning_budget_message"],
"Okay, I have to answer now."
);
assert_eq!(
sent["chat_template_kwargs"],
serde_json::json!({ "enable_thinking": true, "reasoning_effort": "low" })
);
change_loaded(&state, "qwen3:0.6b", |m| m.effort_budget_message = false).await;
send().await;
let sent = last();
assert_eq!(sent["reasoning_budget_tokens"], 30, "{sent}");
assert!(sent.get("reasoning_budget_message").is_none(), "{sent}");
}
async fn change_loaded(state: &AppState, model: &str, change: impl FnOnce(&mut RunningModel)) {
let key = running_key(state, model);
change(state.0.manager.lock().await.running.get_mut(&key).unwrap());
}
#[tokio::test(flavor = "multi_thread")]
async fn ollama_think_levels_get_a_thinking_budget_only_on_a_model_flagged_for_one() {
for (route, input) in [("/api/chat", "messages"), ("/api/generate", "prompt")] {
let standin = RoleStandin::new(None);
let (base, state) = serve_over_state("qwen3:0.6b", standin.router.clone(), false).await;
let send = |think: serde_json::Value| {
let mut body = serde_json::json!({
"model": "qwen3:0.6b",
"stream": false,
"think": think,
"options": { "num_predict": 120 },
});
body[input] = if input == "messages" {
serde_json::json!([{ "role": "user", "content": "hi" }])
} else {
serde_json::json!("hi")
};
post_with(&base, route, body, serde_json::json!({}))
};
let last = || standin.seen.lock().unwrap().last().unwrap().clone();
send(serde_json::json!("low")).await;
let sent = last();
assert!(
sent.get("reasoning_budget_tokens").is_none(),
"{route}: {sent}"
);
change_loaded(&state, "qwen3:0.6b", |m| m.effort_budget = true).await;
send(serde_json::json!("low")).await;
let sent = last();
assert_eq!(sent["reasoning_budget_tokens"], 30, "{route}: {sent}");
assert_eq!(sent["reasoning_budget_message"], EFFORT_BUDGET_MESSAGE);
assert_eq!(
sent["chat_template_kwargs"],
serde_json::json!({ "enable_thinking": true, "reasoning_effort": "low" })
);
send(serde_json::json!("minimal")).await;
assert_eq!(last()["reasoning_budget_tokens"], 15, "{route}");
send(serde_json::json!(true)).await;
let sent = last();
assert!(
sent.get("reasoning_budget_tokens").is_none(),
"{route}: {sent}"
);
change_loaded(&state, "qwen3:0.6b", |m| m.effort_budget_message = false).await;
send(serde_json::json!("low")).await;
let sent = last();
assert_eq!(sent["reasoning_budget_tokens"], 30, "{route}: {sent}");
assert!(
sent.get("reasoning_budget_message").is_none(),
"{route}: {sent}"
);
}
}
#[tokio::test(flavor = "multi_thread")]
async fn ollama_think_a_level_the_model_does_not_take_gets_its_default() {
for (route, input) in [("/api/chat", "messages"), ("/api/generate", "prompt")] {
let standin = RoleStandin::new(None);
let (base, state) = serve_over_state("qwen3:0.6b", standin.router.clone(), false).await;
change_loaded(&state, "qwen3:0.6b", |m| {
m.thinking = Some(crate::chat_template::Thinking {
values: vec![false.into(), "low".into(), "medium".into(), "xhigh".into()],
default: "medium".into(),
});
})
.await;
let body = |think: serde_json::Value| {
let mut body = serde_json::json!({ "model": "qwen3:0.6b", "stream": false });
body["think"] = think;
body[input] = if input == "messages" {
serde_json::json!([{ "role": "user", "content": "hi" }])
} else {
serde_json::json!("hi")
};
body
};
let kwargs = |think: serde_json::Value| {
let (base, standin, body) = (&base, &standin, body(think));
async move {
post_with(base, route, body, serde_json::json!({})).await;
let sent = standin.seen.lock().unwrap().last().unwrap().clone();
sent["chat_template_kwargs"].clone()
}
};
let on =
|level: &str| serde_json::json!({ "enable_thinking": true, "reasoning_effort": level });
assert_eq!(
kwargs(serde_json::json!("xhigh")).await,
on("xhigh"),
"{route}"
);
assert_eq!(
kwargs(serde_json::json!("high")).await,
on("medium"),
"{route}"
);
assert_eq!(kwargs(serde_json::json!("low")).await, on("low"), "{route}");
assert_eq!(
kwargs(serde_json::json!(false)).await,
serde_json::json!({ "enable_thinking": false }),
"{route}"
);
for think in [serde_json::json!(3), serde_json::json!({ "level": "low" })] {
let resp = stand_in_client()
.post(format!("{base}{route}"))
.json(&body(think))
.send()
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST, "{route}");
assert!(resp
.text()
.await
.unwrap()
.contains("think must be a boolean or string"));
}
}
}
const ALTERNATION_REFUSAL: &str = "Unable to generate parser for this template. Automatic parser \
generation failed: Conversation roles must alternate user/assistant/user/assistant/...";
fn turns(pairs: &[(&str, &str)]) -> Vec<(String, String)> {
pairs
.iter()
.map(|(role, content)| (role.to_string(), content.to_string()))
.collect()
}
#[tokio::test(flavor = "multi_thread")]
async fn a_template_that_needs_alternation_is_sent_merged_turns() {
let standin = RoleStandin::new(Some(ALTERNATION_REFUSAL));
let base = serve_over("gemma3:1b", standin.router.clone()).await;
let last = |standin: &RoleStandin| standin.requests().pop().unwrap();
let reply: serde_json::Value = post_with(
&base,
"/v1/chat/completions",
serde_json::json!({
"model": "gemma3:1b",
"messages": [
{ "role": "system", "content": "be terse" },
{ "role": "user", "content": "one" },
{ "role": "user", "content": "two" },
],
}),
serde_json::json!({}),
)
.await
.json()
.await
.unwrap();
assert_eq!(reply["choices"][0]["message"]["content"], "Hello");
assert_eq!(
standin.requests(),
[
turns(&[("system", "be terse"), ("user", "one"), ("user", "two")]),
turns(&[("system", "be terse"), ("user", "one\n\ntwo")]),
]
);
let text = post_with(
&base,
"/v1/chat/completions",
serde_json::json!({
"model": "gemma3:1b",
"messages": [
{ "role": "user", "content": "a" },
{ "role": "assistant", "content": "b" },
{ "role": "user", "content": "c" },
{ "role": "user", "content": "d" },
],
}),
serde_json::json!({ "stream": true }),
)
.await
.text()
.await
.unwrap();
assert!(
text.contains("Hel") && text.ends_with("data: [DONE]\n\n"),
"{text}"
);
assert_eq!(
last(&standin),
turns(&[("user", "a"), ("assistant", "b"), ("user", "c\n\nd")])
);
let reply: serde_json::Value = post_with(
&base,
"/api/chat",
serde_json::json!({
"model": "gemma3:1b",
"messages": [
{ "role": "user", "content": "x" },
{ "role": "user", "content": "y" },
],
}),
serde_json::json!({ "stream": false }),
)
.await
.json()
.await
.unwrap();
assert_eq!(reply["message"]["content"], "Hello");
assert_eq!(reply["done"], true);
assert_eq!(last(&standin), turns(&[("user", "x\n\ny")]));
let reply: serde_json::Value = post_with(
&base,
"/v1/messages",
serde_json::json!({
"model": "gemma3:1b",
"max_tokens": 8,
"messages": [
{ "role": "user", "content": "p" },
{ "role": "user", "content": "q" },
],
}),
serde_json::json!({}),
)
.await
.json()
.await
.unwrap();
assert_eq!(reply["content"][0]["text"], "Hello");
assert_eq!(last(&standin), turns(&[("user", "p\n\nq")]));
}
#[tokio::test(flavor = "multi_thread")]
async fn turns_are_merged_only_for_a_template_that_refused_them() {
let two_user_turns = serde_json::json!({
"model": "qwen3.5:0.8b",
"messages": [
{ "role": "user", "content": "one" },
{ "role": "user", "content": "two" },
],
});
let lenient = RoleStandin::new(None);
let base = serve_over("qwen3.5:0.8b", lenient.router.clone()).await;
post_with(
&base,
"/v1/chat/completions",
two_user_turns.clone(),
serde_json::json!({}),
)
.await;
post_with(
&base,
"/api/chat",
two_user_turns.clone(),
serde_json::json!({ "stream": false }),
)
.await;
assert_eq!(
lenient.requests(),
[
turns(&[("user", "one"), ("user", "two")]),
turns(&[("user", "one"), ("user", "two")]),
]
);
let other = RoleStandin::new(Some("the request exceeds the available context size"));
let base = serve_over("qwen3.5:0.8b", other.router.clone()).await;
for route in ["/v1/chat/completions", "/api/chat"] {
let resp = Client::new()
.post(format!("{base}{route}"))
.json(&two_user_turns)
.send()
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST, "{route}");
let text = resp.text().await.unwrap();
assert!(
text.contains("exceeds the available context size"),
"{route}: {text}"
);
}
assert_eq!(
other.requests().len(),
2,
"no second attempt for another 400"
);
let refusing = RoleStandin::new(Some(ALTERNATION_REFUSAL));
let base = serve_over("gemma3:1b", refusing.router.clone()).await;
let with_tool_calls = serde_json::json!({
"model": "gemma3:1b",
"messages": [
{ "role": "assistant", "content": null,
"tool_calls": [{ "id": "c", "type": "function",
"function": { "name": "f", "arguments": "{}" } }] },
{ "role": "assistant", "content": "again" },
],
});
let resp = Client::new()
.post(format!("{base}/v1/chat/completions"))
.json(&with_tool_calls)
.send()
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
assert!(resp.text().await.unwrap().contains("must alternate"));
assert_eq!(refusing.requests().len(), 1);
}
#[tokio::test(flavor = "multi_thread")]
async fn openai_embeddings_honour_dimensions() {
let base = serve_over("embeddinggemma", llama_server_standin()).await;
let ask = |extra: serde_json::Value| {
let body = serde_json::json!({ "model": "embeddinggemma", "input": "hello" });
let base = base.clone();
async move {
let resp = post_with(&base, "/v1/embeddings", body, extra).await;
resp.json::<serde_json::Value>().await.unwrap()
}
};
let cut = ask(serde_json::json!({ "dimensions": 2 })).await;
assert_eq!(cut["data"][0]["embedding"], serde_json::json!([0.6, 0.8]));
assert_eq!(cut["usage"]["prompt_tokens"], 2);
assert_eq!(cut["data"][0]["index"], 0);
for extra in [
serde_json::json!({}),
serde_json::json!({ "dimensions": 3 }),
serde_json::json!({ "dimensions": 768 }),
serde_json::json!({ "dimensions": 0 }),
] {
let whole = ask(extra.clone()).await;
let want = serde_json::json!([3.0, 4.0, 12.0]);
assert_eq!(whole["data"][0]["embedding"], want, "{extra}");
}
let cut = ask(serde_json::json!({ "dimensions": 2, "encoding_format": "base64" })).await;
let encoded = cut["data"][0]["embedding"].as_str().unwrap();
assert_eq!(decode_f32s(encoded).unwrap(), [0.6, 0.8]);
}
async fn systemone(base: &str, body: serde_json::Value) -> reqwest::Response {
stand_in_client()
.post(format!("{base}/v1/systemone"))
.json(&body)
.send()
.await
.unwrap()
}
fn received(reply: &str) -> String {
let reply: serde_json::Value = serde_json::from_str(reply).unwrap();
reply["received"].as_str().unwrap().to_string()
}
#[tokio::test(flavor = "multi_thread")]
async fn systemone_keeps_the_order_of_questions_and_answers() {
let (base, _) = serve_over_state("clef-flash", decision_server_standin(), true).await;
let text = r#"{"model":"clef-flash","state":"Win a cruise","questions":{"zebra?":{"type":"noul","instructions":"zebra?"},"apple?":{"type":"noul","instructions":"apple?"}}}"#;
let resp = stand_in_client()
.post(format!("{base}/v1/systemone"))
.header("content-type", "application/json")
.body(text)
.send()
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let reply = resp.text().await.unwrap();
let got = received(&reply);
assert!(
got.find("zebra?").unwrap() < got.find("apple?").unwrap(),
"{got}"
);
assert!(reply.find("\"zebra?\":{\"type").unwrap() < reply.find("\"apple?\":{\"type").unwrap());
let v: serde_json::Value = serde_json::from_str(&reply).unwrap();
assert_eq!(v["usage"]["input_tokens"], 3);
}
#[tokio::test(flavor = "multi_thread")]
async fn systemone_speaks_ollamas_images_and_names_the_clients_model() {
let (base, _) = serve_over_state("clef-flash", decision_server_standin(), true).await;
let resp = systemone(
&base,
serde_json::json!({
"model": "clef-flash",
"state": "Win a cruise",
"images": ["AAAA", "data:image/jpeg;base64,BBBB"],
"keep_alive": "30m",
"questions": { "q": { "type": "noul", "instructions": "q" } },
}),
)
.await;
assert_eq!(resp.status(), StatusCode::OK);
let reply = resp.text().await.unwrap();
let v: serde_json::Value = serde_json::from_str(&reply).unwrap();
assert_eq!(v["model"], "clef-flash");
assert!(!reply.contains(BACKEND_MODEL_PATH), "{reply}");
let got: serde_json::Value = serde_json::from_str(&received(&reply)).unwrap();
assert_eq!(
got["images"],
serde_json::json!(["data:image/png;base64,AAAA", "data:image/jpeg;base64,BBBB"])
);
assert!(got.get("keep_alive").is_none(), "{got}");
assert_eq!(got["model"], "docker.io/ai/clef-flash:latest");
}
#[tokio::test(flavor = "multi_thread")]
async fn systemone_applies_keep_alive_to_the_model() {
let (base, state) = serve_over_state("clef-flash", decision_server_standin(), true).await;
let kept = |state: &AppState| {
let state = state.clone();
async move {
let mgr = state.0.manager.lock().await;
mgr.running.values().next().unwrap().keep_alive
}
};
for (keep_alive, want) in [
(serde_json::json!("30m"), Some(Duration::from_secs(1800))),
(serde_json::json!(-1), None),
(serde_json::json!(90), Some(Duration::from_secs(90))),
] {
let resp = systemone(
&base,
serde_json::json!({
"model": "clef-flash", "state": "x", "keep_alive": keep_alive,
"questions": { "q": { "type": "noul", "instructions": "q" } },
}),
)
.await;
assert_eq!(resp.status(), StatusCode::OK);
assert_eq!(kept(&state).await, want, "{keep_alive}");
}
}
#[tokio::test]
async fn systemone_leaves_keep_alive_for_a_peer() {
let (origin, seen) = mock_peer(node(8 << 30, &["docker.io/ai/m:latest"], &[])).await;
let state = state_with_peers(vec![origin], 0);
let body = serde_json::json!({
"model": "docker.io/ai/m", "state": "x", "keep_alive": "1h",
"questions": { "q": { "type": "noul", "instructions": "q" } },
});
let resp = handle_openai_systemone(
State(state),
HeaderMap::new(),
Bytes::from(serde_json::to_vec(&body).unwrap()),
)
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::OK);
let calls = seen.lock().await;
let (path, headers, body) = &calls[0];
assert_eq!(path, "/v1/systemone");
assert_eq!(headers[aggregation::HOP], "1");
let body: serde_json::Value = serde_json::from_slice(body).unwrap();
assert_eq!(body["keep_alive"], "1h");
assert_eq!(body["model"], "docker.io/ai/m:latest");
}
#[tokio::test(flavor = "multi_thread")]
async fn systemone_takes_a_body_larger_than_axums_default_limit() {
let (base, _) = serve_over_state("clef-flash", decision_server_standin(), true).await;
let resp = systemone(
&base,
serde_json::json!({
"model": "clef-flash",
"state": "x".repeat(3 * 1024 * 1024),
"questions": { "q": { "type": "noul", "instructions": "q" } },
}),
)
.await;
assert_eq!(resp.status(), StatusCode::OK);
assert!(received(&resp.text().await.unwrap()).len() > 3 * 1024 * 1024);
}
#[tokio::test(flavor = "multi_thread")]
async fn systemone_explains_a_text_that_does_not_fit_one_batch() {
let body = serde_json::json!({
"model": "clef-flash", "state": "x",
"questions": { "q": { "type": "noul", "instructions": "q" } },
});
let failing = |message: &'static str| {
Router::new().route(
"/v1/systemone",
post(move || async move {
(
StatusCode::INTERNAL_SERVER_ERROR,
Json(serde_json::json!({
"error": { "code": 500, "message": message, "type": "server_error" }
})),
)
}),
)
};
let (base, _) = serve_over_state(
"clef-flash",
failing(
"input (9143 tokens) is too large to process. increase the physical batch size \
(current batch size: 8192)",
),
true,
)
.await;
let resp = systemone(&base, body.clone()).await;
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
let error = resp.json::<serde_json::Value>().await.unwrap()["error"]
.as_str()
.unwrap()
.to_string();
assert!(error.contains("longer than the 8192 tokens"), "{error}");
assert!(error.contains("LLMMAN_CONTEXT_LENGTH"), "{error}");
let (base, _) = serve_over_state("clef-flash", failing("something else"), true).await;
let resp = systemone(&base, body).await;
assert_eq!(resp.status(), StatusCode::INTERNAL_SERVER_ERROR);
let v = resp.json::<serde_json::Value>().await.unwrap();
assert_eq!(v["error"]["message"], "something else");
}
#[tokio::test(flavor = "multi_thread")]
async fn a_decision_model_refuses_to_generate_or_embed() {
let (base, state) = serve_over_state("clef-flash", decision_server_standin(), true).await;
let long_ago = Instant::now() - Duration::from_secs(3600);
for m in state.0.manager.lock().await.running.values_mut() {
m.last_active = long_ago;
}
let chat = serde_json::json!({
"model": "clef-flash", "messages": [{ "role": "user", "content": "Hello" }],
});
let generate = serde_json::json!({ "model": "clef-flash", "prompt": "Hello" });
let embed = serde_json::json!({ "model": "clef-flash", "input": "Hello" });
let responses = serde_json::json!({ "model": "clef-flash", "input": "Hello" });
let messages = serde_json::json!({
"model": "clef-flash", "max_tokens": 8,
"messages": [{ "role": "user", "content": "Hello" }],
});
for (route, body, says) in [
("/api/chat", &chat, "does not generate text"),
("/api/generate", &generate, "does not generate text"),
("/v1/chat/completions", &chat, "does not generate text"),
("/v1/completions", &generate, "does not generate text"),
("/v1/responses", &responses, "does not generate text"),
("/v1/messages", &messages, "does not generate text"),
("/api/embed", &embed, "does not support embeddings"),
(
"/api/embeddings",
&serde_json::json!({ "model": "clef-flash", "prompt": "Hello" }),
"does not support embeddings",
),
("/v1/embeddings", &embed, "does not support embeddings"),
] {
let resp = stand_in_client()
.post(format!("{base}{route}"))
.json(body)
.send()
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST, "{route}");
let text = resp.text().await.unwrap();
assert!(
text.contains("decision model") && text.contains(says),
"{route}: {text}"
);
}
for _ in 0..50 {
if state
.0
.manager
.lock()
.await
.running
.values()
.all(|m| m.in_flight == 0)
{
break;
}
sleep(Duration::from_millis(20)).await;
}
let mgr = state.0.manager.lock().await;
let m = mgr.running.values().next().unwrap();
assert_eq!(m.in_flight, 0);
assert_eq!(m.last_active, long_ago, "a refusal is not activity");
drop(mgr);
let ok = systemone(
&base,
serde_json::json!({
"model": "clef-flash", "state": "x",
"questions": { "q": { "type": "noul", "instructions": "q" } },
}),
)
.await;
assert_eq!(ok.status(), StatusCode::OK);
let base = serve_over("qwen3.5:0.8b", llama_server_standin()).await;
let resp = post_with(
&base,
"/v1/chat/completions",
serde_json::json!({
"model": "qwen3.5:0.8b", "messages": [{ "role": "user", "content": "Hi" }],
}),
serde_json::json!({}),
)
.await;
assert_eq!(resp.status(), StatusCode::OK);
}
#[tokio::test(flavor = "multi_thread")]
async fn a_decision_model_is_refused_before_it_loads() {
let dir = fresh_store_dir("decision-cold");
let state = test_state_at(dir.clone());
let path = crate::gguf::write_test_gguf_with_strings(&[], &[("llama.decision.type", "clef")]);
let bytes = std::fs::read(&path).unwrap();
let _ = std::fs::remove_file(&path);
store_one_model(
&OciStore::open(&dir).unwrap(),
"decisioncold",
"application/vnd.docker.ai.gguf.v3",
"model.gguf",
&bytes,
);
let key = running_key(&state, "docker.io/ai/bystander");
state
.0
.manager
.lock()
.await
.running
.insert(key.clone(), running_model_fixture(None, Duration::ZERO, 0));
for task in [Task::Generate, Task::Embed] {
let Err(err) = ensure_model_for(
&state,
"docker.io/ai/decisioncold",
None,
LoadOptions::default(),
task,
)
.await
else {
panic!("a decision model cannot do {task:?}");
};
assert_eq!(err.1, StatusCode::BAD_REQUEST);
assert!(
format!("{:#}", err.0).contains("is a decision model"),
"{:#}",
err.0
);
}
let running = state.0.manager.lock().await;
assert_eq!(running.running.len(), 1, "nothing was loaded");
assert!(running.running.contains_key(&key), "nothing was evicted");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn is_speech_model_keys_off_the_canary_architecture() {
let info = |strings: &[(&str, &str)]| {
let path = crate::gguf::write_test_gguf_with_strings(&[], strings);
let info = crate::gguf::read_info(&path).unwrap();
let _ = std::fs::remove_file(&path);
info
};
let canary = info(&[("general.architecture", "canary")]);
assert!(is_speech_model(&canary));
assert!(!is_speech_model(&info(&[])));
}
#[tokio::test(flavor = "multi_thread")]
async fn a_speech_model_is_refused_text_and_embeddings_cold_and_loaded() {
let dir = fresh_store_dir("speech-cold");
let state = test_state_at(dir.clone());
let path =
crate::gguf::write_test_gguf_with_strings(&[], &[("general.architecture", "canary")]);
let bytes = std::fs::read(&path).unwrap();
let _ = std::fs::remove_file(&path);
store_one_model(
&OciStore::open(&dir).unwrap(),
"speechcold",
"application/vnd.docker.ai.gguf.v3",
"model.gguf",
&bytes,
);
let refused = |task: Task| {
let state = state.clone();
async move {
let Err(err) = ensure_model_for(
&state,
"docker.io/ai/speechcold",
None,
LoadOptions::default(),
task,
)
.await
else {
panic!("a speech model cannot do {task:?}");
};
assert_eq!(err.1, StatusCode::BAD_REQUEST);
let message = format!("{:#}", err.0);
assert!(message.contains("is a speech model"), "{message}");
assert!(message.contains("/v1/audio/transcriptions"), "{message}");
}
};
for task in [Task::Generate, Task::Embed, Task::Forward] {
refused(task).await;
}
assert!(
state.0.manager.lock().await.running.is_empty(),
"nothing was loaded"
);
mark_loaded(&state, "docker.io/ai/speechcold", Engine::Speech, 1).await;
for task in [Task::Generate, Task::Embed, Task::Forward] {
refused(task).await;
}
let (_, target, guard) = ensure_model_for(
&state,
"docker.io/ai/speechcold",
None,
LoadOptions::default(),
Task::Other,
)
.await
.expect("a transcription reaches the loaded worker");
assert!(matches!(target, Target::Local(1)), "{target:?}");
guard.release_idle();
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn a_pull_the_source_says_found_nothing_is_a_404_and_any_other_failure_a_500() {
let missing =
|| crate::storage::oci::source_not_found(anyhow!("resolve: docker.io/ai/nope: not found"));
let err = pull_failure("nope:1b", missing());
assert_eq!(err.1, StatusCode::NOT_FOUND);
assert_eq!(
format!("{:#}", err.0),
"model 'nope:1b' not found: resolve: docker.io/ai/nope: not found"
);
assert_eq!(
pull_failure("nope:1b", missing().context("a")).1,
StatusCode::NOT_FOUND
);
let err = pull_failure("nope:1b", anyhow!("dial tcp: no such host"));
assert_eq!(err.1, StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(
format!("{:#}", err.0),
"pull failed: dial tcp: no such host"
);
}
fn chat_upstream(status: StatusCode, body: &'static str) -> ChatUpstream {
ChatUpstream {
status,
headers: HeaderMap::new(),
body: futures::stream::once(futures::future::ready(Ok(Bytes::from(body)))).boxed(),
}
}
const OVERFLOW: &str = r#"{"error":{"code":400,"message":"request (72031 tokens) exceeds the available context size (8192 tokens), try increasing it","type":"exceed_context_size_error","n_prompt_tokens":72031,"n_ctx":8192}}"#;
#[tokio::test]
async fn a_local_400_is_a_400_and_other_failures_stay_500() {
let Err(err) = chat_body(
&Target::Local(1),
chat_upstream(StatusCode::BAD_REQUEST, OVERFLOW),
)
.await
else {
panic!("an overflowing prompt is a failure");
};
assert_eq!(err.1, StatusCode::BAD_REQUEST);
assert!(err.0.downcast_ref::<ContextOverflow>().is_some());
assert!(format!("{:#}", err.0).contains("exceeds the available context size (8192 tokens)"));
let template = r#"{"error":{"code":400,"message":"Conversation roles must alternate user/assistant/user/assistant/...","type":"invalid_request_error"}}"#;
let Err(err) = chat_body(
&Target::Local(1),
chat_upstream(StatusCode::BAD_REQUEST, template),
)
.await
else {
panic!("a refused request is a failure");
};
assert_eq!(err.1, StatusCode::BAD_REQUEST);
assert!(err.0.downcast_ref::<ContextOverflow>().is_none());
assert!(format!("{:#}", err.0).contains("Conversation roles must alternate"));
for (status, body) in [
(StatusCode::UNAUTHORIZED, "{}"),
(StatusCode::NOT_FOUND, "{}"),
(StatusCode::INTERNAL_SERVER_ERROR, OVERFLOW),
] {
let Err(err) = chat_body(&Target::Local(1), chat_upstream(status, body)).await else {
panic!("{status} is a failure");
};
assert_eq!(err.1, StatusCode::INTERNAL_SERVER_ERROR, "{status} {body}");
}
let Err(err) = chat_body(
&remote_target("http://provider.invalid"),
chat_upstream(StatusCode::UNAUTHORIZED, "bad key"),
)
.await
else {
panic!("a refusal is a failure");
};
assert_eq!(err.1, StatusCode::UNAUTHORIZED);
}
const EMBEDDING_OVERFLOW: &str = r#"{"error":{"code":500,"message":"input (2603 tokens) is too large to process. increase the physical batch size (current batch size: 2048)","type":"server_error"}}"#;
fn json_response(status: StatusCode, body: &str) -> Response {
(
status,
[(reqwest::header::CONTENT_TYPE, "application/json")],
body.to_string(),
)
.into_response()
}
async fn body_bytes(resp: Response) -> Bytes {
axum::body::to_bytes(resp.into_body(), usize::MAX)
.await
.unwrap()
}
#[tokio::test]
async fn an_embeddings_input_past_the_context_is_a_400_and_nothing_else_changes() {
let resp = openai::explain_embedding_overflow(json_response(
StatusCode::INTERNAL_SERVER_ERROR,
EMBEDDING_OVERFLOW,
))
.await
.unwrap();
assert_eq!(resp.status(), StatusCode::BAD_REQUEST);
let v: serde_json::Value = serde_json::from_slice(&body_bytes(resp).await).unwrap();
assert_eq!(v["error"]["code"], 400);
assert_eq!(v["error"]["type"], "invalid_request_error");
assert_eq!(
v["error"]["message"],
"the input is longer than the 2048 tokens this model reads at once; shorten it"
);
for (status, body) in [
(StatusCode::OK, r#"{"data":[{"embedding":[0.5]}]}"#),
(
StatusCode::INTERNAL_SERVER_ERROR,
r#"{"error":{"message":"out of memory"}}"#,
),
(StatusCode::BAD_REQUEST, EMBEDDING_OVERFLOW),
] {
let resp = openai::explain_embedding_overflow(json_response(status, body))
.await
.unwrap();
assert_eq!(resp.status(), status, "{body}");
assert_eq!(body_bytes(resp).await, body.as_bytes());
}
}
async fn mock_llama_server(routes: &[(&str, serde_json::Value)]) -> u16 {
let mut app = Router::new();
for (path, body) in routes {
let body = body.clone();
app = app.route(
path,
get(move || {
let body = body.clone();
async move { Json(body) }
}),
);
}
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
port
}
async fn mark_loaded(state: &AppState, model: &str, engine: Engine, port: u16) {
let mut loaded = running_model_fixture_with_engine(engine, None);
loaded.port = port;
let key = running_key(state, model);
state.0.manager.lock().await.running.insert(key, loaded);
}
async fn fake_speech_server(reply: Option<(StatusCode, serde_json::Value)>) -> u16 {
let app = Router::new()
.route("/props", get(|| async { Json(serde_json::json!({})) }))
.route(
"/v1/audio/transcriptions",
post(move |headers: HeaderMap, body: Bytes| async move {
match reply {
Some((status, body)) => (status, Json(body)).into_response(),
None => {
let content_type = headers["content-type"].clone();
([("content-type", content_type)], body).into_response()
}
}
}),
);
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
port
}
const TRANSCRIPTION_AUDIO: &[u8] = b"RIFF\x00\xff\r\n--not-the-boundary\r\n\x00\x01WAVE";
async fn post_transcription(base: &str, model: &str) -> reqwest::Response {
let (body, headers) = multipart_upload(&[
("model", None, model.as_bytes()),
("file", Some(("a.wav", "audio/wav")), TRANSCRIPTION_AUDIO),
("language", None, b"en"),
]);
stand_in_client()
.post(format!("{base}/v1/audio/transcriptions"))
.header("content-type", headers["content-type"].clone())
.body(body)
.send()
.await
.unwrap()
}
#[tokio::test(flavor = "multi_thread")]
async fn a_speech_worker_gets_the_transcription_as_sent() {
for (engine, prompt) in [
(Engine::Speech, None),
(Engine::LlamaServer, Some(DEFAULT_TRANSCRIPTION_PROMPT)),
] {
let port = fake_speech_server(None).await;
let state = test_state();
mark_loaded(&state, "canary", engine, port).await;
let base = serve_router(state).await;
let resp = post_transcription(&base, "canary").await;
assert_eq!(resp.status(), StatusCode::OK, "{engine:?}");
let headers = headers_with(&[(
"content-type",
resp.headers()["content-type"].to_str().unwrap(),
)]);
let forwarded = multipart_parts(&resp.bytes().await.unwrap(), &headers).await;
let owned = |s: &str| Some(s.to_string());
let mut expected = vec![
("model".to_string(), None, None, b"canary".to_vec()),
(
"file".to_string(),
owned("a.wav"),
owned("audio/wav"),
TRANSCRIPTION_AUDIO.to_vec(),
),
("language".to_string(), None, None, b"en".to_vec()),
];
if let Some(prompt) = prompt {
expected.push(("prompt".to_string(), None, None, prompt.as_bytes().to_vec()));
}
assert_eq!(forwarded, expected, "{engine:?}");
}
}
#[tokio::test(flavor = "multi_thread")]
async fn a_speech_workers_reply_passes_through_unchanged() {
let error = serde_json::json!({
"error": {
"code": "unsupported_media_type",
"message": "the file is not a PCM16 mono 16 kHz WAV",
"type": "invalid_request_error"
}
});
let transcript =
serde_json::json!({ "text": "However, due to the slow communication channels" });
for (status, body) in [
(StatusCode::UNSUPPORTED_MEDIA_TYPE, error),
(StatusCode::OK, transcript),
] {
let port = fake_speech_server(Some((status, body.clone()))).await;
let state = test_state();
mark_loaded(&state, "canary", Engine::Speech, port).await;
let base = serve_router(state).await;
let resp = post_transcription(&base, "canary").await;
assert_eq!(resp.status(), status);
assert_eq!(resp.json::<serde_json::Value>().await.unwrap(), body);
}
}
async fn get_json(base: &str, path_and_query: &str) -> (StatusCode, serde_json::Value) {
let resp = reqwest::get(format!("{base}{path_and_query}"))
.await
.unwrap();
(resp.status(), resp.json().await.unwrap())
}
#[tokio::test]
async fn props_without_a_model_are_the_routers_own() {
let base = serve_router(test_state()).await;
for query in ["/props", "/props?model=", "/props?model=%20"] {
let (status, v) = get_json(&base, query).await;
assert_eq!(status, StatusCode::OK, "{query}");
assert_eq!(
(&v["role"], &v["total_slots"]),
(&"router".into(), &0.into()),
"{query}"
);
}
}
#[tokio::test]
async fn props_of_a_loaded_model_are_its_llama_servers() {
let props = serde_json::json!({
"total_slots": 4,
"default_generation_settings": { "n_ctx": 262144 },
"modalities": { "vision": true },
});
let port = mock_llama_server(&[("/props", props.clone())]).await;
let state = test_state();
mark_loaded(&state, "qwen3.5:2b", Engine::LlamaServer, port).await;
let key = running_key(&state, "qwen3.5:2b");
let idle_clock = state.0.manager.lock().await.running[&key].last_active;
let base = serve_router(state.clone()).await;
for spelling in ["qwen3.5:2b", "docker.io/ai/qwen3.5:2b"] {
let (status, v) = get_json(&base, &format!("/props?model={spelling}")).await;
assert_eq!(status, StatusCode::OK, "{spelling}");
assert_eq!(v, props, "{spelling}");
}
for _ in 0..100 {
if state.0.manager.lock().await.running[&key].in_flight == 0 {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
let mgr = state.0.manager.lock().await;
assert_eq!(mgr.running[&key].in_flight, 0);
assert_eq!(mgr.running[&key].last_active, idle_clock);
}
#[tokio::test]
async fn props_of_a_model_that_is_not_a_loaded_llama_server_are_a_404() {
let port = mock_llama_server(&[("/props", serde_json::json!({"total_slots": 4}))]).await;
let state = test_state();
mark_loaded(&state, "vllm-model", Engine::Vllm, port).await;
let base = serve_router(state.clone()).await;
for model in ["not-loaded", "vllm-model", "llmman.provider/openai/gpt-5"] {
for autoload in ["", "&autoload=true"] {
let (status, v) = get_json(&base, &format!("/props?model={model}{autoload}")).await;
assert_eq!(status, StatusCode::NOT_FOUND, "{model}{autoload}");
assert!(v["error"].as_str().unwrap().contains(model), "{v}");
}
}
assert_eq!(
state.0.manager.lock().await.running.len(),
1,
"nothing was loaded"
);
}
const GGUF_V3: &str = "application/vnd.docker.ai.gguf.v3";
fn gguf_bytes(u32s: &[(&str, u32)]) -> Vec<u8> {
let path = crate::gguf::write_test_gguf_with(u32s);
let bytes = std::fs::read(&path).unwrap();
let _ = std::fs::remove_file(&path);
bytes
}
async fn serve_store(
name: &str,
models: &[(&str, &str, &str, Vec<u8>)],
) -> (AppState, String, PathBuf) {
let dir = fresh_store_dir(name);
let store = OciStore::open(&dir).unwrap();
for (model, media_type, filepath, bytes) in models {
store_one_model(&store, model, media_type, filepath, bytes);
}
let state = test_state_at(dir.clone());
let base = serve_router(state.clone()).await;
(state, base, dir)
}
fn listed<'a>(v: &'a serde_json::Value, name: &str) -> &'a serde_json::Value {
v["data"]
.as_array()
.unwrap()
.iter()
.find(|m| m["id"] == format!("docker.io/ai/{name}:latest"))
.unwrap_or_else(|| panic!("{name} not listed: {v}"))
}
#[tokio::test]
async fn v1_models_carries_the_trained_context_of_models_that_are_not_loaded() {
let (_, base, dir) = serve_store(
"v1models",
&[
("modelstext", GGUF_V3, "model.gguf", gguf_bytes(&[])),
(
"modelsembed",
GGUF_V3,
"model.gguf",
gguf_bytes(&[("llama.pooling_type", 1)]),
),
(
"modelsst",
"application/vnd.cncf.model.weight.v1.tar",
"model.safetensors",
b"weights".to_vec(),
),
],
)
.await;
let (status, v) = get_json(&base, "/v1/models").await;
assert_eq!(status, StatusCode::OK);
let _ = std::fs::remove_dir_all(&dir);
let text = listed(&v, "modelstext");
assert_eq!(
text["meta"],
serde_json::json!({ "n_ctx": 4096, "n_ctx_train": 4096 })
);
assert_eq!(
text["architecture"],
serde_json::json!({ "input_modalities": ["text"], "output_modalities": ["text"] })
);
assert_eq!(text["status"]["value"], "unloaded");
let embed = listed(&v, "modelsembed");
assert_eq!(
embed["meta"],
serde_json::json!({ "n_ctx": 4096, "n_ctx_train": 4096 })
);
assert_eq!(embed.get("architecture"), None);
let st = listed(&v, "modelsst");
assert_eq!(st.get("meta"), None);
assert_eq!(
st["architecture"]["input_modalities"],
serde_json::json!(["text"])
);
}
#[tokio::test]
async fn v1_models_reports_the_context_an_unloaded_model_would_start_with() {
let dir = fresh_store_dir("v1startctx");
let store = OciStore::open(&dir).unwrap();
store_one_model(&store, "text", GGUF_V3, "model.gguf", &gguf_bytes(&[]));
let embed = gguf_bytes(&[("llama.pooling_type", 1)]);
store_one_model(&store, "embed", GGUF_V3, "model.gguf", &embed);
let image = br#"{"config":{"capabilities":{"outputTypes":["image"]}}}"#;
store_one_model_with_config(
&store,
"image",
GGUF_V3,
"model.gguf",
&gguf_bytes(&[]),
image,
);
let cases = [
((Some(DEFAULT_CTX_SIZE), false, None, None), (4096, 4096)),
((Some(DEFAULT_CTX_SIZE), false, Some(4), None), (4096, 4096)),
((Some(2048), true, None, None), (2048, 2048)),
((Some(65536), true, None, None), (4096, 4096)),
((Some(0), true, None, None), (4096, 4096)),
((Some(0), true, Some(2), None), (2048, 4096)),
((Some(2048), true, Some(4), None), (2048, 2048)),
(
(Some(DEFAULT_CTX_SIZE), false, None, Some(512)),
(4096, 512),
),
((Some(2048), true, None, Some(512)), (2048, 512)),
((Some(2048), true, None, Some(65536)), (2048, 4096)),
((Some(2048), true, None, Some(0)), (2048, 4096)),
];
for ((ctx_size, explicit, num_parallel, embedding_ctx), (text, embedding)) in cases {
let mut inner = test_inner(dir.clone());
inner.ctx_size = ctx_size;
inner.ctx_size_explicit = explicit;
inner.num_parallel = num_parallel;
inner.embedding_ctx_size = embedding_ctx;
let base = serve_router(AppState(Arc::new(inner))).await;
let (_, v) = get_json(&base, "/v1/models").await;
let case = format!(
"ctx_size {ctx_size:?} ({explicit}), num_parallel {num_parallel:?}, \
embedding {embedding_ctx:?}"
);
assert_eq!(listed(&v, "text")["meta"]["n_ctx"], text, "{case}");
assert_eq!(listed(&v, "embed")["meta"]["n_ctx"], embedding, "{case}");
assert_eq!(listed(&v, "image")["meta"].get("n_ctx"), None, "{case}");
}
let _ = std::fs::remove_dir_all(&dir);
}
#[tokio::test]
async fn v1_models_prefers_what_a_loaded_llama_server_reports() {
let live = serde_json::json!({
"data": [{
"id": "/blobs/sha256/abc",
"meta": { "n_ctx": 32768, "n_ctx_train": 262144 },
"architecture": { "input_modalities": ["text", "image"], "output_modalities": ["text"] },
}]
});
let older = serde_json::json!({ "data": [{ "meta": { "n_ctx": 8192 } }] });
let wrong_engine = serde_json::json!({ "data": [{ "meta": { "n_ctx": 1 } }] });
let silent = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
let dead_port = silent.local_addr().unwrap().port();
let models: Vec<_> = ["full", "older", "vllm", "dead"]
.into_iter()
.map(|name| (name, GGUF_V3, "model.gguf", gguf_bytes(&[])))
.collect();
let (state, base, dir) = serve_store("v1live", &models).await;
for (name, engine, port) in [
(
"full",
Engine::LlamaServer,
mock_llama_server(&[("/v1/models", live.clone())]).await,
),
(
"older",
Engine::LlamaServer,
mock_llama_server(&[("/v1/models", older)]).await,
),
(
"vllm",
Engine::Vllm,
mock_llama_server(&[("/v1/models", wrong_engine)]).await,
),
("dead", Engine::LlamaServer, dead_port),
] {
mark_loaded(&state, &format!("docker.io/ai/{name}:latest"), engine, port).await;
}
let (_, v) = get_json(&base, "/v1/models").await;
drop(silent);
let _ = std::fs::remove_dir_all(&dir);
let full = listed(&v, "full");
assert_eq!(full["status"]["value"], "loaded");
assert_eq!(full["meta"], live["data"][0]["meta"]);
assert_eq!(full["architecture"], live["data"][0]["architecture"]);
assert_eq!(full["id"], "docker.io/ai/full:latest");
let older = listed(&v, "older");
assert_eq!(older["meta"], serde_json::json!({ "n_ctx": 8192 }));
assert_eq!(
older["architecture"]["input_modalities"],
serde_json::json!(["text"])
);
for name in ["vllm", "dead"] {
let entry = listed(&v, name);
assert_eq!(entry["status"]["value"], "loaded", "{name}");
assert_eq!(
entry["meta"],
serde_json::json!({ "n_ctx_train": 4096 }),
"{name}"
);
}
}
#[tokio::test]
async fn a_model_loaded_by_digest_is_found_through_its_tag() {
let props = serde_json::json!({ "total_slots": 4 });
let live = serde_json::json!({ "data": [{ "meta": { "n_ctx": 65536 } }] });
let port = mock_llama_server(&[("/props", props.clone()), ("/v1/models", live)]).await;
let (state, base, dir) = serve_store(
"bydigest",
&[("bydigest", GGUF_V3, "model.gguf", gguf_bytes(&[]))],
)
.await;
let digest = OciStore::open(&dir)
.unwrap()
.find("docker.io/ai/bydigest:latest")
.unwrap()
.digest;
let mut loaded = running_model_fixture_with_engine(Engine::LlamaServer, None);
(loaded.port, loaded.digest) = (port, digest.clone());
let key = format!("docker.io/ai/bydigest@{digest}");
state.0.manager.lock().await.running.insert(key, loaded);
let (status, v) = get_json(&base, "/props?model=bydigest").await;
assert_eq!((status, &v), (StatusCode::OK, &props));
let (_, v) = get_json(&base, "/v1/models").await;
let entry = listed(&v, "bydigest");
assert_eq!(entry["status"]["value"], "loaded");
assert_eq!(entry["meta"], serde_json::json!({ "n_ctx": 65536 }));
let _ = std::fs::remove_dir_all(&dir);
}
#[tokio::test]
async fn v1_models_takes_a_peers_live_facts_for_a_model_loaded_only_there() {
let peer_row = |name: &str, n_ctx: u64| {
serde_json::json!({
"id": format!("docker.io/ai/{name}:latest"), "object": "model",
"status": { "value": "loaded" },
"meta": { "n_ctx": n_ctx },
"architecture": { "input_modalities": ["text", "image"] },
})
};
let metaless = serde_json::json!({
"id": "docker.io/ai/metaless:latest", "object": "model",
"status": { "value": "loaded" },
});
let peer = mock_llama_server(&[(
"/v1/models",
serde_json::json!({
"data": [peer_row("remote", 111), peer_row("local", 222), metaless],
}),
)])
.await;
let own = mock_llama_server(&[(
"/v1/models",
serde_json::json!({ "data": [{ "meta": { "n_ctx": 333 } }] }),
)])
.await;
let dir = fresh_store_dir("peerfacts");
let store = OciStore::open(&dir).unwrap();
for name in ["remote", "local", "metaless"] {
store_one_model(&store, name, GGUF_V3, "model.gguf", &gguf_bytes(&[]));
}
let mut inner = test_inner(dir.clone());
inner.peers = vec![format!("http://127.0.0.1:{peer}")];
let state = AppState(Arc::new(inner));
mark_loaded(
&state,
"docker.io/ai/local:latest",
Engine::LlamaServer,
own,
)
.await;
let base = serve_router(state).await;
let (_, v) = get_json(&base, "/v1/models").await;
let _ = std::fs::remove_dir_all(&dir);
let remote = listed(&v, "remote");
assert_eq!(remote["status"]["value"], "loaded");
assert_eq!(remote["meta"], serde_json::json!({ "n_ctx": 111 }));
assert_eq!(
remote["architecture"]["input_modalities"],
serde_json::json!(["text", "image"])
);
assert_eq!(
listed(&v, "local")["meta"],
serde_json::json!({ "n_ctx": 333 })
);
let metaless = listed(&v, "metaless");
assert_eq!(metaless["status"]["value"], "loaded");
assert_eq!(metaless["meta"], serde_json::json!({ "n_ctx_train": 4096 }));
}
#[tokio::test]
async fn handle_tags_reports_the_family_size_and_quantization_of_each_gguf() {
let (state, _, dir) = serve_store(
"tagsdetails",
&[
(
"tagsmeta",
GGUF_V3,
"model.gguf",
gguf_bytes(&[("general.file_type", 15)]),
),
(
"tagsnoheader",
"application/vnd.cncf.model.weight.v1.tar",
"model.safetensors",
b"weights".to_vec(),
),
],
)
.await;
let tags = |state: AppState| async move {
let resp = handle_tags(State(state), HeaderMap::new())
.await
.into_response();
assert_eq!(resp.status(), StatusCode::OK);
serde_json::from_slice::<serde_json::Value>(&body_bytes(resp).await).unwrap()
};
let details = |v: &serde_json::Value, name: &str| {
v["models"]
.as_array()
.unwrap()
.iter()
.find(|m| m["name"] == format!("docker.io/ai/{name}:latest"))
.unwrap_or_else(|| panic!("{name} not listed: {v}"))["details"]
.clone()
};
let first = tags(state.clone()).await;
let gguf = details(&first, "tagsmeta");
assert_eq!(
gguf,
serde_json::json!({
"parent_model": "",
"format": "gguf",
"family": "llama",
"families": ["llama"],
"parameter_size": "32",
"quantization_level": "Q4_K_M",
})
);
let st = details(&first, "tagsnoheader");
assert_eq!(
(
&st["format"],
&st["family"],
&st["families"],
&st["parameter_size"]
),
(
&"safetensors".into(),
&"".into(),
&serde_json::json!([]),
&"".into()
)
);
assert_eq!(ollama::cached_model_facts(&dir), 1);
for entry in std::fs::read_dir(dir.join("blobs").join("sha256")).unwrap() {
let path = entry.unwrap().path();
if std::fs::read(&path).unwrap().starts_with(b"GGUF") {
std::fs::remove_file(&path).unwrap();
}
}
assert_eq!(details(&tags(state.clone()).await, "tagsmeta"), gguf);
OciStore::open(&dir)
.unwrap()
.remove("docker.io/ai/tagsmeta:latest")
.unwrap();
tags(state).await;
assert_eq!(ollama::cached_model_facts(&dir), 0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn pull_only_step_classifies_only_after_a_container_engine_was_picked() {
use crate::container::{ContainerEngine, ContainerManager};
use runtime::{Resolved, Runtime};
let local = Resolved::Local {
source: Runtime::Bin,
bin: std::path::PathBuf::from("/tmp/llama-server"),
};
let container = Resolved::Container(ContainerManager::Docker);
let unstored = || anyhow::bail!("--pull-only: pull first");
assert_eq!(
pull_only_step(&local, unstored).unwrap(),
PullOnlyStep::Done
);
assert_eq!(
pull_only_step(&local, || panic!("a local answer never classifies")).unwrap(),
PullOnlyStep::Done
);
assert!(pull_only_step(&container, unstored)
.unwrap_err()
.to_string()
.contains("pull first"));
assert_eq!(
pull_only_step(&container, || Ok(ContainerEngine::Speech)).unwrap(),
PullOnlyStep::ResolveCpuOnly
);
assert_eq!(
pull_only_step(&container, || Ok(ContainerEngine::LlamaServer)).unwrap(),
PullOnlyStep::Done
);
assert_eq!(
pull_only_step(&container, || Ok(ContainerEngine::Vllm)).unwrap(),
PullOnlyStep::PullEngine(ContainerEngine::Vllm)
);
assert_eq!(
pull_only_step(&container, || Ok(ContainerEngine::Sglang)).unwrap(),
PullOnlyStep::PullEngine(ContainerEngine::Sglang)
);
}