//! Axum-based HTTP server implementation for Ferrum
//!
//! This module provides a concrete implementation of the HttpServer trait
//! using the Axum web framework, with OpenAI-shaped endpoint compatibility.
use crate::{
chat_template::{
render_chat_prompt_with_model_template_options_and_compatibility_with_prefill,
render_chat_prompt_with_tools_and_model_template_compatibility_with_prefill,
ChatTemplateOptions, ModelChatTemplate, ModelReasoningProtocol, ReasoningEffort,
},
model_registry::{LoraAdapterModel, ServedModelKind, ServedModelRegistry},
openai::*,
traits::HttpServer,
types::*,
};
use async_trait::async_trait;
use axum::{
extract::{multipart::MultipartRejection, rejection::JsonRejection, State},
http::{HeaderMap, StatusCode as AxumStatusCode},
response::{sse::Event, IntoResponse, Response, Sse},
routing::{get, post},
Json, Router,
};
use ferrum_bench_core::{
BenchmarkRequestCorrelation, BENCHMARK_CELL_ID_HEADER, BENCHMARK_PHASE_HEADER,
BENCHMARK_REPEAT_INDEX_HEADER, BENCHMARK_REQUEST_INDEX_HEADER, BENCHMARK_RUN_ID_HEADER,
};
use ferrum_interfaces::engine::{EmbedEngine, LlmInferenceEngine, TranscribeEngine, TtsEngine};
use ferrum_types::{
has_unclosed_model_reasoning_block, model_reasoning_markers,
parse_harmony_response_for_finish_reason, parse_model_reasoning_response,
should_defer_model_reasoning_stream_delta, EngineMetrics, EngineStatus, FerrumConfigBuilder,
FerrumError as Error, FerrumProfileEvent, FinishReason, InferenceExecutionEvidence,
InferenceRequest, InferenceResponse, ModelId, ModelOutputProtocol, NativeChatOutputProjector,
ParsedReasoningResponse, Priority, ProcessMemoryObservation, ProcessMemorySample,
ProcessMemorySampler, ProfileEntrypoint, ProfileError, ProfileEventKind, ProfileStatus,
ReplayReference, RequestId, ResolvedFerrumConfig, ResourceAction, ResourceTraceEvent,
ResponseCompletionBoundary, RuntimeConfigSnapshot, SamplingParams, StructuredOutputStart,
TokenId, TokenUsage, DEFAULT_CHAT_REPETITION_PENALTY, DEFAULT_MAX_TOKENS_METADATA_KEY,
OBSERVABILITY_PROFILE_SCHEMA_VERSION, PROMPT_OPENED_REASONING_METADATA_KEY, THINK_END_TAG,
THINK_START_TAG,
};
use sha2::{Digest, Sha256};
use std::{
collections::{BTreeMap, HashMap},
error::Error as StdError,
fs,
path::{Path, PathBuf},
sync::{
atomic::{AtomicBool, Ordering},
Arc, Mutex, OnceLock,
},
time::Instant,
};
use tokio::sync::{mpsc, Notify};
use tokio_stream::StreamExt;
use tower::ServiceBuilder;
use tower_http::{cors::CorsLayer, trace::TraceLayer};
use tracing::{debug, error, info, span, warn, Level};
use uuid::Uuid;
mod responses;
const DEFAULT_SAMPLING_TEMPERATURE: f32 = 0.0;
const DEFAULT_SAMPLING_TOP_P: f32 = 1.0;
const DEFAULT_COMPLETION_MAX_TOKENS: u32 = 4096;
const INITIAL_FORBIDDEN_TOKEN_TEXTS_METADATA_KEY: &str = "ferrum_initial_forbidden_token_texts";
const DEFAULT_GUIDED_TOOL_ARGUMENT_STRING_MAX_LENGTH: u64 = 128;
const MAX_CACHED_JSON_SCHEMA_VALIDATORS: usize = 64;
const INITIAL_STRUCTURED_CALL_FORBIDDEN_TOKEN_TEXTS: &[&str] =
&["<|im_end|>", "<|endoftext|>", "<|eot_id|>", "</s>"];
const FERRUM_SESSION_HEADER: &str = "x-ferrum-session";
static JSON_SCHEMA_VALIDATOR_CACHE: OnceLock<Mutex<HashMap<String, Arc<jsonschema::Validator>>>> =
OnceLock::new();
/// Product defaults used when an OpenAI chat request omits sampling fields.
/// The engine composition root consumes the same typed value so its resolved
/// startup plan cannot describe different defaults from the HTTP endpoint.
pub fn default_chat_sampling_params() -> SamplingParams {
SamplingParams {
max_tokens: DEFAULT_COMPLETION_MAX_TOKENS as usize,
temperature: DEFAULT_SAMPLING_TEMPERATURE,
top_p: DEFAULT_SAMPLING_TOP_P,
repetition_penalty: DEFAULT_CHAT_REPETITION_PENALTY,
..SamplingParams::default()
}
}
#[derive(Debug, Clone)]
struct CachePolicy {
prefix_cache_enabled: bool,
session_cache_mode: String,
session_cache_max_entries: usize,
session_cache_max_tokens: usize,
}
impl CachePolicy {
fn current() -> Self {
Self {
prefix_cache_enabled: env_bool("FERRUM_PREFIX_CACHE_PRODUCT")
.or_else(|| env_bool("FERRUM_PREFIX_CACHE_REQUESTED"))
.or_else(|| env_bool("FERRUM_PREFIX_CACHE"))
.unwrap_or(false),
session_cache_mode: std::env::var("FERRUM_SESSION_CACHE")
.unwrap_or_else(|_| "off".to_string())
.to_ascii_lowercase(),
session_cache_max_entries: env_usize("FERRUM_SESSION_CACHE_MAX_ENTRIES").unwrap_or(128),
session_cache_max_tokens: env_usize("FERRUM_SESSION_CACHE_MAX_TOKENS").unwrap_or(4096),
}
}
fn session_memory_enabled(&self) -> bool {
self.session_cache_mode == "memory"
}
}
fn env_bool(key: &str) -> Option<bool> {
match std::env::var(key).ok()?.to_ascii_lowercase().as_str() {
"1" | "true" | "yes" | "on" => Some(true),
"0" | "false" | "no" | "off" => Some(false),
_ => None,
}
}
fn env_usize(key: &str) -> Option<usize> {
std::env::var(key).ok()?.parse().ok()
}
/// Shared Prometheus recorder handle for rendering metrics.
static PROM_HANDLE: std::sync::OnceLock<metrics_exporter_prometheus::PrometheusHandle> =
std::sync::OnceLock::new();
/// Initialize the Prometheus metrics recorder.
///
/// Must be called once before any `metrics::counter!()` / `histogram!()` calls.
/// Safe to call multiple times — subsequent calls are no-ops.
pub fn init_prometheus_recorder() {
PROM_HANDLE.get_or_init(|| {
let builder = metrics_exporter_prometheus::PrometheusBuilder::new();
let handle = builder
.install_recorder()
.expect("Failed to install Prometheus recorder");
info!("Prometheus metrics recorder installed");
handle
});
}
/// Axum-based server implementation.
///
/// The server is built around [`AppState`], which holds an optional
/// engine per modality. Handlers fault to 503 when the modality they
/// need isn't loaded, instead of running stub error logic.
pub struct AxumServer {
state: AppState,
config: ServerConfig,
lifecycle: Arc<AxumServerLifecycle>,
}
#[derive(Default)]
struct AxumServerLifecycle {
shutdown_requested: AtomicBool,
running: AtomicBool,
engines_stopped: AtomicBool,
shutdown_notify: Notify,
stopped_notify: Notify,
stop_lock: tokio::sync::Mutex<()>,
}
impl AxumServerLifecycle {
fn request_shutdown(&self) {
self.shutdown_requested.store(true, Ordering::Release);
self.shutdown_notify.notify_waiters();
}
async fn wait_for_shutdown(&self) {
while !self.shutdown_requested.load(Ordering::Acquire) {
self.shutdown_notify.notified().await;
}
}
async fn wait_until_stopped(&self) {
while self.running.load(Ordering::Acquire) {
self.stopped_notify.notified().await;
}
}
}
struct AxumServerRunGuard {
lifecycle: Arc<AxumServerLifecycle>,
}
impl Drop for AxumServerRunGuard {
fn drop(&mut self) {
self.lifecycle.running.store(false, Ordering::Release);
self.lifecycle.stopped_notify.notify_waiters();
}
}
fn single_model_registry(engine_model_id: ModelId, kind: ServedModelKind) -> ServedModelRegistry {
let public_name = engine_model_id.to_string();
ServedModelRegistry::try_new(engine_model_id, kind, vec![public_name], vec![])
.expect("engine config must contain a valid model id")
}
impl AxumServer {
/// Create a server with a fully populated AppState.
pub fn from_state(state: AppState) -> Self {
Self {
state,
config: ServerConfig::default(),
lifecycle: Arc::new(AxumServerLifecycle::default()),
}
}
/// Convenience constructor for an LLM-only server (chat / completions).
pub fn from_llm(engine: Arc<dyn LlmInferenceEngine + Send + Sync>) -> Self {
Self::from_state(AppState::default().with_llm(engine))
}
/// Convenience constructor for an embedding-only server (`/v1/embeddings`).
pub fn from_embed(engine: Arc<dyn EmbedEngine + Send + Sync>) -> Self {
Self::from_state(AppState::default().with_embed(engine))
}
/// Convenience constructor for a transcription-only server
/// (`/v1/audio/transcriptions`).
pub fn from_transcribe(engine: Arc<dyn TranscribeEngine + Send + Sync>) -> Self {
Self::from_state(AppState::default().with_transcribe(engine))
}
/// Convenience constructor for a TTS-only server (`/v1/audio/speech`).
pub fn from_tts(engine: Arc<dyn TtsEngine + Send + Sync>) -> Self {
Self::from_state(AppState::default().with_tts(engine))
}
/// Attach the startup auto-configuration decision trace exposed by
/// `/health`. Constructors keep this optional so tests and non-LLM
/// deployments can use the server without a model-specific resolver.
pub fn with_auto_config(mut self, auto_config: ResolvedFerrumConfig) -> Self {
self.state = self.state.with_auto_config(auto_config);
self
}
/// Attach the loaded model's prompt template, if available. This keeps
/// OpenAI request aliases from driving prompt-family selection.
pub fn with_prompt_template(mut self, prompt_template: Option<ModelChatTemplate>) -> Self {
self.state = self.state.with_prompt_template(prompt_template);
self
}
/// Set the server-wide reasoning default. Explicit standard effort or
/// `chat_template_kwargs.enable_thinking` overrides this default.
pub fn with_default_enable_thinking(mut self, enable_thinking: Option<bool>) -> Self {
self.state = self.state.with_default_enable_thinking(enable_thinking);
self
}
/// Control the compatibility retry for model templates that reject
/// non-leading system messages. It is enabled by default.
pub fn with_interleaved_system_coalescing(mut self, enabled: bool) -> Self {
self.state = self.state.with_interleaved_system_coalescing(enabled);
self
}
/// Install the public OpenAI model namespace used for request routing and
/// `/v1/models`. The registry keeps public aliases separate from the
/// engine's internal model id.
pub fn with_served_model_registry(mut self, registry: ServedModelRegistry) -> Self {
self.state = self.state.with_served_model_registry(registry);
self
}
/// Attach startup-loaded LoRA adapter model ids.
pub fn with_lora_adapters(
mut self,
base_model_id: impl Into<String>,
adapters: Vec<LoraAdapterModel>,
) -> ferrum_types::Result<Self> {
let base_model_id = base_model_id.into();
let registry = if self.state.served_model_registry.is_empty() {
ServedModelRegistry::try_new(
base_model_id.clone(),
ServedModelKind::Llm,
vec![base_model_id],
adapters,
)
} else {
self.state
.served_model_registry
.try_with_lora_adapters(&base_model_id, adapters)
}
.map_err(|error| Error::config(error.to_string()))?;
self.state = self.state.with_served_model_registry(registry);
Ok(self)
}
async fn shutdown_loaded_engines(&self) -> ferrum_types::Result<()> {
let mut first_error = None;
if let Some(engine) = &self.state.llm {
if let Err(error) = engine.shutdown().await {
first_error = Some(error);
}
}
if let Some(engine) = &self.state.embed {
if let Err(error) = engine.shutdown().await {
if first_error.is_none() {
first_error = Some(error);
}
}
}
if let Some(engine) = &self.state.transcribe {
if let Err(error) = engine.shutdown().await {
if first_error.is_none() {
first_error = Some(error);
}
}
}
if let Some(engine) = &self.state.tts {
if let Err(error) = engine.shutdown().await {
if first_error.is_none() {
first_error = Some(error);
}
}
}
first_error.map_or(Ok(()), Err)
}
/// Build the router with all routes
#[allow(dead_code)]
fn build_router(&self) -> Router {
self.build_router_with_state(self.state.clone())
}
fn build_router_with_state(&self, app_state: AppState) -> Router {
Router::new()
// OpenAI API routes
.route("/v1/chat/completions", post(chat_completions_handler))
.route("/v1/responses", post(responses::responses_handler))
.route("/v1/completions", post(completions_handler))
.route("/v1/embeddings", post(embeddings_handler))
.route("/v1/audio/transcriptions", post(transcriptions_handler))
.route("/v1/audio/speech", post(speech_handler))
.route("/v1/models", get(models_handler))
// Health & observability
.route("/health", get(health_handler))
.route("/metrics", get(metrics_handler))
.route("/", get(root_handler))
// Apply middleware
.layer(
ServiceBuilder::new()
.layer(TraceLayer::new_for_http())
.layer(CorsLayer::permissive()), // For MVP, allow all origins
)
.with_state(app_state)
}
}
/// Application state shared across handlers — one optional engine per
/// modality. Handlers reach into the field they need and 503 when it's
/// not loaded.
#[derive(Clone, Default)]
pub struct AppState {
pub llm: Option<Arc<dyn LlmInferenceEngine + Send + Sync>>,
pub embed: Option<Arc<dyn EmbedEngine + Send + Sync>>,
pub transcribe: Option<Arc<dyn TranscribeEngine + Send + Sync>>,
pub tts: Option<Arc<dyn TtsEngine + Send + Sync>>,
pub auto_config: Option<ResolvedFerrumConfig>,
pub prompt_template: Option<Arc<ModelChatTemplate>>,
pub default_enable_thinking: Option<bool>,
interleaved_system_coalescing: Option<bool>,
pub served_model_registry: Arc<ServedModelRegistry>,
pub request_dump_dir: Option<Arc<PathBuf>>,
pub profile_jsonl: Option<Arc<PathBuf>>,
pub profile_detail: ferrum_types::ObservabilityProfileDetail,
pub memory_profile_jsonl: Option<Arc<PathBuf>>,
pub first_request_memory_recorded: Arc<AtomicBool>,
cache: Arc<CacheRuntimeState>,
}
impl AppState {
pub fn with_llm(mut self, engine: Arc<dyn LlmInferenceEngine + Send + Sync>) -> Self {
if self.served_model_registry.is_empty() {
self.served_model_registry = Arc::new(single_model_registry(
engine.config().model.model_id.clone(),
ServedModelKind::Llm,
));
}
self.llm = Some(engine);
self
}
pub fn with_embed(mut self, engine: Arc<dyn EmbedEngine + Send + Sync>) -> Self {
if self.served_model_registry.is_empty() {
self.served_model_registry = Arc::new(single_model_registry(
engine.config().model.model_id.clone(),
ServedModelKind::Embedding,
));
}
self.embed = Some(engine);
self
}
pub fn with_transcribe(mut self, engine: Arc<dyn TranscribeEngine + Send + Sync>) -> Self {
if self.served_model_registry.is_empty() {
self.served_model_registry = Arc::new(single_model_registry(
engine.config().model.model_id.clone(),
ServedModelKind::Transcription,
));
}
self.transcribe = Some(engine);
self
}
pub fn with_tts(mut self, engine: Arc<dyn TtsEngine + Send + Sync>) -> Self {
if self.served_model_registry.is_empty() {
self.served_model_registry = Arc::new(single_model_registry(
engine.config().model.model_id.clone(),
ServedModelKind::Speech,
));
}
self.tts = Some(engine);
self
}
pub fn with_auto_config(mut self, auto_config: ResolvedFerrumConfig) -> Self {
self.auto_config = Some(auto_config);
self
}
pub fn with_prompt_template(mut self, prompt_template: Option<ModelChatTemplate>) -> Self {
self.prompt_template = prompt_template.map(Arc::new);
self
}
pub fn with_default_enable_thinking(mut self, enable_thinking: Option<bool>) -> Self {
self.default_enable_thinking = enable_thinking;
self
}
pub fn with_interleaved_system_coalescing(mut self, enabled: bool) -> Self {
self.interleaved_system_coalescing = Some(enabled);
self
}
pub fn with_served_model_registry(mut self, registry: ServedModelRegistry) -> Self {
self.served_model_registry = Arc::new(registry);
self
}
pub fn with_request_dump_dir(mut self, request_dump_dir: Option<PathBuf>) -> Self {
self.request_dump_dir = request_dump_dir.map(Arc::new);
self
}
pub fn with_profile_jsonl(mut self, profile_jsonl: Option<PathBuf>) -> Self {
self.profile_jsonl = profile_jsonl.map(Arc::new);
self
}
pub fn with_profile_detail(
mut self,
profile_detail: ferrum_types::ObservabilityProfileDetail,
) -> Self {
self.profile_detail = profile_detail;
self
}
pub fn with_memory_profile_jsonl(mut self, memory_profile_jsonl: Option<PathBuf>) -> Self {
self.memory_profile_jsonl = memory_profile_jsonl.map(Arc::new);
self
}
/// Async aggregated status across whichever modality is loaded.
/// In single-modality deployments (current CLI), exactly one is Some.
async fn status(&self) -> EngineStatus {
if let Some(e) = &self.llm {
return e.status().await;
}
if let Some(e) = &self.embed {
return e.status().await;
}
if let Some(e) = &self.transcribe {
return e.status().await;
}
if let Some(e) = &self.tts {
return e.status().await;
}
EngineStatus {
is_ready: false,
loaded_models: vec![],
active_requests: 0,
queued_requests: 0,
memory_usage: ferrum_types::MemoryUsage {
total_bytes: 0,
used_bytes: 0,
free_bytes: 0,
gpu_memory_bytes: None,
cpu_memory_bytes: None,
cache_memory_bytes: 0,
utilization_percent: 0.0,
},
uptime_seconds: 0,
last_heartbeat: chrono::Utc::now(),
version: env!("CARGO_PKG_VERSION").to_string(),
}
}
fn metrics(&self) -> EngineMetrics {
if let Some(e) = &self.llm {
return e.metrics();
}
if let Some(e) = &self.embed {
return e.metrics();
}
if let Some(e) = &self.transcribe {
return e.metrics();
}
if let Some(e) = &self.tts {
return e.metrics();
}
EngineMetrics {
total_requests: 0,
successful_requests: 0,
failed_requests: 0,
avg_request_latency_ms: 0.0,
p95_request_latency_ms: 0.0,
p99_request_latency_ms: 0.0,
throughput_rps: 0.0,
tokens_per_second: 0.0,
queue_metrics: Default::default(),
resource_utilization: Default::default(),
error_stats: Default::default(),
performance_breakdown: Default::default(),
}
}
}
#[derive(Default)]
struct CacheRuntimeState {
stats: Mutex<CacheStats>,
prefix_prompts: Mutex<HashMap<String, usize>>,
sessions: Mutex<HashMap<String, Vec<ChatMessage>>>,
}
#[derive(Debug, Clone, Default)]
struct CacheStats {
prefix_hits: u64,
prefix_misses: u64,
prefix_evictions: u64,
prefix_saved_prefill_tokens: u64,
prefix_entries: u64,
prefix_bytes: u64,
session_hits: u64,
session_misses: u64,
session_evictions: u64,
session_entries: u64,
session_tokens: u64,
}
#[derive(Clone)]
struct SessionContext {
id: String,
prior_messages: Vec<ChatMessage>,
incoming_messages: Vec<ChatMessage>,
}
impl CacheRuntimeState {
fn record_prefix_prompt(&self, prompt: &str, policy: &CachePolicy) {
if !policy.prefix_cache_enabled {
return;
}
let prompt_tokens = approx_tokens(prompt);
let mut prompts = self.prefix_prompts.lock().expect("prefix cache lock");
let saved_tokens = prompts
.keys()
.map(|seen| approx_tokens_for_chars(longest_common_prefix_chars(seen, prompt)))
.max()
.unwrap_or(0);
let mut stats = self.stats.lock().expect("cache stats lock");
if saved_tokens > 0 {
stats.prefix_hits += 1;
stats.prefix_saved_prefill_tokens += saved_tokens as u64;
} else {
stats.prefix_misses += 1;
}
let max_entries = policy.session_cache_max_entries.max(1);
if !prompts.contains_key(prompt) && prompts.len() >= max_entries {
if let Some(key) = prompts.keys().next().cloned() {
prompts.remove(&key);
stats.prefix_evictions += 1;
}
}
prompts.insert(prompt.to_string(), prompt_tokens);
stats.prefix_entries = prompts.len() as u64;
stats.prefix_bytes = prompts.keys().map(|key| key.len() as u64).sum();
}
fn prepare_session_request(
&self,
request: &mut ChatCompletionsRequest,
headers: &HeaderMap,
policy: &CachePolicy,
) -> Option<SessionContext> {
let session_id = request_session_id(headers, request)?;
if !policy.session_memory_enabled() {
return None;
}
let incoming_messages = request.messages.clone();
let prior_messages = {
let sessions = self.sessions.lock().expect("session cache lock");
sessions.get(&session_id).cloned().unwrap_or_default()
};
{
let mut stats = self.stats.lock().expect("cache stats lock");
if prior_messages.is_empty() {
stats.session_misses += 1;
} else {
stats.session_hits += 1;
let mut merged = prior_messages.clone();
merged.extend(request.messages.clone());
request.messages = merged;
}
}
Some(SessionContext {
id: session_id,
prior_messages,
incoming_messages,
})
}
fn update_session(
&self,
context: Option<SessionContext>,
assistant_message: ChatMessage,
policy: &CachePolicy,
) {
let Some(context) = context else {
return;
};
if !policy.session_memory_enabled() {
return;
}
let mut history = context.prior_messages;
history.extend(context.incoming_messages);
history.push(assistant_message);
trim_messages_to_token_budget(&mut history, policy.session_cache_max_tokens);
let mut sessions = self.sessions.lock().expect("session cache lock");
if !sessions.contains_key(&context.id)
&& sessions.len() >= policy.session_cache_max_entries.max(1)
{
if let Some(evict_key) = sessions.keys().next().cloned() {
sessions.remove(&evict_key);
self.stats
.lock()
.expect("cache stats lock")
.session_evictions += 1;
}
}
sessions.insert(context.id, history);
let entries = sessions.len() as u64;
let tokens = sessions
.values()
.map(|messages| {
messages
.iter()
.map(|msg| approx_tokens(&msg.content))
.sum::<usize>()
})
.sum::<usize>() as u64;
let mut stats = self.stats.lock().expect("cache stats lock");
stats.session_entries = entries;
stats.session_tokens = tokens;
}
fn stats(&self) -> CacheStats {
let mut stats = self.stats.lock().expect("cache stats lock").clone();
stats.prefix_entries = self.prefix_prompts.lock().expect("prefix cache lock").len() as u64;
let sessions = self.sessions.lock().expect("session cache lock");
stats.session_entries = sessions.len() as u64;
stats.session_tokens = sessions
.values()
.map(|messages| {
messages
.iter()
.map(|msg| approx_tokens(&msg.content))
.sum::<usize>()
})
.sum::<usize>() as u64;
stats
}
fn health_json(
&self,
policy: &CachePolicy,
engine_prefix_cache: Option<&serde_json::Value>,
) -> serde_json::Value {
let stats = self.stats();
let prefix_hits = engine_u64(engine_prefix_cache, "hits").unwrap_or(stats.prefix_hits);
let prefix_misses =
engine_u64(engine_prefix_cache, "misses").unwrap_or(stats.prefix_misses);
let prefix_evictions =
engine_u64(engine_prefix_cache, "evictions").unwrap_or(stats.prefix_evictions);
let prefix_saved = engine_u64(engine_prefix_cache, "saved_prefill_tokens")
.unwrap_or(stats.prefix_saved_prefill_tokens);
let prefix_entries =
engine_u64(engine_prefix_cache, "entries").unwrap_or(stats.prefix_entries);
let prefix_bytes = engine_u64(engine_prefix_cache, "bytes").unwrap_or(stats.prefix_bytes);
let mut prefix_cache = serde_json::json!({
"enabled": engine_bool(engine_prefix_cache, "enabled").unwrap_or(policy.prefix_cache_enabled),
"position": engine_str(engine_prefix_cache, "position").unwrap_or("product-observability"),
"source": engine_str(engine_prefix_cache, "source").unwrap_or("server-prompt-lcp-observability"),
"entries": prefix_entries,
"hits": prefix_hits,
"misses": prefix_misses,
"evictions": prefix_evictions,
"saved_prefill_tokens": prefix_saved,
"bytes": prefix_bytes,
"block_size": engine_u64(engine_prefix_cache, "block_size"),
"kv_dtype": engine_str(engine_prefix_cache, "kv_dtype"),
});
if let (Some(engine), Some(prefix)) = (
engine_prefix_cache.and_then(|value| value.as_object()),
prefix_cache.as_object_mut(),
) {
for (key, value) in engine {
prefix.entry(key.clone()).or_insert_with(|| value.clone());
}
}
serde_json::json!({
"prefix_cache": prefix_cache,
"session_cache": {
"mode": policy.session_cache_mode,
"entries": stats.session_entries,
"hits": stats.session_hits,
"misses": stats.session_misses,
"evictions": stats.session_evictions,
"tokens": stats.session_tokens,
"max_entries": policy.session_cache_max_entries,
"max_tokens": policy.session_cache_max_tokens,
}
})
}
fn prometheus_metrics(&self, engine_prefix_cache: Option<&serde_json::Value>) -> String {
let stats = self.stats();
let prefix_hits = engine_u64(engine_prefix_cache, "hits").unwrap_or(stats.prefix_hits);
let prefix_misses =
engine_u64(engine_prefix_cache, "misses").unwrap_or(stats.prefix_misses);
let prefix_evictions =
engine_u64(engine_prefix_cache, "evictions").unwrap_or(stats.prefix_evictions);
let prefix_saved = engine_u64(engine_prefix_cache, "saved_prefill_tokens")
.unwrap_or(stats.prefix_saved_prefill_tokens);
let prefix_entries =
engine_u64(engine_prefix_cache, "entries").unwrap_or(stats.prefix_entries);
let prefix_bytes = engine_u64(engine_prefix_cache, "bytes").unwrap_or(stats.prefix_bytes);
format!(
concat!(
"ferrum_prefix_cache_hits_total {}\n",
"ferrum_prefix_cache_misses_total {}\n",
"ferrum_prefix_cache_evictions_total {}\n",
"ferrum_prefix_cache_saved_prefill_tokens_total {}\n",
"ferrum_prefix_cache_entries {}\n",
"ferrum_prefix_cache_bytes {}\n",
"ferrum_session_cache_hits_total {}\n",
"ferrum_session_cache_misses_total {}\n",
"ferrum_session_cache_evictions_total {}\n",
"ferrum_session_cache_entries {}\n",
"ferrum_session_cache_tokens {}\n"
),
prefix_hits,
prefix_misses,
prefix_evictions,
prefix_saved,
prefix_entries,
prefix_bytes,
stats.session_hits,
stats.session_misses,
stats.session_evictions,
stats.session_entries,
stats.session_tokens,
)
}
}
fn engine_u64(snapshot: Option<&serde_json::Value>, key: &str) -> Option<u64> {
snapshot?.get(key)?.as_u64()
}
fn engine_bool(snapshot: Option<&serde_json::Value>, key: &str) -> Option<bool> {
snapshot?.get(key)?.as_bool()
}
fn engine_str<'a>(snapshot: Option<&'a serde_json::Value>, key: &str) -> Option<&'a str> {
snapshot?.get(key)?.as_str()
}
fn auto_config_health_value(auto_config: Option<&ResolvedFerrumConfig>) -> serde_json::Value {
match auto_config {
Some(auto_config) => auto_config.effective_config_document(),
None => {
match FerrumConfigBuilder::new(RuntimeConfigSnapshot::capture_current()).resolve() {
Ok(auto_config) => auto_config.effective_config_document(),
Err(err) => serde_json::json!({
"schema_version": 1,
"error": err.to_string(),
}),
}
}
}
}
fn admission_health_json(
engine_status: &EngineStatus,
scheduler_metrics: &EngineMetrics,
auto_config: &serde_json::Value,
runtime_snapshot: Option<&ferrum_types::ExecutorAdmissionSnapshot>,
runtime_error: Option<&str>,
) -> serde_json::Value {
let configured = auto_config
.get("admission")
.and_then(|value| value.as_object());
let preflight_effective_max_concurrent = configured
.and_then(|value| value.get("effective_max_concurrent"))
.and_then(|value| value.as_u64());
let effective_max_concurrent = if runtime_error.is_some() {
None
} else {
Some(
runtime_snapshot
.map(|snapshot| u64::from(snapshot.maximum_active_sequences()))
.or(preflight_effective_max_concurrent)
.unwrap_or_else(|| {
(engine_status.active_requests + engine_status.queued_requests)
.max(1)
.try_into()
.unwrap_or(u64::MAX)
}),
)
};
let active_sequences = runtime_error.is_none().then(|| {
runtime_snapshot
.map(|snapshot| u64::from(snapshot.active_sequences()))
.unwrap_or_else(|| engine_status.active_requests as u64)
});
let waiting_requests = runtime_error.is_none().then(|| {
runtime_snapshot
.map(|snapshot| u64::from(snapshot.waiting_requests()))
.unwrap_or_else(|| engine_status.queued_requests as u64)
});
serde_json::json!({
"schema_version": 2,
"source": if runtime_error.is_some() {
"runtime_error"
} else if runtime_snapshot.is_some() {
"runtime_executor"
} else {
"startup_preflight_and_engine_status"
},
"runtime_snapshot_available": runtime_snapshot.is_some(),
"runtime_contract_error": runtime_error,
"resource_authority": runtime_snapshot
.and_then(|snapshot| serde_json::to_value(snapshot.resource_authority()).ok())
.unwrap_or(serde_json::Value::Null),
"effective_max_concurrent": effective_max_concurrent,
"maximum_active_sequences": runtime_snapshot
.map(|snapshot| u64::from(snapshot.maximum_active_sequences())),
"maximum_scheduled_tokens": runtime_snapshot
.map(|snapshot| snapshot.maximum_scheduled_tokens()),
"preflight_effective_max_concurrent": preflight_effective_max_concurrent,
"queue_depth": waiting_requests,
"active_sequences": active_sequences,
"active_prefill": runtime_snapshot
.map(|snapshot| u64::from(snapshot.active_prefill_sequences())),
"active_decode": runtime_snapshot
.map(|snapshot| u64::from(snapshot.active_decode_sequences())),
"current_batch_size": runtime_snapshot
.and_then(|snapshot| snapshot.current_batch_size())
.map(u64::from),
"capacity_blocked_requests": runtime_snapshot
.and_then(|snapshot| snapshot.capacity_blocked_requests())
.map(u64::from),
"rejected_requests_total": 0u64,
"failed_requests_total": scheduler_metrics.failed_requests,
"completed_requests_total": scheduler_metrics.successful_requests,
"avg_queue_wait_time_ms": scheduler_metrics.queue_metrics.avg_queue_wait_time_ms,
"scheduler_policy": configured
.and_then(|value| value.get("scheduler_policy"))
.and_then(|value| value.as_str())
.unwrap_or("unknown"),
"phase_detail_source": if runtime_snapshot.is_some() {
"scheduler_request_index_single_read"
} else {
"unavailable"
},
})
}
fn admission_prometheus_metrics(admission: &serde_json::Value) -> String {
let snapshot_available = u8::from(
admission
.get("runtime_snapshot_available")
.and_then(serde_json::Value::as_bool)
.unwrap_or(false),
);
let mut output = format!("ferrum_admission_runtime_snapshot_available {snapshot_available}\n");
for (field, metric) in [
(
"effective_max_concurrent",
"ferrum_admission_effective_max_concurrent",
),
(
"maximum_active_sequences",
"ferrum_admission_maximum_active_sequences",
),
(
"maximum_scheduled_tokens",
"ferrum_admission_maximum_scheduled_tokens",
),
("queue_depth", "ferrum_admission_queue_depth"),
(
"capacity_blocked_requests",
"ferrum_admission_capacity_blocked_requests",
),
("active_sequences", "ferrum_admission_active_sequences"),
("active_prefill", "ferrum_admission_active_prefill"),
("active_decode", "ferrum_admission_active_decode"),
("current_batch_size", "ferrum_admission_current_batch_size"),
(
"rejected_requests_total",
"ferrum_admission_rejected_requests_total",
),
(
"failed_requests_total",
"ferrum_admission_failed_requests_total",
),
(
"completed_requests_total",
"ferrum_admission_completed_requests_total",
),
] {
if let Some(value) = admission.get(field).and_then(serde_json::Value::as_u64) {
output.push_str(&format!("{metric} {value}\n"));
}
}
output
}
fn request_session_id(headers: &HeaderMap, request: &ChatCompletionsRequest) -> Option<String> {
headers
.get(FERRUM_SESSION_HEADER)
.and_then(|value| value.to_str().ok())
.map(str::trim)
.filter(|value| !value.is_empty())
.map(str::to_string)
.or_else(|| {
request
.metadata
.as_ref()
.and_then(|metadata| metadata.get("ferrum_session_id"))
.and_then(|value| value.as_str())
.map(str::trim)
.filter(|value| !value.is_empty())
.map(str::to_string)
})
}
fn benchmark_request_correlation(
headers: &HeaderMap,
) -> std::result::Result<Option<BenchmarkRequestCorrelation>, ServerError> {
let header_value = |name: &'static str| {
headers
.get(name)
.map(|value| {
value.to_str().map_err(|_| {
ServerError::invalid_request(
format!("{name} must contain visible ASCII text"),
Some(name),
)
})
})
.transpose()
};
BenchmarkRequestCorrelation::from_header_values(
header_value(BENCHMARK_RUN_ID_HEADER)?,
header_value(BENCHMARK_CELL_ID_HEADER)?,
header_value(BENCHMARK_REPEAT_INDEX_HEADER)?,
header_value(BENCHMARK_PHASE_HEADER)?,
header_value(BENCHMARK_REQUEST_INDEX_HEADER)?,
)
.map_err(|error| ServerError::invalid_request(error, Some(BENCHMARK_RUN_ID_HEADER)))
}
fn extend_benchmark_profile_attributes(
attributes: &mut BTreeMap<String, serde_json::Value>,
correlation: Option<&BenchmarkRequestCorrelation>,
) {
let Some(correlation) = correlation else {
return;
};
attributes.extend([
(
"benchmark_run_id".to_string(),
serde_json::json!(correlation.benchmark_run_id),
),
(
"cell_id".to_string(),
serde_json::json!(correlation.cell_id),
),
(
"repeat_index".to_string(),
serde_json::json!(correlation.repeat_index),
),
(
"phase".to_string(),
serde_json::json!(correlation.phase.as_str()),
),
(
"request_index".to_string(),
serde_json::json!(correlation.request_index),
),
]);
}
fn approx_tokens(text: &str) -> usize {
approx_tokens_for_chars(text.chars().count())
}
fn approx_tokens_for_chars(chars: usize) -> usize {
(chars / 4).max(1)
}
fn longest_common_prefix_chars(a: &str, b: &str) -> usize {
a.chars().zip(b.chars()).take_while(|(a, b)| a == b).count()
}
fn trim_messages_to_token_budget(messages: &mut Vec<ChatMessage>, max_tokens: usize) {
let max_tokens = max_tokens.max(1);
while messages.len() > 1
&& messages
.iter()
.map(|msg| approx_tokens(&msg.content))
.sum::<usize>()
> max_tokens
{
messages.remove(0);
}
}
#[async_trait]
impl HttpServer for AxumServer {
async fn start(&self, config: &ServerConfig) -> ferrum_types::Result<()> {
if self.lifecycle.shutdown_requested.load(Ordering::Acquire) {
return Err(Error::internal(
"cannot start Axum server after shutdown was requested",
));
}
self.lifecycle
.running
.compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
.map_err(|_| Error::internal("Axum server is already running"))?;
let _run_guard = AxumServerRunGuard {
lifecycle: Arc::clone(&self.lifecycle),
};
let addr = format!("{}:{}", config.host, config.port);
info!("Starting Axum server on {}", addr);
let app = self.build_router_with_state(
self.state
.clone()
.with_request_dump_dir(config.request_dump_dir.clone())
.with_profile_jsonl(config.profile_jsonl.clone())
.with_profile_detail(config.profile_detail)
.with_memory_profile_jsonl(config.memory_profile_jsonl.clone()),
);
let listener = tokio::net::TcpListener::bind(&addr)
.await
.map_err(|e| Error::internal(format!("Failed to bind to {}: {}", addr, e)))?;
info!("Server listening on {}", addr);
let lifecycle = Arc::clone(&self.lifecycle);
axum::serve(listener, app)
.with_graceful_shutdown(async move { lifecycle.wait_for_shutdown().await })
.await
.map_err(|e| Error::internal(format!("Server error: {}", e)))?;
Ok(())
}
async fn stop(&self, timeout: std::time::Duration) -> ferrum_types::Result<()> {
let _stop_guard = self.lifecycle.stop_lock.lock().await;
info!("Stopping Axum server");
self.lifecycle.request_shutdown();
let mut first_error = None;
if self.lifecycle.running.load(Ordering::Acquire) {
if tokio::time::timeout(timeout, self.lifecycle.wait_until_stopped())
.await
.is_err()
{
first_error = Some(Error::internal(format!(
"Axum server did not drain within {} ms",
timeout.as_millis()
)));
}
}
if !self.lifecycle.engines_stopped.load(Ordering::Acquire) {
match tokio::time::timeout(timeout, self.shutdown_loaded_engines()).await {
Ok(Ok(())) => {
self.lifecycle
.engines_stopped
.store(true, Ordering::Release);
}
Ok(Err(error)) => {
if first_error.is_none() {
first_error = Some(error);
}
}
Err(_) => {
if first_error.is_none() {
first_error = Some(Error::internal(format!(
"engine shutdown did not complete within {} ms",
timeout.as_millis()
)));
}
}
}
}
first_error.map_or(Ok(()), Err)
}
fn is_running(&self) -> bool {
self.lifecycle.running.load(Ordering::Acquire)
}
fn address(&self) -> Option<std::net::SocketAddr> {
// For MVP, return configured address
format!("{}:{}", self.config.host, self.config.port)
.parse()
.ok()
}
fn register_handler(
&mut self,
_path: &str,
_method: HttpMethod,
_handler: Box<dyn crate::traits::RequestHandler>,
) {
// For MVP, routes are static
unimplemented!("Dynamic handler registration not implemented in MVP")
}
fn register_middleware(&mut self, _middleware: Box<dyn crate::traits::Middleware>) {
// For MVP, middleware is static
unimplemented!("Dynamic middleware registration not implemented in MVP")
}
fn get_metrics(&self) -> ServerMetrics {
// Return empty metrics for MVP
ServerMetrics {
total_requests: 0,
requests_by_endpoint: std::collections::HashMap::new(),
requests_by_status: std::collections::HashMap::new(),
avg_response_time_ms: 0.0,
p95_response_time_ms: 0.0,
p99_response_time_ms: 0.0,
active_connections: 0,
bytes_sent: 0,
bytes_received: 0,
error_rate: 0.0,
uptime_seconds: 0,
}
}
async fn health_check(&self) -> HealthStatus {
HealthStatus::Healthy
}
}
/// Main chat completions handler
async fn chat_completions_handler(
State(state): State<AppState>,
headers: HeaderMap,
request: std::result::Result<Json<ChatCompletionsRequest>, JsonRejection>,
) -> std::result::Result<Response, ServerError> {
chat_completions_handler_with_phases(State(state), headers, request, None).await
}
async fn chat_completions_handler_with_phases(
State(state): State<AppState>,
headers: HeaderMap,
request: std::result::Result<Json<ChatCompletionsRequest>, JsonRejection>,
mut message_phases: Option<Vec<Option<AssistantMessagePhase>>>,
) -> std::result::Result<Response, ServerError> {
let Json(mut request) = request.map_err(|error| {
ServerError::invalid_request(
format!(
"invalid chat completions request: {}",
json_rejection_detail(&error)
),
None,
)
})?;
let benchmark_correlation = benchmark_request_correlation(&headers)?;
let cache_policy = CachePolicy::current();
if message_phases
.as_ref()
.is_some_and(|phases| phases.len() != request.messages.len())
{
return Err(ServerError::InternalError(
"Responses message phase metadata did not match input history".to_string(),
));
}
let session_context =
state
.cache
.prepare_session_request(&mut request, &headers, &cache_policy);
if let Some(phases) = &mut message_phases {
let prepended = request
.messages
.len()
.checked_sub(phases.len())
.ok_or_else(|| {
ServerError::InternalError(
"session preparation shortened Responses input history".to_string(),
)
})?;
phases.splice(0..0, std::iter::repeat(None).take(prepended));
}
let span = span!(Level::INFO, "chat_completions", model = %request.model);
let _enter = span.enter();
info!(
"Received chat completions request for model: {}",
request.model
);
debug!("Request: {:?}", request);
// OpenAI spec requires at least one message. Reject empty arrays at
// the boundary rather than synthesising a fake prompt downstream.
validate_chat_request(&request)?;
let (engine_model_id, lora_adapter) = resolve_request_model(
&state.served_model_registry,
&request.model,
ServedModelKind::Llm,
)?;
// Convert OpenAI request to internal format
let mut inference_request = convert_chat_request_with_template_model_and_default(
&request,
&engine_model_id.0,
state.prompt_template.as_deref(),
state.default_enable_thinking,
state.interleaved_system_coalescing.unwrap_or(true),
message_phases.as_deref(),
)
.map_err(server_error_from_ferrum_error)?;
apply_served_model_resolution(&mut inference_request, engine_model_id, lora_adapter);
if state.request_dump_dir.is_some() {
inference_request.evidence_request.capture_prompt_token_ids = true;
}
inference_request
.evidence_request
.capture_engine_token_timing = state.profile_detail.captures_engine_token_timing();
state
.cache
.record_prefix_prompt(&inference_request.prompt, &cache_policy);
if let Err(err) =
write_chat_request_replay_bundle(&state, &headers, &request, &inference_request)
{
warn!("failed to write chat request replay bundle: {}", err);
}
// Check if streaming is requested
if request.stream.unwrap_or(false) {
handle_chat_completions_stream(state, request, inference_request, benchmark_correlation)
.await
} else {
handle_chat_completions_sync(
state,
request,
inference_request,
session_context,
benchmark_correlation,
)
.await
}
}
fn json_rejection_detail(rejection: &JsonRejection) -> String {
const MAX_DETAIL_CHARS: usize = 512;
let mut details = Vec::new();
let mut current: Option<&(dyn StdError + 'static)> = Some(rejection);
while let Some(error) = current {
let detail = error.to_string();
if !detail.is_empty() && details.last() != Some(&detail) {
details.push(detail);
}
current = error.source();
}
details.join(": ").chars().take(MAX_DETAIL_CHARS).collect()
}
fn write_chat_request_replay_bundle(
state: &AppState,
headers: &HeaderMap,
openai_request: &ChatCompletionsRequest,
inference_request: &InferenceRequest,
) -> std::result::Result<(), String> {
let Some(root) = state.request_dump_dir.as_ref() else {
return Ok(());
};
let request_id = inference_request.id.to_string();
let bundle_dir = root.join(&request_id);
fs::create_dir_all(&bundle_dir).map_err(|err| err.to_string())?;
let sanitized_body = sanitized_chat_request_body(openai_request);
let replay_body_path = bundle_dir.join("replay_body.json");
write_json_value(&replay_body_path, &sanitized_body)?;
let engine_replay_argv = replay_bundle_argv(&bundle_dir);
let output_text_body = format!(
"[server request replay emitted before response]\nsha256={}\nchars=0\n",
sha256_hex(b"")
);
let request = serde_json::json!({
"schema_version": OBSERVABILITY_PROFILE_SCHEMA_VERSION,
"entrypoint": "serve",
"request_id": request_id,
"model": openai_request.model.clone(),
"backend": "actual",
"endpoint": "/v1/chat/completions",
"method": "POST",
"stream": openai_request.stream.unwrap_or(false),
"actual_model_smoke": true,
"sanitized": true,
"http": {
"method": "POST",
"path": "/v1/chat/completions",
"headers": sanitized_replay_headers(headers),
"body": sanitized_body
}
});
let files = [
("request.json", request),
(
"prompt_token_ids.json",
serde_json::json!({
"schema_version": OBSERVABILITY_PROFILE_SCHEMA_VERSION,
"request_id": request_id,
"model": openai_request.model.clone(),
"tokenizer_or_model": openai_request.model.clone(),
"token_ids": null,
"token_count": null,
"unavailable_reason": "server request replay captures the OpenAI body before prompt token ids are retained",
"sanitized": true
}),
),
(
"sampling_params.json",
serde_json::json!({
"schema_version": OBSERVABILITY_PROFILE_SCHEMA_VERSION,
"request_id": request_id,
"sampling_params": inference_request.sampling_params.clone(),
"unavailable_reason": null
}),
),
(
"runtime_effective_config.json",
serde_json::json!({
"schema_version": OBSERVABILITY_PROFILE_SCHEMA_VERSION,
"request_id": request_id,
"entrypoint": "serve",
"endpoint": "/v1/chat/completions",
"stream": openai_request.stream.unwrap_or(false),
"request_dump_dir": root.to_string_lossy(),
"sanitized": true
}),
),
(
"backend_selection.json",
serde_json::json!({
"schema_version": OBSERVABILITY_PROFILE_SCHEMA_VERSION,
"request_id": request_id,
"backend": "actual",
"model": openai_request.model.clone(),
"actual_model_smoke": true
}),
),
(
"output_token_ids.json",
serde_json::json!({
"schema_version": OBSERVABILITY_PROFILE_SCHEMA_VERSION,
"request_id": request_id,
"token_ids": [],
"token_count": 0,
"finish_reason": null,
"unavailable_reason": "server request replay bundle is emitted at request admission in this WP9 slice"
}),
),
(
"bad_output_scan.json",
serde_json::json!({
"schema_version": OBSERVABILITY_PROFILE_SCHEMA_VERSION,
"request_id": request_id,
"bad_output": false,
"bad_text_count": 0,
"reasons": [],
"first_bad_text_span": null,
"failure_kind": null,
"output_chars": 0,
"classified_output_sha256": sha256_hex(b""),
"output_sha256": sha256_hex(output_text_body.as_bytes())
}),
),
(
"replay.command.json",
serde_json::json!({
"schema_version": OBSERVABILITY_PROFILE_SCHEMA_VERSION,
"request_id": request_id,
"entrypoint": "serve",
"command": replay_curl_command(&bundle_dir),
"argv": replay_curl_argv(&bundle_dir),
"bundle_dir": bundle_dir.to_string_lossy(),
"requires_running_server": true,
"engine_replay": {
"mode": "bundle_offline",
"requires_http_server": false,
"command": shell_command(&engine_replay_argv),
"argv": engine_replay_argv
},
"sanitized": true
}),
),
];
for (name, value) in files {
write_json_value(&bundle_dir.join(name), &value)?;
}
fs::write(bundle_dir.join("output_text.txt"), output_text_body)
.map_err(|err| err.to_string())?;
Ok(())
}
fn write_chat_request_failure_diagnostics(
state: &AppState,
request_id: &str,
failure_kind: &str,
phase: &str,
error_kind: &str,
message: &str,
engine_status: Option<&EngineStatus>,
) -> std::result::Result<(), String> {
let admission_summary = state
.auto_config
.as_ref()
.map(|config| config.admission_summary_document());
write_chat_request_failure_diagnostics_at_root(
state.request_dump_dir.as_ref().map(|root| root.as_path()),
admission_summary.as_ref(),
engine_status,
request_id,
failure_kind,
phase,
error_kind,
message,
)
}
fn write_chat_request_completion_replay_bundle(
request_dump_dir: Option<&Path>,
request_id: &str,
output_text: &str,
output_token_ids: &[TokenId],
finish_reason: Option<&str>,
) -> std::result::Result<(), String> {
let Some(root) = request_dump_dir else {
return Ok(());
};
let bundle_dir = root.join(request_id);
fs::create_dir_all(&bundle_dir).map_err(|err| err.to_string())?;
let token_ids = output_token_ids
.iter()
.map(|token| token.get())
.collect::<Vec<_>>();
let output_text_body = format!(
"[redacted actual output]\nsha256={}\nchars={}\n",
sha256_hex(output_text.as_bytes()),
output_text.chars().count()
);
write_json_value(
&bundle_dir.join("output_token_ids.json"),
&serde_json::json!({
"schema_version": OBSERVABILITY_PROFILE_SCHEMA_VERSION,
"request_id": request_id,
"token_ids": token_ids,
"token_count": output_token_ids.len(),
"finish_reason": finish_reason,
"unavailable_reason": null
}),
)?;
write_json_value(
&bundle_dir.join("bad_output_scan.json"),
&bad_output_scan_json(request_id, output_text, None, output_text_body.as_bytes()),
)?;
fs::write(bundle_dir.join("output_text.txt"), output_text_body)
.map_err(|err| err.to_string())?;
Ok(())
}
fn write_chat_prompt_token_evidence(
request_dump_dir: Option<&Path>,
request_id: &str,
model: &str,
execution_evidence: Option<&InferenceExecutionEvidence>,
) -> std::result::Result<(), String> {
let (Some(root), Some(evidence)) = (request_dump_dir, execution_evidence) else {
return Ok(());
};
let bundle_dir = root.join(request_id);
fs::create_dir_all(&bundle_dir).map_err(|err| err.to_string())?;
let prompt_token_ids = evidence
.prompt_token_ids
.iter()
.map(|token| token.get())
.collect::<Vec<_>>();
write_json_value(
&bundle_dir.join("prompt_token_ids.json"),
&serde_json::json!({
"schema_version": OBSERVABILITY_PROFILE_SCHEMA_VERSION,
"request_id": request_id,
"model": model,
"tokenizer_or_model": model,
"token_ids": prompt_token_ids,
"token_count": evidence.prompt_token_ids.len(),
"unavailable_reason": null,
"sanitized": true
}),
)
}
#[derive(Clone, Copy, Default)]
struct ChatRequestProfileTiming<'a> {
engine_evidence: Option<&'a InferenceExecutionEvidence>,
first_engine_chunk_received_us: Option<u64>,
first_sse_enqueue_us: Option<u64>,
}
#[allow(clippy::too_many_arguments)]
fn write_chat_request_profile_event(
state: &AppState,
request_id: &str,
benchmark_correlation: Option<&BenchmarkRequestCorrelation>,
model: &str,
stream: bool,
phase: &str,
started_at: Instant,
timing: ChatRequestProfileTiming<'_>,
output_token_count: usize,
usage: Option<&TokenUsage>,
finish_reason: Option<&str>,
error: Option<ProfileError>,
) -> std::result::Result<(), String> {
let Some(path) = state.profile_jsonl.as_ref() else {
return Ok(());
};
let timestamp = chrono::Utc::now();
let status = if error.is_some() {
ProfileStatus::Failure
} else {
ProfileStatus::Ok
};
let duration_us = elapsed_us_since(started_at);
let mut attributes = BTreeMap::from([
("actual_model_smoke".to_string(), serde_json::json!(true)),
(
"diagnostic_only".to_string(),
serde_json::json!(state.profile_detail.diagnostic_only()),
),
(
"endpoint".to_string(),
serde_json::json!("/v1/chat/completions"),
),
(
"e2e_duration_us".to_string(),
serde_json::json!(duration_us),
),
("l0_only".to_string(), serde_json::json!(false)),
(
"profile_detail".to_string(),
serde_json::json!(state.profile_detail.as_str()),
),
("stream".to_string(), serde_json::json!(stream)),
(
"output_token_count".to_string(),
serde_json::json!(output_token_count),
),
(
"execution_request_id".to_string(),
serde_json::json!(format!("request.product.{request_id}")),
),
]);
extend_benchmark_profile_attributes(&mut attributes, benchmark_correlation);
if let Some(usage) = usage {
attributes.insert(
"prompt_token_count".to_string(),
serde_json::json!(usage.prompt_tokens),
);
attributes.insert(
"completion_token_count".to_string(),
serde_json::json!(usage.completion_tokens),
);
attributes.insert(
"total_token_count".to_string(),
serde_json::json!(usage.total_tokens),
);
attributes.insert("token_count_source".to_string(), serde_json::json!("usage"));
} else {
attributes.insert(
"completion_token_count".to_string(),
serde_json::json!(output_token_count),
);
attributes.insert(
"total_token_count".to_string(),
serde_json::json!(output_token_count),
);
attributes.insert(
"token_count_source".to_string(),
serde_json::json!("generated_tokens"),
);
}
if let Some(engine_timing) = timing
.engine_evidence
.and_then(|evidence| evidence.engine_token_timing.as_ref())
{
engine_timing
.validate(output_token_count)
.map_err(|error| format!("invalid engine token timing evidence: {error}"))?;
attributes.extend(ferrum_types::engine_token_timing_profile_attributes(
engine_timing,
));
} else if status == ProfileStatus::Ok && state.profile_detail.captures_engine_token_timing() {
return Err(format!(
"{} profile completed without required engine token timing evidence",
state.profile_detail.as_str()
));
}
if let Some(received_us) = timing.first_engine_chunk_received_us {
attributes.insert(
"engine_stream_first_chunk_received_us".to_string(),
serde_json::json!(received_us),
);
}
if let Some(enqueue_us) = timing.first_sse_enqueue_us {
attributes.insert(
"http_first_sse_enqueue_us".to_string(),
serde_json::json!(enqueue_us),
);
}
if stream {
attributes.insert(
"http_stream_flush_unavailable_reason".to_string(),
serde_json::json!(
"socket flush completion is outside the axum handler observation boundary"
),
);
}
if let Some(reason) = finish_reason {
attributes.insert("finish_reason".to_string(), serde_json::json!(reason));
}
if let Some(error) = error.as_ref() {
attributes.insert(
if error.blocking {
"first_failure_event"
} else {
"terminal_failure_event"
}
.to_string(),
serde_json::json!(true),
);
}
let replay = state.request_dump_dir.as_ref().map(|root| {
let bundle_dir = root.join(request_id);
ReplayReference {
command: replay_curl_command(&bundle_dir),
bundle_dir: Some(root.to_string_lossy().to_string()),
}
});
let resource = error.as_ref().map(|error| ResourceTraceEvent {
owner_kind: "request".to_string(),
owner_id: request_id.to_string(),
resource_kind: "chat_request".to_string(),
action: ResourceAction::Reject,
amount: None,
before: None,
after: None,
capacity: Some(1),
underflow_amount: None,
reason: Some(error.message.clone()),
error_kind: Some(error.kind.clone()),
message: Some(error.message.clone()),
resource_error_kind: Some(error.kind.clone()),
});
let event = FerrumProfileEvent {
schema_version: OBSERVABILITY_PROFILE_SCHEMA_VERSION,
ts_unix_nanos: timestamp
.timestamp_nanos_opt()
.unwrap_or_else(|| timestamp.timestamp_micros() * 1_000),
event_id: format!(
"evt-server-chat-{}-{request_id}",
if stream { "stream" } else { "sync" }
),
request_id: request_id.to_string(),
correlation_id: Some(request_id.to_string()),
entrypoint: ProfileEntrypoint::Serve,
backend: "actual".to_string(),
runtime_preset_hash: state
.auto_config
.as_ref()
.map(ResolvedFerrumConfig::runtime_env_hash)
.unwrap_or_else(|| format!("sha256:{}", sha256_hex(b"serve-profile"))),
phase: phase.to_string(),
event_kind: ProfileEventKind::TimedSpan,
timestamp,
status,
model: Some(model.to_string()),
duration_us: Some(duration_us),
memory: None,
resource,
error,
replay,
shape: BTreeMap::from([("batch_size".to_string(), serde_json::json!(1))]),
backend_detail: None,
attributes,
};
append_profile_event(path.as_path(), &event)
}
fn maybe_write_first_request_memory_stage(
state: &AppState,
request_id: &str,
benchmark_correlation: Option<&BenchmarkRequestCorrelation>,
model: &str,
stream: bool,
started_at: Instant,
before: Option<ProcessMemorySample>,
) -> std::result::Result<(), String> {
if state.profile_jsonl.is_none() && state.memory_profile_jsonl.is_none() {
return Ok(());
}
if state
.first_request_memory_recorded
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
.is_err()
{
return Ok(());
}
let after = ProcessMemorySampler.sample();
let memory = after.map(|after| ProcessMemoryObservation::from_samples(before, after));
let timestamp = chrono::Utc::now();
let mut attributes = BTreeMap::from([
("actual_model_smoke".to_string(), serde_json::json!(true)),
(
"diagnostic_only".to_string(),
serde_json::json!(state.profile_detail.diagnostic_only()),
),
(
"endpoint".to_string(),
serde_json::json!("/v1/chat/completions"),
),
("l0_only".to_string(), serde_json::json!(false)),
(
"memory_stage".to_string(),
serde_json::json!("first_request_done"),
),
(
"profile_detail".to_string(),
serde_json::json!(state.profile_detail.as_str()),
),
("stream".to_string(), serde_json::json!(stream)),
]);
extend_benchmark_profile_attributes(&mut attributes, benchmark_correlation);
let memory_snapshot = if let Some(memory) = &memory {
attributes.insert(
"memory_measurement".to_string(),
serde_json::json!("process_rss"),
);
attributes.insert(
"process_memory_source".to_string(),
serde_json::json!(memory.source),
);
memory.to_snapshot("process", Some("actual"))
} else {
attributes.insert(
"memory_measurement".to_string(),
serde_json::json!("not_collected"),
);
ferrum_types::MemorySnapshot {
scope: "process".to_string(),
backend: Some("actual".to_string()),
before_bytes: Some(0),
after_bytes: Some(0),
current_bytes: Some(0),
high_water_bytes: Some(0),
available_bytes: None,
}
};
let replay = state.request_dump_dir.as_ref().map(|root| {
let bundle_dir = root.join(request_id);
ReplayReference {
command: replay_curl_command(&bundle_dir),
bundle_dir: Some(root.to_string_lossy().to_string()),
}
});
let event = FerrumProfileEvent {
schema_version: OBSERVABILITY_PROFILE_SCHEMA_VERSION,
ts_unix_nanos: timestamp
.timestamp_nanos_opt()
.unwrap_or_else(|| timestamp.timestamp_micros() * 1_000),
event_id: format!("evt-server-chat-memory-first-request-{request_id}"),
request_id: request_id.to_string(),
correlation_id: Some(request_id.to_string()),
entrypoint: ProfileEntrypoint::Serve,
backend: "actual".to_string(),
runtime_preset_hash: state
.auto_config
.as_ref()
.map(ResolvedFerrumConfig::runtime_env_hash)
.unwrap_or_else(|| format!("sha256:{}", sha256_hex(b"serve-profile"))),
phase: "actual_serve_first_request_done".to_string(),
event_kind: ProfileEventKind::Memory,
timestamp,
status: ProfileStatus::Ok,
model: Some(model.to_string()),
duration_us: Some(elapsed_us_since(started_at)),
memory: Some(memory_snapshot),
resource: None,
error: None,
replay,
shape: BTreeMap::from([("batch_size".to_string(), serde_json::json!(1))]),
backend_detail: None,
attributes,
};
if let Some(path) = &state.profile_jsonl {
append_profile_event(path.as_path(), &event)?;
}
if let Some(path) = &state.memory_profile_jsonl {
append_profile_event(path.as_path(), &event)?;
}
Ok(())
}
fn request_memory_sample_before(state: &AppState) -> Option<ProcessMemorySample> {
(state.profile_jsonl.is_some() || state.memory_profile_jsonl.is_some())
.then(|| ProcessMemorySampler.sample())
.flatten()
}
fn append_profile_event(
path: &Path,
event: &FerrumProfileEvent,
) -> std::result::Result<(), String> {
event.validate().map_err(|err| err.to_string())?;
ferrum_bench_core::write_jsonl_records(
path,
ferrum_bench_core::JsonlJournalOpenMode::Append,
std::slice::from_ref(event),
)
.map_err(|error| error.to_string())
}
fn elapsed_us_since(started_at: Instant) -> u64 {
started_at
.elapsed()
.as_micros()
.max(1)
.try_into()
.unwrap_or(u64::MAX)
}
fn write_chat_request_failure_diagnostics_at_root(
request_dump_dir: Option<&Path>,
admission_summary: Option<&serde_json::Value>,
engine_status: Option<&EngineStatus>,
request_id: &str,
failure_kind: &str,
phase: &str,
error_kind: &str,
message: &str,
) -> std::result::Result<(), String> {
let Some(root) = request_dump_dir else {
return Ok(());
};
let bundle_dir = root.join(request_id);
fs::create_dir_all(&bundle_dir).map_err(|err| err.to_string())?;
let message = sanitize_diagnostic_text(message);
let now = chrono::Utc::now();
let bad_scan_path = bundle_dir.join("bad_output_scan.json");
let mut bad_scan = fs::read_to_string(&bad_scan_path)
.ok()
.and_then(|text| serde_json::from_str::<serde_json::Value>(&text).ok())
.filter(|value| value.is_object())
.unwrap_or_else(|| serde_json::json!({}));
let bad_scan_obj = bad_scan
.as_object_mut()
.expect("bad scan fallback should be an object");
bad_scan_obj.insert(
"schema_version".to_string(),
serde_json::json!(OBSERVABILITY_PROFILE_SCHEMA_VERSION),
);
bad_scan_obj.insert("request_id".to_string(), serde_json::json!(request_id));
bad_scan_obj
.entry("bad_output".to_string())
.or_insert_with(|| serde_json::json!(false));
bad_scan_obj
.entry("bad_text_count".to_string())
.or_insert_with(|| serde_json::json!(0));
bad_scan_obj
.entry("reasons".to_string())
.or_insert_with(|| serde_json::json!([]));
bad_scan_obj
.entry("first_bad_text_span".to_string())
.or_insert(serde_json::Value::Null);
bad_scan_obj.insert("failure_kind".to_string(), serde_json::json!(failure_kind));
bad_scan_obj.insert("failure_phase".to_string(), serde_json::json!(phase));
bad_scan_obj.insert("error_kind".to_string(), serde_json::json!(error_kind));
bad_scan_obj
.entry("output_chars".to_string())
.or_insert_with(|| serde_json::json!(0));
bad_scan_obj
.entry("output_sha256".to_string())
.or_insert_with(|| serde_json::json!(sha256_hex(b"")));
write_json_value(&bad_scan_path, &bad_scan)?;
let diagnostics = if chat_resource_failure_kind(failure_kind) {
chat_resource_failure_diagnostics(
request_id,
failure_kind,
phase,
error_kind,
&message,
now.timestamp_millis(),
admission_summary,
engine_status,
)
} else {
serde_json::json!({
"schema_version": OBSERVABILITY_PROFILE_SCHEMA_VERSION,
"entrypoint": "serve",
"request_id": request_id,
"failure_kind": failure_kind,
"phase": phase,
"first_failure_event": {
"schema_version": OBSERVABILITY_PROFILE_SCHEMA_VERSION,
"entrypoint": "serve",
"request_id": request_id,
"phase": phase,
"error_kind": error_kind,
"message": message,
"timestamp_unix_ms": now.timestamp_millis()
},
"nearest_request_id": request_id,
"log_excerpt": format!("{phase}: {message}"),
"backtrace_excerpt": null,
"nearest_resource_event": null,
"nearest_memory_snapshot": null
})
};
write_json_value(&bundle_dir.join("failure_diagnostics.json"), &diagnostics)?;
Ok(())
}
fn chat_resource_failure_diagnostics(
request_id: &str,
failure_kind: &str,
phase: &str,
error_kind: &str,
message: &str,
timestamp_unix_ms: i64,
admission_summary: Option<&serde_json::Value>,
engine_status: Option<&EngineStatus>,
) -> serde_json::Value {
let resource_kind = chat_resource_kind_for_failure(failure_kind);
let memory = engine_status
.map(|status| &status.memory_usage)
.map(|memory| {
let current = memory.used_bytes as i64;
let high_water = current.max(0);
serde_json::json!({
"scope": "serve_failure",
"backend": "engine_status",
"current_bytes": current.max(0),
"high_water_bytes": high_water,
"total_bytes": memory.total_bytes,
"free_bytes": memory.free_bytes,
"gpu_memory_bytes": memory.gpu_memory_bytes,
"cpu_memory_bytes": memory.cpu_memory_bytes,
"source": "engine_status"
})
})
.unwrap_or_else(|| {
serde_json::json!({
"scope": "serve_failure",
"backend": "engine_status",
"current_bytes": 0,
"high_water_bytes": 0,
"source": "not_collected"
})
});
let capacity = chat_failure_capacity(resource_kind, admission_summary, engine_status, message);
let needed = capacity
.get("needed")
.and_then(|value| value.as_i64())
.unwrap_or(1)
.max(1);
let capacity_value = capacity
.get("capacity")
.and_then(|value| value.as_i64())
.unwrap_or(0)
.max(0);
serde_json::json!({
"schema_version": OBSERVABILITY_PROFILE_SCHEMA_VERSION,
"entrypoint": "serve",
"request_id": request_id,
"failure_kind": failure_kind,
"phase": phase,
"first_failure_event": {
"schema_version": OBSERVABILITY_PROFILE_SCHEMA_VERSION,
"entrypoint": "serve",
"request_id": request_id,
"phase": phase,
"error_kind": error_kind,
"message": message,
"timestamp_unix_ms": timestamp_unix_ms
},
"nearest_request_id": request_id,
"log_excerpt": format!("{phase}: {message}"),
"capacity": capacity,
"nearest_resource_event": {
"owner_kind": "request",
"owner_id": request_id,
"resource_kind": resource_kind,
"action": "reject",
"amount": needed,
"before": capacity_value,
"after": capacity_value,
"capacity": capacity_value,
"reason": message
},
"nearest_memory_snapshot": memory
})
}
fn chat_failure_capacity(
resource_kind: &str,
admission_summary: Option<&serde_json::Value>,
engine_status: Option<&EngineStatus>,
reason: &str,
) -> serde_json::Value {
if resource_kind == "device_memory" {
let (needed, available, capacity) = engine_status
.map(|status| {
let memory = &status.memory_usage;
let used = memory.used_bytes as i64;
let available = memory.free_bytes as i64;
let capacity = memory.total_bytes as i64;
(
used.saturating_add(1).max(1),
available.max(0),
capacity.max(0),
)
})
.unwrap_or((1, 0, 0));
return serde_json::json!({
"resource_kind": resource_kind,
"needed": needed,
"available": available,
"capacity": capacity,
"reason": reason
});
}
let capacity = admission_summary
.and_then(|summary| summary.get("effective_max_concurrent"))
.and_then(|value| {
value
.as_i64()
.or_else(|| value.as_u64().map(|value| value as i64))
})
.unwrap_or_else(|| {
engine_status
.map(|status| {
(status.active_requests as i64)
.saturating_add(status.queued_requests as i64)
.saturating_add(1)
})
.unwrap_or(0)
})
.max(0);
let used = engine_status
.map(|status| (status.active_requests as i64).saturating_add(status.queued_requests as i64))
.unwrap_or(0)
.max(0);
serde_json::json!({
"resource_kind": resource_kind,
"needed": 1,
"available": capacity.saturating_sub(used),
"capacity": capacity,
"reason": reason
})
}
fn chat_resource_failure_kind(failure_kind: &str) -> bool {
matches!(
failure_kind,
"oom" | "prevented_oom" | "admission" | "admission_reject" | "oom_admission"
)
}
fn chat_resource_kind_for_failure(failure_kind: &str) -> &'static str {
match failure_kind {
"oom" | "prevented_oom" => "device_memory",
_ => "admission_capacity",
}
}
fn sanitize_diagnostic_text(message: &str) -> String {
let trimmed = message.trim();
if trimmed.is_empty() {
return "generation failed without an error message".to_string();
}
let lower = trimmed.to_ascii_lowercase();
if lower.contains("authorization")
|| lower.contains("cookie")
|| lower.contains("api_key")
|| lower.contains("access_token")
|| lower.contains("refresh_token")
|| lower.contains("password")
|| trimmed.contains("sk-")
{
return "[redacted diagnostic message]".to_string();
}
trimmed.chars().take(2048).collect()
}
fn sanitized_replay_headers(headers: &HeaderMap) -> serde_json::Value {
let mut result = serde_json::Map::new();
for key in ["content-type", "traceparent", "tracestate"] {
if let Some(value) = headers.get(key).and_then(|value| value.to_str().ok()) {
result.insert(key.to_string(), serde_json::json!(value));
}
}
result.insert("authorization".to_string(), serde_json::json!("[redacted]"));
result.insert("cookie".to_string(), serde_json::json!("[redacted]"));
serde_json::Value::Object(result)
}
fn sanitized_chat_request_body(request: &ChatCompletionsRequest) -> serde_json::Value {
let mut value = serde_json::to_value(request).unwrap_or_else(|_| {
serde_json::json!({
"model": request.model.clone(),
"stream": request.stream.unwrap_or(false),
"messages": []
})
});
redact_json_value(&mut value, None);
value
}
fn redact_json_value(value: &mut serde_json::Value, key: Option<&str>) {
if key.is_some_and(is_secret_key) {
*value = serde_json::json!("[redacted]");
return;
}
if matches!(key, Some("content" | "arguments")) && value.is_string() {
*value = serde_json::json!("[redacted]");
return;
}
match value {
serde_json::Value::Object(map) => {
for field in ["content", "arguments"] {
if let Some(chars) = map
.get(field)
.and_then(|child| child.as_str())
.map(|text| text.chars().count())
{
map.insert(field.to_string(), serde_json::json!("[redacted]"));
map.insert(format!("{field}_redacted"), serde_json::json!(true));
map.insert(format!("{field}_chars"), serde_json::json!(chars));
}
}
for (child_key, child) in map.iter_mut() {
redact_json_value(child, Some(child_key.as_str()));
}
}
serde_json::Value::Array(items) => {
for child in items {
redact_json_value(child, None);
}
}
_ => {}
}
}
fn is_secret_key(key: &str) -> bool {
let normalized = key
.chars()
.filter(|ch| *ch != '-' && *ch != '_')
.flat_map(char::to_lowercase)
.collect::<String>();
matches!(
normalized.as_str(),
"authorization"
| "cookie"
| "secret"
| "apikey"
| "password"
| "accesstoken"
| "refreshtoken"
| "idtoken"
)
}
fn replay_curl_argv(bundle_dir: &Path) -> Vec<String> {
vec![
"curl".to_string(),
"-sS".to_string(),
"-X".to_string(),
"POST".to_string(),
"http://127.0.0.1:8000/v1/chat/completions".to_string(),
"-H".to_string(),
"content-type: application/json".to_string(),
"--data-binary".to_string(),
format!("@{}", bundle_dir.join("replay_body.json").display()),
]
}
fn replay_curl_command(bundle_dir: &Path) -> String {
shell_command(&replay_curl_argv(bundle_dir))
}
fn replay_bundle_argv(bundle_dir: &Path) -> Vec<String> {
vec![
"cargo".to_string(),
"run".to_string(),
"-p".to_string(),
"ferrum-cli".to_string(),
"--".to_string(),
"replay-bundle".to_string(),
bundle_dir.to_string_lossy().to_string(),
"--out".to_string(),
bundle_dir
.join("engine_replay")
.to_string_lossy()
.to_string(),
"--json".to_string(),
]
}
fn shell_command(argv: &[String]) -> String {
argv.iter()
.map(|part| shell_quote(part))
.collect::<Vec<_>>()
.join(" ")
}
fn shell_quote(value: &str) -> String {
if value
.chars()
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '/' | '.' | '_' | '-' | ':' | '@'))
{
value.to_string()
} else {
format!("'{}'", value.replace('\'', "'\\''"))
}
}
fn write_json_value(path: &Path, value: &serde_json::Value) -> std::result::Result<(), String> {
let bytes = serde_json::to_vec_pretty(value).map_err(|err| err.to_string())?;
fs::write(path, [bytes, b"\n".to_vec()].concat()).map_err(|err| err.to_string())
}
fn bad_output_scan_json(
request_id: &str,
text: &str,
failure_kind: Option<&str>,
output_artifact_bytes: &[u8],
) -> serde_json::Value {
let mut reasons = Vec::new();
let mut first_span: Option<serde_json::Value> = None;
for (needle, reason) in [
("<unk>", "reserved_token"),
("[PAD", "reserved_token"),
("<pad>", "reserved_token"),
("<|endoftext|>", "reserved_token"),
("<|im_start|>", "reserved_token"),
("<|im_end|>", "reserved_token"),
("<|reserved_special_token", "reserved_token"),
("\u{fffd}", "invalid_utf8"),
] {
if let Some(index) = text.find(needle) {
reasons.push(reason);
first_span.get_or_insert_with(|| {
serde_json::json!({
"byte_start": index,
"byte_end": index + needle.len(),
"text": needle,
"reason": reason
})
});
}
}
if let Some(index) = first_mojibake_index(text) {
reasons.push("mojibake");
first_span.get_or_insert_with(|| {
serde_json::json!({
"byte_start": index,
"byte_end": index + 1,
"reason": "mojibake"
})
});
}
reasons.sort_unstable();
reasons.dedup();
let bad_output = !reasons.is_empty();
serde_json::json!({
"schema_version": OBSERVABILITY_PROFILE_SCHEMA_VERSION,
"request_id": request_id,
"bad_output": bad_output,
"bad_text_count": if bad_output { 1 } else { 0 },
"reasons": reasons,
"first_bad_text_span": first_span,
"failure_kind": failure_kind,
"output_chars": text.chars().count(),
"classified_output_sha256": sha256_hex(text.as_bytes()),
"output_sha256": sha256_hex(output_artifact_bytes)
})
}
fn first_mojibake_index(text: &str) -> Option<usize> {
let mut chars = text.char_indices().peekable();
while let Some((index, ch)) = chars.next() {
match ch {
'\u{00c2}' | '\u{00c3}' => {
if chars.peek().is_some_and(|(_, next)| !next.is_ascii()) {
return Some(index);
}
}
'\u{00e2}' => {
if chars.peek().is_some_and(|(_, next)| *next == '\u{20ac}') {
return Some(index);
}
}
_ => {}
}
}
None
}
fn sha256_hex(bytes: &[u8]) -> String {
let mut hasher = Sha256::new();
hasher.update(bytes);
format!("{:x}", hasher.finalize())
}
struct ParsedChatModelOutput {
visible: ParsedReasoningResponse,
harmony_response: Option<ferrum_types::ApiChatResponse>,
}
fn parse_chat_model_output(
protocol: ModelOutputProtocol,
text: &str,
started_in_think: bool,
finish_reason: FinishReason,
) -> std::result::Result<ParsedChatModelOutput, ServerError> {
match protocol {
ModelOutputProtocol::Text | ModelOutputProtocol::GemmaThought => {
Ok(ParsedChatModelOutput {
visible: parse_model_reasoning_response(protocol, text, started_in_think)
.map_err(|error| ServerError::InternalError(error.to_string()))?,
harmony_response: None,
})
}
ModelOutputProtocol::HarmonyGptOss => {
let parsed = parse_harmony_response_for_finish_reason(text, Some(finish_reason))
.map_err(|error| {
ServerError::InternalError(format!(
"model output did not satisfy the GPT-OSS Harmony protocol: {error}"
))
})?;
let harmony_response =
parsed
.tool_call
.map(|tool_call| ferrum_types::ApiChatResponse {
message: ferrum_types::ApiChatMessage {
role: ferrum_types::ApiMessageRole::Assistant,
content: String::new(),
name: None,
tool_calls: vec![ferrum_types::ApiToolCall {
id: format!("call_{}", Uuid::new_v4().simple()),
tool_type: "function".to_string(),
function: ferrum_types::ApiFunctionCall {
name: tool_call.name,
arguments: tool_call.arguments_json,
},
}],
tool_call_id: None,
function_call: None,
},
finish_reason: Some("tool_calls".to_string()),
});
Ok(ParsedChatModelOutput {
visible: ParsedReasoningResponse {
content: parsed.content,
reasoning: parsed.reasoning_content,
},
harmony_response,
})
}
}
}
/// Handle streaming chat completions
async fn handle_chat_completions_stream(
state: AppState,
openai_request: ChatCompletionsRequest,
inference_request: InferenceRequest,
benchmark_correlation: Option<BenchmarkRequestCorrelation>,
) -> std::result::Result<Response, ServerError> {
let (tx, rx) = mpsc::unbounded_channel::<std::result::Result<Event, axum::Error>>();
// Spawn task to generate tokens
let engine = state.llm.clone().ok_or_else(|| {
ServerError::ServiceUnavailable("LLM engine not loaded; chat unavailable".into())
})?;
let request_id = inference_request.id.to_string();
let include_stream_usage = openai_request
.stream_options
.as_ref()
.and_then(|opts| opts.include_usage)
.unwrap_or(false);
let output_contract = EffectiveChatOutputContract::resolve(&openai_request);
let buffer_json_object_stream = matches!(
output_contract,
EffectiveChatOutputContract::JsonObjectContent
);
let buffer_strict_json_schema_stream = matches!(
output_contract,
EffectiveChatOutputContract::StrictJsonSchemaContent
);
let stream_api_request = match inference_request.api_request.as_ref() {
Some(ferrum_types::ApiRequest::Chat(request)) => request.clone(),
_ => api_chat_request(
&openai_request,
openai_request.tool_choice.as_ref(),
ferrum_types::ApiToolCallProtocol::default(),
),
};
let buffer_structured_api_stream =
ferrum_types::chat_api_may_emit_tool_or_function_call(&stream_api_request);
let model_output_protocol = inference_request.sampling_params.model_output_protocol;
let buffer_stream_output = buffer_json_object_stream
|| buffer_strict_json_schema_stream
|| buffer_structured_api_stream
|| model_output_protocol == ModelOutputProtocol::HarmonyGptOss;
// R1-distill-style templates open the think block inside the prompt;
// the parser must know generation starts mid-think.
let started_in_think = request_started_in_reasoning(&inference_request);
let mut native_projector = NativeChatOutputProjector::for_request(&inference_request);
let replay_request_id = inference_request.id.to_string();
let profile_request_model = openai_request.model.clone();
let profile_started_at = Instant::now();
let request_memory_before = request_memory_sample_before(&state);
let mut stream = match engine.infer_stream(inference_request).await {
Ok(stream) => stream,
Err(e) => {
let failure_kind = e.observability_failure_kind();
let error_kind = e.observability_error_kind();
let error_message = e.to_string();
if let Err(err) = write_chat_request_profile_event(
&state,
&replay_request_id,
benchmark_correlation.as_ref(),
&profile_request_model,
true,
"chat_completions_stream_start",
profile_started_at,
ChatRequestProfileTiming::default(),
0,
None,
Some("error"),
Some(ProfileError {
kind: error_kind.to_string(),
message: error_message.clone(),
blocking: false,
}),
) {
warn!("failed to write chat stream failure profile event: {}", err);
}
let engine_status = if chat_resource_failure_kind(failure_kind) {
Some(engine.status().await)
} else {
None
};
error!(
"Stream generation failed before first chunk: {}",
error_message
);
if let Err(err) = write_chat_request_failure_diagnostics(
&state,
&replay_request_id,
failure_kind,
"chat_completions_stream_start",
error_kind,
&error_message,
engine_status.as_ref(),
) {
warn!("failed to write chat stream failure diagnostics: {}", err);
}
return Err(server_error_from_ferrum_error(e));
}
};
let request_dump_dir = state.request_dump_dir.clone();
let admission_summary = state
.auto_config
.as_ref()
.map(|config| config.admission_summary_document());
let diagnostics_engine = engine.clone();
let profile_state = state.clone();
tokio::spawn(async move {
let mut current_text = String::new();
let mut output_token_ids = Vec::new();
let mut first_engine_chunk_received_us = None;
let mut first_sse_enqueue_us = None;
let mut sent_reasoning_len = 0usize;
let mut sent_content_len = 0usize;
loop {
let next = tokio::select! {
biased;
_ = tx.closed() => break,
next = stream.next() => next,
};
let Some(result) = next else {
break;
};
match result {
Ok(chunk) => {
if first_engine_chunk_received_us.is_none()
&& (chunk.token.is_some() || !chunk.text.is_empty())
{
first_engine_chunk_received_us = Some(elapsed_us_since(profile_started_at));
}
if let Some(token) = chunk.token {
output_token_ids.push(token);
}
if !chunk.text.is_empty() {
current_text.push_str(&chunk.text);
if let Some(projector) = native_projector.as_mut() {
projector.push(&chunk.text);
}
if native_projector.is_some()
|| (!buffer_stream_output
&& !should_defer_model_reasoning_stream_delta(
model_output_protocol,
¤t_text,
))
{
let parsed_result = if let Some(projector) = native_projector.as_ref() {
Ok(ParsedReasoningResponse {
content: projector.visible_prefix().to_owned(),
reasoning: projector.reasoning_prefix().map(str::to_owned),
})
} else {
parse_model_reasoning_response(
model_output_protocol,
¤t_text,
started_in_think,
)
};
let parsed = match parsed_result {
Ok(parsed) => parsed,
Err(error) => {
let _ = tx.send(Ok(openai_error_sse_event(
error.to_string(),
"internal_server_error",
Some("model_output"),
)));
let _ = tx.send(Ok(Event::default().data("[DONE]")));
break;
}
};
let full_reasoning = parsed.reasoning.as_deref().unwrap_or("");
let reasoning_delta =
stream_text_delta(full_reasoning, &mut sent_reasoning_len);
let content_delta =
stream_text_delta(&parsed.content, &mut sent_content_len);
if !reasoning_delta.is_empty() || !content_delta.is_empty() {
// Create streaming response chunk
let response_chunk = ChatCompletionsResponse {
id: request_id.clone(),
object: "chat.completion.chunk".to_string(),
created: chrono::Utc::now().timestamp() as u64,
model: openai_request.model.clone(),
choices: vec![ChatChoice {
index: 0,
message: None,
delta: Some(ChatMessage {
role: MessageRole::Assistant,
content: content_delta,
reasoning: (!reasoning_delta.is_empty())
.then_some(reasoning_delta),
name: None,
tool_calls: None,
tool_call_id: None,
function_call: None,
}),
finish_reason: None,
}],
usage: None,
};
let sse_event = Event::default()
.json_data(&response_chunk)
.unwrap_or_else(|_| Event::default().data("error"));
if tx.send(Ok(sse_event)).is_err() {
break;
}
first_sse_enqueue_us
.get_or_insert_with(|| elapsed_us_since(profile_started_at));
}
}
}
if chunk.finish_reason.is_some() {
let terminal_finish_reason = chunk
.finish_reason
.expect("finish_reason presence checked above");
if let Err(err) = write_chat_prompt_token_evidence(
request_dump_dir.as_ref().map(|root| root.as_path()),
&replay_request_id,
&profile_request_model,
chunk.execution_evidence.as_ref(),
) {
warn!("failed to write chat stream prompt-token evidence: {}", err);
}
let usage = chunk.usage.as_ref().map(openai_usage_from_token_usage);
let native_projected = native_projector
.take()
.map(|projector| projector.finish(terminal_finish_reason));
let parsed_output_result =
if let Some(projected) = native_projected.as_ref() {
Ok(ParsedChatModelOutput {
visible: projected.visible.clone(),
harmony_response: None,
})
} else {
parse_chat_model_output(
model_output_protocol,
¤t_text,
started_in_think,
terminal_finish_reason,
)
};
let parsed_model_output = match parsed_output_result {
Ok(parsed) => parsed,
Err(error) => {
let error_event = openai_error_sse_event(
stream_validation_error_message(error),
"internal_server_error",
Some("model_output"),
);
let _ = tx.send(Ok(error_event));
let _ = tx.send(Ok(Event::default().data("[DONE]")));
break;
}
};
let mut parsed_final = parsed_model_output.visible;
parsed_final.content = normalize_structured_response_content(
&openai_request,
&parsed_final.content,
);
let mut structured_chat_response =
finish_reason_allows_structured_api_response(terminal_finish_reason)
.then(|| match chunk.api_response.as_ref() {
// The native envelope owns the recipient and whether this is
// a call at all, including a final answer containing JSON.
_ if model_output_protocol
== ModelOutputProtocol::HarmonyGptOss =>
{
parsed_model_output.harmony_response.clone()
}
Some(ferrum_types::ApiResponse::Chat(response)) => {
Some(response.clone())
}
_ if native_projected.is_some() => native_projected
.as_ref()
.and_then(|projected| projected.api_response.clone()),
_ if buffer_structured_api_stream => {
chat_api_response_from_parsed_generated_text(
&stream_api_request,
&parsed_final,
terminal_finish_reason,
)
}
_ => None,
})
.flatten();
if native_projected.is_none()
&& matches!(
chunk.api_response,
Some(ferrum_types::ApiResponse::Chat(_))
)
{
if let Some(response) = structured_chat_response.as_mut() {
if let Err(error) = project_typed_tool_response_content(
response,
model_output_protocol,
started_in_think,
) {
let _ = tx.send(Ok(openai_error_sse_event(
stream_validation_error_message(error),
"internal_server_error",
Some("model_output"),
)));
let _ = tx.send(Ok(Event::default().data("[DONE]")));
break;
}
}
}
if let Some(chat_response) = structured_chat_response.as_ref() {
if let Err(e) =
validate_structured_tool_response(&openai_request, chat_response)
{
let error_event = openai_error_sse_event(
stream_validation_error_message(e),
"internal_server_error",
Some("tool_choice"),
);
let _ = tx.send(Ok(error_event));
let _ = tx.send(Ok(Event::default().data("[DONE]")));
break;
}
} else if tool_choice_required(&openai_request) {
log_required_tool_choice_failure(
&openai_request,
&parsed_final.content,
parsed_final.reasoning.as_deref(),
);
let error_event = openai_error_sse_event(
"model output did not satisfy required tool_choice",
"invalid_request_error",
Some("tool_choice"),
);
let _ = tx.send(Ok(error_event));
let _ = tx.send(Ok(Event::default().data("[DONE]")));
break;
}
if let Err(e) = validate_hard_structured_response(
&openai_request,
&parsed_final.content,
structured_chat_response.as_ref(),
) {
let error_event = openai_error_sse_event(
stream_validation_error_message(e),
"internal_server_error",
structured_response_error_param(output_contract),
);
let _ = tx.send(Ok(error_event));
let _ = tx.send(Ok(Event::default().data("[DONE]")));
break;
}
if let Some(chat_response) = structured_chat_response.as_ref() {
let mut delta = openai_chat_delta_from_api(&chat_response.message);
if native_projected.is_some() {
delta.content =
stream_text_delta(&delta.content, &mut sent_content_len);
}
if delta.reasoning.is_none() {
delta.reasoning = parsed_final.reasoning.clone();
}
if native_projected.is_some() {
let reasoning = stream_text_delta(
delta.reasoning.as_deref().unwrap_or(""),
&mut sent_reasoning_len,
);
delta.reasoning = (!reasoning.is_empty()).then_some(reasoning);
}
let response_chunk = ChatCompletionsResponse {
id: request_id.clone(),
object: "chat.completion.chunk".to_string(),
created: chrono::Utc::now().timestamp() as u64,
model: openai_request.model.clone(),
choices: vec![ChatChoice {
index: 0,
message: None,
delta: Some(delta),
finish_reason: None,
}],
usage: None,
};
let sse_event = Event::default()
.json_data(&response_chunk)
.unwrap_or_else(|_| Event::default().data("error"));
if tx.send(Ok(sse_event)).is_err() {
break;
}
first_sse_enqueue_us
.get_or_insert_with(|| elapsed_us_since(profile_started_at));
} else if buffer_structured_api_stream
&& parsed_final.content.trim().is_empty()
// A token-limited response may contain only reasoning.
// Preserve its terminal reason and usage, as sync does;
// required tool and hard content contracts were checked above.
&& terminal_finish_reason != FinishReason::Length
{
let error_event = openai_error_sse_event(
"model output did not satisfy tool/function call request",
"internal_server_error",
Some("tool_choice"),
);
let _ = tx.send(Ok(error_event));
let _ = tx.send(Ok(Event::default().data("[DONE]")));
break;
} else if !current_text.is_empty() {
// Flush safe text held with an incomplete trailing
// marker, including when text and finish share a chunk.
// Buffered streams have sent lengths of zero.
let content_delta =
stream_text_delta(&parsed_final.content, &mut sent_content_len);
let reasoning_delta = stream_text_delta(
parsed_final.reasoning.as_deref().unwrap_or(""),
&mut sent_reasoning_len,
);
if !content_delta.is_empty() || !reasoning_delta.is_empty() {
let response_chunk = ChatCompletionsResponse {
id: request_id.clone(),
object: "chat.completion.chunk".to_string(),
created: chrono::Utc::now().timestamp() as u64,
model: openai_request.model.clone(),
choices: vec![ChatChoice {
index: 0,
message: None,
delta: Some(ChatMessage {
role: MessageRole::Assistant,
content: content_delta,
reasoning: (!reasoning_delta.is_empty())
.then_some(reasoning_delta),
name: None,
tool_calls: None,
tool_call_id: None,
function_call: None,
}),
finish_reason: None,
}],
usage: None,
};
let sse_event = Event::default()
.json_data(&response_chunk)
.unwrap_or_else(|_| Event::default().data("error"));
if tx.send(Ok(sse_event)).is_err() {
break;
}
first_sse_enqueue_us
.get_or_insert_with(|| elapsed_us_since(profile_started_at));
}
}
// Send final chunk. OpenAI-style streaming
// clients (e.g. `vllm bench serve`) blindly
// read `choices[0]["delta"]` on every chunk
// that has any `choices` entries, so the
// last chunk must include `delta` even when
// empty. Skipping it triggers
// `KeyError: 'delta'` on the client side
// and the request is reported as failed
// despite returning a 200 with content.
let final_finish_reason = structured_chat_response
.as_ref()
.and_then(|response| response.finish_reason.clone())
.or_else(|| chunk.finish_reason.as_ref().map(finish_reason_to_string))
.or(Some("length".to_string()));
let final_chunk = ChatCompletionsResponse {
id: request_id.clone(),
object: "chat.completion.chunk".to_string(),
created: chrono::Utc::now().timestamp() as u64,
model: openai_request.model.clone(),
choices: vec![ChatChoice {
index: 0,
message: None,
delta: Some(ChatMessage {
role: MessageRole::Assistant,
content: String::new(),
reasoning: None,
name: None,
tool_calls: None,
tool_call_id: None,
function_call: None,
}),
finish_reason: final_finish_reason.clone(),
}],
usage: None,
};
let final_event = Event::default()
.json_data(&final_chunk)
.unwrap_or_else(|_| Event::default().data("error"));
if tx.send(Ok(final_event)).is_ok() {
first_sse_enqueue_us
.get_or_insert_with(|| elapsed_us_since(profile_started_at));
}
let completion_token_count = chunk
.usage
.as_ref()
.map(|usage| usage.completion_tokens)
.unwrap_or(output_token_ids.len());
let replay_output_token_ids = chunk
.execution_evidence
.as_ref()
.map(|evidence| evidence.output_token_ids.as_slice())
.filter(|tokens| tokens.len() == completion_token_count)
.unwrap_or(output_token_ids.as_slice());
if let Err(err) = write_chat_request_completion_replay_bundle(
request_dump_dir.as_ref().map(|root| root.as_path()),
&replay_request_id,
&parsed_final.content,
replay_output_token_ids,
final_finish_reason.as_deref(),
) {
warn!("failed to write chat stream replay bundle: {}", err);
}
if let Err(err) = write_chat_request_profile_event(
&profile_state,
&replay_request_id,
benchmark_correlation.as_ref(),
&profile_request_model,
true,
"chat_completions_stream_complete",
profile_started_at,
ChatRequestProfileTiming {
engine_evidence: chunk.execution_evidence.as_ref(),
first_engine_chunk_received_us,
first_sse_enqueue_us,
},
completion_token_count,
chunk.usage.as_ref(),
final_finish_reason.as_deref(),
None,
) {
warn!("failed to write chat stream profile event: {}", err);
}
if let Err(err) = maybe_write_first_request_memory_stage(
&profile_state,
&replay_request_id,
benchmark_correlation.as_ref(),
&profile_request_model,
true,
profile_started_at,
request_memory_before,
) {
warn!("failed to write chat stream memory profile event: {}", err);
}
if include_stream_usage && usage.is_some() {
let usage_chunk = ChatCompletionsResponse {
id: request_id.clone(),
object: "chat.completion.chunk".to_string(),
created: chrono::Utc::now().timestamp() as u64,
model: openai_request.model.clone(),
choices: vec![],
usage,
};
let usage_event = Event::default()
.json_data(&usage_chunk)
.unwrap_or_else(|_| Event::default().data("error"));
let _ = tx.send(Ok(usage_event));
}
let _ = tx.send(Ok(Event::default().data("[DONE]")));
break;
}
}
Err(e) => {
let failure_kind = e.observability_failure_kind();
let error_kind = e.observability_error_kind();
let error_message = e.to_string();
let engine_status = if chat_resource_failure_kind(failure_kind) {
Some(diagnostics_engine.status().await)
} else {
None
};
error!("Stream generation error: {}", error_message);
if let Err(err) = write_chat_request_profile_event(
&profile_state,
&replay_request_id,
benchmark_correlation.as_ref(),
&profile_request_model,
true,
"chat_completions_stream_next",
profile_started_at,
ChatRequestProfileTiming {
engine_evidence: None,
first_engine_chunk_received_us,
first_sse_enqueue_us,
},
output_token_ids.len(),
None,
Some("error"),
Some(ProfileError {
kind: error_kind.to_string(),
message: error_message.clone(),
blocking: false,
}),
) {
warn!("failed to write chat stream chunk profile event: {}", err);
}
if let Err(err) = write_chat_request_failure_diagnostics_at_root(
request_dump_dir.as_ref().map(|root| root.as_path()),
admission_summary.as_ref(),
engine_status.as_ref(),
&replay_request_id,
failure_kind,
"chat_completions_stream_next",
error_kind,
&error_message,
) {
warn!(
"failed to write chat stream chunk failure diagnostics: {}",
err
);
}
let _ = tx.send(Ok(openai_error_sse_event(
error_message,
"internal_server_error",
None,
)));
let _ = tx.send(Ok(Event::default().data("[DONE]")));
break;
}
}
}
});
let stream = tokio_stream::wrappers::UnboundedReceiverStream::new(rx);
let sse_stream = Sse::new(stream);
Ok(sse_stream.into_response())
}
/// Handle non-streaming chat completions
async fn handle_chat_completions_sync(
state: AppState,
openai_request: ChatCompletionsRequest,
inference_request: InferenceRequest,
session_context: Option<SessionContext>,
benchmark_correlation: Option<BenchmarkRequestCorrelation>,
) -> std::result::Result<Response, ServerError> {
info!("Processing non-streaming chat completion");
let engine = state.llm.clone().ok_or_else(|| {
ServerError::ServiceUnavailable("LLM engine not loaded; chat unavailable".into())
})?;
let request_chat_api = inference_request
.api_request
.as_ref()
.and_then(|api_request| match api_request {
ferrum_types::ApiRequest::Chat(chat_request) => {
ferrum_types::chat_api_may_emit_tool_or_function_call(chat_request)
.then(|| chat_request.clone())
}
_ => None,
});
let model_output_protocol = inference_request.sampling_params.model_output_protocol;
// R1-distill-style templates open the think block inside the prompt.
let started_in_think = request_started_in_reasoning(&inference_request);
let mut native_projector = NativeChatOutputProjector::for_request(&inference_request);
let replay_request_id = inference_request.id.to_string();
let profile_request_model = openai_request.model.clone();
let profile_started_at = Instant::now();
let request_memory_before = request_memory_sample_before(&state);
match engine.infer(inference_request).await {
Ok(output) => {
let InferenceResponse {
text: output_text,
tokens,
finish_reason,
usage,
api_response,
execution_evidence,
..
} = output;
if let Err(err) = write_chat_prompt_token_evidence(
state.request_dump_dir.as_ref().map(|root| root.as_path()),
&replay_request_id,
&profile_request_model,
execution_evidence.as_ref(),
) {
warn!("failed to write chat prompt-token evidence: {}", err);
}
// OpenAI stop strings mark a boundary and are not included in the
// returned completion. Hard structured formats are never repaired
// here; malformed output must fail the response contract below.
let stop_sequences = openai_request.stop.clone().unwrap_or_default();
let content = strip_after_stop(&output_text, &stop_sequences);
let native_projected = native_projector.take().map(|mut projector| {
projector.push(&content);
projector.finish(finish_reason)
});
let parsed_model_output = if let Some(projected) = native_projected.as_ref() {
ParsedChatModelOutput {
visible: projected.visible.clone(),
harmony_response: None,
}
} else {
parse_chat_model_output(
model_output_protocol,
&content,
started_in_think,
finish_reason,
)?
};
let parsed = parsed_model_output.visible;
let visible_content =
normalize_structured_response_content(&openai_request, &parsed.content);
let mut message = ChatMessage {
role: MessageRole::Assistant,
content: visible_content,
reasoning: parsed.reasoning.clone(),
name: None,
tool_calls: None,
tool_call_id: None,
function_call: None,
};
let mut openai_finish_reason = finish_reason_to_string(&finish_reason);
let mut structured_chat_response =
finish_reason_allows_structured_api_response(finish_reason)
.then(|| match api_response.as_ref() {
// The native envelope owns the recipient and whether this is
// a call at all, including a final answer containing JSON.
_ if model_output_protocol == ModelOutputProtocol::HarmonyGptOss => {
parsed_model_output.harmony_response.clone()
}
Some(ferrum_types::ApiResponse::Chat(chat_response)) => {
Some(chat_response.clone())
}
_ if native_projected.is_some() => native_projected
.as_ref()
.and_then(|projected| projected.api_response.clone()),
_ => match request_chat_api.as_ref() {
Some(chat_request) => chat_api_response_from_parsed_generated_text(
chat_request,
&parsed,
finish_reason,
),
_ => None,
},
})
.flatten();
if native_projected.is_none()
&& matches!(api_response, Some(ferrum_types::ApiResponse::Chat(_)))
{
if let Some(response) = structured_chat_response.as_mut() {
project_typed_tool_response_content(
response,
model_output_protocol,
started_in_think,
)?;
}
}
if let Some(chat_response) = structured_chat_response.as_ref() {
if let Err(error) =
validate_structured_tool_response(&openai_request, chat_response)
{
if let Err(err) = write_chat_request_profile_event(
&state,
&replay_request_id,
benchmark_correlation.as_ref(),
&profile_request_model,
false,
"chat_completions_sync_tool_contract",
profile_started_at,
ChatRequestProfileTiming {
engine_evidence: execution_evidence.as_ref(),
..Default::default()
},
tokens.len(),
Some(&usage),
Some("error"),
Some(ProfileError {
kind: "tool_contract_failure".to_string(),
message: format!("{error:?}"),
blocking: true,
}),
) {
warn!("failed to write chat tool-contract profile event: {}", err);
}
return Err(error);
}
message = openai_chat_message_from_api(&chat_response.message);
if message.reasoning.is_none() {
message.reasoning = parsed.reasoning.clone();
}
if let Some(reason) = &chat_response.finish_reason {
openai_finish_reason = reason.clone();
}
} else if tool_choice_required(&openai_request) {
log_required_tool_choice_failure(
&openai_request,
&parsed.content,
parsed.reasoning.as_deref(),
);
if let Err(err) = write_chat_request_profile_event(
&state,
&replay_request_id,
benchmark_correlation.as_ref(),
&profile_request_model,
false,
"chat_completions_sync_tool_choice",
profile_started_at,
ChatRequestProfileTiming {
engine_evidence: execution_evidence.as_ref(),
..Default::default()
},
tokens.len(),
Some(&usage),
Some("error"),
Some(ProfileError {
kind: "required_tool_failure".to_string(),
message: "model output did not satisfy required tool_choice".to_string(),
blocking: true,
}),
) {
warn!("failed to write chat tool-choice profile event: {}", err);
}
return Err(ServerError::invalid_request(
"model output did not satisfy required tool_choice",
Some("tool_choice"),
));
}
if let Err(error) = validate_hard_structured_response(
&openai_request,
&message.content,
structured_chat_response.as_ref(),
) {
if let Err(err) = write_chat_request_profile_event(
&state,
&replay_request_id,
benchmark_correlation.as_ref(),
&profile_request_model,
false,
"chat_completions_sync_structured_output",
profile_started_at,
ChatRequestProfileTiming {
engine_evidence: execution_evidence.as_ref(),
..Default::default()
},
tokens.len(),
Some(&usage),
Some("error"),
Some(ProfileError {
kind: "structured_output_failure".to_string(),
message: format!("{error:?}"),
blocking: true,
}),
) {
warn!("failed to write chat strict-schema profile event: {}", err);
}
return Err(error);
}
if let Err(err) = write_chat_request_completion_replay_bundle(
state.request_dump_dir.as_ref().map(|root| root.as_path()),
&replay_request_id,
&message.content,
&tokens,
Some(&openai_finish_reason),
) {
warn!("failed to write chat completion replay bundle: {}", err);
}
if let Err(err) = write_chat_request_profile_event(
&state,
&replay_request_id,
benchmark_correlation.as_ref(),
&profile_request_model,
false,
"chat_completions_sync_complete",
profile_started_at,
ChatRequestProfileTiming {
engine_evidence: execution_evidence.as_ref(),
..Default::default()
},
tokens.len(),
Some(&usage),
Some(&openai_finish_reason),
None,
) {
warn!("failed to write chat sync profile event: {}", err);
}
if let Err(err) = maybe_write_first_request_memory_stage(
&state,
&replay_request_id,
benchmark_correlation.as_ref(),
&profile_request_model,
false,
profile_started_at,
request_memory_before,
) {
warn!("failed to write chat sync memory profile event: {}", err);
}
state
.cache
.update_session(session_context, message.clone(), &CachePolicy::current());
let response = ChatCompletionsResponse {
id: replay_request_id,
object: "chat.completion".to_string(),
created: chrono::Utc::now().timestamp() as u64,
model: openai_request.model,
choices: vec![ChatChoice {
index: 0,
message: Some(message),
delta: None,
finish_reason: Some(openai_finish_reason),
}],
usage: Some(openai_usage_from_token_usage(&usage)),
};
Ok(Json(response).into_response())
}
Err(e) => {
let failure_kind = e.observability_failure_kind();
let error_kind = e.observability_error_kind();
let error_message = e.to_string();
let engine_status = if chat_resource_failure_kind(failure_kind) {
Some(engine.status().await)
} else {
None
};
error!("Generation failed: {}", error_message);
if let Err(err) = write_chat_request_profile_event(
&state,
&replay_request_id,
benchmark_correlation.as_ref(),
&profile_request_model,
false,
"chat_completions_sync",
profile_started_at,
ChatRequestProfileTiming::default(),
0,
None,
Some("error"),
Some(ProfileError {
kind: error_kind.to_string(),
message: error_message.clone(),
blocking: false,
}),
) {
warn!("failed to write chat sync failure profile event: {}", err);
}
if let Err(err) = write_chat_request_failure_diagnostics(
&state,
&replay_request_id,
failure_kind,
"chat_completions_sync",
error_kind,
&error_message,
engine_status.as_ref(),
) {
warn!(
"failed to write chat generation failure diagnostics: {}",
err
);
}
Err(server_error_from_ferrum_error(e))
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum EffectiveChatOutputContract {
RequiredToolCall,
StrictJsonSchemaContent,
JsonObjectContent,
BestEffortJsonSchemaContent,
Text,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ChatOutputBudget {
AutoCeiling(u32),
Explicit(u32),
}
impl ChatOutputBudget {
fn resolve(request: &ChatCompletionsRequest) -> Self {
request
.max_completion_tokens
.or(request.max_tokens)
.map(Self::Explicit)
.unwrap_or(Self::AutoCeiling(DEFAULT_COMPLETION_MAX_TOKENS))
}
const fn ceiling(self) -> u32 {
match self {
Self::AutoCeiling(value) | Self::Explicit(value) => value,
}
}
const fn is_auto(self) -> bool {
matches!(self, Self::AutoCeiling(_))
}
}
impl EffectiveChatOutputContract {
fn resolve(request: &ChatCompletionsRequest) -> Self {
if tool_choice_required(request) {
return Self::RequiredToolCall;
}
let Some(format) = request.response_format.as_ref() else {
return Self::Text;
};
match format.format_type.as_str() {
"json_schema"
if format
.json_schema
.as_ref()
.and_then(|schema| schema.strict)
.unwrap_or(false) =>
{
Self::StrictJsonSchemaContent
}
"json_schema" => Self::BestEffortJsonSchemaContent,
"json_object" => Self::JsonObjectContent,
_ => Self::Text,
}
}
fn accepts_requested_response_format(self) -> bool {
!matches!(self, Self::RequiredToolCall)
}
}
/// Convert OpenAI chat request to internal inference request
#[allow(dead_code)]
fn convert_chat_request(
request: &ChatCompletionsRequest,
) -> ferrum_types::Result<InferenceRequest> {
convert_chat_request_with_template_model(request, &request.model, None)
}
fn request_started_in_reasoning(request: &InferenceRequest) -> bool {
request
.metadata
.get(PROMPT_OPENED_REASONING_METADATA_KEY)
.and_then(serde_json::Value::as_bool)
.unwrap_or_else(|| {
// Requests constructed outside the chat converter retain their
// legacy parser behavior. Rendered chat requests carry the actual
// generation-suffix state, independent of user/schema text.
has_unclosed_model_reasoning_block(
request.sampling_params.model_output_protocol,
&request.prompt,
)
})
}
/// Convert OpenAI chat request to internal inference request.
///
/// `template_model_id` is the loaded model id used for prompt-template family
/// detection. The request `model` field may be an OpenAI-compatible alias such
/// as "ferrum"; using it for template selection can feed a fallback prompt to
/// a Qwen/Llama model.
fn convert_chat_request_with_template_model(
request: &ChatCompletionsRequest,
template_model_id: &str,
model_template: Option<&ModelChatTemplate>,
) -> ferrum_types::Result<InferenceRequest> {
convert_chat_request_with_template_model_and_default(
request,
template_model_id,
model_template,
None,
true,
None,
)
}
fn convert_chat_request_with_template_model_and_default(
request: &ChatCompletionsRequest,
template_model_id: &str,
model_template: Option<&ModelChatTemplate>,
default_enable_thinking: Option<bool>,
interleaved_system_coalescing: bool,
message_phases: Option<&[Option<AssistantMessagePhase>]>,
) -> ferrum_types::Result<InferenceRequest> {
let no_tools: &[ChatTool] = &[];
let tools = if tool_choice_none_hides_tools(request.tool_choice.as_ref(), model_template) {
no_tools
} else {
request.tools.as_deref().unwrap_or_default()
};
let default_tool_choice =
default_auto_tool_choice_for_tools(tools, request.tool_choice.as_ref());
let effective_tool_choice = request
.tool_choice
.as_ref()
.or(default_tool_choice.as_ref());
let functions = request.functions.as_deref().unwrap_or_default();
let model_output_protocol = model_template
.map(|template| template.output_protocol)
.unwrap_or(ModelOutputProtocol::Text);
let output_contract = EffectiveChatOutputContract::resolve(request);
let output_budget = ChatOutputBudget::resolve(request);
let tool_call_protocol = model_template
.map(|template| {
// Harmony owns its channel/recipient envelope independently of the
// generic native JSON tool grammar.
if model_output_protocol == ModelOutputProtocol::HarmonyGptOss
&& template.tool_call_protocol == ferrum_types::ApiToolCallProtocol::NativeJson
{
ferrum_types::ApiToolCallProtocol::Json
} else {
template.tool_call_protocol
}
})
.unwrap_or_default();
let api_chat = api_chat_request(request, effective_tool_choice, tool_call_protocol);
let native_tool_call_contract = api_chat.requires_native_tool_call();
// Harmony tool calls own their complete channel/message/call envelope.
// Applying the generic tool-argument JSON grammar at token zero would
// mask that envelope and force the model to emit bare arguments instead.
let forced_response_format = (model_output_protocol != ModelOutputProtocol::HarmonyGptOss
&& !native_tool_call_contract)
.then(|| forced_tool_choice_response_format(request))
.flatten();
let hard_tool_call_contract = forced_response_format.is_some() || native_tool_call_contract;
let requested_response_format = output_contract
.accepts_requested_response_format()
.then(|| requested_response_format_for_sampling(request))
.transpose()?
.flatten();
let chat_template_options =
chat_template_options_for_request(request, model_template, default_enable_thinking)?;
let response_format = forced_response_format
.or(requested_response_format)
.unwrap_or(ferrum_types::ResponseFormat::Text);
let model_generated_thinking = model_template.is_some_and(|template| {
template.reasoning_protocol == ModelReasoningProtocol::ModelGenerated
&& template.reasoning_enabled(chat_template_options.enable_thinking)
});
let reasoning_enabled = model_template
.is_some_and(|template| template.reasoning_enabled(chat_template_options.enable_thinking));
let (render_messages, render_message_phases) = render_messages_with_response_format_instruction(
request,
output_contract,
reasoning_enabled,
model_template,
message_phases,
);
let rendered_prompt = if tools.is_empty() && functions.is_empty() {
render_chat_prompt_with_model_template_options_and_compatibility_with_prefill(
&render_messages,
template_model_id,
model_template,
&chat_template_options,
interleaved_system_coalescing,
Some(&render_message_phases),
)?
} else {
render_chat_prompt_with_tools_and_model_template_compatibility_with_prefill(
&render_messages,
template_model_id,
model_template,
&chat_template_options,
tools,
effective_tool_choice,
functions,
request.function_call.as_ref(),
interleaved_system_coalescing,
Some(&render_message_phases),
)?
};
let prompt = rendered_prompt.text;
let prompt_opened_thinking = rendered_prompt.reasoning_prefill;
let mut metadata = HashMap::new();
metadata.insert(
PROMPT_OPENED_REASONING_METADATA_KEY.to_string(),
serde_json::Value::Bool(prompt_opened_thinking),
);
metadata.insert(
"openai_messages".to_string(),
serde_json::to_value(&request.messages)?,
);
if let Some(tools) = &request.tools {
metadata.insert("openai_tools".to_string(), serde_json::to_value(tools)?);
}
if let Some(tool_choice) = effective_tool_choice {
metadata.insert(
"openai_tool_choice".to_string(),
serde_json::to_value(tool_choice)?,
);
}
if let Some(functions) = &request.functions {
metadata.insert(
"openai_legacy_functions".to_string(),
serde_json::to_value(functions)?,
);
}
if let Some(function_call) = &request.function_call {
metadata.insert(
"openai_legacy_function_call".to_string(),
serde_json::to_value(function_call)?,
);
}
if request.ignore_eos.unwrap_or(false) {
metadata.insert("ferrum_ignore_eos".to_string(), serde_json::json!(true));
}
if output_budget.is_auto() {
metadata.insert(
DEFAULT_MAX_TOKENS_METADATA_KEY.to_string(),
serde_json::json!(true),
);
}
let reasoning_markers = model_reasoning_markers(model_output_protocol);
if !prompt_opened_thinking {
let mut forbidden = reasoning_markers
.map(|(_, close)| vec![close.to_string()])
.unwrap_or_default();
if hard_tool_call_contract {
for token_text in INITIAL_STRUCTURED_CALL_FORBIDDEN_TOKEN_TEXTS {
push_unique_forbidden_token_text(&mut forbidden, token_text);
}
if let Some(eos) = model_template.as_ref().and_then(|template| {
template
.eos_token
.as_deref()
.filter(|token| !token.is_empty())
}) {
push_unique_forbidden_token_text(&mut forbidden, eos);
}
}
if model_output_protocol == ModelOutputProtocol::Text
&& chat_template_options.enable_thinking == Some(false)
&& model_template
.is_some_and(|template| template.reasoning_protocol.supports_reasoning())
{
push_unique_forbidden_token_text(&mut forbidden, THINK_START_TAG);
}
metadata.insert(
INITIAL_FORBIDDEN_TOKEN_TEXTS_METADATA_KEY.to_string(),
serde_json::json!(forbidden),
);
}
let structured_output =
!matches!(response_format, ferrum_types::ResponseFormat::Text) || native_tool_call_contract;
let structured_output_after_reasoning = structured_output
&& model_output_protocol == ModelOutputProtocol::Text
&& (prompt_opened_thinking || model_generated_thinking);
let structured_output_start =
if structured_output && model_output_protocol == ModelOutputProtocol::HarmonyGptOss {
StructuredOutputStart::HarmonyFinal
} else if structured_output && model_output_protocol == ModelOutputProtocol::GemmaThought {
let (opening, closing) = reasoning_markers.expect("Gemma thought markers");
if prompt_opened_thinking {
StructuredOutputStart::AfterDelimiter(closing.to_string())
} else if prompt.trim_end().ends_with(closing) {
StructuredOutputStart::Immediate
} else {
StructuredOutputStart::AfterReasoningEnvelope {
opening: opening.to_string(),
closing: closing.to_string(),
allow_reasoning: reasoning_enabled,
}
}
} else if structured_output_after_reasoning {
StructuredOutputStart::AfterDelimiter(THINK_END_TAG.to_string())
} else {
StructuredOutputStart::Immediate
};
let delayed_grammar = matches!(
structured_output_start,
StructuredOutputStart::AfterDelimiter(_)
| StructuredOutputStart::AfterReasoningEnvelope { .. }
);
let response_completion_boundary = if let Some((_, closing)) =
reasoning_markers.filter(|_| prompt_opened_thinking || delayed_grammar)
{
ResponseCompletionBoundary::AfterDelimiterAndPayload {
delimiter: closing.to_string(),
alternate_envelope: api_chat.generated_response_envelope(),
}
} else {
ResponseCompletionBoundary::Immediate
};
Ok(InferenceRequest {
id: RequestId(Uuid::new_v4()),
model_id: ModelId(request.model.clone()),
prompt,
sampling_params: SamplingParams {
max_tokens: output_budget.ceiling() as usize,
temperature: request.temperature.unwrap_or(DEFAULT_SAMPLING_TEMPERATURE),
top_p: request.top_p.unwrap_or(DEFAULT_SAMPLING_TOP_P),
top_k: request
.top_k
.filter(|value| *value > 0)
.and_then(|value| usize::try_from(value).ok()),
repetition_penalty: request
.repetition_penalty
.unwrap_or(DEFAULT_CHAT_REPETITION_PENALTY),
presence_penalty: request.presence_penalty.unwrap_or(0.0),
frequency_penalty: request.frequency_penalty.unwrap_or(0.0),
stop_sequences: request.stop.clone().unwrap_or_default(),
seed: request.seed,
min_p: request.min_p.filter(|value| *value > 0.0),
tfs: None,
typical_p: None,
mirostat: None,
response_format,
structured_output_start,
response_completion_boundary,
model_output_protocol,
},
stream: request.stream.unwrap_or(false),
priority: Priority::Normal, // Default priority
client_id: None,
session_id: None,
created_at: chrono::Utc::now(),
api_request: Some(ferrum_types::ApiRequest::Chat(api_chat)),
evidence_request: Default::default(),
metadata,
})
}
fn push_unique_forbidden_token_text(tokens: &mut Vec<String>, token: &str) {
if !token.is_empty() && !tokens.iter().any(|existing| existing == token) {
tokens.push(token.to_string());
}
}
fn default_auto_tool_choice_for_tools(
tools: &[ChatTool],
choice: Option<&ToolChoice>,
) -> Option<ToolChoice> {
if choice.is_none() && !tools.is_empty() {
Some(ToolChoice::Mode("auto".to_string()))
} else {
None
}
}
fn tool_choice_none(choice: Option<&ToolChoice>) -> bool {
matches!(choice, Some(ToolChoice::Mode(mode)) if mode.eq_ignore_ascii_case("none"))
}
fn tool_choice_none_hides_tools(
choice: Option<&ToolChoice>,
model_template: Option<&ModelChatTemplate>,
) -> bool {
tool_choice_none(choice)
&& model_template
.map(|template| template.template.contains("tools_in_user_message"))
.unwrap_or(false)
}
fn chat_template_options_for_request(
request: &ChatCompletionsRequest,
model_template: Option<&ModelChatTemplate>,
default_enable_thinking: Option<bool>,
) -> ferrum_types::Result<ChatTemplateOptions> {
let mut options = ChatTemplateOptions::default_for_template(model_template);
let kwargs = request.chat_template_kwargs.as_ref();
let explicit_thinking = kwargs
.and_then(|values| values.get("enable_thinking"))
.filter(|value| !value.is_null())
.map(|value| {
value.as_bool().ok_or_else(|| {
Error::invalid_request("chat_template_kwargs.enable_thinking must be a boolean")
})
})
.transpose()?;
let extension_effort = kwargs
.and_then(|values| values.get("reasoning_effort"))
.filter(|value| !value.is_null())
.map(|value| {
serde_json::from_value::<ReasoningEffort>(value.clone()).map_err(|error| {
Error::invalid_request(format!("chat_template_kwargs.reasoning_effort: {error}"))
})
})
.transpose()?;
if let (Some(standard), Some(extension)) = (request.reasoning_effort, extension_effort) {
if standard != extension {
return Err(Error::invalid_request(
"reasoning_effort conflicts with chat_template_kwargs.reasoning_effort",
));
}
}
options.reasoning_effort = request.reasoning_effort.or(extension_effort);
let effort_thinking = request
.reasoning_effort
.map(|effort| effort != ReasoningEffort::None);
if let (Some(explicit), Some(derived)) = (explicit_thinking, effort_thinking) {
if explicit != derived {
return Err(Error::invalid_request(
"reasoning_effort conflicts with chat_template_kwargs.enable_thinking",
));
}
}
// Standard effort overrides product defaults and supplies both variables.
// Legacy-only kwargs keep their template-level behavior: an effort alone
// must not change their thinking switch or acquire new capability checks.
options.enable_thinking = explicit_thinking
.or(effort_thinking)
.or(default_enable_thinking);
if let (Some(template), Some(effort)) = (model_template, request.reasoning_effort) {
template.validate_reasoning_effort(effort)?;
}
Ok(options)
}
fn render_messages_with_response_format_instruction(
request: &ChatCompletionsRequest,
output_contract: EffectiveChatOutputContract,
reasoning_enabled: bool,
model_template: Option<&ModelChatTemplate>,
message_phases: Option<&[Option<AssistantMessagePhase>]>,
) -> (Vec<ChatMessage>, Vec<Option<AssistantMessagePhase>>) {
let mut phases = message_phases
.map(ToOwned::to_owned)
.unwrap_or_else(|| vec![None; request.messages.len()]);
debug_assert_eq!(phases.len(), request.messages.len());
let Some(instruction) = response_format_prompt_instruction(
request,
output_contract,
reasoning_enabled,
model_template,
) else {
return (request.messages.clone(), phases);
};
let mut messages = request.messages.clone();
let leading_systems = messages
.iter()
.take_while(|message| message.role == MessageRole::System)
.count();
let mut system_parts = Vec::with_capacity(leading_systems + 1);
system_parts.push(instruction);
system_parts.extend(
messages
.drain(..leading_systems)
.map(|message| message.content)
.filter(|content| !content.is_empty()),
);
phases.drain(..leading_systems);
messages.insert(
0,
ChatMessage {
role: MessageRole::System,
content: system_parts.join("\n\n"),
reasoning: None,
name: None,
tool_calls: None,
tool_call_id: None,
function_call: None,
},
);
phases.insert(0, None);
(messages, phases)
}
fn response_format_prompt_instruction(
request: &ChatCompletionsRequest,
output_contract: EffectiveChatOutputContract,
reasoning_enabled: bool,
model_template: Option<&ModelChatTemplate>,
) -> Option<String> {
if !output_contract.accepts_requested_response_format() {
return None;
}
let automatic_tools = request
.tools
.as_ref()
.is_some_and(|tools| !tools.is_empty())
&& match request.tool_choice.as_ref() {
None => true,
Some(ToolChoice::Mode(mode)) => mode.eq_ignore_ascii_case("auto"),
_ => false,
}
&& matches!(
output_contract,
EffectiveChatOutputContract::StrictJsonSchemaContent
| EffectiveChatOutputContract::JsonObjectContent
);
let native_tool_template =
model_template.is_some_and(crate::chat_template::model_template_supports_tools);
let tool_instruction = "If a tool is needed, put a JSON object with the declared function name in the name field and its argument object in the arguments field inside <tool_call>...</tool_call>. This envelope distinguishes a tool call from a final JSON answer.";
if let Some(format) = request.response_format.as_ref() {
return match format.format_type.as_str() {
"json_object" if automatic_tools && native_tool_template => {
Some(native_tool_final_format_instruction(r#"{"type":"object"}"#))
}
"json_object" if automatic_tools => Some(format!(
"{tool_instruction} The response_format applies only to the final answer: output a single valid JSON object, with no markdown fences or extra text."
)),
"json_object" => Some(
"The response_format requires a single valid JSON object. Output only JSON, with no markdown fences, no explanation, no chain-of-thought, and no extra text."
.to_string(),
),
"json_schema" => {
let schema = format.json_schema.as_ref()?.schema.as_ref()?;
let schema_text = serde_json::to_string(schema).ok()?;
Some(if automatic_tools && native_tool_template {
native_tool_final_format_instruction(&schema_text)
} else if automatic_tools {
format!(
"{tool_instruction} Complete any enabled reasoning before the final answer. The response_format applies only to the final answer: output a single valid JSON value satisfying this JSON Schema, with no markdown fences or extra text. Schema: {schema_text}"
)
} else if reasoning_enabled {
format!(
"The response_format requires a single valid JSON value satisfying this JSON Schema. Complete any enabled reasoning before the final answer. In the final answer, output only JSON, with no markdown fences, no explanation, and no extra text. Schema: {schema_text}"
)
} else {
format!(
"The response_format requires a single valid JSON value satisfying this JSON Schema. Output only JSON, with no markdown fences, no explanation, no chain-of-thought, and no extra text. Schema: {schema_text}"
)
})
}
_ => None,
};
}
None
}
fn native_tool_final_format_instruction(schema: &str) -> String {
// The model template owns tool selection and wire instructions. Give the
// final-answer schema its own section so it is not folded into those tool
// instructions, and leave the native call format unchanged.
format!(
"# Response Format\n\nYour final response should be a JSON value that conforms to the following schema:\n\n{schema}\n\nDo not wrap your JSON response in Markdown code blocks."
)
}
fn forced_tool_choice_response_format(
request: &ChatCompletionsRequest,
) -> Option<ferrum_types::ResponseFormat> {
let selected_tool = selected_tool_for_forced_tool_choice(request)?;
let schema = guided_tool_arguments_schema(selected_tool.function.parameters.as_ref())?;
serde_json::to_string(&schema)
.ok()
.map(ferrum_types::ResponseFormat::JsonSchema)
}
fn requested_response_format_for_sampling(
request: &ChatCompletionsRequest,
) -> ferrum_types::Result<Option<ferrum_types::ResponseFormat>> {
let Some(format) = request.response_format.as_ref() else {
return Ok(None);
};
match format.format_type.as_str() {
"json_object" => Ok(Some(ferrum_types::ResponseFormat::JsonObject)),
"json_schema" => {
let Some(schema) = format.json_schema.as_ref() else {
return Err(Error::invalid_request(
"response_format.json_schema.schema is required",
));
};
if !schema.strict.unwrap_or(false) {
return Ok(None);
}
let Some(schema_value) = schema.schema.as_ref() else {
return Err(Error::invalid_request(
"response_format.json_schema.schema is required",
));
};
serde_json::to_string(schema_value)
.map(|schema| Some(ferrum_types::ResponseFormat::JsonSchema(schema)))
.map_err(|err| Error::invalid_request(err.to_string()))
}
_ => Ok(None),
}
}
fn selected_tool_for_forced_tool_choice(request: &ChatCompletionsRequest) -> Option<&ChatTool> {
match request.tool_choice.as_ref()? {
ToolChoice::Function {
tool_type,
function,
} if tool_type == "function" => request
.tools
.as_ref()?
.iter()
.find(|tool| tool.function.name == function.name),
ToolChoice::Mode(mode) if mode.eq_ignore_ascii_case("required") => {
single_function_tool(request.tools.as_deref()?)
}
_ => None,
}
}
fn guided_tool_arguments_schema(
parameters: Option<&serde_json::Value>,
) -> Option<serde_json::Value> {
let mut schema = parameters?.clone();
bound_unconstrained_tool_argument_strings(
&mut schema,
DEFAULT_GUIDED_TOOL_ARGUMENT_STRING_MAX_LENGTH,
);
Some(schema)
}
fn bound_unconstrained_tool_argument_strings(value: &mut serde_json::Value, default_max: u64) {
match value {
serde_json::Value::Object(map) => {
let is_string = map
.get("type")
.and_then(serde_json::Value::as_str)
.is_some_and(|ty| ty == "string");
let has_finite_string_shape = map.contains_key("enum") || map.contains_key("maxLength");
if is_string && !has_finite_string_shape {
map.insert(
"maxLength".to_string(),
serde_json::Value::Number(default_max.into()),
);
}
if let Some(properties) = map
.get_mut("properties")
.and_then(serde_json::Value::as_object_mut)
{
for property in properties.values_mut() {
bound_unconstrained_tool_argument_strings(property, default_max);
}
}
if let Some(items) = map.get_mut("items") {
bound_unconstrained_tool_argument_strings(items, default_max);
}
}
serde_json::Value::Array(items) => {
for item in items {
bound_unconstrained_tool_argument_strings(item, default_max);
}
}
_ => {}
}
}
fn single_function_tool(tools: &[ChatTool]) -> Option<&ChatTool> {
let mut function_tools = tools.iter().filter(|tool| tool.tool_type == "function");
let tool = function_tools.next()?;
function_tools.next().is_none().then_some(tool)
}
fn stream_text_delta(text: &str, sent_len: &mut usize) -> String {
if *sent_len <= text.len() && text.is_char_boundary(*sent_len) {
let delta = text[*sent_len..].to_string();
*sent_len = text.len();
return delta;
}
*sent_len = text.len();
String::new()
}
fn project_typed_tool_response_content(
response: &mut ferrum_types::ApiChatResponse,
protocol: ModelOutputProtocol,
started_in_think: bool,
) -> std::result::Result<(), ServerError> {
if protocol == ModelOutputProtocol::HarmonyGptOss
|| response.message.tool_calls.is_empty()
|| response.message.content.is_empty()
{
return Ok(());
}
// Engine tool parsing can precede reasoning projection. Its structural
// parser already removed the envelopes and owns every call and argument;
// project only the outside text, never a payload or classified final JSON.
response.message.content =
parse_model_reasoning_response(protocol, &response.message.content, started_in_think)
.map_err(|error| ServerError::InternalError(error.to_string()))?
.content;
Ok(())
}
fn chat_api_response_from_parsed_generated_text(
chat_request: &ferrum_types::ApiChatRequest,
parsed: &ParsedReasoningResponse,
finish_reason: FinishReason,
) -> Option<ferrum_types::ApiChatResponse> {
parsed
.reasoning
.as_deref()
.and_then(|reasoning| {
ferrum_types::chat_api_response_from_generated_text(
chat_request,
reasoning,
finish_reason,
)
})
.map(|mut response| {
// Reasoning is published through its own channel. Detecting a tool
// there must not promote its surrounding text to visible content.
response.message.content.clear();
response
})
.or_else(|| {
ferrum_types::chat_api_response_from_generated_text(
chat_request,
&parsed.content,
finish_reason,
)
})
}
fn finish_reason_allows_structured_api_response(finish_reason: FinishReason) -> bool {
matches!(finish_reason, FinishReason::Stop | FinishReason::EOS)
}
fn log_required_tool_choice_failure(
request: &ChatCompletionsRequest,
content: &str,
reasoning: Option<&str>,
) {
warn!(
model = %request.model,
content_len = content.len(),
content_head = %log_excerpt(content, 512),
reasoning_len = reasoning.map(str::len).unwrap_or(0),
reasoning_head = %reasoning.map(|value| log_excerpt(value, 512)).unwrap_or_default(),
"model output did not satisfy required tool_choice"
);
}
fn log_excerpt(value: &str, max_chars: usize) -> String {
let mut out = value.chars().take(max_chars).collect::<String>();
if value.chars().count() > max_chars {
out.push_str("...");
}
out
}
fn normalize_structured_response_content(
request: &ChatCompletionsRequest,
content: &str,
) -> String {
match EffectiveChatOutputContract::resolve(request) {
EffectiveChatOutputContract::BestEffortJsonSchemaContent => {
extract_json_object_text(content)
.unwrap_or_else(|| strip_markdown_json_fence(content).to_string())
}
EffectiveChatOutputContract::RequiredToolCall
| EffectiveChatOutputContract::StrictJsonSchemaContent
| EffectiveChatOutputContract::JsonObjectContent
| EffectiveChatOutputContract::Text => content.to_string(),
}
}
fn extract_json_object_text(text: &str) -> Option<String> {
let text = strip_markdown_json_fence(text.trim());
if serde_json::from_str::<serde_json::Value>(&text)
.ok()
.filter(|value| value.is_object())
.is_some()
{
return Some(text.to_string());
}
let start = text.find('{')?;
let mut depth = 0usize;
let mut in_string = false;
let mut escaped = false;
for (offset, ch) in text[start..].char_indices() {
if in_string {
if escaped {
escaped = false;
} else if ch == '\\' {
escaped = true;
} else if ch == '"' {
in_string = false;
}
continue;
}
match ch {
'"' => in_string = true,
'{' => depth += 1,
'}' => {
depth = depth.saturating_sub(1);
if depth == 0 {
let end = start + offset + ch.len_utf8();
let candidate = &text[start..end];
if serde_json::from_str::<serde_json::Value>(candidate)
.ok()
.filter(|value| value.is_object())
.is_some()
{
return Some(candidate.to_string());
}
}
}
_ => {}
}
}
None
}
fn api_chat_request(
request: &ChatCompletionsRequest,
effective_tool_choice: Option<&ToolChoice>,
tool_call_protocol: ferrum_types::ApiToolCallProtocol,
) -> ferrum_types::ApiChatRequest {
ferrum_types::ApiChatRequest {
messages: request.messages.iter().map(api_chat_message).collect(),
tools: request
.tools
.as_deref()
.unwrap_or_default()
.iter()
.map(api_tool)
.collect(),
tool_choice: effective_tool_choice.map(api_tool_choice),
tool_call_protocol,
legacy_functions: request
.functions
.as_deref()
.unwrap_or_default()
.iter()
.map(api_function)
.collect(),
legacy_function_call: request.function_call.as_ref().map(api_function_call_choice),
response_format: request.response_format.as_ref().map(api_response_format),
stream_options: request.stream_options.as_ref().map(|opts| {
ferrum_types::ApiStreamOptions {
include_usage: opts.include_usage,
}
}),
}
}
fn api_chat_message(message: &ChatMessage) -> ferrum_types::ApiChatMessage {
ferrum_types::ApiChatMessage {
role: match message.role {
MessageRole::System => ferrum_types::ApiMessageRole::System,
MessageRole::User => ferrum_types::ApiMessageRole::User,
MessageRole::Assistant => ferrum_types::ApiMessageRole::Assistant,
MessageRole::Function => ferrum_types::ApiMessageRole::Function,
MessageRole::Tool => ferrum_types::ApiMessageRole::Tool,
},
content: message.content.clone(),
name: message.name.clone(),
tool_calls: message
.tool_calls
.as_deref()
.unwrap_or_default()
.iter()
.map(api_tool_call)
.collect(),
tool_call_id: message.tool_call_id.clone(),
function_call: message.function_call.as_ref().map(api_function_call),
}
}
fn api_tool(tool: &ChatTool) -> ferrum_types::ApiTool {
ferrum_types::ApiTool {
tool_type: tool.tool_type.clone(),
function: api_function(&tool.function),
}
}
fn api_function(function: &ChatFunction) -> ferrum_types::ApiFunction {
ferrum_types::ApiFunction {
name: function.name.clone(),
description: function.description.clone(),
parameters: function.parameters.clone(),
strict: function.strict,
}
}
fn api_tool_choice(choice: &ToolChoice) -> ferrum_types::ApiToolChoice {
match choice {
ToolChoice::Mode(mode) => ferrum_types::ApiToolChoice::Mode(mode.clone()),
ToolChoice::Function {
tool_type,
function,
} => ferrum_types::ApiToolChoice::Function {
tool_type: tool_type.clone(),
function: ferrum_types::ApiToolChoiceFunction {
name: function.name.clone(),
},
},
}
}
fn api_function_call_choice(choice: &FunctionCallChoice) -> ferrum_types::ApiFunctionCallChoice {
match choice {
FunctionCallChoice::Mode(mode) => ferrum_types::ApiFunctionCallChoice::Mode(mode.clone()),
FunctionCallChoice::Function { name } => {
ferrum_types::ApiFunctionCallChoice::Function { name: name.clone() }
}
}
}
fn api_tool_call(tool_call: &ChatToolCall) -> ferrum_types::ApiToolCall {
ferrum_types::ApiToolCall {
id: tool_call.id.clone(),
tool_type: tool_call.tool_type.clone(),
function: api_function_call(&tool_call.function),
}
}
fn api_function_call(function_call: &ChatFunctionCall) -> ferrum_types::ApiFunctionCall {
ferrum_types::ApiFunctionCall {
name: function_call.name.clone(),
arguments: function_call.arguments.clone(),
}
}
fn openai_chat_message_from_api(message: &ferrum_types::ApiChatMessage) -> ChatMessage {
ChatMessage {
role: openai_message_role_from_api(message.role),
content: message.content.clone(),
reasoning: None,
name: message.name.clone(),
tool_calls: if message.tool_calls.is_empty() {
None
} else {
Some(
message
.tool_calls
.iter()
.map(openai_tool_call_from_api)
.collect(),
)
},
tool_call_id: message.tool_call_id.clone(),
function_call: message
.function_call
.as_ref()
.map(openai_function_call_from_api),
}
}
fn openai_message_role_from_api(role: ferrum_types::ApiMessageRole) -> MessageRole {
match role {
ferrum_types::ApiMessageRole::System => MessageRole::System,
ferrum_types::ApiMessageRole::User => MessageRole::User,
ferrum_types::ApiMessageRole::Assistant => MessageRole::Assistant,
ferrum_types::ApiMessageRole::Function => MessageRole::Function,
ferrum_types::ApiMessageRole::Tool => MessageRole::Tool,
}
}
fn openai_tool_call_from_api(tool_call: &ferrum_types::ApiToolCall) -> ChatToolCall {
ChatToolCall {
index: None,
id: tool_call.id.clone(),
tool_type: tool_call.tool_type.clone(),
function: openai_function_call_from_api(&tool_call.function),
}
}
fn openai_tool_call_delta_from_api(
index: usize,
tool_call: &ferrum_types::ApiToolCall,
) -> ChatToolCall {
ChatToolCall {
index: Some(usize_to_u32_saturating(index)),
id: tool_call.id.clone(),
tool_type: tool_call.tool_type.clone(),
function: openai_function_call_from_api(&tool_call.function),
}
}
fn openai_chat_delta_from_api(message: &ferrum_types::ApiChatMessage) -> ChatMessage {
let mut delta = openai_chat_message_from_api(message);
if !message.tool_calls.is_empty() {
delta.tool_calls = Some(
message
.tool_calls
.iter()
.enumerate()
.map(|(index, call)| openai_tool_call_delta_from_api(index, call))
.collect(),
);
}
delta
}
fn openai_function_call_from_api(
function_call: &ferrum_types::ApiFunctionCall,
) -> ChatFunctionCall {
ChatFunctionCall {
name: function_call.name.clone(),
arguments: function_call.arguments.clone(),
}
}
fn api_response_format(format: &OpenAiResponseFormat) -> ferrum_types::ApiResponseFormat {
ferrum_types::ApiResponseFormat {
format_type: format.format_type.clone(),
json_schema: format
.json_schema
.as_ref()
.map(|schema| ferrum_types::ApiJsonSchema {
name: schema.name.clone(),
schema: schema.schema.clone().unwrap_or(serde_json::Value::Null),
strict: schema.strict,
}),
}
}
fn validate_chat_request(request: &ChatCompletionsRequest) -> std::result::Result<(), ServerError> {
if request.messages.is_empty() {
return Err(ServerError::invalid_request(
"messages array must not be empty",
Some("messages"),
));
}
if let Some(n) = request.n {
if n != 1 {
return Err(ServerError::unsupported_feature(
"only n=1 is supported for chat completions",
Some("n"),
));
}
}
if request
.logit_bias
.as_ref()
.is_some_and(|bias| !bias.is_empty())
{
return Err(ServerError::unsupported_feature(
"logit_bias is not supported",
Some("logit_bias"),
));
}
if request.logprobs.unwrap_or(false) {
return Err(ServerError::unsupported_feature(
"logprobs is not supported",
Some("logprobs"),
));
}
if request.top_logprobs.unwrap_or(0) > 0 {
return Err(ServerError::unsupported_feature(
"top_logprobs is not supported",
Some("top_logprobs"),
));
}
if let Some(top_k) = request.top_k {
if top_k < -1 {
return Err(ServerError::invalid_request(
"top_k must be -1, 0, or a positive integer",
Some("top_k"),
));
}
}
if let Some(min_p) = request.min_p {
if !min_p.is_finite() || !(0.0..=1.0).contains(&min_p) {
return Err(ServerError::invalid_request(
"min_p must be in range [0, 1]",
Some("min_p"),
));
}
}
if let Some(repetition_penalty) = request.repetition_penalty {
if !repetition_penalty.is_finite() || repetition_penalty <= 0.0 {
return Err(ServerError::invalid_request(
"repetition_penalty must be positive",
Some("repetition_penalty"),
));
}
}
if let Some(presence_penalty) = request.presence_penalty {
if !presence_penalty.is_finite() || !(-2.0..=2.0).contains(&presence_penalty) {
return Err(ServerError::invalid_request(
"presence_penalty must be in range [-2, 2]",
Some("presence_penalty"),
));
}
}
if let Some(frequency_penalty) = request.frequency_penalty {
if !frequency_penalty.is_finite() || !(-2.0..=2.0).contains(&frequency_penalty) {
return Err(ServerError::invalid_request(
"frequency_penalty must be in range [-2, 2]",
Some("frequency_penalty"),
));
}
}
if request.stream_options.is_some() && !request.stream.unwrap_or(false) {
return Err(ServerError::invalid_request(
"stream_options is only valid when stream=true",
Some("stream_options"),
));
}
ensure_response_format_supported(request)?;
if let Some(tools) = &request.tools {
for tool in tools {
if tool.tool_type != "function" {
return Err(ServerError::unsupported_feature(
"only function tools are supported",
Some("tools"),
));
}
}
}
if let Some(choice) = &request.tool_choice {
match choice {
ToolChoice::Mode(mode)
if mode.eq_ignore_ascii_case("auto") || mode.eq_ignore_ascii_case("none") => {}
ToolChoice::Mode(mode) if mode.eq_ignore_ascii_case("required") => {
if request.tools.as_deref().unwrap_or_default().is_empty() {
return Err(ServerError::invalid_request(
"tool_choice=required requires at least one function tool",
Some("tool_choice"),
));
}
}
ToolChoice::Mode(_) => {
return Err(ServerError::unsupported_feature(
"unsupported tool_choice mode",
Some("tool_choice"),
));
}
ToolChoice::Function {
tool_type,
function,
} => {
if tool_type != "function" {
return Err(ServerError::unsupported_feature(
"only function tool_choice is supported",
Some("tool_choice"),
));
}
let declared = request
.tools
.as_deref()
.unwrap_or_default()
.iter()
.any(|tool| tool.function.name == function.name);
if !declared {
return Err(ServerError::invalid_request(
"tool_choice selects a function that is not declared in tools",
Some("tool_choice"),
));
}
}
}
}
if let Some(choice) = &request.function_call {
match choice {
FunctionCallChoice::Mode(mode)
if mode.eq_ignore_ascii_case("auto") || mode.eq_ignore_ascii_case("none") => {}
FunctionCallChoice::Mode(_) => {
return Err(ServerError::unsupported_feature(
"unsupported function_call mode",
Some("function_call"),
));
}
FunctionCallChoice::Function { name } => {
let declared = request
.functions
.as_deref()
.unwrap_or_default()
.iter()
.any(|function| function.name == *name);
if !declared {
return Err(ServerError::invalid_request(
"function_call selects a function that is not declared in functions",
Some("function_call"),
));
}
}
}
}
Ok(())
}
fn tool_choice_required(request: &ChatCompletionsRequest) -> bool {
match request.tool_choice.as_ref() {
Some(ToolChoice::Mode(mode)) if mode.eq_ignore_ascii_case("required") => true,
Some(ToolChoice::Function {
tool_type,
function,
}) => {
tool_type == "function"
&& request
.tools
.as_deref()
.unwrap_or_default()
.iter()
.any(|tool| tool.function.name == function.name)
}
_ => false,
}
}
fn openai_usage_from_token_usage(usage: &TokenUsage) -> Usage {
let prompt_tokens = usize_to_u32_saturating(usage.prompt_tokens);
let completion_tokens = usize_to_u32_saturating(usage.completion_tokens);
let total_tokens = usize_to_u32_saturating(usage.total_tokens);
Usage {
prompt_tokens,
completion_tokens,
total_tokens,
}
}
fn usize_to_u32_saturating(value: usize) -> u32 {
u32::try_from(value).unwrap_or(u32::MAX)
}
fn ensure_response_format_supported(
request: &ChatCompletionsRequest,
) -> std::result::Result<(), ServerError> {
if let Some(rf) = &request.response_format {
match rf.format_type.as_str() {
"text" | "json_object" => {}
"json_schema" => {
let Some(schema_config) = rf.json_schema.as_ref() else {
return Err(ServerError::invalid_request(
"response_format.json_schema.schema is required",
Some("response_format.json_schema"),
));
};
let Some(schema) = schema_config.schema.as_ref() else {
return Err(ServerError::invalid_request(
"response_format.json_schema.schema is required",
Some("response_format.json_schema"),
));
};
if schema_config.strict.unwrap_or(false) {
compiled_json_schema_validator(schema).map_err(|reason| {
ServerError::invalid_request(
format!("unsupported strict json_schema: {reason}"),
Some("response_format.json_schema"),
)
})?;
}
}
_ => {
return Err(ServerError::invalid_request(
"unsupported response_format.type",
Some("response_format.type"),
));
}
}
}
Ok(())
}
fn strict_json_schema_string(
request: &ChatCompletionsRequest,
) -> std::result::Result<Option<String>, ServerError> {
let Some(rf) = &request.response_format else {
return Ok(None);
};
if rf.format_type != "json_schema" {
return Ok(None);
}
let Some(schema) = &rf.json_schema else {
return Err(ServerError::invalid_request(
"response_format.json_schema.schema is required",
Some("response_format.json_schema"),
));
};
let Some(schema_value) = schema.schema.as_ref() else {
return Err(ServerError::invalid_request(
"response_format.json_schema.schema is required",
Some("response_format.json_schema"),
));
};
if !schema.strict.unwrap_or(false) {
return Ok(None);
}
serde_json::to_string(schema_value).map(Some).map_err(|e| {
ServerError::invalid_request(e.to_string(), Some("response_format.json_schema"))
})
}
fn validate_hard_structured_response(
request: &ChatCompletionsRequest,
content: &str,
validated_chat_response: Option<&ferrum_types::ApiChatResponse>,
) -> std::result::Result<(), ServerError> {
// Both terminal paths validate tool names, arguments and choice before
// reaching this point. The final-answer schema applies to the content
// branch; a tool call has its own independently validated argument schema.
if validated_chat_response.is_some_and(|response| !response.message.tool_calls.is_empty()) {
return Ok(());
}
// Validate the same content that the typed response will publish. A text
// fallback may interpret marker-like strings inside valid JSON as framing.
let content = validated_chat_response
.map(|response| response.message.content.as_str())
.unwrap_or(content);
match EffectiveChatOutputContract::resolve(request) {
EffectiveChatOutputContract::JsonObjectContent => {
let value = serde_json::from_str::<serde_json::Value>(content).map_err(|error| {
ServerError::InternalError(format!(
"model output did not satisfy response_format.json_object: invalid JSON: {error}"
))
})?;
if !value.is_object() {
return Err(ServerError::InternalError(
"model output did not satisfy response_format.json_object: root must be an object"
.to_string(),
));
}
Ok(())
}
EffectiveChatOutputContract::StrictJsonSchemaContent => {
let Some(schema_json) = strict_json_schema_string(request)? else {
return Ok(());
};
let schema: serde_json::Value = serde_json::from_str(&schema_json).map_err(|e| {
ServerError::InternalError(format!(
"strict json_schema could not be reconstructed after request validation: {e}"
))
})?;
validate_json_text_against_schema(&schema, content).map_err(|reason| {
ServerError::InternalError(format!(
"model output did not satisfy response_format.json_schema.strict: {reason}"
))
})
}
EffectiveChatOutputContract::RequiredToolCall
| EffectiveChatOutputContract::BestEffortJsonSchemaContent
| EffectiveChatOutputContract::Text => Ok(()),
}
}
fn structured_response_error_param(contract: EffectiveChatOutputContract) -> Option<&'static str> {
match contract {
EffectiveChatOutputContract::JsonObjectContent => Some("response_format"),
EffectiveChatOutputContract::StrictJsonSchemaContent => Some("response_format.json_schema"),
_ => None,
}
}
fn validate_structured_tool_response(
request: &ChatCompletionsRequest,
response: &ferrum_types::ApiChatResponse,
) -> std::result::Result<(), ServerError> {
let required = tool_choice_required(request);
if response.message.tool_calls.is_empty() {
if required {
return Err(ServerError::invalid_request(
"model output did not satisfy required tool_choice",
Some("tool_choice"),
));
}
return Ok(());
}
if tool_choice_none(request.tool_choice.as_ref()) {
return Err(ServerError::InternalError(
"model emitted a tool call while tool_choice is 'none'".to_string(),
));
}
if required {
if !response.message.content.trim().is_empty() {
return Err(ServerError::InternalError(
"required tool response contained assistant content".to_string(),
));
}
if response.finish_reason.as_deref() != Some("tool_calls") {
return Err(ServerError::InternalError(
"required tool response did not finish with tool_calls".to_string(),
));
}
}
let tools = request.tools.as_deref().unwrap_or_default();
for call in &response.message.tool_calls {
if call.tool_type != "function" {
return Err(ServerError::InternalError(format!(
"model emitted unsupported tool call type '{}'",
call.tool_type
)));
}
let Some(tool) = tools
.iter()
.find(|tool| tool.tool_type == "function" && tool.function.name == call.function.name)
else {
return Err(ServerError::InternalError(format!(
"model emitted undeclared tool call '{}'",
call.function.name
)));
};
if let Some(ToolChoice::Function {
tool_type,
function,
}) = request.tool_choice.as_ref()
{
if tool_type != "function" || function.name != call.function.name {
return Err(ServerError::InternalError(format!(
"model emitted tool '{}' instead of selected tool '{}'",
call.function.name, function.name
)));
}
}
let arguments: serde_json::Value =
serde_json::from_str(&call.function.arguments).map_err(|e| {
ServerError::InternalError(format!(
"model emitted invalid JSON arguments for tool '{}': {e}",
call.function.name
))
})?;
if !arguments.is_object() {
return Err(ServerError::InternalError(format!(
"model emitted non-object arguments for tool '{}'",
call.function.name
)));
}
// Automatic Chat Completions tools are best effort unless the tool
// opts into strict mode. Keep non-strict arguments intact so clients
// can report validation errors through the normal tool-result turn.
// Required/forced calls retain their existing hard output contract.
if let Some(schema) = tool
.function
.parameters
.as_ref()
.filter(|_| required || tool.function.strict.unwrap_or(false))
{
validate_json_text_against_schema(schema, &call.function.arguments).map_err(
|reason| {
ServerError::InternalError(format!(
"model arguments for tool '{}' did not satisfy its schema: {reason}",
call.function.name
))
},
)?;
}
}
Ok(())
}
fn validate_json_text_against_schema(
schema: &serde_json::Value,
content: &str,
) -> std::result::Result<(), String> {
let value = serde_json::from_str::<serde_json::Value>(content)
.map_err(|e| format!("invalid JSON: {e}"))?;
compiled_json_schema_validator(schema)?
.validate(&value)
.map_err(|error| error.to_string())
}
fn compiled_json_schema_validator(
schema: &serde_json::Value,
) -> std::result::Result<Arc<jsonschema::Validator>, String> {
let cache_key = serde_json::to_string(schema)
.map_err(|error| format!("could not serialize JSON Schema: {error}"))?;
let cache = JSON_SCHEMA_VALIDATOR_CACHE.get_or_init(|| Mutex::new(HashMap::new()));
let mut validators = cache
.lock()
.map_err(|_| "JSON Schema validator cache lock was poisoned".to_string())?;
if let Some(validator) = validators.get(&cache_key) {
return Ok(Arc::clone(validator));
}
let validator = Arc::new(
jsonschema::validator_for(schema)
.map_err(|error| format!("could not compile JSON Schema: {error}"))?,
);
if validators.len() >= MAX_CACHED_JSON_SCHEMA_VALIDATORS {
validators.clear();
}
validators.insert(cache_key, Arc::clone(&validator));
Ok(validator)
}
fn stream_validation_error_message(error: ServerError) -> String {
match error {
ServerError::InternalError(message)
| ServerError::NotImplemented(message)
| ServerError::ServiceUnavailable(message)
| ServerError::ContextLengthExceeded(message)
| ServerError::InvalidRequest { message, .. }
| ServerError::UnsupportedFeature { message, .. } => message,
}
}
fn server_error_from_ferrum_error(error: Error) -> ServerError {
match error {
Error::RequestValidation { message } => ServerError::invalid_request(message, None),
error @ Error::ContextLengthExceeded { .. } => {
ServerError::ContextLengthExceeded(error.to_string())
}
Error::ResourceExhausted { message } => ServerError::ServiceUnavailable(message),
other => ServerError::InternalError(other.to_string()),
}
}
fn stream_error_payload(
message: impl Into<String>,
error_type: &str,
param: Option<&str>,
) -> OpenAiError {
OpenAiError {
error: OpenAiErrorDetail {
message: message.into(),
error_type: error_type.to_string(),
param: param.map(str::to_string),
code: None,
},
}
}
fn openai_error_sse_event(
message: impl Into<String>,
error_type: &str,
param: Option<&str>,
) -> Event {
Event::default()
.json_data(&stream_error_payload(message, error_type, param))
.unwrap_or_else(|_| Event::default().data("error"))
}
fn convert_completion_request(request: &CompletionsRequest) -> InferenceRequest {
let prompt = request
.prompt
.as_text()
.expect("completion prompt validated before conversion");
InferenceRequest {
id: RequestId(Uuid::new_v4()),
model_id: ModelId(request.model.clone()),
prompt: prompt.to_string(),
sampling_params: SamplingParams {
max_tokens: request.max_tokens.unwrap_or(DEFAULT_COMPLETION_MAX_TOKENS) as usize,
temperature: request.temperature.unwrap_or(DEFAULT_SAMPLING_TEMPERATURE),
top_p: request.top_p.unwrap_or(DEFAULT_SAMPLING_TOP_P),
top_k: None,
repetition_penalty: 1.0,
presence_penalty: 0.0,
frequency_penalty: 0.0,
stop_sequences: request.stop.clone().unwrap_or_default(),
seed: None,
min_p: None,
tfs: None,
typical_p: None,
mirostat: None,
response_format: ferrum_types::ResponseFormat::Text,
structured_output_start: StructuredOutputStart::Immediate,
response_completion_boundary: ResponseCompletionBoundary::Immediate,
model_output_protocol: ferrum_types::ModelOutputProtocol::Text,
},
stream: request.stream.unwrap_or(false),
priority: Priority::Normal,
client_id: None,
session_id: None,
created_at: chrono::Utc::now(),
api_request: Some(ferrum_types::ApiRequest::Completion(
ferrum_types::ApiCompletionRequest {
prompt: prompt.to_string(),
response_format: None,
},
)),
evidence_request: Default::default(),
metadata: if request.max_tokens.is_none() {
HashMap::from([(
DEFAULT_MAX_TOKENS_METADATA_KEY.to_string(),
serde_json::json!(true),
)])
} else {
HashMap::new()
},
}
}
fn resolve_request_model<'a>(
registry: &'a ServedModelRegistry,
request_model: &str,
required_kind: ServedModelKind,
) -> std::result::Result<(ModelId, Option<&'a LoraAdapterModel>), ServerError> {
if registry.is_empty() {
return Ok((ModelId::new(request_model), None));
}
let entry = registry
.resolve(request_model, required_kind)
.ok_or_else(|| {
ServerError::invalid_request(format!("unknown model: {request_model}"), Some("model"))
})?;
Ok((entry.engine_model_id().clone(), entry.adapter()))
}
fn apply_served_model_resolution(
inference_request: &mut InferenceRequest,
engine_model_id: ModelId,
adapter: Option<&LoraAdapterModel>,
) {
inference_request.model_id = engine_model_id;
if let Some(adapter) = adapter {
inference_request.metadata.insert(
"ferrum_lora_adapter".to_string(),
serde_json::json!(adapter.name),
);
inference_request.metadata.insert(
"ferrum_lora_model_id".to_string(),
serde_json::json!(adapter.model_id),
);
inference_request.metadata.insert(
"ferrum_lora_path".to_string(),
serde_json::json!(adapter.path),
);
}
}
async fn handle_completions_sync(
state: AppState,
openai_request: CompletionsRequest,
inference_request: InferenceRequest,
) -> std::result::Result<Response, ServerError> {
let engine = state.llm.clone().ok_or_else(|| {
ServerError::ServiceUnavailable("LLM engine not loaded; completions unavailable".into())
})?;
match engine.infer(inference_request).await {
Ok(output) => {
let InferenceResponse {
text: output_text,
finish_reason,
usage,
api_response,
..
} = output;
let stop_sequences = openai_request.stop.clone().unwrap_or_default();
let mut text = strip_after_stop(&output_text, &stop_sequences);
let mut openai_finish_reason = finish_reason_to_string(&finish_reason);
if let Some(ferrum_types::ApiResponse::Completion(completion_response)) =
api_response.as_ref()
{
text = strip_after_stop(&completion_response.text, &stop_sequences);
if let Some(reason) = &completion_response.finish_reason {
openai_finish_reason = reason.clone();
}
}
let response = CompletionsResponse {
id: Uuid::new_v4().to_string(),
object: "text_completion".to_string(),
created: chrono::Utc::now().timestamp() as u64,
model: openai_request.model,
choices: vec![CompletionChoice {
text,
index: 0,
finish_reason: Some(openai_finish_reason),
}],
usage: Some(openai_usage_from_token_usage(&usage)),
};
Ok(Json(response).into_response())
}
Err(e) => {
error!("Completion generation failed: {}", e);
Err(server_error_from_ferrum_error(e))
}
}
}
async fn handle_completions_stream(
state: AppState,
openai_request: CompletionsRequest,
inference_request: InferenceRequest,
) -> std::result::Result<Response, ServerError> {
let (tx, rx) = mpsc::unbounded_channel::<std::result::Result<Event, axum::Error>>();
let engine = state.llm.clone().ok_or_else(|| {
ServerError::ServiceUnavailable("LLM engine not loaded; completions unavailable".into())
})?;
let request_id = Uuid::new_v4().to_string();
// Resolve startup rejection before sending SSE headers, as on the Chat
// route. Once a stream exists, later failures remain SSE error events.
let mut stream = engine.infer_stream(inference_request).await.map_err(|e| {
error!("Failed to start completion stream: {}", e);
server_error_from_ferrum_error(e)
})?;
tokio::spawn(async move {
while let Some(result) = stream.next().await {
match result {
Ok(chunk) => {
let response_chunk = CompletionsResponse {
id: request_id.clone(),
object: "text_completion".to_string(),
created: chrono::Utc::now().timestamp() as u64,
model: openai_request.model.clone(),
choices: vec![CompletionChoice {
text: chunk.text.clone(),
index: 0,
finish_reason: chunk
.finish_reason
.as_ref()
.map(finish_reason_to_string),
}],
usage: None,
};
let event = Event::default()
.json_data(&response_chunk)
.unwrap_or_else(|_| Event::default().data("error"));
if tx.send(Ok(event)).is_err() {
break;
}
if chunk.finish_reason.is_some() {
if let Some(usage) = chunk.usage.as_ref().map(openai_usage_from_token_usage)
{
let final_chunk = CompletionsResponse {
id: request_id.clone(),
object: "text_completion".to_string(),
created: chrono::Utc::now().timestamp() as u64,
model: openai_request.model.clone(),
choices: vec![],
usage: Some(usage),
};
let event = Event::default()
.json_data(&final_chunk)
.unwrap_or_else(|_| Event::default().data("error"));
let _ = tx.send(Ok(event));
}
let _ = tx.send(Ok(Event::default().data("[DONE]")));
break;
}
}
Err(e) => {
error!("Completion stream generation error: {}", e);
let _ = tx.send(Ok(openai_error_sse_event(
e.to_string(),
"internal_server_error",
None,
)));
let _ = tx.send(Ok(Event::default().data("[DONE]")));
break;
}
}
}
});
let stream = tokio_stream::wrappers::UnboundedReceiverStream::new(rx);
Ok(Sse::new(stream).into_response())
}
/// Other handlers
async fn completions_handler(
State(state): State<AppState>,
request: std::result::Result<Json<CompletionsRequest>, JsonRejection>,
) -> std::result::Result<Response, ServerError> {
let Json(request) = request.map_err(|e| {
ServerError::invalid_request(format!("invalid completions request: {e}"), None)
})?;
validate_completion_request(&request)?;
let (engine_model_id, lora_adapter) = resolve_request_model(
&state.served_model_registry,
&request.model,
ServedModelKind::Llm,
)?;
let mut inference_request = convert_completion_request(&request);
apply_served_model_resolution(&mut inference_request, engine_model_id, lora_adapter);
if request.stream.unwrap_or(false) {
handle_completions_stream(state, request, inference_request).await
} else {
handle_completions_sync(state, request, inference_request).await
}
}
fn validate_completion_request(
request: &CompletionsRequest,
) -> std::result::Result<(), ServerError> {
if request.prompt.as_text().is_none() {
return Err(ServerError::invalid_request(
"only string prompt is supported for completions",
Some("prompt"),
));
}
if let Some(n) = request.n {
if n != 1 {
return Err(ServerError::unsupported_feature(
"only n=1 is supported for completions",
Some("n"),
));
}
}
if request.logprobs.is_some() {
return Err(ServerError::unsupported_feature(
"logprobs is not supported for completions",
Some("logprobs"),
));
}
if request
.logit_bias
.as_ref()
.is_some_and(|bias| !bias.is_empty())
{
return Err(ServerError::unsupported_feature(
"logit_bias is not supported",
Some("logit_bias"),
));
}
Ok(())
}
/// Embeddings handler — text and image embedding via OpenAI-compatible API.
async fn embeddings_handler(
State(state): State<AppState>,
request: std::result::Result<Json<EmbeddingsRequest>, JsonRejection>,
) -> std::result::Result<Response, ServerError> {
let Json(request) = request.map_err(|e| {
ServerError::invalid_request(format!("invalid embeddings request: {e}"), None)
})?;
let span = span!(Level::INFO, "embeddings", model = %request.model);
let _enter = span.enter();
validate_embeddings_request(&request)?;
resolve_request_model(
&state.served_model_registry,
&request.model,
ServedModelKind::Embedding,
)?;
// Flatten input into individual items
let items: Vec<EmbeddingItem> = match request.input {
EmbeddingInput::Single(text) => vec![EmbeddingItem {
text: Some(text),
image: None,
}],
EmbeddingInput::Batch(texts) => texts
.into_iter()
.map(|t| EmbeddingItem {
text: Some(t),
image: None,
})
.collect(),
EmbeddingInput::SingleObject(item) => vec![item],
EmbeddingInput::BatchObjects(items) => items,
};
if items.is_empty() {
return Err(ServerError::invalid_request(
"input must not be empty",
Some("input"),
));
}
let mut data = Vec::with_capacity(items.len());
let mut total_tokens = 0u32;
let engine = state.embed.as_ref().ok_or_else(|| {
ServerError::NotImplemented("Embed engine not loaded; embeddings unavailable".into())
})?;
for (idx, item) in items.iter().enumerate() {
let embedding = if let Some(ref image) = item.image {
engine
.embed_image(image)
.await
.map_err(|e| ServerError::InternalError(format!("embed_image: {e}")))?
} else if let Some(ref text) = item.text {
total_tokens += text.len() as u32;
engine
.embed_text(text)
.await
.map_err(|e| ServerError::InternalError(format!("embed_text: {e}")))?
} else {
return Err(ServerError::invalid_request(
"each input item must have either text or image",
Some("input"),
));
};
data.push(EmbeddingData {
object: "embedding".to_string(),
embedding,
index: idx,
});
}
let response = EmbeddingsResponse {
object: "list".to_string(),
data,
model: request.model,
usage: EmbeddingUsage {
prompt_tokens: total_tokens,
total_tokens,
},
};
Ok(Json(response).into_response())
}
fn validate_embeddings_request(
request: &EmbeddingsRequest,
) -> std::result::Result<(), ServerError> {
if let Some(format) = request.encoding_format.as_deref() {
if !format.eq_ignore_ascii_case("float") {
return Err(ServerError::unsupported_feature(
"only encoding_format=float is supported for embeddings",
Some("encoding_format"),
));
}
}
Ok(())
}
/// Audio transcription handler (OpenAI-compatible multipart form).
async fn transcriptions_handler(
State(state): State<AppState>,
multipart: std::result::Result<axum::extract::Multipart, MultipartRejection>,
) -> std::result::Result<Response, ServerError> {
let mut multipart = multipart.map_err(|e| {
ServerError::invalid_request(format!("invalid transcriptions request: {e}"), None)
})?;
let span = span!(Level::INFO, "transcription");
let _enter = span.enter();
let mut file_data: Option<Vec<u8>> = None;
let mut language: Option<String> = None;
let mut response_format: Option<String> = None;
while let Some(field) = multipart
.next_field()
.await
.map_err(|e| ServerError::invalid_request(format!("multipart: {e}"), None))?
{
let name = field.name().unwrap_or("").to_string();
match name.as_str() {
"file" => {
file_data = Some(
field
.bytes()
.await
.map_err(|e| {
ServerError::invalid_request(format!("read file: {e}"), Some("file"))
})?
.to_vec(),
);
}
"language" => {
language = field.text().await.ok().filter(|s| !s.is_empty());
}
"response_format" => {
response_format = field.text().await.ok().filter(|s| !s.is_empty());
}
_ => {} // ignore model and other optional multipart fields for now
}
}
validate_transcription_response_format(response_format.as_deref())?;
let data = file_data
.ok_or_else(|| ServerError::invalid_request("missing file field", Some("file")))?;
let engine = state.transcribe.as_ref().ok_or_else(|| {
ServerError::NotImplemented("Transcribe engine not loaded; ASR unavailable".into())
})?;
let text = engine
.transcribe_bytes(&data, language.as_deref())
.await
.map_err(|e| ServerError::InternalError(format!("transcribe: {e}")))?;
Ok(Json(TranscriptionResponse { text }).into_response())
}
fn validate_transcription_response_format(
response_format: Option<&str>,
) -> std::result::Result<(), ServerError> {
if let Some(format) = response_format {
if !format.eq_ignore_ascii_case("json") {
return Err(ServerError::unsupported_feature(
"only response_format=json is supported for transcriptions",
Some("response_format"),
));
}
}
Ok(())
}
/// TTS speech synthesis handler (OpenAI-compatible /v1/audio/speech)
async fn speech_handler(
State(state): State<AppState>,
request: std::result::Result<Json<SpeechRequest>, JsonRejection>,
) -> std::result::Result<Response, ServerError> {
let Json(request) = request
.map_err(|e| ServerError::invalid_request(format!("invalid speech request: {e}"), None))?;
let response_format = speech_output_format(&request)?;
resolve_request_model(
&state.served_model_registry,
&request.model,
ServedModelKind::Speech,
)?;
let span = span!(Level::INFO, "speech");
let _guard = span.enter();
let language = if request.language.is_empty() || request.language == "auto" {
None
} else {
Some(request.language.as_str())
};
let chunk_frames = 10usize;
let tts = state.tts.as_ref().ok_or_else(|| {
ServerError::NotImplemented("TTS engine not loaded; speech unavailable".into())
})?;
let sample_rate = tts.tts_sample_rate();
if request.stream {
// Streaming: chunked transfer encoding with WAV audio
let (tx, rx) =
mpsc::unbounded_channel::<std::result::Result<axum::body::Bytes, std::io::Error>>();
let engine = tts.clone();
let text = request.input.clone();
let lang = request.language.clone();
tokio::task::spawn_blocking(move || {
let lang_opt = if lang.is_empty() || lang == "auto" {
None
} else {
Some(lang.as_str())
};
let rt = tokio::runtime::Handle::current();
match rt.block_on(engine.synthesize_speech(&text, lang_opt, chunk_frames)) {
Ok(chunks) => {
for chunk in &chunks {
let audio_bytes = encode_speech_audio(chunk, sample_rate, response_format);
let _ = tx.send(Ok(axum::body::Bytes::from(audio_bytes)));
}
}
Err(e) => {
error!("TTS error: {e}");
}
}
});
let stream = tokio_stream::wrappers::UnboundedReceiverStream::new(rx);
let body = axum::body::Body::from_stream(stream);
Ok(Response::builder()
.status(200)
.header("content-type", speech_content_type(response_format))
.header("transfer-encoding", "chunked")
.body(body)
.unwrap())
} else {
// Non-streaming: return complete WAV
let chunks = tts
.synthesize_speech(&request.input, language, chunk_frames)
.await
.map_err(|e| ServerError::InternalError(format!("TTS: {e}")))?;
let all_samples: Vec<f32> = chunks.into_iter().flatten().collect();
let audio_bytes = encode_speech_audio(&all_samples, sample_rate, response_format);
Ok(Response::builder()
.status(200)
.header("content-type", speech_content_type(response_format))
.header("content-length", audio_bytes.len().to_string())
.body(axum::body::Body::from(audio_bytes))
.unwrap())
}
}
#[derive(Clone, Copy)]
enum SpeechOutputFormat {
Wav,
Pcm,
}
fn speech_output_format(
request: &SpeechRequest,
) -> std::result::Result<SpeechOutputFormat, ServerError> {
if request.response_format.eq_ignore_ascii_case("wav") {
Ok(SpeechOutputFormat::Wav)
} else if request.response_format.eq_ignore_ascii_case("pcm") {
Ok(SpeechOutputFormat::Pcm)
} else {
Err(ServerError::unsupported_feature(
"only response_format=wav or response_format=pcm is supported for speech",
Some("response_format"),
))
}
}
fn speech_content_type(format: SpeechOutputFormat) -> &'static str {
match format {
SpeechOutputFormat::Wav => "audio/wav",
SpeechOutputFormat::Pcm => "audio/pcm",
}
}
fn encode_speech_audio(samples: &[f32], sample_rate: u32, format: SpeechOutputFormat) -> Vec<u8> {
match format {
SpeechOutputFormat::Wav => pcm_to_wav_bytes(samples, sample_rate),
SpeechOutputFormat::Pcm => pcm_to_s16le_bytes(samples),
}
}
/// Convert PCM f32 samples to WAV bytes (16-bit, mono).
fn pcm_to_wav_bytes(samples: &[f32], sample_rate: u32) -> Vec<u8> {
let num_samples = samples.len();
let data_size = num_samples * 2; // 16-bit = 2 bytes per sample
let file_size = 44 + data_size;
let mut buf = Vec::with_capacity(file_size);
// RIFF header
buf.extend_from_slice(b"RIFF");
buf.extend_from_slice(&((file_size - 8) as u32).to_le_bytes());
buf.extend_from_slice(b"WAVE");
// fmt chunk
buf.extend_from_slice(b"fmt ");
buf.extend_from_slice(&16u32.to_le_bytes()); // chunk size
buf.extend_from_slice(&1u16.to_le_bytes()); // PCM
buf.extend_from_slice(&1u16.to_le_bytes()); // mono
buf.extend_from_slice(&sample_rate.to_le_bytes());
buf.extend_from_slice(&(sample_rate * 2).to_le_bytes()); // byte rate
buf.extend_from_slice(&2u16.to_le_bytes()); // block align
buf.extend_from_slice(&16u16.to_le_bytes()); // bits per sample
// data chunk
buf.extend_from_slice(b"data");
buf.extend_from_slice(&(data_size as u32).to_le_bytes());
buf.extend_from_slice(&pcm_to_s16le_bytes(samples));
buf
}
fn pcm_to_s16le_bytes(samples: &[f32]) -> Vec<u8> {
let mut buf = Vec::with_capacity(samples.len() * 2);
for &s in samples {
let i16_val = (s.clamp(-1.0, 1.0) * 32767.0) as i16;
buf.extend_from_slice(&i16_val.to_le_bytes());
}
buf
}
async fn models_handler(
State(state): State<AppState>,
) -> std::result::Result<Response, ServerError> {
let now = chrono::Utc::now().timestamp() as u64;
let reasoning = state.prompt_template.as_ref().and_then(|template| {
let supported_efforts = template
.reasoning_effort_support
.declared_efforts()
.map(|efforts| efforts.iter().copied().collect());
let thinking =
template
.supports_thinking_control()
.then(|| crate::openai::ModelThinkingCapability {
default_enabled: state
.default_enable_thinking
.unwrap_or(template.reasoning_default_enabled),
});
(supported_efforts.is_some() || thinking.is_some()).then_some(
crate::openai::ModelReasoningCapabilities {
supported_efforts,
thinking,
},
)
});
let data = state
.served_model_registry
.entries()
.iter()
.map(|entry| crate::openai::ModelInfo {
id: entry.public_name().to_string(),
object: "model".to_string(),
created: now,
owned_by: "ferrum".to_string(),
max_model_len: match entry.kind() {
ServedModelKind::Llm => state.llm.as_ref().and_then(|llm| llm.context_capacity()),
_ => None,
},
reasoning: if entry.kind() == ServedModelKind::Llm
&& state
.llm
.as_ref()
.is_some_and(|llm| &llm.config().model.model_id == entry.engine_model_id())
{
reasoning.clone()
} else {
None
},
modalities: entry
.kind()
.modalities()
.iter()
.map(ToString::to_string)
.collect(),
permission: vec![],
root: entry.parent_public_name().map(ToString::to_string),
parent: entry.parent_public_name().map(ToString::to_string),
})
.collect();
let models = ModelListResponse {
object: "list".to_string(),
data,
};
Ok(Json(models).into_response())
}
async fn health_handler(
State(state): State<AppState>,
) -> std::result::Result<Response, ServerError> {
let engine_status = state.status().await;
let scheduler_metrics = state.metrics();
let runtime_config = RuntimeConfigSnapshot::capture_current();
let cache_policy = CachePolicy::current();
let engine_cache = state
.llm
.as_ref()
.and_then(|engine| engine.cache_metrics_snapshot());
let execution_attribution = state
.llm
.as_ref()
.and_then(|engine| engine.execution_attribution_snapshot());
let engine_lora = state
.llm
.as_ref()
.and_then(|engine| engine.lora_metrics_snapshot());
let auto_config = auto_config_health_value(state.auto_config.as_ref());
let runtime_admission = match state.llm.as_ref() {
Some(engine) => engine.admission_snapshot(),
None => Ok(None),
};
let (runtime_admission_snapshot, runtime_admission_error) = match &runtime_admission {
Ok(snapshot) => (snapshot.as_ref(), None),
Err(error) => (None, Some(error.to_string())),
};
let admission = admission_health_json(
&engine_status,
&scheduler_metrics,
&auto_config,
runtime_admission_snapshot,
runtime_admission_error.as_deref(),
);
let health = serde_json::json!({
"status": if runtime_admission_error.is_some() { "unhealthy" } else { "healthy" },
"reasoning_protocol": state.prompt_template.as_deref().map(ModelChatTemplate::reasoning_capability).unwrap_or_default(),
"timestamp": chrono::Utc::now().to_rfc3339(),
"version": env!("CARGO_PKG_VERSION"),
"engine": {
"active_requests": engine_status.active_requests,
"queued_requests": engine_status.queued_requests,
},
"scheduler": {
"total_requests": scheduler_metrics.total_requests,
"successful_requests": scheduler_metrics.successful_requests,
"failed_requests": scheduler_metrics.failed_requests,
"throughput_rps": scheduler_metrics.throughput_rps,
"avg_wait_time_ms": scheduler_metrics.queue_metrics.avg_queue_wait_time_ms,
"scheduling_time_ms": scheduler_metrics.performance_breakdown.scheduling_time_ms,
"model_execution_time_ms": scheduler_metrics
.performance_breakdown
.model_execution_time_ms,
"iteration_lock_wait_time_ms": scheduler_metrics
.performance_breakdown
.other_overhead_time_ms,
},
"config": runtime_config,
"auto_config": auto_config,
"admission": admission,
"numerical_execution": engine_cache.as_ref().and_then(|snapshot| snapshot.get("numerical_execution")),
"kv_storage": engine_cache.as_ref().and_then(|snapshot| snapshot.get("kv_storage")),
"cache": state.cache.health_json(&cache_policy, engine_cache.as_ref()),
"execution_attribution": execution_attribution,
"lora": engine_lora.unwrap_or_else(|| serde_json::json!({
"enabled": state.served_model_registry.adapter_count() > 0,
"adapter_count": state.served_model_registry.adapter_count() as u64,
"active_cache_bindings": 0u64,
"projection_applications": 0u64,
"position": "startup-routing",
"source": "server-lora-registry",
})),
});
Ok(Json(health).into_response())
}
/// Prometheus metrics endpoint — returns metrics in Prometheus text format.
async fn metrics_handler(
State(state): State<AppState>,
) -> std::result::Result<Response, ServerError> {
let mut body = match PROM_HANDLE.get() {
Some(handle) => handle.render(),
None => "# Prometheus recorder not initialized\n".to_string(),
};
if !body.ends_with('\n') {
body.push('\n');
}
let engine_cache = state
.llm
.as_ref()
.and_then(|engine| engine.cache_metrics_snapshot());
body.push_str(&state.cache.prometheus_metrics(engine_cache.as_ref()));
let engine_status = state.status().await;
let scheduler_metrics = state.metrics();
let auto_config = auto_config_health_value(state.auto_config.as_ref());
let runtime_admission = match state.llm.as_ref() {
Some(engine) => engine.admission_snapshot(),
None => Ok(None),
};
let (runtime_admission_snapshot, runtime_admission_error) = match &runtime_admission {
Ok(snapshot) => (snapshot.as_ref(), None),
Err(error) => (None, Some(error.to_string())),
};
let admission = admission_health_json(
&engine_status,
&scheduler_metrics,
&auto_config,
runtime_admission_snapshot,
runtime_admission_error.as_deref(),
);
body.push_str(&admission_prometheus_metrics(&admission));
Ok((
[(
axum::http::header::CONTENT_TYPE,
"text/plain; version=0.0.4; charset=utf-8",
)],
body,
)
.into_response())
}
async fn root_handler() -> std::result::Result<Response, ServerError> {
let info = serde_json::json!({
"name": "Ferrum Inference Server",
"version": env!("CARGO_PKG_VERSION"),
"api_version": "v1",
"status": "running"
});
Ok(Json(info).into_response())
}
/// Server error type for HTTP responses
#[derive(Debug)]
enum ServerError {
InvalidRequest {
message: String,
param: Option<String>,
},
UnsupportedFeature {
message: String,
param: Option<String>,
},
InternalError(String),
ContextLengthExceeded(String),
NotImplemented(String),
ServiceUnavailable(String),
}
impl ServerError {
fn invalid_request(message: impl Into<String>, param: Option<&str>) -> Self {
Self::InvalidRequest {
message: message.into(),
param: param.map(str::to_string),
}
}
fn unsupported_feature(message: impl Into<String>, param: Option<&str>) -> Self {
Self::UnsupportedFeature {
message: message.into(),
param: param.map(str::to_string),
}
}
}
impl IntoResponse for ServerError {
fn into_response(self) -> Response {
let code = matches!(&self, ServerError::ContextLengthExceeded(_))
.then(|| "context_length_exceeded".to_owned());
let (status, message, error_type, param) = match self {
ServerError::ContextLengthExceeded(message) => (
AxumStatusCode::BAD_REQUEST,
message,
"invalid_request_error",
None,
),
ServerError::InvalidRequest { message, param } => (
AxumStatusCode::BAD_REQUEST,
message,
"invalid_request_error",
param,
),
ServerError::UnsupportedFeature { message, param } => (
AxumStatusCode::BAD_REQUEST,
message,
"invalid_request_error",
param,
),
ServerError::InternalError(msg) => (
AxumStatusCode::INTERNAL_SERVER_ERROR,
msg,
"internal_server_error",
None,
),
ServerError::NotImplemented(msg) => (
AxumStatusCode::SERVICE_UNAVAILABLE,
msg,
"service_unavailable_error",
None,
),
ServerError::ServiceUnavailable(msg) => (
AxumStatusCode::SERVICE_UNAVAILABLE,
msg,
"service_unavailable_error",
None,
),
};
let error = OpenAiError {
error: OpenAiErrorDetail {
message,
error_type: error_type.to_string(),
param,
code,
},
};
(status, Json(error)).into_response()
}
}
impl std::fmt::Display for MessageRole {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
MessageRole::System => write!(f, "system"),
MessageRole::User => write!(f, "user"),
MessageRole::Assistant => write!(f, "assistant"),
MessageRole::Function => write!(f, "function"),
MessageRole::Tool => write!(f, "tool"),
}
}
}
/// Strip model output at the first user-supplied stop sequence.
/// OpenAI-compatible `stop` strings are generation boundaries and must not
/// be returned to the caller, even if the model continued after the boundary.
fn strip_after_stop(text: &str, stops: &[String]) -> String {
let mut first: Option<usize> = None;
for stop in stops {
if stop.is_empty() {
continue;
}
if let Some(idx) = text.find(stop.as_str()) {
first = Some(first.map_or(idx, |current| current.min(idx)));
}
}
match first {
Some(idx) => text[..idx].to_string(),
None => text.to_string(),
}
}
/// Compatibility cleanup for explicitly best-effort structured output.
/// Hard `json_object` and strict schema contracts never call this helper.
fn strip_markdown_json_fence(text: &str) -> String {
let trimmed = text.trim();
// Try the most specific marker first.
for prefix in ["```json\n", "```json", "```\n", "```"] {
if let Some(rest) = trimmed.strip_prefix(prefix) {
if let Some(inner) = rest.strip_suffix("```") {
return inner.trim().to_string();
}
}
}
text.to_string()
}
/// Convert FinishReason to OpenAI API string
fn finish_reason_to_string(reason: &FinishReason) -> String {
match reason {
FinishReason::Length => "length".to_string(),
FinishReason::Stop => "stop".to_string(),
FinishReason::EOS => "stop".to_string(),
FinishReason::Cancelled => "cancelled".to_string(),
FinishReason::Error => "error".to_string(),
FinishReason::ContentFilter => "content_filter".to_string(),
}
}
#[cfg(test)]
mod tests {
mod auto_tools_json;
mod engine_stop_contract;
mod gemma_thought;
mod harmony_stops;
mod model_reasoning_metadata;
mod native_tool_stream;
mod reasoning_controls;
mod tool_argument_strictness;
mod tool_length;
use super::*;
use async_trait::async_trait;
use axum::{
body::{to_bytes, Body},
http::{header, Request},
response::Response,
};
use ferrum_interfaces::engine::{
EmbedEngine, InferenceEngine, LlmInferenceEngine, TranscribeEngine, TtsEngine,
};
use ferrum_types::{
has_unclosed_thinking_block, parse_reasoning_response_started_in_think, EngineConfig,
EngineMetrics, EngineStatus, EngineTokenTimingEvidence, FinishReason,
HealthStatus as EngineHealthStatus, InferenceRequest, InferenceResponse, MemoryUsage,
ModelId, StreamChunk, TokenId, TokenUsage,
};
use futures::{stream, Stream};
use serde_json::{json, Value};
use std::{
collections::HashMap,
pin::Pin,
sync::{atomic::AtomicUsize, Arc, Mutex},
};
use tower::ServiceExt;
#[test]
fn strip_after_stop_removes_first_boundary() {
assert_eq!(
strip_after_stop(
"KS0214Z\nS0225\nEND0214Z0214Z\nS0225\n",
&["END0214Z".to_string()]
),
"KS0214Z\nS0225\n"
);
}
#[test]
fn gpt_oss_harmony_final_is_split_into_reasoning_and_visible_content() {
let parsed = parse_chat_model_output(
ModelOutputProtocol::HarmonyGptOss,
"<|channel|>analysis<|message|>Reason.<|end|>\
<|start|>assistant<|channel|>final<|message|>Answer.<|return|>",
false,
FinishReason::Stop,
)
.unwrap();
assert_eq!(parsed.visible.content, "Answer.");
assert_eq!(parsed.visible.reasoning.as_deref(), Some("Reason."));
assert!(parsed.harmony_response.is_none());
}
#[test]
fn gpt_oss_harmony_tool_call_becomes_openai_structured_response() {
let parsed = parse_chat_model_output(
ModelOutputProtocol::HarmonyGptOss,
"<|channel|>analysis<|message|>Need weather.<|end|>\
<|start|>assistant<|channel|>commentary to=functions.weather\
<|constrain|>json<|message|>{\"city\":\"Paris\"}<|call|>",
false,
FinishReason::Stop,
)
.unwrap();
let response = parsed.harmony_response.unwrap();
assert_eq!(response.finish_reason.as_deref(), Some("tool_calls"));
assert_eq!(response.message.tool_calls.len(), 1);
assert_eq!(response.message.tool_calls[0].function.name, "weather");
assert_eq!(
response.message.tool_calls[0].function.arguments,
"{\"city\":\"Paris\"}"
);
assert!(response.message.tool_calls[0].id.starts_with("call_"));
}
#[test]
fn gpt_oss_harmony_accepts_missing_text_terminal_only_for_explicit_truncation() {
let output = "<|channel|>analysis<|message|>Still reasoning";
for finish_reason in [FinishReason::Stop, FinishReason::Length] {
let parsed = parse_chat_model_output(
ModelOutputProtocol::HarmonyGptOss,
output,
false,
finish_reason,
)
.unwrap();
assert_eq!(parsed.visible.reasoning.as_deref(), Some("Still reasoning"));
assert!(parsed.visible.content.is_empty());
}
for finish_reason in [
FinishReason::EOS,
FinishReason::Cancelled,
FinishReason::Error,
FinishReason::ContentFilter,
] {
assert!(parse_chat_model_output(
ModelOutputProtocol::HarmonyGptOss,
output,
false,
finish_reason,
)
.is_err());
}
}
#[tokio::test]
async fn stop_drains_running_server_and_shuts_down_loaded_engine_once() {
let engine = Arc::new(StubLlm::new("ok"));
let server = Arc::new(AxumServer::from_llm(engine.clone()));
let config = ServerConfig {
host: "127.0.0.1".to_string(),
port: 0,
..ServerConfig::default()
};
let server_task = {
let server = Arc::clone(&server);
tokio::spawn(async move { server.start(&config).await })
};
tokio::time::timeout(std::time::Duration::from_secs(1), async {
while !server.is_running() {
tokio::task::yield_now().await;
}
})
.await
.unwrap();
server
.stop(std::time::Duration::from_secs(1))
.await
.unwrap();
server
.stop(std::time::Duration::from_secs(1))
.await
.unwrap();
server_task.await.unwrap().unwrap();
assert_eq!(engine.shutdown_count.load(Ordering::Acquire), 1);
assert!(!server.is_running());
}
struct StubLlm {
config: EngineConfig,
context_capacity: Option<usize>,
text: String,
stream_chunks: Option<Vec<String>>,
stream_final_chunk_separate: bool,
stream_tail_without_token: bool,
stream_usage: Option<TokenUsage>,
api_response: Option<ferrum_types::ApiResponse>,
finish_reason: FinishReason,
execution_attribution: Option<Value>,
lora_metrics: Option<Value>,
pending_stream_drop_notify: Option<Arc<Notify>>,
stream_after_first_gate: Option<Arc<StreamGate>>,
stream_terminal_error: bool,
shutdown_count: AtomicUsize,
}
impl StubLlm {
fn new(text: &str) -> Self {
let mut config = EngineConfig::default();
config.model.model_id = ModelId::new("stub-model");
Self {
config,
text: text.to_string(),
context_capacity: None,
stream_chunks: None,
stream_final_chunk_separate: false,
stream_tail_without_token: false,
stream_usage: Some(TokenUsage::new(5, 1)),
api_response: None,
finish_reason: FinishReason::EOS,
execution_attribution: None,
lora_metrics: None,
pending_stream_drop_notify: None,
stream_after_first_gate: None,
stream_terminal_error: false,
shutdown_count: AtomicUsize::new(0),
}
}
fn without_stream_usage(text: &str) -> Self {
Self {
stream_usage: None,
..Self::new(text)
}
}
fn with_stream_chunks(chunks: &[&str]) -> Self {
Self {
text: chunks.concat(),
stream_chunks: Some(chunks.iter().map(|chunk| (*chunk).to_string()).collect()),
stream_usage: Some(TokenUsage::new(5, chunks.len())),
..Self::new("")
}
}
fn with_separate_final_stream_chunk(chunks: &[&str]) -> Self {
Self {
text: chunks.concat(),
stream_chunks: Some(chunks.iter().map(|chunk| (*chunk).to_string()).collect()),
stream_final_chunk_separate: true,
stream_usage: Some(TokenUsage::new(5, chunks.len())),
..Self::new("")
}
}
fn with_tokenless_tail(chunks: &[&str]) -> Self {
Self {
stream_tail_without_token: true,
..Self::with_separate_final_stream_chunk(chunks)
}
}
fn with_api_response(text: &str, api_response: ferrum_types::ApiResponse) -> Self {
Self {
api_response: Some(api_response),
..Self::new(text)
}
}
fn with_api_response_and_finish_reason(
text: &str,
api_response: ferrum_types::ApiResponse,
finish_reason: FinishReason,
) -> Self {
Self {
api_response: Some(api_response),
finish_reason,
..Self::new(text)
}
}
fn with_lora_metrics(text: &str, lora_metrics: Value) -> Self {
Self {
lora_metrics: Some(lora_metrics),
..Self::new(text)
}
}
fn with_execution_attribution(text: &str, execution_attribution: Value) -> Self {
Self {
execution_attribution: Some(execution_attribution),
..Self::new(text)
}
}
fn with_pending_stream(drop_notify: Arc<Notify>) -> Self {
Self {
pending_stream_drop_notify: Some(drop_notify),
..Self::new("")
}
}
}
struct PendingDropStream {
drop_notify: Arc<Notify>,
}
#[derive(Default)]
struct StreamGate {
entered: Notify,
resume: Notify,
}
impl Stream for PendingDropStream {
type Item = ferrum_types::Result<StreamChunk>;
fn poll_next(
self: Pin<&mut Self>,
_cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Option<Self::Item>> {
std::task::Poll::Pending
}
}
impl Drop for PendingDropStream {
fn drop(&mut self) {
self.drop_notify.notify_one();
}
}
struct StubEmbed {
config: EngineConfig,
}
impl StubEmbed {
fn new() -> Self {
let mut config = EngineConfig::default();
config.model.model_id = ModelId::new("stub-embed");
Self { config }
}
}
struct StubTranscribe {
config: EngineConfig,
}
impl StubTranscribe {
fn new() -> Self {
let mut config = EngineConfig::default();
config.model.model_id = ModelId::new("stub-transcribe");
Self { config }
}
}
struct StubTts {
config: EngineConfig,
}
impl StubTts {
fn new() -> Self {
let mut config = EngineConfig::default();
config.model.model_id = ModelId::new("stub-tts");
Self { config }
}
}
struct FailingLlm {
config: EngineConfig,
fail_after_stream_start: bool,
infer_failure: ferrum_types::FerrumError,
stream_start_failure: ferrum_types::FerrumError,
stream_chunk_failure: ferrum_types::FerrumError,
}
impl FailingLlm {
fn new() -> Self {
let mut config = EngineConfig::default();
config.model.model_id = ModelId::new("failing-model");
Self {
config,
fail_after_stream_start: false,
infer_failure: ferrum_types::FerrumError::internal("stub generation failed"),
stream_start_failure: ferrum_types::FerrumError::internal("stub stream failed"),
stream_chunk_failure: ferrum_types::FerrumError::internal(
"stub stream chunk failed",
),
}
}
fn after_stream_start() -> Self {
Self {
fail_after_stream_start: true,
..Self::new()
}
}
fn resource_exhausted() -> Self {
let failure = ferrum_types::FerrumError::resource_exhausted(
"admission capacity exhausted while reserving request resources",
);
Self {
infer_failure: failure.clone(),
stream_start_failure: failure.clone(),
stream_chunk_failure: failure,
..Self::new()
}
}
fn context_length_exceeded() -> Self {
let failure = ferrum_types::FerrumError::ContextLengthExceeded {
capacity: 512,
input_tokens: 500,
output_tokens: 100,
};
Self {
infer_failure: failure.clone(),
stream_start_failure: failure,
..Self::new()
}
}
}
struct CapturingLlm {
config: EngineConfig,
last_request: Mutex<Option<InferenceRequest>>,
}
impl CapturingLlm {
fn new() -> Self {
let mut config = EngineConfig::default();
config.model.model_id = ModelId::new("qwen3");
Self {
config,
last_request: Mutex::new(None),
}
}
fn last_request(&self) -> InferenceRequest {
self.last_request
.lock()
.expect("capture lock")
.clone()
.expect("request captured")
}
fn has_captured_request(&self) -> bool {
self.last_request.lock().expect("capture lock").is_some()
}
}
#[async_trait]
impl InferenceEngine for StubLlm {
async fn status(&self) -> EngineStatus {
EngineStatus {
is_ready: true,
loaded_models: vec![self.config.model.model_id.clone()],
active_requests: 0,
queued_requests: 0,
memory_usage: MemoryUsage {
total_bytes: 0,
used_bytes: 0,
free_bytes: 0,
gpu_memory_bytes: None,
cpu_memory_bytes: None,
cache_memory_bytes: 0,
utilization_percent: 0.0,
},
uptime_seconds: 0,
last_heartbeat: chrono::Utc::now(),
version: "test".to_string(),
}
}
async fn shutdown(&self) -> ferrum_types::Result<()> {
self.shutdown_count.fetch_add(1, Ordering::AcqRel);
Ok(())
}
fn config(&self) -> &EngineConfig {
&self.config
}
fn metrics(&self) -> EngineMetrics {
EngineMetrics::default()
}
async fn health_check(&self) -> EngineHealthStatus {
EngineHealthStatus::healthy()
}
fn execution_attribution_snapshot(&self) -> Option<Value> {
self.execution_attribution.clone()
}
fn lora_metrics_snapshot(&self) -> Option<Value> {
self.lora_metrics.clone()
}
}
#[async_trait]
impl InferenceEngine for StubEmbed {
async fn status(&self) -> EngineStatus {
EngineStatus {
is_ready: true,
loaded_models: vec![self.config.model.model_id.clone()],
active_requests: 0,
queued_requests: 0,
memory_usage: MemoryUsage {
total_bytes: 0,
used_bytes: 0,
free_bytes: 0,
gpu_memory_bytes: None,
cpu_memory_bytes: None,
cache_memory_bytes: 0,
utilization_percent: 0.0,
},
uptime_seconds: 0,
last_heartbeat: chrono::Utc::now(),
version: "test".to_string(),
}
}
async fn shutdown(&self) -> ferrum_types::Result<()> {
Ok(())
}
fn config(&self) -> &EngineConfig {
&self.config
}
fn metrics(&self) -> EngineMetrics {
EngineMetrics::default()
}
async fn health_check(&self) -> EngineHealthStatus {
EngineHealthStatus::healthy()
}
}
#[async_trait]
impl EmbedEngine for StubEmbed {
async fn embed_text(&self, text: &str) -> ferrum_types::Result<Vec<f32>> {
Ok(vec![text.len() as f32, 1.0, 0.0])
}
async fn embed_image(&self, image: &str) -> ferrum_types::Result<Vec<f32>> {
Ok(vec![image.len() as f32, 0.0, 1.0])
}
fn embedding_dim(&self) -> usize {
3
}
}
#[async_trait]
impl InferenceEngine for StubTranscribe {
async fn status(&self) -> EngineStatus {
EngineStatus {
is_ready: true,
loaded_models: vec![self.config.model.model_id.clone()],
active_requests: 0,
queued_requests: 0,
memory_usage: MemoryUsage {
total_bytes: 0,
used_bytes: 0,
free_bytes: 0,
gpu_memory_bytes: None,
cpu_memory_bytes: None,
cache_memory_bytes: 0,
utilization_percent: 0.0,
},
uptime_seconds: 0,
last_heartbeat: chrono::Utc::now(),
version: "test".to_string(),
}
}
async fn shutdown(&self) -> ferrum_types::Result<()> {
Ok(())
}
fn config(&self) -> &EngineConfig {
&self.config
}
fn metrics(&self) -> EngineMetrics {
EngineMetrics::default()
}
async fn health_check(&self) -> EngineHealthStatus {
EngineHealthStatus::healthy()
}
}
#[async_trait]
impl TranscribeEngine for StubTranscribe {
async fn transcribe_file(
&self,
path: &str,
language: Option<&str>,
) -> ferrum_types::Result<String> {
Ok(format!("file:{path}:{}", language.unwrap_or("auto")))
}
async fn transcribe_bytes(
&self,
data: &[u8],
language: Option<&str>,
) -> ferrum_types::Result<String> {
Ok(format!(
"bytes:{}:{}",
data.len(),
language.unwrap_or("auto")
))
}
}
#[async_trait]
impl InferenceEngine for StubTts {
async fn status(&self) -> EngineStatus {
EngineStatus {
is_ready: true,
loaded_models: vec![self.config.model.model_id.clone()],
active_requests: 0,
queued_requests: 0,
memory_usage: MemoryUsage {
total_bytes: 0,
used_bytes: 0,
free_bytes: 0,
gpu_memory_bytes: None,
cpu_memory_bytes: None,
cache_memory_bytes: 0,
utilization_percent: 0.0,
},
uptime_seconds: 0,
last_heartbeat: chrono::Utc::now(),
version: "test".to_string(),
}
}
async fn shutdown(&self) -> ferrum_types::Result<()> {
Ok(())
}
fn config(&self) -> &EngineConfig {
&self.config
}
fn metrics(&self) -> EngineMetrics {
EngineMetrics::default()
}
async fn health_check(&self) -> EngineHealthStatus {
EngineHealthStatus::healthy()
}
}
#[async_trait]
impl TtsEngine for StubTts {
async fn synthesize_speech(
&self,
_text: &str,
_language: Option<&str>,
_chunk_frames: usize,
) -> ferrum_types::Result<Vec<Vec<f32>>> {
Ok(vec![vec![0.0, 0.5, -0.5]])
}
fn tts_sample_rate(&self) -> u32 {
16_000
}
}
#[async_trait]
impl InferenceEngine for FailingLlm {
async fn status(&self) -> EngineStatus {
EngineStatus {
is_ready: true,
loaded_models: vec![self.config.model.model_id.clone()],
active_requests: 0,
queued_requests: 0,
memory_usage: MemoryUsage {
total_bytes: 0,
used_bytes: 0,
free_bytes: 0,
gpu_memory_bytes: None,
cpu_memory_bytes: None,
cache_memory_bytes: 0,
utilization_percent: 0.0,
},
uptime_seconds: 0,
last_heartbeat: chrono::Utc::now(),
version: "test".to_string(),
}
}
async fn shutdown(&self) -> ferrum_types::Result<()> {
Ok(())
}
fn config(&self) -> &EngineConfig {
&self.config
}
fn metrics(&self) -> EngineMetrics {
EngineMetrics::default()
}
async fn health_check(&self) -> EngineHealthStatus {
EngineHealthStatus::healthy()
}
}
#[async_trait]
impl InferenceEngine for CapturingLlm {
async fn status(&self) -> EngineStatus {
EngineStatus {
is_ready: true,
loaded_models: vec![self.config.model.model_id.clone()],
active_requests: 0,
queued_requests: 0,
memory_usage: MemoryUsage {
total_bytes: 0,
used_bytes: 0,
free_bytes: 0,
gpu_memory_bytes: None,
cpu_memory_bytes: None,
cache_memory_bytes: 0,
utilization_percent: 0.0,
},
uptime_seconds: 0,
last_heartbeat: chrono::Utc::now(),
version: "test".to_string(),
}
}
async fn shutdown(&self) -> ferrum_types::Result<()> {
Ok(())
}
fn config(&self) -> &EngineConfig {
&self.config
}
fn metrics(&self) -> EngineMetrics {
EngineMetrics::default()
}
async fn health_check(&self) -> EngineHealthStatus {
EngineHealthStatus::healthy()
}
}
fn stub_execution_evidence(
request: &InferenceRequest,
output_token_count: usize,
) -> Option<InferenceExecutionEvidence> {
let requested = &request.evidence_request;
if !requested.capture_prompt_token_ids && !requested.capture_engine_token_timing {
return None;
}
Some(InferenceExecutionEvidence {
prompt_token_ids: requested
.capture_prompt_token_ids
.then(|| vec![TokenId::new(101), TokenId::new(202), TokenId::new(303)])
.unwrap_or_default(),
output_token_ids: (0..output_token_count)
.map(|index| TokenId::new(11 + index as u32))
.collect(),
engine_token_timing: requested.capture_engine_token_timing.then(|| {
EngineTokenTimingEvidence {
clock_source: "rust_std_instant".to_string(),
wall_anchor_unix_nanos: 1_700_000_000_000_000_000,
wall_anchor_max_error_nanos: 500,
decode_ready_nanos_since_request_start: Some(1_000_000),
token_commit_nanos_since_request_start: (1..=output_token_count)
.map(|ordinal| ordinal as u64 * 1_000_000)
.collect(),
decode_stage_intervals: Vec::new(),
}
}),
})
}
#[async_trait]
impl LlmInferenceEngine for StubLlm {
fn context_capacity(&self) -> Option<usize> {
self.context_capacity
}
async fn infer(
&self,
request: InferenceRequest,
) -> ferrum_types::Result<InferenceResponse> {
let execution_evidence = stub_execution_evidence(&request, 2);
Ok(InferenceResponse {
request_id: request.id,
text: self.text.clone(),
tokens: vec![TokenId::new(11), TokenId::new(12)],
finish_reason: self.finish_reason,
usage: TokenUsage::new(7, 2),
latency_ms: 1,
created_at: chrono::Utc::now(),
metadata: HashMap::new(),
api_response: self.api_response.clone(),
execution_evidence,
})
}
async fn infer_stream(
&self,
request: InferenceRequest,
) -> ferrum_types::Result<
Pin<Box<dyn Stream<Item = ferrum_types::Result<StreamChunk>> + Send>>,
> {
if let Some(drop_notify) = self.pending_stream_drop_notify.as_ref() {
return Ok(Box::pin(PendingDropStream {
drop_notify: Arc::clone(drop_notify),
}));
}
if let Some(chunks) = &self.stream_chunks {
let completion_token_count = self
.stream_usage
.as_ref()
.map(|usage| usage.completion_tokens)
.unwrap_or(chunks.len());
let execution_evidence = stub_execution_evidence(&request, completion_token_count);
let request_id = request.id;
let mut stream_chunks = Vec::with_capacity(
chunks.len() + usize::from(self.stream_final_chunk_separate),
);
let last = chunks.len().saturating_sub(1);
for (index, text) in chunks.iter().enumerate() {
let is_final_text_chunk = index == last && !self.stream_final_chunk_separate;
stream_chunks.push(Ok(StreamChunk {
request_id: request_id.clone(),
text: text.clone(),
token: (!(self.stream_tail_without_token && index == last))
.then_some(TokenId::new(11 + index as u32)),
finish_reason: is_final_text_chunk.then_some(self.finish_reason),
usage: is_final_text_chunk
.then(|| self.stream_usage.clone())
.flatten(),
created_at: chrono::Utc::now(),
metadata: HashMap::new(),
api_response: is_final_text_chunk
.then(|| self.api_response.clone())
.flatten(),
execution_evidence: is_final_text_chunk
.then(|| execution_evidence.clone())
.flatten(),
}));
}
if self.stream_final_chunk_separate {
stream_chunks.push(Ok(StreamChunk {
request_id,
text: String::new(),
token: None,
finish_reason: Some(self.finish_reason),
usage: self.stream_usage.clone(),
created_at: chrono::Utc::now(),
metadata: HashMap::new(),
api_response: self.api_response.clone(),
execution_evidence,
}));
}
if self.stream_terminal_error {
*stream_chunks.last_mut().expect("nonempty fixture stream") = Err(
ferrum_types::FerrumError::internal("fixture generation failed"),
);
}
if let Some(gate) = self.stream_after_first_gate.clone() {
return Ok(Box::pin(stream::unfold(
(stream_chunks.into_iter().enumerate(), gate),
|(mut chunks, gate)| async move {
let (index, chunk) = chunks.next()?;
if index == 1 {
gate.entered.notify_one();
gate.resume.notified().await;
}
Some((chunk, (chunks, gate)))
},
)));
}
return Ok(Box::pin(stream::iter(stream_chunks)));
}
let execution_evidence = stub_execution_evidence(&request, 1);
let chunk = StreamChunk {
request_id: request.id,
text: self.text.clone(),
token: Some(TokenId::new(11)),
finish_reason: Some(self.finish_reason),
usage: self.stream_usage.clone(),
created_at: chrono::Utc::now(),
metadata: HashMap::new(),
api_response: self.api_response.clone(),
execution_evidence,
};
Ok(Box::pin(stream::iter(vec![Ok(chunk)])))
}
}
#[async_trait]
impl LlmInferenceEngine for FailingLlm {
async fn infer(
&self,
_request: InferenceRequest,
) -> ferrum_types::Result<InferenceResponse> {
Err(self.infer_failure.clone())
}
async fn infer_stream(
&self,
request: InferenceRequest,
) -> ferrum_types::Result<
Pin<Box<dyn Stream<Item = ferrum_types::Result<StreamChunk>> + Send>>,
> {
if self.fail_after_stream_start {
let _request_id = request.id;
return Ok(Box::pin(stream::iter(vec![Err(self
.stream_chunk_failure
.clone())])));
}
Err(self.stream_start_failure.clone())
}
}
#[async_trait]
impl LlmInferenceEngine for CapturingLlm {
async fn infer(
&self,
request: InferenceRequest,
) -> ferrum_types::Result<InferenceResponse> {
*self.last_request.lock().expect("capture lock") = Some(request.clone());
Ok(InferenceResponse {
request_id: request.id,
text: "captured".to_string(),
tokens: vec![TokenId::new(21)],
finish_reason: FinishReason::Stop,
usage: TokenUsage::new(9, 1),
latency_ms: 1,
created_at: chrono::Utc::now(),
metadata: HashMap::new(),
api_response: None,
execution_evidence: None,
})
}
async fn infer_stream(
&self,
request: InferenceRequest,
) -> ferrum_types::Result<
Pin<Box<dyn Stream<Item = ferrum_types::Result<StreamChunk>> + Send>>,
> {
*self.last_request.lock().expect("capture lock") = Some(request.clone());
let chunk = StreamChunk {
request_id: request.id,
text: "captured".to_string(),
token: Some(TokenId::new(21)),
finish_reason: Some(FinishReason::Stop),
usage: Some(TokenUsage::new(9, 1)),
created_at: chrono::Utc::now(),
metadata: HashMap::new(),
api_response: None,
execution_evidence: None,
};
Ok(Box::pin(stream::iter(vec![Ok(chunk)])))
}
}
fn state_with_stub(text: &str) -> AppState {
AppState::default().with_llm(Arc::new(StubLlm::new(text)))
}
fn router_with_stub(text: &str) -> Router {
AxumServer::from_llm(Arc::new(StubLlm::new(text))).build_router()
}
fn router_with_stub_and_template(text: &str, template: ModelChatTemplate) -> Router {
AxumServer::from_llm(Arc::new(StubLlm::new(text)))
.with_prompt_template(Some(template))
.build_router()
}
fn router_with_stub_and_request_dump_dir(text: &str, request_dump_dir: PathBuf) -> Router {
AxumServer::from_state(
AppState::default()
.with_llm(Arc::new(StubLlm::new(text)))
.with_request_dump_dir(Some(request_dump_dir)),
)
.build_router()
}
fn router_with_stub_request_dump_and_profile(
text: &str,
request_dump_dir: PathBuf,
profile_jsonl: PathBuf,
) -> Router {
AxumServer::from_state(
AppState::default()
.with_llm(Arc::new(StubLlm::new(text)))
.with_request_dump_dir(Some(request_dump_dir))
.with_profile_detail(ferrum_types::ObservabilityProfileDetail::Latency)
.with_profile_jsonl(Some(profile_jsonl)),
)
.build_router()
}
fn router_with_stub_stream_chunks(chunks: &[&str]) -> Router {
AxumServer::from_llm(Arc::new(StubLlm::with_stream_chunks(chunks))).build_router()
}
fn router_with_stub_finish_reason(text: &str, finish_reason: FinishReason) -> Router {
AxumServer::from_llm(Arc::new(StubLlm {
finish_reason,
..StubLlm::new(text)
}))
.build_router()
}
fn router_with_stub_stream_chunks_and_request_dump_dir(
chunks: &[&str],
request_dump_dir: PathBuf,
) -> Router {
AxumServer::from_state(
AppState::default()
.with_llm(Arc::new(StubLlm::with_stream_chunks(chunks)))
.with_request_dump_dir(Some(request_dump_dir)),
)
.build_router()
}
fn router_with_stub_stream_request_dump_and_profile(
chunks: &[&str],
request_dump_dir: PathBuf,
profile_jsonl: PathBuf,
) -> Router {
AxumServer::from_state(
AppState::default()
.with_llm(Arc::new(StubLlm::with_stream_chunks(chunks)))
.with_request_dump_dir(Some(request_dump_dir))
.with_profile_detail(ferrum_types::ObservabilityProfileDetail::Latency)
.with_profile_jsonl(Some(profile_jsonl)),
)
.build_router()
}
fn router_with_stub_separate_final_stream_chunk(chunks: &[&str]) -> Router {
AxumServer::from_llm(Arc::new(StubLlm::with_separate_final_stream_chunk(chunks)))
.build_router()
}
fn router_with_stub_api_response(
text: &str,
api_response: ferrum_types::ApiResponse,
) -> Router {
AxumServer::from_llm(Arc::new(StubLlm::with_api_response(text, api_response)))
.build_router()
}
fn router_with_stub_api_response_and_finish_reason(
text: &str,
api_response: ferrum_types::ApiResponse,
finish_reason: FinishReason,
) -> Router {
AxumServer::from_llm(Arc::new(StubLlm::with_api_response_and_finish_reason(
text,
api_response,
finish_reason,
)))
.build_router()
}
fn weather_tool_api_response() -> ferrum_types::ApiResponse {
ferrum_types::ApiResponse::Chat(ferrum_types::ApiChatResponse {
message: ferrum_types::ApiChatMessage {
role: ferrum_types::ApiMessageRole::Assistant,
content: String::new(),
name: None,
tool_calls: vec![ferrum_types::ApiToolCall {
id: "call_1".to_string(),
tool_type: "function".to_string(),
function: ferrum_types::ApiFunctionCall {
name: "weather".to_string(),
arguments: "{\"city\":\"Paris\"}".to_string(),
},
}],
tool_call_id: None,
function_call: None,
},
finish_reason: Some("tool_calls".to_string()),
})
}
fn weather_tool_api_response_with_commentary() -> ferrum_types::ApiResponse {
let mut response = weather_tool_api_response();
let ferrum_types::ApiResponse::Chat(chat) = &mut response else {
unreachable!("weather response is chat")
};
chat.message.content = "I will check.".to_string();
response
}
fn namespaced_tool_api_response() -> ferrum_types::ApiResponse {
ferrum_types::ApiResponse::Chat(ferrum_types::ApiChatResponse {
message: ferrum_types::ApiChatMessage {
role: ferrum_types::ApiMessageRole::Assistant,
content: String::new(),
name: None,
tool_calls: vec![ferrum_types::ApiToolCall {
id: "call_ns_1".to_string(),
tool_type: "function".to_string(),
function: ferrum_types::ApiFunctionCall {
name: "collaboration__wait_agent".to_string(),
arguments: "{}".to_string(),
},
}],
tool_call_id: None,
function_call: None,
},
finish_reason: Some("tool_calls".to_string()),
})
}
fn two_tool_api_response() -> ferrum_types::ApiResponse {
ferrum_types::ApiResponse::Chat(ferrum_types::ApiChatResponse {
message: ferrum_types::ApiChatMessage {
role: ferrum_types::ApiMessageRole::Assistant,
content: String::new(),
name: None,
tool_calls: vec![
ferrum_types::ApiToolCall {
id: "call_1".to_string(),
tool_type: "function".to_string(),
function: ferrum_types::ApiFunctionCall {
name: "weather".to_string(),
arguments: "{}".to_string(),
},
},
ferrum_types::ApiToolCall {
id: "call_2".to_string(),
tool_type: "function".to_string(),
function: ferrum_types::ApiFunctionCall {
name: "clock".to_string(),
arguments: "{}".to_string(),
},
},
],
tool_call_id: None,
function_call: None,
},
finish_reason: Some("tool_calls".to_string()),
})
}
fn router_with_stub_without_stream_usage(text: &str) -> Router {
AxumServer::from_llm(Arc::new(StubLlm::without_stream_usage(text))).build_router()
}
fn router_without_llm() -> Router {
AxumServer::from_state(AppState::default()).build_router()
}
fn router_with_failing_llm() -> Router {
AxumServer::from_llm(Arc::new(FailingLlm::new())).build_router()
}
fn router_with_failing_llm_and_request_dump_dir(request_dump_dir: PathBuf) -> Router {
AxumServer::from_state(
AppState::default()
.with_llm(Arc::new(FailingLlm::new()))
.with_request_dump_dir(Some(request_dump_dir)),
)
.build_router()
}
fn router_with_failing_llm_request_dump_and_profile(
request_dump_dir: PathBuf,
profile_jsonl: PathBuf,
) -> Router {
AxumServer::from_state(
AppState::default()
.with_llm(Arc::new(FailingLlm::new()))
.with_request_dump_dir(Some(request_dump_dir))
.with_profile_detail(ferrum_types::ObservabilityProfileDetail::Latency)
.with_profile_jsonl(Some(profile_jsonl)),
)
.build_router()
}
fn router_with_resource_exhausted_llm_and_request_dump_dir(
request_dump_dir: PathBuf,
) -> Router {
AxumServer::from_state(
AppState::default()
.with_llm(Arc::new(FailingLlm::resource_exhausted()))
.with_request_dump_dir(Some(request_dump_dir)),
)
.build_router()
}
fn router_with_stream_chunk_failing_llm() -> Router {
AxumServer::from_llm(Arc::new(FailingLlm::after_stream_start())).build_router()
}
fn router_with_stream_chunk_failing_llm_and_request_dump_dir(
request_dump_dir: PathBuf,
) -> Router {
AxumServer::from_state(
AppState::default()
.with_llm(Arc::new(FailingLlm::after_stream_start()))
.with_request_dump_dir(Some(request_dump_dir)),
)
.build_router()
}
fn router_with_capturing_llm() -> (Router, Arc<CapturingLlm>) {
let engine = Arc::new(CapturingLlm::new());
let registry = ServedModelRegistry::try_new(
"qwen3",
ServedModelKind::Llm,
vec![
"qwen3".to_string(),
"stub-model".to_string(),
"served-alias".to_string(),
],
vec![],
)
.unwrap();
let router = AxumServer::from_llm(engine.clone())
.with_served_model_registry(registry)
.build_router();
(router, engine)
}
fn unique_request_dump_dir(test_name: &str) -> PathBuf {
let path =
std::env::temp_dir().join(format!("ferrum-server-{test_name}-{}", Uuid::new_v4()));
fs::create_dir_all(&path).expect("create request dump dir");
path
}
fn unique_profile_jsonl(test_name: &str) -> PathBuf {
std::env::temp_dir().join(format!(
"ferrum-server-{test_name}-{}.jsonl",
Uuid::new_v4()
))
}
fn only_replay_bundle(root: &Path) -> PathBuf {
let mut dirs = fs::read_dir(root)
.expect("read request dump dir")
.filter_map(|entry| {
let path = entry.expect("dir entry").path();
path.is_dir().then_some(path)
})
.collect::<Vec<_>>();
dirs.sort();
assert_eq!(
dirs.len(),
1,
"expected exactly one replay bundle in {root:?}"
);
dirs.remove(0)
}
fn read_json_file(path: impl AsRef<Path>) -> Value {
let path = path.as_ref();
let text = fs::read_to_string(path).unwrap_or_else(|err| {
panic!("failed to read {}: {}", path.display(), err);
});
serde_json::from_str(&text).unwrap_or_else(|err| {
panic!("failed to parse {}: {}", path.display(), err);
})
}
fn read_profile_events(path: &Path) -> Vec<Value> {
let text = fs::read_to_string(path)
.unwrap_or_else(|err| panic!("failed to read {}: {}", path.display(), err));
text.lines()
.filter(|line| !line.trim().is_empty())
.map(|line| serde_json::from_str::<Value>(line).expect("profile event json"))
.collect()
}
fn assert_chat_failure_replay_bundle(
root: &Path,
expected_phase: &str,
expected_error_kind: &str,
expected_message: &str,
) {
let bundle = only_replay_bundle(root);
let request = read_json_file(bundle.join("request.json"));
let request_id = request["request_id"]
.as_str()
.expect("request id")
.to_string();
let bad_scan = read_json_file(bundle.join("bad_output_scan.json"));
assert_eq!(bad_scan["request_id"], request_id);
assert_eq!(bad_scan["failure_kind"], "error");
assert_eq!(bad_scan["failure_phase"], expected_phase);
assert_eq!(bad_scan["error_kind"], expected_error_kind);
let diagnostics = read_json_file(bundle.join("failure_diagnostics.json"));
assert_eq!(diagnostics["request_id"], request_id);
assert_eq!(diagnostics["failure_kind"], "error");
assert_eq!(diagnostics["first_failure_event"]["phase"], expected_phase);
assert_eq!(
diagnostics["first_failure_event"]["error_kind"],
expected_error_kind
);
assert_eq!(diagnostics["nearest_request_id"], request_id);
assert!(diagnostics["log_excerpt"]
.as_str()
.expect("log excerpt")
.contains(expected_message));
assert!(bundle.join("replay.command.json").is_file());
}
fn assert_chat_success_replay_bundle(
root: &Path,
expected_token_ids: &[u32],
expected_finish_reason: &str,
expected_output_text: &str,
) {
let bundle = only_replay_bundle(root);
let request = read_json_file(bundle.join("request.json"));
let request_id = request["request_id"]
.as_str()
.expect("request id")
.to_string();
let prompt_tokens = read_json_file(bundle.join("prompt_token_ids.json"));
assert_eq!(prompt_tokens["request_id"], request_id);
assert_eq!(prompt_tokens["token_ids"], json!([101, 202, 303]));
assert_eq!(prompt_tokens["token_count"], 3);
assert!(prompt_tokens["unavailable_reason"].is_null());
let output_tokens = read_json_file(bundle.join("output_token_ids.json"));
assert_eq!(output_tokens["request_id"], request_id);
assert_eq!(output_tokens["token_ids"], json!(expected_token_ids));
assert_eq!(output_tokens["token_count"], expected_token_ids.len());
assert_eq!(output_tokens["finish_reason"], expected_finish_reason);
assert!(output_tokens["unavailable_reason"].is_null());
let bad_scan = read_json_file(bundle.join("bad_output_scan.json"));
assert_eq!(bad_scan["request_id"], request_id);
assert_eq!(bad_scan["bad_output"], false);
assert_eq!(bad_scan["failure_kind"], serde_json::Value::Null);
assert_eq!(
bad_scan["output_chars"],
expected_output_text.chars().count()
);
assert_eq!(
bad_scan["classified_output_sha256"],
sha256_hex(expected_output_text.as_bytes())
);
let output_text_bytes = fs::read(bundle.join("output_text.txt")).unwrap();
assert_eq!(bad_scan["output_sha256"], sha256_hex(&output_text_bytes));
let output_text = String::from_utf8(output_text_bytes).unwrap();
assert!(output_text.contains("[redacted actual output]"));
assert!(output_text.contains(&format!(
"sha256={}",
sha256_hex(expected_output_text.as_bytes())
)));
assert!(output_text.contains(&format!("chars={}", expected_output_text.chars().count())));
let replay_body = read_json_file(bundle.join("replay_body.json"));
assert_eq!(replay_body["messages"][0]["role"], "user");
assert_eq!(replay_body["messages"][0]["content"], "[redacted]");
assert_eq!(replay_body["messages"][0]["content_redacted"], true);
let replay = read_json_file(bundle.join("replay.command.json"));
assert_eq!(replay["requires_running_server"], true);
let argv = replay["argv"].as_array().expect("replay argv");
assert!(argv.iter().any(|item| item == "--data-binary"));
assert!(argv.iter().any(|item| {
item.as_str()
.is_some_and(|value| value.starts_with('@') && value.ends_with("replay_body.json"))
}));
assert_eq!(replay["engine_replay"]["requires_http_server"], false);
let engine_argv = replay["engine_replay"]["argv"]
.as_array()
.expect("engine replay argv");
assert!(engine_argv.iter().any(|item| item == "replay-bundle"));
}
fn router_with_capturing_llm_and_template(
template: ModelChatTemplate,
) -> (Router, Arc<CapturingLlm>) {
router_with_capturing_llm_and_template_default(template, None)
}
fn router_with_capturing_llm_and_template_default(
template: ModelChatTemplate,
default_enable_thinking: Option<bool>,
) -> (Router, Arc<CapturingLlm>) {
let engine = Arc::new(CapturingLlm::new());
let registry = ServedModelRegistry::try_new(
"qwen3",
ServedModelKind::Llm,
vec!["served-alias".to_string()],
vec![],
)
.unwrap();
let router = AxumServer::from_llm(engine.clone())
.with_served_model_registry(registry)
.with_prompt_template(Some(template))
.with_default_enable_thinking(default_enable_thinking)
.build_router();
(router, engine)
}
fn qwen36_chat_template() -> ModelChatTemplate {
ModelChatTemplate::new(
include_str!("../tests/fixtures/chat_template/Qwen__Qwen3.6-35B-A3B/template.jinja"),
"Qwen/Qwen3.6-35B-A3B",
)
}
async fn capture_qwen36_tool_history_request(
reasoning_fields: Value,
stream: bool,
) -> InferenceRequest {
let mut assistant = json!({
"role": "assistant",
"content": null,
"tool_calls": [{
"id": "call_1",
"type": "function",
"function": {"name": "weather", "arguments": "{\"city\":\"Paris\"}"}
}]
});
assistant
.as_object_mut()
.expect("assistant message object")
.extend(
reasoning_fields
.as_object()
.expect("reasoning fields object")
.clone(),
);
let (router, engine) = router_with_capturing_llm_and_template(qwen36_chat_template());
let response = post_json(
router,
"/v1/chat/completions",
json!({
"model": "served-alias",
"messages": [
{"role": "user", "content": "Use the weather tool."},
assistant,
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"}
],
"tools": [{
"type": "function",
"function": {
"name": "weather",
"description": "Get weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"]
}
}
}],
"stream": stream
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
if stream {
assert!(response_text(response).await.contains("[DONE]"));
}
engine.last_request()
}
fn router_with_capturing_lora_llm() -> (Router, Arc<CapturingLlm>) {
let engine = Arc::new(CapturingLlm::new());
let router = AxumServer::from_llm(engine.clone())
.with_lora_adapters(
"qwen3",
vec![LoraAdapterModel::new(
"sql",
"qwen3:sql",
"/tmp/sql-adapter",
)],
)
.unwrap()
.build_router();
(router, engine)
}
fn router_with_stub_embed() -> Router {
AxumServer::from_embed(Arc::new(StubEmbed::new())).build_router()
}
fn router_with_stub_transcribe() -> Router {
AxumServer::from_transcribe(Arc::new(StubTranscribe::new())).build_router()
}
fn router_with_stub_tts() -> Router {
AxumServer::from_tts(Arc::new(StubTts::new())).build_router()
}
async fn post_json(app: Router, path: &str, body: Value) -> Response {
app.oneshot(
Request::builder()
.method("POST")
.uri(path)
.header(header::CONTENT_TYPE, "application/json")
.body(Body::from(body.to_string()))
.expect("request"),
)
.await
.expect("route response")
}
async fn post_json_with_benchmark_correlation(
app: Router,
path: &str,
body: Value,
correlation: &BenchmarkRequestCorrelation,
) -> Response {
app.oneshot(
Request::builder()
.method("POST")
.uri(path)
.header(header::CONTENT_TYPE, "application/json")
.header(BENCHMARK_RUN_ID_HEADER, &correlation.benchmark_run_id)
.header(BENCHMARK_CELL_ID_HEADER, &correlation.cell_id)
.header(
BENCHMARK_REPEAT_INDEX_HEADER,
correlation.repeat_index.to_string(),
)
.header(BENCHMARK_PHASE_HEADER, correlation.phase.as_str())
.header(
BENCHMARK_REQUEST_INDEX_HEADER,
correlation.request_index.to_string(),
)
.body(Body::from(body.to_string()))
.expect("request"),
)
.await
.expect("route response")
}
async fn post_raw_json(app: Router, path: &str, body: &str) -> Response {
app.oneshot(
Request::builder()
.method("POST")
.uri(path)
.header(header::CONTENT_TYPE, "application/json")
.body(Body::from(body.to_string()))
.expect("request"),
)
.await
.expect("route response")
}
async fn post_multipart(app: Router, path: &str, boundary: &str, body: &str) -> Response {
app.oneshot(
Request::builder()
.method("POST")
.uri(path)
.header(
header::CONTENT_TYPE,
format!("multipart/form-data; boundary={boundary}"),
)
.body(Body::from(body.to_string()))
.expect("request"),
)
.await
.expect("route response")
}
async fn get(app: Router, path: &str) -> Response {
app.oneshot(
Request::builder()
.method("GET")
.uri(path)
.body(Body::empty())
.expect("request"),
)
.await
.expect("route response")
}
async fn response_json(response: Response) -> Value {
let bytes = to_bytes(response.into_body(), usize::MAX)
.await
.expect("body bytes");
serde_json::from_slice(&bytes).expect("json body")
}
async fn response_text(response: Response) -> String {
let bytes = to_bytes(response.into_body(), usize::MAX)
.await
.expect("body bytes");
String::from_utf8(bytes.to_vec()).expect("utf8 body")
}
fn responses_sse_json_events(body: &str) -> Vec<Value> {
body.lines()
.filter_map(|line| line.strip_prefix("data: "))
.filter(|data| *data != "[DONE]")
.map(|data| serde_json::from_str(data).expect("Responses SSE JSON event"))
.collect()
}
async fn response_bytes(response: Response) -> Vec<u8> {
to_bytes(response.into_body(), usize::MAX)
.await
.expect("body bytes")
.to_vec()
}
async fn error_json(error: ServerError) -> (AxumStatusCode, Value) {
let response = error.into_response();
let status = response.status();
(status, response_json(response).await)
}
fn assert_openai_stream_error(body: &str, expected_message: &str) {
assert!(body.contains("data: [DONE]"), "missing DONE: {body}");
assert!(
body.contains("\"error\":{\"message\":\""),
"stream failure should emit OpenAI error envelope: {body}"
);
assert!(
body.contains(expected_message),
"stream failure should include engine error message {expected_message:?}: {body}"
);
assert!(
body.contains("\"type\":\"internal_server_error\""),
"stream failure should use internal_server_error: {body}"
);
assert!(
!body.contains("{\"error\":\""),
"stream failure must not use legacy bare error payload: {body}"
);
}
fn chat_request(extra: Value) -> ChatCompletionsRequest {
let mut value = json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "hello"}],
"max_tokens": 8
});
let obj = value.as_object_mut().unwrap();
for (k, v) in extra.as_object().unwrap() {
obj.insert(k.clone(), v.clone());
}
serde_json::from_value(value).expect("chat request")
}
#[tokio::test]
async fn responses_route_returns_sync_text_and_usage() {
let response = post_json(
router_with_stub("hello from ferrum"),
"/v1/responses",
json!({
"model": "stub-model",
"input": "hello",
"store": false
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["object"], "response");
assert_eq!(body["status"], "completed");
assert_eq!(body["store"], false);
assert_eq!(body["output"][0]["type"], "message");
assert_eq!(body["output"][0]["phase"], "final_answer");
assert_eq!(body["output"][0]["content"][0]["text"], "hello from ferrum");
assert_eq!(body["usage"]["input_tokens"], 7);
assert_eq!(body["usage"]["output_tokens"], 2);
assert_eq!(body["usage"]["total_tokens"], 9);
assert_eq!(body["presence_penalty"], 0.0);
assert_eq!(body["frequency_penalty"], 0.0);
}
#[tokio::test]
async fn responses_route_streams_ordered_text_events_once() {
let response = post_json(
router_with_stub_stream_chunks(&["he", "llo"]),
"/v1/responses",
json!({
"model": "stub-model",
"input": [{"role": "user", "content": "say hello"}],
"stream": true
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
for event in [
"response.created",
"response.output_item.added",
"response.output_text.delta",
"response.output_text.done",
"response.output_item.done",
"response.completed",
] {
assert!(
body.contains(&format!("event: {event}")),
"missing {event}: {body}"
);
}
assert_eq!(
body.matches("event: response.completed").count(),
1,
"completed must be emitted exactly once: {body}"
);
assert!(
body.contains("\"delta\":\"he\""),
"missing first delta: {body}"
);
assert!(
body.contains("\"delta\":\"llo\""),
"missing second delta: {body}"
);
assert!(body.contains("\"input_tokens\":5"), "missing usage: {body}");
let events = responses_sse_json_events(&body);
let message_added = events
.iter()
.find(|event| {
event["type"] == "response.output_item.added" && event["item"]["type"] == "message"
})
.expect("message item added");
assert!(
message_added["item"].get("phase").is_none(),
"stream must not guess phase before later tool calls are known: {body}"
);
let message_done = events
.iter()
.find(|event| {
event["type"] == "response.output_item.done" && event["item"]["type"] == "message"
})
.expect("message item done");
assert_eq!(message_done["item"]["phase"], "final_answer");
let terminal = events
.iter()
.find(|event| event["type"] == "response.completed")
.expect("completed response");
assert_eq!(terminal["response"]["output"][0]["phase"], "final_answer");
let completed = body
.find("event: response.completed")
.expect("completed event");
let done = body.find("data: [DONE]").expect("terminal DONE marker");
assert!(
completed < done,
"DONE must follow response.completed: {body}"
);
}
#[tokio::test]
async fn responses_route_supports_stateless_function_round_trip() {
let tool = json!({
"type": "function",
"name": "weather",
"description": "Get weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"]
}
});
let first = post_json(
router_with_stub_api_response("", weather_tool_api_response()),
"/v1/responses",
json!({
"model": "stub-model",
"input": "Use the weather tool",
"tools": [tool.clone()],
"tool_choice": "auto"
}),
)
.await;
assert_eq!(first.status(), AxumStatusCode::OK);
let first_body = response_json(first).await;
let call = first_body["output"][0].clone();
assert_eq!(call["type"], "function_call");
assert_eq!(call["call_id"], "call_1");
assert_eq!(call["name"], "weather");
assert_eq!(call["arguments"], "{\"city\":\"Paris\"}");
let second = post_json(
router_with_stub("weather received"),
"/v1/responses",
json!({
"model": "stub-model",
"input": [
{"role": "user", "content": "Use the weather tool"},
call,
{"type": "function_call_output", "call_id": "call_1", "output": "sunny"}
],
"tools": [tool]
}),
)
.await;
assert_eq!(second.status(), AxumStatusCode::OK);
let second_body = response_json(second).await;
assert_eq!(
second_body["output"][0]["content"][0]["text"],
"weather received"
);
}
#[tokio::test]
async fn responses_route_marks_text_before_calls_as_commentary() {
let request = || {
json!({
"model": "stub-model",
"input": "Use the weather tool",
"stream": false,
"tools": [{
"type": "function",
"name": "weather",
"parameters": {"type": "object"}
}]
})
};
let sync = post_json(
router_with_stub_api_response("", weather_tool_api_response_with_commentary()),
"/v1/responses",
request(),
)
.await;
assert_eq!(sync.status(), AxumStatusCode::OK);
let sync = response_json(sync).await;
assert_eq!(sync["output"][0]["type"], "message");
assert_eq!(sync["output"][0]["phase"], "commentary");
assert_eq!(sync["output"][1]["type"], "function_call");
let mut stream_request = request();
stream_request["stream"] = json!(true);
let stream = post_json(
router_with_stub_api_response("", weather_tool_api_response_with_commentary()),
"/v1/responses",
stream_request,
)
.await;
assert_eq!(stream.status(), AxumStatusCode::OK);
let body = response_text(stream).await;
let events = responses_sse_json_events(&body);
let message_added = events
.iter()
.find(|event| {
event["type"] == "response.output_item.added" && event["item"]["type"] == "message"
})
.expect("message item added");
assert!(message_added["item"].get("phase").is_none());
let message_done = events
.iter()
.find(|event| {
event["type"] == "response.output_item.done" && event["item"]["type"] == "message"
})
.expect("message item done");
assert_eq!(message_done["item"]["phase"], "commentary");
let terminal = events
.iter()
.find(|event| event["type"] == "response.completed")
.expect("completed response");
assert_eq!(terminal["response"]["output"][0]["phase"], "commentary");
assert_eq!(terminal["response"]["output"][1]["type"], "function_call");
}
#[tokio::test]
async fn responses_route_accepts_real_caller_owned_second_turn_shape() {
let response = post_json(
router_with_stub("You first said hello."),
"/v1/responses",
json!({
"model": "stub-model",
"instructions": "Answer from the supplied history.",
"input": [
{
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "Hello"}]
},
{
"type": "message",
"role": "assistant",
"content": [{"type": "output_text", "text": "Hi there!"}]
},
{
"type": "reasoning",
"encrypted_content": null,
"summary": []
},
{
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "What did I say first?"}]
}
],
"store": false,
"stream": false,
"include": ["reasoning.encrypted_content"],
"parallel_tool_calls": false,
"prompt_cache_key": "thread-1",
"reasoning": {"effort": "high", "summary": "auto"}
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(
body["output"][0]["content"][0]["text"],
"You first said hello."
);
assert_eq!(body["parallel_tool_calls"], false);
assert_eq!(body["prompt_cache_key"], "thread-1");
assert_eq!(body["reasoning"]["effort"], "high");
}
#[tokio::test]
async fn responses_route_merges_instructions_with_leading_developer_message() {
let template = ModelChatTemplate::new(
"{% for message in messages %}{% if message.role == 'system' and not loop.first %}{{ raise_exception('System message must be at the beginning.') }}{% endif %}[{{ message.role }}]{{ message.content }}{% endfor %}",
"strict-leading-system-template",
);
let response = post_json(
router_with_stub_and_template("ok", template),
"/v1/responses",
json!({
"model": "stub-model",
"instructions": "Top-level instructions",
"input": [
{"type": "message", "role": "developer", "content": "Developer instructions"},
{"type": "message", "role": "user", "content": "Hello"}
]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
}
#[tokio::test]
async fn responses_route_adapts_interleaved_system_for_strict_template() {
let template = ModelChatTemplate::new(
"{% for message in messages %}{% if message.role == 'system' and not loop.first %}{{ raise_exception('System message must be at the beginning.') }}{% endif %}{% endfor %}{% if messages|length != 2 %}{{ raise_exception('expected two messages') }}{% endif %}{% if messages[0].role != 'system' %}{{ raise_exception('system must be first') }}{% endif %}{% if messages[0].content != 'Initial instructions\\n\\nDeferred tool instructions' %}{{ raise_exception('system instructions were not preserved') }}{% endif %}[{{ messages[0].role }}]{{ messages[0].content }}[{{ messages[1].role }}]{{ messages[1].content }}",
"strict-leading-system-template",
);
let response = post_json(
router_with_stub_and_template("ok", template),
"/v1/responses",
json!({
"model": "stub-model",
"input": [
{"role": "system", "content": "Initial instructions"},
{"role": "user", "content": "Use the available tool"},
{"role": "developer", "content": "Deferred tool instructions"}
]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
}
#[tokio::test]
async fn responses_route_keeps_phase_aligned_through_system_injection() {
let template = ModelChatTemplate::new(
"{% if messages|length != 3 %}{{ raise_exception('expected three messages') }}{% endif %}{% if messages[0].role != 'system' %}{{ raise_exception('system must be first') }}{% endif %}{% if messages[1].role != 'assistant' or messages[1].phase != 'commentary' %}{{ raise_exception('assistant phase was not preserved') }}{% endif %}{% if messages[2].role != 'user' or messages[2].phase is defined %}{{ raise_exception('phase metadata shifted') }}{% endif %}[assistant]",
"phase-alignment-template",
);
let response = post_json(
router_with_stub_and_template(r#"{"ok":true}"#, template),
"/v1/responses",
json!({
"model": "stub-model",
"instructions": "Top-level instructions",
"input": [
{
"type": "message",
"role": "assistant",
"phase": "commentary",
"content": "I will inspect."
},
{"type": "message", "role": "user", "content": "Continue"}
],
"text": {"format": {"type": "json_object"}}
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
}
#[tokio::test]
async fn responses_route_can_disable_interleaved_system_coalescing() {
let template = ModelChatTemplate::new(
"{% for message in messages %}{% if message.role == 'system' and not loop.first %}{{ raise_exception('System message must be at the beginning.') }}{% endif %}[{{ message.role }}]{{ message.content }}{% endfor %}",
"strict-leading-system-template",
);
let mut engine = CapturingLlm::new();
engine.config.model.model_id = ModelId::new("stub-model");
let engine = Arc::new(engine);
let router = AxumServer::from_state(
AppState::default()
.with_llm(engine.clone())
.with_prompt_template(Some(template))
.with_interleaved_system_coalescing(false),
)
.build_router();
let response = post_json(
router,
"/v1/responses",
json!({
"model": "stub-model",
"input": [
{"role": "system", "content": "Initial instructions"},
{"role": "user", "content": "Use the available tool"},
{"role": "developer", "content": "Deferred tool instructions"}
]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert!(!engine.has_captured_request());
assert!(
body.to_string()
.contains("System message must be at the beginning."),
"{body}"
);
}
#[tokio::test]
async fn chat_route_applies_and_can_disable_interleaved_system_coalescing() {
let template = || {
ModelChatTemplate::new(
"{% for message in messages %}{% if message.role == 'system' and not loop.first %}{{ raise_exception('System message must be at the beginning.') }}{% endif %}[{{ message.role }}]{{ message.content }}{% endfor %}",
"strict-leading-system-template",
)
};
let request = || {
json!({
"model": "stub-model",
"messages": [
{"role": "system", "content": "Initial instructions"},
{"role": "user", "content": "Use the available tool"},
{"role": "system", "content": "Deferred tool instructions"}
]
})
};
let enabled = post_json(
router_with_stub_and_template("ok", template()),
"/v1/chat/completions",
request(),
)
.await;
assert_eq!(enabled.status(), AxumStatusCode::OK);
let consecutive = post_json(
router_with_stub_and_template("ok", template()),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [
{"role": "system", "content": "Initial instructions"},
{"role": "system", "content": "Deferred tool instructions"},
{"role": "user", "content": "Use the available tool"}
]
}),
)
.await;
assert_eq!(consecutive.status(), AxumStatusCode::OK);
let mut engine = CapturingLlm::new();
engine.config.model.model_id = ModelId::new("stub-model");
let engine = Arc::new(engine);
let disabled_router = AxumServer::from_state(
AppState::default()
.with_llm(engine.clone())
.with_prompt_template(Some(template()))
.with_interleaved_system_coalescing(false),
)
.build_router();
let disabled = post_json(disabled_router, "/v1/chat/completions", request()).await;
assert_eq!(disabled.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(disabled).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert!(!engine.has_captured_request());
assert!(
body.to_string()
.contains("System message must be at the beginning."),
"{body}"
);
}
#[tokio::test]
async fn responses_route_keeps_structured_output_to_one_leading_system_message() {
let template = ModelChatTemplate::new(
"{% for message in messages %}{% if message.role == 'system' and not loop.first %}{{ raise_exception('System message must be at the beginning.') }}{% endif %}[{{ message.role }}]{{ message.content }}{% endfor %}",
"strict-leading-system-template",
);
let response = post_json(
router_with_stub_and_template(r#"{"ok":true}"#, template),
"/v1/responses",
json!({
"model": "stub-model",
"instructions": "Top-level instructions",
"input": [
{"type": "message", "role": "developer", "content": "Developer instructions"},
{"type": "message", "role": "user", "content": "Return JSON"}
],
"text": {"format": {"type": "json_object"}}
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["output"][0]["content"][0]["text"], r#"{"ok":true}"#);
}
#[tokio::test]
async fn responses_route_output_can_be_replayed_with_readable_reasoning() {
let first = post_json(
router_with_stub("<think>Checked the supplied facts.</think>\nFirst answer"),
"/v1/responses",
json!({
"model": "stub-model",
"input": "First question",
"include": ["reasoning.encrypted_content"]
}),
)
.await;
assert_eq!(first.status(), AxumStatusCode::OK);
let first_body = response_json(first).await;
assert_eq!(first_body["output"][0]["type"], "reasoning");
assert_eq!(
first_body["output"][0]["content"][0]["text"],
"Checked the supplied facts."
);
assert_eq!(first_body["output"][0]["encrypted_content"], Value::Null);
assert_eq!(first_body["output"][1]["type"], "message");
let mut input = vec![json!({
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "First question"}]
})];
input.extend(first_body["output"].as_array().unwrap().iter().cloned());
input.push(json!({
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": "Continue"}]
}));
let second = post_json(
router_with_stub("Second answer"),
"/v1/responses",
json!({"model": "stub-model", "input": input}),
)
.await;
assert_eq!(second.status(), AxumStatusCode::OK);
let second_body = response_json(second).await;
assert_eq!(
second_body["output"][0]["content"][0]["text"],
"Second answer"
);
}
#[tokio::test]
async fn responses_route_streams_reasoning_before_text_with_stable_indices() {
let response = post_json(
router_with_stub_stream_chunks(&["<think>inspect", " history</think>\nfinal"]),
"/v1/responses",
json!({
"model": "stub-model",
"input": "answer",
"stream": true,
"include": ["reasoning.encrypted_content"]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
let events = responses_sse_json_events(&body);
for (sequence, event) in events.iter().enumerate() {
assert_eq!(
event["sequence_number"], sequence,
"Responses sequence numbers must be contiguous: {body}"
);
}
for event in [
"response.reasoning_text.delta",
"response.reasoning_text.done",
"response.output_text.delta",
"response.completed",
] {
assert!(
body.contains(&format!("event: {event}")),
"missing {event}: {body}"
);
}
let reasoning_done = body
.find("event: response.reasoning_text.done")
.expect("reasoning done");
let text_added = body[reasoning_done..]
.find("event: response.output_item.added")
.map(|offset| reasoning_done + offset)
.expect("text item added");
assert!(
reasoning_done < text_added,
"reasoning must finish before text: {body}"
);
assert!(
body.contains("\"output_index\":0,\"content_index\":0,\"delta\":\"inspect"),
"reasoning must use output index 0: {body}"
);
assert!(
body.contains("\"output_index\":1,\"content_index\":0,\"delta\":\"final"),
"text must use output index 1: {body}"
);
let reasoning_added = events
.iter()
.find(|event| {
event["type"] == "response.output_item.added"
&& event["item"]["type"] == "reasoning"
})
.expect("reasoning item added");
assert_eq!(reasoning_added["item"]["status"], "in_progress");
let reasoning_part_added = events
.iter()
.find(|event| {
event["type"] == "response.content_part.added"
&& event["part"]["type"] == "reasoning_text"
})
.expect("reasoning content part added");
assert_eq!(reasoning_part_added["output_index"], 0);
let reasoning_item_done = events
.iter()
.find(|event| {
event["type"] == "response.output_item.done" && event["item"]["type"] == "reasoning"
})
.expect("reasoning item done");
assert_eq!(reasoning_item_done["item"]["status"], "completed");
let terminal = events
.iter()
.find(|event| event["type"] == "response.completed")
.expect("terminal response");
assert_eq!(
terminal["response"]["output"][0],
reasoning_item_done["item"]
);
assert!(
body.contains("data: [DONE]"),
"missing terminal marker: {body}"
);
}
#[tokio::test]
async fn responses_route_streams_function_call_events() {
let response = post_json(
router_with_stub_api_response("", weather_tool_api_response()),
"/v1/responses",
json!({
"model": "stub-model",
"input": "Use the weather tool",
"stream": true,
"tools": [{
"type": "function",
"name": "weather",
"parameters": {"type": "object"}
}]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(
body.contains("event: response.function_call_arguments.delta"),
"missing function delta: {body}"
);
assert!(
body.contains("event: response.function_call_arguments.done"),
"missing function done: {body}"
);
assert!(
body.contains("\"call_id\":\"call_1\""),
"missing call id: {body}"
);
assert_eq!(body.matches("event: response.completed").count(), 1);
}
#[tokio::test]
async fn responses_route_round_trips_namespace_identity_without_leaking_chat_alias() {
let namespace_tool = json!({
"type": "namespace",
"name": "collaboration",
"description": "Agent coordination tools",
"tools": [{
"type": "function",
"name": "wait_agent",
"parameters": {"type": "object"}
}]
});
let sync = post_json(
router_with_stub_api_response("", namespaced_tool_api_response()),
"/v1/responses",
json!({
"model": "stub-model",
"input": "Wait for the agent",
"tools": [namespace_tool.clone()]
}),
)
.await;
assert_eq!(sync.status(), AxumStatusCode::OK);
let sync_body = response_json(sync).await;
assert_eq!(sync_body["output"][0]["type"], "function_call");
assert_eq!(sync_body["output"][0]["namespace"], "collaboration");
assert_eq!(sync_body["output"][0]["name"], "wait_agent");
let stream = post_json(
router_with_stub_api_response("", namespaced_tool_api_response()),
"/v1/responses",
json!({
"model": "stub-model",
"input": "Wait for the agent",
"tools": [namespace_tool],
"stream": true
}),
)
.await;
assert_eq!(stream.status(), AxumStatusCode::OK);
let stream_body = response_text(stream).await;
let events = responses_sse_json_events(&stream_body);
let function_events = events
.iter()
.filter(|event| {
event["item"]["type"] == "function_call"
|| event["type"] == "response.function_call_arguments.done"
})
.collect::<Vec<_>>();
assert!(!function_events.is_empty());
for event in function_events {
let value = event.get("item").unwrap_or(event);
assert_eq!(value["namespace"], "collaboration");
assert_eq!(value["name"], "wait_agent");
}
assert!(stream_body.contains("data: [DONE]"));
assert!(!stream_body.contains("collaboration__wait_agent"));
}
#[tokio::test]
async fn responses_route_enforces_parallel_tool_call_constraint() {
let tools = json!([
{"type": "function", "name": "weather", "parameters": {"type": "object"}},
{"type": "function", "name": "clock", "parameters": {"type": "object"}}
]);
let sync = post_json(
router_with_stub_api_response("", two_tool_api_response()),
"/v1/responses",
json!({
"model": "stub-model",
"input": "Use both tools",
"tools": tools.clone(),
"parallel_tool_calls": false
}),
)
.await;
assert_eq!(sync.status(), AxumStatusCode::INTERNAL_SERVER_ERROR);
let stream = post_json(
router_with_stub_api_response("", two_tool_api_response()),
"/v1/responses",
json!({
"model": "stub-model",
"input": "Use both tools",
"tools": tools,
"parallel_tool_calls": false,
"stream": true
}),
)
.await;
assert_eq!(stream.status(), AxumStatusCode::OK);
let body = response_text(stream).await;
assert!(
body.contains("event: response.failed"),
"missing failure: {body}"
);
assert!(
!body.contains("event: response.completed"),
"must not complete: {body}"
);
assert!(
body.contains("data: [DONE]"),
"missing terminal marker: {body}"
);
}
#[tokio::test]
async fn responses_route_streams_incomplete_terminal_event() {
let response = post_json(
router_with_stub_finish_reason("partial", FinishReason::Length),
"/v1/responses",
json!({"model": "stub-model", "input": "answer", "stream": true}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(
body.contains("event: response.incomplete"),
"missing incomplete terminal event: {body}"
);
assert!(
!body.contains("event: response.completed"),
"incomplete response must not emit completed: {body}"
);
assert!(
body.contains("data: [DONE]"),
"missing terminal marker: {body}"
);
let events = responses_sse_json_events(&body);
let output_done = events
.iter()
.find(|event| event["type"] == "response.output_item.done")
.expect("incomplete output item done event");
assert_eq!(output_done["item"]["status"], "incomplete");
let terminal = events
.iter()
.find(|event| event["type"] == "response.incomplete")
.expect("incomplete terminal event");
assert_eq!(terminal["response"]["output"][0]["status"], "incomplete");
}
#[tokio::test]
async fn responses_route_marks_sync_length_output_incomplete() {
let response = post_json(
router_with_stub_finish_reason("partial", FinishReason::Length),
"/v1/responses",
json!({"model": "stub-model", "input": "answer"}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["status"], "incomplete");
assert_eq!(body["output"][0]["status"], "incomplete");
}
#[tokio::test]
async fn responses_route_rejects_state_and_non_function_tools() {
for (extra, param) in [
(json!({"store": true}), "store"),
(
json!({"previous_response_id": "resp_previous"}),
"previous_response_id",
),
(
json!({"tools": [{"type": "mcp", "server_label": "docs"}]}),
"tools[0].type",
),
] {
let mut body = json!({"model": "stub-model", "input": "hello"});
body.as_object_mut()
.unwrap()
.extend(extra.as_object().unwrap().clone());
let response = post_json(router_with_stub("unused"), "/v1/responses", body).await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let error = response_json(response).await;
assert_eq!(error["error"]["param"], param, "error: {error}");
}
}
#[tokio::test]
async fn responses_mvp_keeps_chat_completions_route_working() {
let response = post_json(
router_with_stub("chat still works"),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "hello"}]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["choices"][0]["message"]["content"], "chat still works");
}
#[test]
fn sanitized_chat_request_body_redacts_user_text_and_secret_metadata() {
let request = chat_request(json!({
"messages": [{"role": "user", "content": "private prompt"}],
"metadata": {"api_key": "should-not-survive"},
"stream": true
}));
let body = sanitized_chat_request_body(&request);
assert_eq!(body["model"], "stub-model");
assert_eq!(body["stream"], true);
assert_eq!(body["messages"][0]["role"], "user");
assert_eq!(body["messages"][0]["content"], "[redacted]");
assert_eq!(body["messages"][0]["content_redacted"], true);
assert_eq!(body["messages"][0]["content_chars"], 14);
assert_eq!(body["metadata"]["api_key"], "[redacted]");
}
#[test]
fn admission_health_prefers_runtime_authority_over_preflight_estimate() {
let engine_status = EngineStatus {
is_ready: true,
loaded_models: Vec::new(),
active_requests: 2,
queued_requests: 1,
memory_usage: MemoryUsage {
total_bytes: 0,
used_bytes: 0,
free_bytes: 0,
gpu_memory_bytes: None,
cpu_memory_bytes: None,
cache_memory_bytes: 0,
utilization_percent: 0.0,
},
uptime_seconds: 0,
last_heartbeat: chrono::Utc::now(),
version: "test".to_owned(),
};
let runtime = ferrum_types::ExecutorAdmissionSnapshot::new(
ferrum_types::ExecutionResourceAuthority::PlanRuntime,
ferrum_types::ExecutorAdmissionLimits::new(32, 4096).unwrap(),
2,
7,
23,
None,
Some(3),
)
.unwrap();
let admission = admission_health_json(
&engine_status,
&EngineMetrics::default(),
&json!({
"admission": {
"effective_max_concurrent": 16,
"scheduler_policy": "continuous"
}
}),
Some(&runtime),
None,
);
assert_eq!(admission["source"], "runtime_executor");
assert_eq!(admission["runtime_snapshot_available"], true);
assert_eq!(admission["resource_authority"], "plan_runtime");
assert_eq!(admission["effective_max_concurrent"], 32);
assert_eq!(admission["maximum_active_sequences"], 32);
assert_eq!(admission["maximum_scheduled_tokens"], 4096);
assert_eq!(admission["preflight_effective_max_concurrent"], 16);
assert_eq!(admission["active_sequences"], 30);
assert_eq!(admission["active_prefill"], 7);
assert_eq!(admission["active_decode"], 23);
assert!(admission["current_batch_size"].is_null());
assert_eq!(admission["queue_depth"], 2);
assert_eq!(admission["capacity_blocked_requests"], 3);
}
#[test]
fn admission_health_surfaces_runtime_contract_failure_without_preflight_fallback() {
let engine_status = EngineStatus {
is_ready: true,
loaded_models: Vec::new(),
active_requests: 32,
queued_requests: 1,
memory_usage: MemoryUsage {
total_bytes: 0,
used_bytes: 0,
free_bytes: 0,
gpu_memory_bytes: None,
cpu_memory_bytes: None,
cache_memory_bytes: 0,
utilization_percent: 0.0,
},
uptime_seconds: 0,
last_heartbeat: chrono::Utc::now(),
version: "test".to_owned(),
};
let admission = admission_health_json(
&engine_status,
&EngineMetrics::default(),
&json!({
"admission": {
"effective_max_concurrent": 16,
"scheduler_policy": "continuous"
}
}),
None,
Some("active phase count exceeded the runtime ceiling"),
);
assert_eq!(admission["source"], "runtime_error");
assert_eq!(admission["runtime_snapshot_available"], false);
assert_eq!(admission["preflight_effective_max_concurrent"], 16);
assert!(admission["effective_max_concurrent"].is_null());
assert!(admission["queue_depth"].is_null());
assert_eq!(
admission["runtime_contract_error"],
"active phase count exceeded the runtime ceiling"
);
}
#[tokio::test]
async fn route_health_includes_runtime_config_snapshot() {
let response = get(router_with_stub("ok"), "/health").await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["status"], "healthy");
assert!(body["config"]["entries"].is_array(), "body: {body}");
assert_eq!(body["auto_config"]["schema_version"], 1);
assert!(body["auto_config"]["entries"].is_array(), "body: {body}");
assert!(body["auto_config"]["admission"].is_object(), "body: {body}");
assert_eq!(body["admission"]["schema_version"], 2);
assert!(body["admission"]["effective_max_concurrent"].is_number());
assert!(body["admission"]["queue_depth"].is_number());
assert!(body["admission"]["active_sequences"].is_number());
assert!(body["admission"]["active_prefill"].is_null());
assert!(body["admission"]["active_decode"].is_null());
assert!(body["admission"]["current_batch_size"].is_null());
assert!(body["admission"]["rejected_requests_total"].is_number());
assert!(body["admission"]["failed_requests_total"].is_number());
assert!(body["admission"]["completed_requests_total"].is_number());
assert!(body["admission"]["avg_queue_wait_time_ms"].is_number());
assert!(body["scheduler"]["avg_wait_time_ms"].is_number());
assert!(body["scheduler"]["scheduling_time_ms"].is_number());
assert!(body["scheduler"]["model_execution_time_ms"].is_number());
assert!(body["scheduler"]["iteration_lock_wait_time_ms"].is_number());
assert!(
body["auto_config"]["decisions"].is_array() || body["auto_config"]["error"].is_string(),
"body: {body}"
);
}
#[tokio::test]
async fn route_metrics_includes_admission_counters() {
let response = get(router_with_stub("ok"), "/metrics").await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
for metric in [
"ferrum_admission_runtime_snapshot_available",
"ferrum_admission_effective_max_concurrent",
"ferrum_admission_queue_depth",
"ferrum_admission_active_sequences",
"ferrum_admission_rejected_requests_total",
"ferrum_admission_failed_requests_total",
"ferrum_admission_completed_requests_total",
] {
assert!(body.contains(metric), "missing {metric}:\n{body}");
}
for unavailable_metric in [
"ferrum_admission_maximum_active_sequences ",
"ferrum_admission_maximum_scheduled_tokens ",
"ferrum_admission_capacity_blocked_requests ",
"ferrum_admission_active_prefill ",
"ferrum_admission_active_decode ",
"ferrum_admission_current_batch_size ",
] {
assert!(
!body.contains(unavailable_metric),
"unknown metric was encoded as a real value: {unavailable_metric}\n{body}"
);
}
}
#[tokio::test]
async fn route_health_includes_engine_lora_metrics_snapshot() {
let router = AxumServer::from_llm(Arc::new(StubLlm::with_lora_metrics(
"ok",
json!({
"enabled": true,
"adapter_count": 1,
"active_cache_bindings": 0,
"projection_applications": 7,
"position": "real-inference",
"source": "test-lora",
}),
)))
.with_lora_adapters(
"stub-model",
vec![LoraAdapterModel::new(
"sql",
"stub-model:sql",
"/tmp/sql-adapter",
)],
)
.unwrap()
.build_router();
let response = get(router, "/health").await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["lora"]["enabled"], true);
assert_eq!(body["lora"]["adapter_count"], 1);
assert_eq!(body["lora"]["projection_applications"], 7);
assert_eq!(body["lora"]["position"], "real-inference");
assert_eq!(body["lora"]["source"], "test-lora");
}
#[tokio::test]
async fn route_health_includes_engine_execution_attribution_snapshot() {
let router = AxumServer::from_llm(Arc::new(StubLlm::with_execution_attribution(
"ok",
json!({
"schema": "ferrum.vnext.provider-attribution.v1",
"attribution_basis": "resolved_plan_and_completed_static_initialization",
"provider_attribution": {
"expected_quant_tensor_count": 400,
"attributed_quant_tensor_count": 400,
"expected_operation_count": 3,
"attributed_operation_count": 3,
"expected_item_count": 403,
"attributed_item_count": 403,
"percent": 100.0,
"denominator_sha256": "5e366997e15e1a94d90b1ae07281269e8a46f75904306564d56354c8ebea2e4e"
},
"fallback_counts": {"silent": 0, "dense": 0, "legacy": 0}
}),
)))
.build_router();
let response = get(router, "/health").await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(
body["execution_attribution"]["provider_attribution"]["expected_item_count"],
403
);
assert_eq!(
body["execution_attribution"]["provider_attribution"]["denominator_sha256"],
"5e366997e15e1a94d90b1ae07281269e8a46f75904306564d56354c8ebea2e4e"
);
assert_eq!(
body["execution_attribution"]["fallback_counts"],
json!({"silent": 0, "dense": 0, "legacy": 0})
);
}
#[tokio::test]
async fn route_models_lists_loaded_stub_model() {
let response = get(router_with_stub("ok"), "/v1/models").await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["object"], "list");
let data = body["data"].as_array().expect("models data array");
assert_eq!(data.len(), 1, "body: {body}");
assert_eq!(data[0]["id"], "stub-model");
assert_eq!(data[0]["object"], "model");
assert_eq!(data[0]["owned_by"], "ferrum");
assert!(data[0]["created"].as_u64().unwrap_or_default() > 0);
assert_eq!(data[0]["modalities"], json!(["text"]));
assert!(data[0]["permission"].as_array().unwrap().is_empty());
assert!(data[0]["root"].is_null());
assert!(data[0]["parent"].is_null());
assert!(data[0].get("max_model_len").is_none());
}
#[tokio::test]
async fn route_chat_public_alias_maps_to_internal_model_and_is_echoed() {
let engine = Arc::new(CapturingLlm::new());
let registry = ServedModelRegistry::try_new(
"qwen3",
ServedModelKind::Llm,
vec!["served-alias".to_string(), "secondary-alias".to_string()],
vec![],
)
.unwrap();
let router = AxumServer::from_llm(engine.clone())
.with_served_model_registry(registry)
.build_router();
let response = post_json(
router,
"/v1/chat/completions",
json!({
"model": "secondary-alias",
"messages": [{"role": "user", "content": "Say hi"}],
"max_tokens": 8
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["model"], "secondary-alias");
assert_eq!(engine.last_request().model_id, ModelId::new("qwen3"));
}
#[tokio::test]
async fn route_models_lists_public_aliases_without_internal_model_id() {
let registry = ServedModelRegistry::try_new(
"qwen3",
ServedModelKind::Llm,
vec!["served-alias".to_string(), "secondary-alias".to_string()],
vec![],
)
.unwrap();
let router = AxumServer::from_llm(Arc::new(CapturingLlm::new()))
.with_served_model_registry(registry)
.build_router();
let body = response_json(get(router, "/v1/models").await).await;
let ids = body["data"]
.as_array()
.unwrap()
.iter()
.map(|entry| entry["id"].as_str().unwrap())
.collect::<Vec<_>>();
assert_eq!(ids, vec!["served-alias", "secondary-alias"]);
assert!(!ids.contains(&"qwen3"));
assert!(body["data"]
.as_array()
.unwrap()
.iter()
.all(|entry| entry["modalities"] == json!(["text"])));
}
#[tokio::test]
async fn route_models_lists_embedding_registry_capabilities() {
let body = response_json(get(router_with_stub_embed(), "/v1/models").await).await;
let data = body["data"].as_array().unwrap();
assert_eq!(data.len(), 1);
assert_eq!(data[0]["id"], "stub-embed");
assert_eq!(data[0]["modalities"], json!(["text", "image"]));
assert!(data[0].get("max_model_len").is_none());
}
#[tokio::test]
async fn route_models_reports_engine_capacity_for_public_aliases_and_adapters() {
let capacity = 3072;
let engine = StubLlm {
context_capacity: Some(capacity),
..StubLlm::new("ok")
};
let registry = ServedModelRegistry::try_new(
"stub-model",
ServedModelKind::Llm,
vec!["public-model".to_owned(), "second-alias".to_owned()],
vec![LoraAdapterModel::new(
"sql",
"public-model:sql",
"/tmp/adapter",
)],
)
.unwrap();
let router = AxumServer::from_llm(Arc::new(engine))
.with_served_model_registry(registry)
.build_router();
let body = response_json(get(router, "/v1/models").await).await;
let entries = body["data"].as_array().unwrap();
assert_eq!(entries.len(), 3);
for entry in entries {
assert_eq!(entry["max_model_len"], capacity);
}
}
#[tokio::test]
async fn route_models_lists_startup_lora_adapters() {
let router = AxumServer::from_llm(Arc::new(StubLlm::new("ok")))
.with_lora_adapters(
"stub-model",
vec![LoraAdapterModel::new(
"sql",
"stub-model:sql",
"/tmp/sql-adapter",
)],
)
.unwrap()
.build_router();
let response = get(router, "/v1/models").await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
let data = body["data"].as_array().expect("models data array");
let ids: Vec<_> = data
.iter()
.map(|item| item["id"].as_str().unwrap_or_default())
.collect();
assert!(ids.contains(&"stub-model"), "body: {body}");
assert!(ids.contains(&"stub-model:sql"), "body: {body}");
let adapter = data
.iter()
.find(|item| item["id"] == "stub-model:sql")
.expect("adapter model");
assert_eq!(adapter["root"], "stub-model");
assert_eq!(adapter["parent"], "stub-model");
assert_eq!(adapter["modalities"], json!(["text"]));
}
#[tokio::test]
async fn route_chat_lora_adapter_maps_internal_request_to_base_model() {
let (router, engine) = router_with_capturing_lora_llm();
let response = post_json(
router,
"/v1/chat/completions",
json!({
"model": "qwen3:sql",
"messages": [{"role": "user", "content": "Say hi"}],
"max_tokens": 8,
"temperature": 0.0
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["model"], "qwen3:sql");
let captured = engine.last_request();
assert_eq!(captured.model_id, ModelId::new("qwen3"));
assert_eq!(captured.metadata["ferrum_lora_adapter"], "sql");
assert_eq!(captured.metadata["ferrum_lora_model_id"], "qwen3:sql");
}
#[tokio::test]
async fn route_chat_base_model_still_uses_base_path_with_lora_loaded() {
let (router, engine) = router_with_capturing_lora_llm();
let response = post_json(
router,
"/v1/chat/completions",
json!({
"model": "qwen3",
"messages": [{"role": "user", "content": "Say hi"}],
"max_tokens": 8,
"temperature": 0.0
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let captured = engine.last_request();
assert_eq!(captured.model_id, ModelId::new("qwen3"));
assert!(!captured.metadata.contains_key("ferrum_lora_adapter"));
}
#[tokio::test]
async fn route_chat_unknown_lora_adapter_returns_openai_model_error() {
let (router, _) = router_with_capturing_lora_llm();
let response = post_json(
router,
"/v1/chat/completions",
json!({
"model": "qwen3:missing",
"messages": [{"role": "user", "content": "Say hi"}],
"max_tokens": 8
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "model");
assert!(
body["error"]["message"]
.as_str()
.unwrap_or_default()
.contains("unknown model"),
"body: {body}"
);
}
#[tokio::test]
async fn route_chat_unknown_served_model_returns_openai_model_error() {
let engine = Arc::new(CapturingLlm::new());
let router = AxumServer::from_llm(engine.clone()).build_router();
let response = post_json(
router,
"/v1/chat/completions",
json!({
"model": "not-a-loaded-model",
"messages": [{"role": "user", "content": "Say hi"}],
"max_tokens": 8
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "model");
assert!(
body["error"]["message"]
.as_str()
.unwrap_or_default()
.contains("unknown model"),
"body: {body}"
);
assert!(!engine.has_captured_request());
}
#[tokio::test]
async fn route_models_without_engine_returns_empty_list() {
let response = get(router_without_llm(), "/v1/models").await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["object"], "list");
assert!(body["data"].as_array().unwrap().is_empty(), "body: {body}");
}
#[tokio::test]
async fn route_basic_chat_contract_uses_stub_engine() {
let response = post_json(
router_with_stub("hello"),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Say hi"}],
"max_tokens": 8,
"temperature": 0.0
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["object"], "chat.completion");
assert_eq!(body["choices"][0]["message"]["role"], "assistant");
assert_eq!(body["choices"][0]["message"]["content"], "hello");
assert_eq!(body["usage"]["prompt_tokens"], 7);
assert_eq!(body["usage"]["completion_tokens"], 2);
}
#[tokio::test]
async fn route_chat_serializes_structured_tool_call_response() {
let response = post_json(
router_with_stub_api_response(
"",
ferrum_types::ApiResponse::Chat(ferrum_types::ApiChatResponse {
message: ferrum_types::ApiChatMessage {
role: ferrum_types::ApiMessageRole::Assistant,
content: String::new(),
name: None,
tool_calls: vec![ferrum_types::ApiToolCall {
id: "call_1".to_string(),
tool_type: "function".to_string(),
function: ferrum_types::ApiFunctionCall {
name: "weather".to_string(),
arguments: "{\"city\":\"Paris\"}".to_string(),
},
}],
tool_call_id: None,
function_call: None,
},
finish_reason: Some("tool_calls".to_string()),
}),
),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Use the weather tool."}],
"tools": [{
"type": "function",
"function": {
"name": "weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"]
}
}
}],
"tool_choice": "auto"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["choices"][0]["finish_reason"], "tool_calls");
assert_eq!(
body["choices"][0]["message"]["tool_calls"][0]["function"]["name"],
"weather"
);
assert_eq!(
body["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"],
"{\"city\":\"Paris\"}"
);
}
#[tokio::test]
async fn route_chat_preserves_length_over_structured_tool_response() {
let generated = r#"{"name":"weather","arguments":{"city":"Paris"}}"#;
let response = post_json(
router_with_stub_api_response_and_finish_reason(
generated,
weather_tool_api_response(),
FinishReason::Length,
),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Use the weather tool."}],
"tools": [{
"type": "function",
"function": {"name": "weather", "parameters": {"type": "object"}}
}],
"tool_choice": "auto"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["choices"][0]["finish_reason"], "length");
assert_eq!(body["choices"][0]["message"]["content"], generated);
assert!(body["choices"][0]["message"]["tool_calls"].is_null());
}
#[tokio::test]
async fn route_chat_serializes_generated_tool_call_json_when_engine_returns_text_only() {
let response = post_json(
router_with_stub(
r#"{"tool_calls":[{"id":"call_1","type":"function","function":{"name":"weather","arguments":{"city":"Paris"}}}]}"#,
),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Use the weather tool."}],
"tools": [{
"type": "function",
"function": {
"name": "weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"]
}
}
}],
"tool_choice": "auto"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["choices"][0]["finish_reason"], "tool_calls");
assert_eq!(body["choices"][0]["message"]["content"], "");
assert_eq!(
body["choices"][0]["message"]["tool_calls"][0]["id"],
"call_1"
);
assert_eq!(
body["choices"][0]["message"]["tool_calls"][0]["function"]["name"],
"weather"
);
assert_eq!(
body["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"],
"{\"city\":\"Paris\"}"
);
}
#[tokio::test]
async fn route_chat_serializes_qwen3_function_parameters_tool_json() {
let response = post_json(
router_with_stub(
r#"{"function":"get_weather","parameters":{"city":"北京","unit":"c"}}"#,
),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "北京现在天气怎么样？"}],
"tools": [{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {
"city": {"type": "string"},
"unit": {"type": "string", "enum": ["c", "f"]}
},
"required": ["city"]
}
}
}],
"tool_choice": "auto"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["choices"][0]["finish_reason"], "tool_calls");
assert_eq!(body["choices"][0]["message"]["content"], "");
assert_eq!(
body["choices"][0]["message"]["tool_calls"][0]["function"]["name"],
"get_weather"
);
assert_eq!(
body["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"],
"{\"city\":\"北京\",\"unit\":\"c\"}"
);
}
#[tokio::test]
async fn route_chat_uses_template_tool_protocol_for_function_parameter_xml() {
let template = ModelChatTemplate::new(
"{% if tools %}<tools>{{ tools | tojson }}</tools>Use <tool_call><function=name><parameter=key>value</parameter></function></tool_call>{% endif %}{% for message in messages %}{{ message.content }}{% endfor %}{% if add_generation_prompt %}<assistant>{% endif %}",
"function-parameter-xml-template",
);
let response = post_json(
router_with_stub_and_template(
"<tool_call>\n<function=get_weather>\n<parameter=city>\n北京\n</parameter>\n<parameter=unit>\ncelsius\n</parameter>\n</function>\n</tool_call>",
template,
),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "请调用 get_weather 查询北京天气。"}],
"tools": [{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {
"city": {"type": "string"},
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}
},
"required": ["city"]
}
}
}]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["choices"][0]["finish_reason"], "tool_calls");
assert_eq!(
body["choices"][0]["message"]["tool_calls"][0]["function"]["name"],
"get_weather"
);
assert_eq!(
body["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"],
"{\"city\":\"北京\",\"unit\":\"celsius\"}"
);
}
fn xml_object_argument_tool_request(stream: bool) -> Value {
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Weather in Berlin with a forecast."}],
"stream": stream,
"tools": [{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"$defs": {
"WeatherOptions": {
"type": "object",
"properties": {
"unit": {"type": "string"},
"include_forecast": {"type": "boolean"}
},
"required": ["unit", "include_forecast"],
"additionalProperties": false
}
},
"properties": {
"city": {"type": "string"},
"options": {"$ref": "#/$defs/WeatherOptions"}
},
"required": ["city", "options"],
"additionalProperties": false
}
}
}]
})
}
#[tokio::test]
async fn route_chat_decodes_xml_object_argument_through_local_schema_ref() {
let template = ModelChatTemplate::new(
"{% if tools %}<tool_call><function=name><parameter=key>value</parameter></function></tool_call>{% endif %}{% for message in messages %}{{ message.content }}{% endfor %}",
"function-parameter-xml-template",
);
let response = post_json(
router_with_stub_and_template(
concat!(
"<tool_call>\n",
"<function=get_weather>\n",
"<parameter=city>\nBerlin\n</parameter>\n",
"<parameter=options>\n",
"{\"unit\":\"celsius\",\"include_forecast\":true}\n",
"</parameter>\n",
"</function>\n",
"</tool_call>",
),
template,
),
"/v1/chat/completions",
xml_object_argument_tool_request(false),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["choices"][0]["finish_reason"], "tool_calls");
let arguments = body["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"]
.as_str()
.and_then(|arguments| serde_json::from_str::<Value>(arguments).ok())
.expect("tool arguments must contain one-decode structured JSON");
assert_eq!(arguments["city"], json!("Berlin"));
assert_eq!(
arguments["options"],
json!({"unit": "celsius", "include_forecast": true})
);
}
#[tokio::test]
async fn route_chat_rejects_malformed_native_xml_object_argument() {
let template = ModelChatTemplate::new(
"{% if tools %}<tool_call><function=name><parameter=key>value</parameter></function></tool_call>{% endif %}{% for message in messages %}{{ message.content }}{% endfor %}",
"function-parameter-xml-template",
);
let mut request = xml_object_argument_tool_request(false);
request["tools"][0]["function"]["strict"] = json!(true);
let response = post_json(
router_with_stub_and_template(
concat!(
"<tool_call><function=get_weather>",
"<parameter=city>Berlin</parameter>",
"<parameter=options>{\"unit\":\"celsius\",</parameter>",
"</function></tool_call>",
),
template,
),
"/v1/chat/completions",
request,
)
.await;
assert_eq!(response.status(), AxumStatusCode::INTERNAL_SERVER_ERROR);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "internal_server_error");
assert!(
body["error"]["message"]
.as_str()
.is_some_and(|message| message.contains("did not satisfy its schema")),
"body: {body}"
);
}
#[tokio::test]
async fn route_streaming_chat_rejects_malformed_native_xml_before_tool_delta() {
let template = ModelChatTemplate::new(
"{% if tools %}<tool_call><function=name><parameter=key>value</parameter></function></tool_call>{% endif %}{% for message in messages %}{{ message.content }}{% endfor %}",
"function-parameter-xml-template",
);
let mut request = xml_object_argument_tool_request(true);
request["tools"][0]["function"]["strict"] = json!(true);
let response = post_json(
router_with_stub_and_template(
concat!(
"<tool_call><function=get_weather>",
"<parameter=city>Berlin</parameter>",
"<parameter=options>{\"unit\":\"celsius\",</parameter>",
"</function></tool_call>",
),
template,
),
"/v1/chat/completions",
request,
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert_eq!(body.matches("data: [DONE]").count(), 1, "body: {body}");
assert!(
body.contains(r#""error":{"#) && body.contains("did not satisfy its schema"),
"stream must return a controlled schema error: {body}"
);
assert!(
!body.contains(r#""tool_calls":[{"#),
"invalid native arguments must not leak a tool delta: {body}"
);
}
#[tokio::test]
async fn route_chat_parses_tool_call_from_reasoning_before_fake_tool_result_content() {
let response = post_json(
router_with_stub(
"kaza\n\
{\"name\":\"get_weather\",\"arguments\":{\"city\":\"北京\",\"unit\":\"celsius\"}}\n\
</think>\n\
{\"name\":\"get_weather\",\"content\":{\"temperature\":25,\"condition\":\"晴\"}}\n\
{\"temperature\":25,\"condition\":\"晴\"}",
),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "北京现在天气怎么样？请先调用工具。"}],
"tools": [{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {
"city": {"type": "string"},
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}
},
"required": ["city"]
}
}
}]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["choices"][0]["finish_reason"], "tool_calls");
assert_eq!(body["choices"][0]["message"]["content"], "");
assert_eq!(
body["choices"][0]["message"]["tool_calls"][0]["function"]["name"],
"get_weather"
);
assert_eq!(
body["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"],
"{\"city\":\"北京\",\"unit\":\"celsius\"}"
);
}
#[tokio::test]
async fn route_chat_prefers_reasoning_tool_call_over_empty_visible_arguments() {
let response = post_json(
router_with_stub(
"{\"name\":\"get_weather\",\"arguments\":{\"city\":\"北京\",\"unit\":\"celsius\"}}\n\
</think>\n\
{\"name\":\"get_weather\",\"arguments\":{}}",
),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "北京现在天气怎么样？请先调用 get_weather 工具。"}],
"tools": [{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {
"city": {"type": "string"},
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}
},
"required": ["city"]
}
}
}]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["choices"][0]["finish_reason"], "tool_calls");
assert_eq!(
body["choices"][0]["message"]["tool_calls"][0]["function"]["name"],
"get_weather"
);
assert_eq!(
body["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"],
"{\"city\":\"北京\",\"unit\":\"celsius\"}"
);
}
#[tokio::test]
async fn route_chat_honors_specific_tool_choice_for_generated_tool_call_json() {
let response = post_json(
router_with_stub(r#"{"name":"weather","arguments":{"city":"Paris"}}"#),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Use the selected tool."}],
"tools": [
{
"type": "function",
"function": {"name": "weather", "parameters": {"type": "object"}}
},
{
"type": "function",
"function": {"name": "calendar", "parameters": {"type": "object"}}
}
],
"tool_choice": {
"type": "function",
"function": {"name": "weather"}
}
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["choices"][0]["finish_reason"], "tool_calls");
assert_eq!(
body["choices"][0]["message"]["tool_calls"][0]["function"]["name"],
"weather"
);
let response = post_json(
router_with_stub(r#"{"name":"calendar","arguments":{}}"#),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Use the selected tool."}],
"tools": [
{
"type": "function",
"function": {"name": "weather", "parameters": {"type": "object"}}
},
{
"type": "function",
"function": {"name": "calendar", "parameters": {"type": "object"}}
}
],
"tool_choice": {
"type": "function",
"function": {"name": "weather"}
}
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["param"], "tool_choice");
assert_eq!(body["error"]["type"], "invalid_request_error");
}
#[tokio::test]
async fn route_chat_specific_tool_choice_wraps_generated_arguments() {
let response = post_json(
router_with_stub(r#"{"city":"Paris"}"#),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Use the selected tool."}],
"tools": [{
"type": "function",
"function": {
"name": "weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"]
}
}
}],
"tool_choice": {
"type": "function",
"function": {"name": "weather"}
}
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["choices"][0]["finish_reason"], "tool_calls");
assert_eq!(body["choices"][0]["message"]["content"], "");
assert_eq!(
body["choices"][0]["message"]["tool_calls"][0]["function"]["name"],
"weather"
);
assert_eq!(
body["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"],
"{\"city\":\"Paris\"}"
);
}
#[tokio::test]
async fn route_chat_tool_choice_none_keeps_generated_tool_json_as_content() {
let generated = r#"{"name":"weather","arguments":{"city":"Paris"}}"#;
let response = post_json(
router_with_stub(generated),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Do not use tools."}],
"tools": [{
"type": "function",
"function": {"name": "weather", "parameters": {"type": "object"}}
}],
"tool_choice": "none"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["choices"][0]["finish_reason"], "stop");
assert_eq!(body["choices"][0]["message"]["content"], generated);
assert!(body["choices"][0]["message"]["tool_calls"].is_null());
}
#[tokio::test]
async fn route_chat_tool_choice_required_wraps_generated_arguments() {
let response = post_json(
router_with_stub(r#"{"city":"Paris"}"#),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Use a tool."}],
"tools": [{
"type": "function",
"function": {
"name": "weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"]
}
}
}],
"tool_choice": "required"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["choices"][0]["finish_reason"], "tool_calls");
assert_eq!(body["choices"][0]["message"]["content"], "");
assert_eq!(
body["choices"][0]["message"]["tool_calls"][0]["function"]["name"],
"weather"
);
assert_eq!(
body["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"],
"{\"city\":\"Paris\"}"
);
}
fn required_tool_with_strict_response_format_request(stream: bool) -> Value {
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Use the weather tool."}],
"stream": stream,
"stream_options": stream.then_some(json!({"include_usage": true})),
"tools": [{
"type": "function",
"function": {
"name": "weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string", "const": "Paris"}},
"required": ["city"],
"additionalProperties": false
}
}
}],
"tool_choice": "required",
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "content_answer",
"strict": true,
"schema": {
"type": "object",
"properties": {"answer": {"type": "string", "const": "IGNORED"}},
"required": ["answer"],
"additionalProperties": false
}
}
}
})
}
#[tokio::test]
async fn route_chat_required_tool_takes_priority_over_strict_response_format() {
let response = post_json(
router_with_stub(r#"{"city":"Paris"}"#),
"/v1/chat/completions",
required_tool_with_strict_response_format_request(false),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["choices"][0]["finish_reason"], "tool_calls");
assert_eq!(body["choices"][0]["message"]["content"], "");
assert_eq!(
body["choices"][0]["message"]["tool_calls"][0]["function"]["name"],
"weather"
);
assert_eq!(
body["choices"][0]["message"]["tool_calls"][0]["function"]["arguments"],
r#"{"city":"Paris"}"#
);
}
#[tokio::test]
async fn route_chat_required_tool_rejects_arguments_that_violate_const_schema() {
let response = post_json(
router_with_stub(r#"{"city":"London"}"#),
"/v1/chat/completions",
required_tool_with_strict_response_format_request(false),
)
.await;
assert_eq!(response.status(), AxumStatusCode::INTERNAL_SERVER_ERROR);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "internal_server_error");
assert!(
body["error"]["message"]
.as_str()
.is_some_and(|message| message.contains("did not satisfy its schema")),
"body: {body}"
);
}
#[tokio::test]
async fn route_streaming_required_tool_takes_priority_over_strict_response_format() {
let response = post_json(
router_with_stub(r#"{"city":"Paris"}"#),
"/v1/chat/completions",
required_tool_with_strict_response_format_request(true),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert_eq!(body.matches("data: [DONE]").count(), 1, "body: {body}");
assert!(
body.contains(r#""finish_reason":"tool_calls""#),
"tool priority must finish with tool_calls: {body}"
);
assert!(
body.contains(r#""name":"weather""#)
&& body.contains(r#""arguments":"{\"city\":\"Paris\"}""#),
"stream must carry the reconstructed tool call: {body}"
);
assert_eq!(
body.matches(r#""usage":{"#).count(),
1,
"stream must carry exactly one usage row: {body}"
);
assert!(
!body.contains("strict json_schema") && !body.contains("invalid JSON"),
"dormant content schema must not reject a required tool call: {body}"
);
}
#[tokio::test]
async fn dropping_buffered_http_response_drops_the_engine_stream() {
let stream_dropped = Arc::new(Notify::new());
let response = post_json(
AxumServer::from_llm(Arc::new(StubLlm::with_pending_stream(Arc::clone(
&stream_dropped,
))))
.build_router(),
"/v1/chat/completions",
required_tool_with_strict_response_format_request(true),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
drop(response);
tokio::time::timeout(std::time::Duration::from_secs(1), stream_dropped.notified())
.await
.expect("client disconnect must stop a buffered structured stream promptly");
}
#[tokio::test]
async fn route_chat_tool_choice_required_errors_without_valid_tool_call() {
let response = post_json(
router_with_stub("plain answer"),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Use a tool."}],
"tools": [{
"type": "function",
"function": {"name": "weather", "parameters": {"type": "object"}}
}],
"tool_choice": "required"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "tool_choice");
assert!(
body["error"]["message"]
.as_str()
.is_some_and(|message| message.contains("required tool_choice")),
"body: {body}"
);
}
#[tokio::test]
async fn route_streaming_chat_serializes_generated_tool_call_delta() {
let response = post_json(
router_with_stub(
r#"{"tool_calls":[{"id":"call_1","type":"function","function":{"name":"weather","arguments":{"city":"Paris"}}}]}"#,
),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Use the weather tool."}],
"stream": true,
"tools": [{
"type": "function",
"function": {
"name": "weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"]
}
}
}],
"tool_choice": "auto"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(body.contains("data: [DONE]"), "missing DONE: {body}");
assert!(
body.contains(r#""finish_reason":"tool_calls""#),
"stream should finish with tool_calls: {body}"
);
assert!(
body.contains(r#""tool_calls":[{"index":0,"id":"call_1","type":"function","function":{"name":"weather""#),
"stream should emit OpenAI tool_calls delta with index: {body}"
);
assert!(
body.contains(r#""arguments":"{\"city\":\"Paris\"}""#),
"tool arguments should be serialized as JSON string: {body}"
);
assert!(
!body.contains(r#""content":"{\"tool_calls\""#),
"raw tool-call JSON should not be streamed as assistant content: {body}"
);
}
#[tokio::test]
async fn route_streaming_chat_serializes_qwen3_function_parameters_tool_delta() {
let response = post_json(
router_with_stub(
r#"{"function":"get_weather","parameters":{"city":"深圳","unit":"c"}}"#,
),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "深圳天气？"}],
"stream": true,
"tools": [{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {
"city": {"type": "string"},
"unit": {"type": "string", "enum": ["c", "f"]}
},
"required": ["city"]
}
}
}],
"tool_choice": "auto"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(body.contains("data: [DONE]"), "missing DONE: {body}");
assert!(
body.contains(r#""finish_reason":"tool_calls""#),
"stream should finish with tool_calls: {body}"
);
assert!(
body.contains(r#""function":{"name":"get_weather","arguments":"{\"city\":\"深圳\",\"unit\":\"c\"}"}"#),
"stream should emit parsed Qwen3 function parameters as tool args: {body}"
);
assert!(
!body.contains(r#""content":"{\"function\""#),
"raw Qwen3 tool JSON should not leak as assistant content: {body}"
);
}
#[tokio::test]
async fn route_streaming_chat_preserves_opencode_edit_xml_whitespace() {
let template = ModelChatTemplate::new(
"{% if tools %}<tool_call><function=name><parameter=key>value</parameter></function></tool_call>{% endif %}{% for message in messages %}{{ message.content }}{% endfor %}",
"function-parameter-xml-template",
);
let generated = concat!(
"<tool_call>\n",
"<function=edit>\n",
"<parameter=filePath>\n",
"/workspace/src/main.rs\n",
"</parameter>\n",
"<parameter=oldString>\n",
" if x:\n",
" return 1\n",
"\n",
"</parameter>\n",
"<parameter=newString>\n",
" if x:\n",
" return 2\n",
"\n",
"</parameter>\n",
"<parameter=replaceAll>\n",
"true\n",
"</parameter>\n",
"</function>\n",
"</tool_call>",
);
let response = post_json(
router_with_stub_and_template(generated, template),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Replace the code."}],
"stream": true,
"tools": [{
"type": "function",
"function": {
"name": "edit",
"parameters": {
"type": "object",
"properties": {
"filePath": {"type": "string"},
"oldString": {"type": "string"},
"newString": {"type": "string"},
"replaceAll": {"type": "boolean"}
},
"required": ["filePath", "oldString", "newString"]
}
}
}]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(body.contains("data: [DONE]"), "missing DONE: {body}");
assert!(
body.contains(r#"\"oldString\":\" if x:\\n return 1\\n\""#),
"stream must preserve exact code whitespace in tool arguments: {body}"
);
assert!(
body.contains(r#"\"replaceAll\":true"#),
"stream must preserve the boolean tool argument type: {body}"
);
}
#[tokio::test]
async fn route_streaming_chat_honors_specific_tool_choice_for_generated_tool_call_delta() {
let request = |generated: &'static str| {
post_json(
router_with_stub(generated),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Use the selected tool."}],
"stream": true,
"tools": [
{
"type": "function",
"function": {"name": "weather", "parameters": {"type": "object"}}
},
{
"type": "function",
"function": {"name": "calendar", "parameters": {"type": "object"}}
}
],
"tool_choice": {
"type": "function",
"function": {"name": "weather"}
}
}),
)
};
let response = request(r#"{"name":"weather","arguments":{"city":"Paris"}}"#).await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(body.contains("data: [DONE]"), "missing DONE: {body}");
assert!(
body.contains(r#""finish_reason":"tool_calls""#),
"selected tool should finish with tool_calls: {body}"
);
assert!(
body.contains(r#""function":{"name":"weather","arguments":"{\"city\":\"Paris\"}"}"#),
"selected tool should stream as tool_calls delta: {body}"
);
let response = request(r#"{"name":"calendar","arguments":{}}"#).await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(
body.contains(
r#""error":{"message":"model output did not satisfy required tool_choice""#
),
"selected-tool stream should reject unselected tool output: {body}"
);
assert!(
!body.contains(r#""finish_reason":"tool_calls""#),
"unselected tool JSON must not become tool_calls: {body}"
);
}
#[tokio::test]
async fn route_streaming_chat_prefers_chunk_api_response_for_tool_delta() {
let response = post_json(
router_with_stub_api_response(
"raw text that should not stream",
ferrum_types::ApiResponse::Chat(ferrum_types::ApiChatResponse {
message: ferrum_types::ApiChatMessage {
role: ferrum_types::ApiMessageRole::Assistant,
content: String::new(),
name: None,
tool_calls: vec![ferrum_types::ApiToolCall {
id: "call_1".to_string(),
tool_type: "function".to_string(),
function: ferrum_types::ApiFunctionCall {
name: "weather".to_string(),
arguments: "{\"city\":\"Paris\"}".to_string(),
},
}],
tool_call_id: None,
function_call: None,
},
finish_reason: Some("tool_calls".to_string()),
}),
),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Use the weather tool."}],
"stream": true,
"tools": [{
"type": "function",
"function": {"name": "weather", "parameters": {"type": "object"}}
}],
"tool_choice": "auto"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(body.contains("data: [DONE]"), "missing DONE: {body}");
assert!(
body.contains(r#""finish_reason":"tool_calls""#),
"stream should finish with tool_calls: {body}"
);
assert!(
body.contains(r#""tool_calls":[{"index":0,"id":"call_1""#),
"stream should emit tool_calls from chunk api_response: {body}"
);
assert!(
!body.contains("raw text that should not stream"),
"structured api_response should suppress raw generated text in tool-call stream: {body}"
);
}
#[tokio::test]
async fn route_streaming_chat_preserves_length_over_structured_tool_response() {
let generated = r#"{"name":"weather","arguments":{"city":"Paris"}}"#;
let response = post_json(
router_with_stub_api_response_and_finish_reason(
generated,
weather_tool_api_response(),
FinishReason::Length,
),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Use the weather tool."}],
"stream": true,
"tools": [{
"type": "function",
"function": {"name": "weather", "parameters": {"type": "object"}}
}],
"tool_choice": "auto"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(body.contains("data: [DONE]"), "missing DONE: {body}");
assert!(
body.contains(r#""finish_reason":"length""#),
"stream must preserve the engine terminal reason: {body}"
);
assert!(
!body.contains(r#""finish_reason":"tool_calls""#),
"length must not be relabeled as tool_calls: {body}"
);
}
#[tokio::test]
async fn route_streaming_chat_tool_choice_required_errors_without_leaking_content() {
let response = post_json(
router_with_stub("plain answer"),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Use a tool."}],
"stream": true,
"tools": [{
"type": "function",
"function": {"name": "weather", "parameters": {"type": "object"}}
}],
"tool_choice": "required"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert_eq!(body.matches("data: [DONE]").count(), 1, "body: {body}");
assert!(
body.contains(
r#""error":{"message":"model output did not satisfy required tool_choice""#
),
"stream should emit OpenAI error envelope: {body}"
);
assert!(
body.contains(r#""type":"invalid_request_error""#),
"stream should use invalid_request_error: {body}"
);
assert!(
body.contains(r#""param":"tool_choice""#),
"stream should include tool_choice param: {body}"
);
assert!(
!body.contains(r#""content":"plain answer""#),
"required stream must not leak invalid content before validation: {body}"
);
}
#[tokio::test]
async fn route_streaming_chat_tool_request_falls_back_to_content_when_no_tool_call() {
let response = post_json(
router_with_stub("plain answer"),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Use the weather tool if needed."}],
"stream": true,
"tools": [{
"type": "function",
"function": {"name": "weather", "parameters": {"type": "object"}}
}],
"tool_choice": "auto"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(
body.contains(r#""content":"plain answer""#),
"plain content should still stream when no tool call is generated: {body}"
);
assert!(
body.contains(r#""finish_reason":"stop""#),
"plain content should keep normal finish reason: {body}"
);
assert!(
!body.contains(r#""tool_calls""#),
"fallback content should not synthesize tool_calls: {body}"
);
}
#[tokio::test]
async fn route_streaming_chat_serializes_generated_legacy_function_call_delta() {
let response = post_json(
router_with_stub(
r#"{"function_call":{"name":"weather","arguments":{"city":"Paris"}}}"#,
),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Use the weather function."}],
"stream": true,
"functions": [{
"name": "weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"]
}
}],
"function_call": "auto"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(body.contains("data: [DONE]"), "missing DONE: {body}");
assert!(
body.contains(r#""finish_reason":"function_call""#),
"stream should finish with function_call: {body}"
);
assert!(
body.contains(
r#""function_call":{"name":"weather","arguments":"{\"city\":\"Paris\"}"}"#
),
"stream should emit OpenAI legacy function_call delta: {body}"
);
assert!(
!body.contains(r#""content":"{\"function_call\""#),
"raw function-call JSON should not be streamed as assistant content: {body}"
);
}
#[tokio::test]
async fn route_streaming_chat_honors_specific_legacy_function_call_delta() {
let request = |generated: &'static str| {
post_json(
router_with_stub(generated),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Use the selected function."}],
"stream": true,
"functions": [
{"name": "weather", "parameters": {"type": "object"}},
{"name": "calendar", "parameters": {"type": "object"}}
],
"function_call": {"name": "weather"}
}),
)
};
let response =
request(r#"{"function_call":{"name":"weather","arguments":{"city":"Paris"}}}"#).await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(body.contains("data: [DONE]"), "missing DONE: {body}");
assert!(
body.contains(r#""finish_reason":"function_call""#),
"selected function should finish with function_call: {body}"
);
assert!(
body.contains(
r#""function_call":{"name":"weather","arguments":"{\"city\":\"Paris\"}"}"#
),
"selected function should stream as function_call delta: {body}"
);
let response = request(r#"{"function_call":{"name":"calendar","arguments":{}}}"#).await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(
body.contains(
r#""content":"{\"function_call\":{\"name\":\"calendar\",\"arguments\":{}}}""#
),
"unselected function JSON should stream as ordinary content: {body}"
);
assert!(
body.contains(r#""finish_reason":"stop""#),
"unselected function JSON should keep normal stop finish: {body}"
);
assert!(
!body.contains(r#""finish_reason":"function_call""#),
"unselected function JSON must not become function_call: {body}"
);
}
#[tokio::test]
async fn route_chat_serializes_generated_legacy_function_call_when_engine_returns_text_only() {
let response = post_json(
router_with_stub(
r#"{"function_call":{"name":"weather","arguments":{"city":"Paris"}}}"#,
),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Use the weather function."}],
"functions": [{
"name": "weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"]
}
}],
"function_call": "auto"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["choices"][0]["finish_reason"], "function_call");
assert_eq!(body["choices"][0]["message"]["content"], "");
assert_eq!(
body["choices"][0]["message"]["function_call"]["name"],
"weather"
);
assert_eq!(
body["choices"][0]["message"]["function_call"]["arguments"],
"{\"city\":\"Paris\"}"
);
}
#[tokio::test]
async fn route_chat_serializes_legacy_function_call_response() {
let response = post_json(
router_with_stub_api_response(
"",
ferrum_types::ApiResponse::Chat(ferrum_types::ApiChatResponse {
message: ferrum_types::ApiChatMessage {
role: ferrum_types::ApiMessageRole::Assistant,
content: String::new(),
name: None,
tool_calls: vec![],
tool_call_id: None,
function_call: Some(ferrum_types::ApiFunctionCall {
name: "weather".to_string(),
arguments: "{\"city\":\"Paris\"}".to_string(),
}),
},
finish_reason: Some("function_call".to_string()),
}),
),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Use the weather function."}],
"functions": [{
"name": "weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"]
}
}],
"function_call": "auto"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["choices"][0]["finish_reason"], "function_call");
assert_eq!(
body["choices"][0]["message"]["function_call"]["name"],
"weather"
);
assert_eq!(
body["choices"][0]["message"]["function_call"]["arguments"],
"{\"city\":\"Paris\"}"
);
}
#[tokio::test]
async fn route_streaming_chat_include_usage_contract() {
let response = post_json(
router_with_stub("ok"),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Say ok"}],
"stream": true,
"stream_options": {"include_usage": true}
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(body.contains("data: [DONE]"), "missing DONE: {body}");
assert!(
body.contains("\"object\":\"chat.completion.chunk\""),
"missing chat chunk: {body}"
);
assert!(
body.contains("\"usage\":{\"prompt_tokens\""),
"missing final usage chunk: {body}"
);
assert!(
body.contains("\"choices\":[],\"usage\""),
"usage should be emitted as a separate chunk: {body}"
);
assert!(
body.contains("\"prompt_tokens\":5"),
"stream usage should come from engine token usage: {body}"
);
}
#[tokio::test]
async fn route_streaming_chat_waits_for_separate_final_usage_at_max_tokens() {
let response = post_json(
router_with_stub_separate_final_stream_chunk(&["he", "llo"]),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Say hello"}],
"max_tokens": 2,
"stream": true,
"stream_options": {"include_usage": true}
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert_eq!(body.matches("data: [DONE]").count(), 1, "body: {body}");
assert!(
body.contains("\"content\":\"he\""),
"missing first chunk: {body}"
);
assert!(
body.contains("\"content\":\"llo\""),
"missing second chunk: {body}"
);
assert!(
body.contains("\"choices\":[],\"usage\""),
"missing separate usage chunk from final engine chunk: {body}"
);
assert!(
body.contains("\"prompt_tokens\":5"),
"stream usage should come from engine final usage: {body}"
);
}
#[tokio::test]
async fn route_streaming_preserves_tokenless_tail_before_terminal() {
for (path, chunks, expected_content, expected_reasoning) in [
("/v1/chat/completions", ["hello ", "尾"], "hello 尾", ""),
(
"/v1/chat/completions",
["<think>reason", "</think>"],
"",
"reason",
),
("/v1/completions", ["hello ", "尾"], "hello 尾", ""),
] {
let chat = path == "/v1/chat/completions";
let mut request = json!({"model": "stub-model", "stream": true});
if chat {
request["messages"] = json!([{"role": "user", "content": "hello"}]);
request["stream_options"] = json!({"include_usage": true});
} else {
request["prompt"] = json!("hello");
}
let router = AxumServer::from_llm(Arc::new(StubLlm::with_tokenless_tail(&chunks)))
.build_router();
let response = post_json(router, path, request).await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
let events = responses_sse_json_events(&body);
let content: String = events
.iter()
.filter_map(|event| {
if chat {
event["choices"][0]["delta"]["content"].as_str()
} else {
event["choices"][0]["text"].as_str()
}
})
.collect();
let reasoning: String = events
.iter()
.filter_map(|event| event["choices"][0]["delta"]["reasoning"].as_str())
.collect();
assert_eq!(content, expected_content, "body: {body}");
assert_eq!(reasoning, expected_reasoning, "body: {body}");
assert_eq!(body.matches("data: [DONE]").count(), 1, "body: {body}");
assert_eq!(
events
.iter()
.filter(|event| event["choices"][0]["finish_reason"] == "stop")
.count(),
1,
"body: {body}"
);
let usage: Vec<_> = events
.iter()
.filter_map(|event| event["usage"].as_object())
.collect();
assert_eq!(usage.len(), 1, "body: {body}");
assert_eq!(usage[0]["prompt_tokens"], 5);
assert_eq!(usage[0]["completion_tokens"], 2);
}
}
#[tokio::test]
async fn route_rejects_multimodal_content_with_400() {
for content in [
json!([{"type": "image_url", "image_url": {"url": "https://example.com/image.png"}}]),
json!([{"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}}]),
json!([{"type": "video_url", "video_url": {"url": "https://example.com/video.mp4"}}]),
json!([
{"type": "text", "text": "describe this"},
{"type": "image_url", "image_url": {"url": "https://example.com/image.png"}}
]),
] {
let response = post_json(
router_with_stub("unused"),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": content}]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
let message = body["error"]["message"].as_str().unwrap();
assert!(message.contains("invalid chat completions request"));
assert!(
message.contains("unsupported message content part type"),
"body: {body}"
);
}
}
#[tokio::test]
async fn route_rejects_non_object_stream_options() {
for stream_options in [json!([]), json!("yes"), json!(42), json!(true)] {
let response = post_json(
router_with_stub("unused"),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "hello"}],
"stream": true,
"stream_options": stream_options
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert!(
body["error"]["message"]
.as_str()
.unwrap_or_default()
.contains("stream_options must be a JSON object"),
"body: {body}"
);
}
}
#[tokio::test]
async fn route_accepts_text_only_content_array() {
let response = post_json(
router_with_stub("ok"),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{
"role": "user",
"content": [
{"type": "text", "text": "say"},
{"type": "text", "text": "ok"}
]
}]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["choices"][0]["message"]["content"], "ok");
}
#[tokio::test]
async fn route_chat_invalid_json_maps_to_openai_error() {
let response = post_raw_json(router_with_stub("unused"), "/v1/chat/completions", "{").await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], Value::Null);
assert!(body["error"]["message"]
.as_str()
.unwrap()
.contains("invalid chat completions request"));
}
#[tokio::test]
async fn route_rejects_logit_bias_with_openai_error_param() {
let response = post_json(
router_with_stub("unused"),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "hello"}],
"logit_bias": {"1": 42.0}
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "logit_bias");
}
#[tokio::test]
async fn route_tool_request_reaches_engine_structured_boundary() {
for stream in [false, true] {
let (router, engine) = router_with_capturing_llm();
let response = post_json(
router,
"/v1/chat/completions",
json!({
"model": "qwen3",
"messages": [
{"role": "user", "content": "Use the weather tool."},
{
"role": "assistant",
"content": null,
"tool_calls": [{
"id": "call_1",
"type": "function",
"function": {"name": "weather", "arguments": "{\"city\":\"Paris\"}"}
}]
},
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"}
],
"tools": [{
"type": "function",
"function": {
"name": "weather",
"description": "Get weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"]
}
}
}],
"tool_choice": "auto",
"functions": [{
"name": "legacy_weather",
"parameters": {"type": "object", "properties": {}}
}],
"function_call": "auto",
"stream": stream
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
if stream {
let body = response_text(response).await;
assert!(body.contains("[DONE]"), "{body}");
assert!(body.contains("captured"), "{body}");
} else {
let body = response_json(response).await;
assert_eq!(body["choices"][0]["message"]["content"], "captured");
assert_eq!(body["choices"][0]["finish_reason"], "stop");
}
let request = engine.last_request();
assert!(request.prompt.contains("\"tools\":[{"));
assert!(request.prompt.contains("\"type\":\"function\""));
assert!(request.prompt.contains("\"name\":\"weather\""));
assert!(request.prompt.contains("<|im_start|>assistant\n{"));
assert!(request.prompt.contains("\"tool_calls\":[{"));
assert!(request.prompt.contains("\"id\":\"call_1\""));
assert!(request.prompt.contains("<|im_start|>tool\nsunny<|im_end|>"));
assert_eq!(
request.metadata["openai_tools"][0]["function"]["name"],
"weather"
);
assert_eq!(request.metadata["openai_tool_choice"], "auto");
assert_eq!(
request.metadata["openai_legacy_functions"][0]["name"],
"legacy_weather"
);
assert_eq!(request.metadata["openai_legacy_function_call"], "auto");
let Some(ferrum_types::ApiRequest::Chat(api)) = request.api_request.as_ref() else {
panic!("expected structured chat api_request");
};
assert_eq!(api.messages.len(), 3);
assert_eq!(api.messages[2].role, ferrum_types::ApiMessageRole::Tool);
assert_eq!(api.messages[2].tool_call_id.as_deref(), Some("call_1"));
assert_eq!(api.messages[1].tool_calls[0].id, "call_1");
assert_eq!(api.messages[1].tool_calls[0].function.name, "weather");
assert_eq!(api.messages[2].content, "sunny");
assert_eq!(api.tools[0].function.name, "weather");
assert_eq!(api.legacy_functions[0].name, "legacy_weather");
assert_eq!(
api.messages[1].tool_calls[0].function.arguments,
"{\"city\":\"Paris\"}"
);
}
}
#[tokio::test]
async fn route_replays_reasoning_content_in_qwen36_tool_history_sync() {
let compatibility = capture_qwen36_tool_history_request(
json!({"reasoning_content": "opencode-reasoning-marker"}),
false,
)
.await;
let canonical = capture_qwen36_tool_history_request(
json!({"reasoning": "opencode-reasoning-marker"}),
false,
)
.await;
assert_eq!(compatibility.prompt, canonical.prompt);
assert!(
compatibility.prompt.contains("opencode-reasoning-marker"),
"Qwen3.6 prompt dropped assistant reasoning history: {}",
compatibility.prompt
);
let message = &compatibility.metadata["openai_messages"][1];
assert_eq!(message["reasoning"], "opencode-reasoning-marker");
assert!(message.get("reasoning_content").is_none());
}
#[tokio::test]
async fn route_replays_reasoning_content_in_qwen36_tool_history_stream() {
let request = capture_qwen36_tool_history_request(
json!({"reasoning_content": "opencode-stream-reasoning-marker"}),
true,
)
.await;
assert!(
request.prompt.contains("opencode-stream-reasoning-marker"),
"Qwen3.6 streaming prompt dropped assistant reasoning history: {}",
request.prompt
);
}
#[tokio::test]
async fn route_prefers_canonical_reasoning_in_qwen36_tool_history() {
for stream in [false, true] {
for reasoning in ["canonical-history-marker", ""] {
let request = capture_qwen36_tool_history_request(
json!({
"reasoning": reasoning,
"reasoning_content": "alias-history-marker"
}),
stream,
)
.await;
let canonical =
capture_qwen36_tool_history_request(json!({"reasoning": reasoning}), stream)
.await;
assert_eq!(request.prompt, canonical.prompt);
assert!(!request.prompt.contains("alias-history-marker"));
let message = &request.metadata["openai_messages"][1];
assert_eq!(message["reasoning"], reasoning);
assert!(message.get("reasoning_content").is_none());
}
}
}
#[tokio::test]
async fn route_does_not_force_reasoning_into_templates_that_ignore_it() {
let template = ModelChatTemplate::new(
"{% for message in messages %}[{{ message.role }}]{{ message.content }}{% endfor %}",
"content-only-template",
);
let (router, engine) = router_with_capturing_llm_and_template(template);
let response = post_json(
router,
"/v1/chat/completions",
json!({
"model": "served-alias",
"messages": [
{"role": "user", "content": "hello"},
{
"role": "assistant",
"content": "visible answer",
"reasoning_content": "hidden-reasoning-marker"
},
{"role": "user", "content": "continue"}
]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let request = engine.last_request();
assert!(request.prompt.contains("visible answer"));
assert!(!request.prompt.contains("hidden-reasoning-marker"));
}
#[tokio::test]
async fn route_tool_request_prefers_model_chat_template() {
for stream in [false, true] {
let template = ModelChatTemplate::new(
"{% if tools %}<tools>{% for tool in tools %}{{ tool.function.name }}{% endfor %}</tools>{% endif %}{% for message in messages %}[{{ message.role }}]{{ message.content }}{% if message.tool_calls %}{% for tool_call in message.tool_calls %}<tool_call id=\"{{ tool_call.id }}\">{{ tool_call.function.name }}:{{ tool_call.function.arguments }}</tool_call>{% endfor %}{% endif %}{% if message.tool_call_id %}<tool_response id=\"{{ message.tool_call_id }}\">{{ message.content }}</tool_response>{% endif %}{% endfor %}{% if add_generation_prompt %}[assistant]{% endif %}",
"tool-template",
);
let (router, engine) = router_with_capturing_llm_and_template(template);
let response = post_json(
router,
"/v1/chat/completions",
json!({
"model": "served-alias",
"messages": [
{"role": "user", "content": "Use the weather tool."},
{
"role": "assistant",
"content": null,
"tool_calls": [{
"id": "weather_paris",
"type": "function",
"function": {"name": "weather", "arguments": "{\"city\":\"Paris\"}"}
}, {
"id": "weather_rome",
"type": "function",
"function": {"name": "weather", "arguments": "{\"city\":\"Rome\"}"}
}]
},
// Results may arrive in a different order than calls. IDs,
// rather than positions or function names, preserve pairing.
{"role": "tool", "tool_call_id": "weather_rome", "content": "rainy"},
{"role": "tool", "tool_call_id": "weather_paris", "content": "sunny"}
],
"tools": [{
"type": "function",
"function": {
"name": "weather",
"description": "Get weather",
"parameters": {"type": "object", "properties": {"city": {"type": "string"}}}
}
}],
"tool_choice": "auto",
"stream": stream
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
if stream {
let body = response_text(response).await;
assert!(body.contains("[DONE]"), "{body}");
assert!(body.contains("captured"), "{body}");
} else {
let body = response_json(response).await;
assert_eq!(body["choices"][0]["message"]["content"], "captured");
assert_eq!(body["choices"][0]["finish_reason"], "stop");
}
let request = engine.last_request();
assert!(request.prompt.contains("<tools>weather</tools>"));
assert!(
request
.prompt
.contains("<tool_call id=\"weather_paris\">weather:"),
"{}",
request.prompt
);
assert!(request.prompt.contains("\"city\""), "{}", request.prompt);
assert!(request.prompt.contains("Paris"), "{}", request.prompt);
assert!(request
.prompt
.contains("<tool_response id=\"weather_paris\">sunny</tool_response>"));
assert!(request
.prompt
.contains("<tool_call id=\"weather_rome\">weather:"));
assert!(request.prompt.contains("Rome"));
assert!(request
.prompt
.contains("<tool_response id=\"weather_rome\">rainy</tool_response>"));
assert!(request.prompt.ends_with("[assistant]"));
let Some(ferrum_types::ApiRequest::Chat(api)) = request.api_request.as_ref() else {
panic!("expected structured continuation request");
};
assert_eq!(api.messages.len(), 4);
assert_eq!(api.messages[1].tool_calls.len(), 2);
for (call, id, city) in [
(&api.messages[1].tool_calls[0], "weather_paris", "Paris"),
(&api.messages[1].tool_calls[1], "weather_rome", "Rome"),
] {
assert_eq!(call.id, id);
assert_eq!(call.function.name, "weather");
let args: Value = serde_json::from_str(&call.function.arguments).unwrap();
assert_eq!(args, json!({"city": city}));
let prefix = format!("<tool_call id=\"{id}\">weather:");
let rendered_arguments = request
.prompt
.split_once(&prefix)
.unwrap()
.1
.split_once("</tool_call>")
.unwrap()
.0;
let rendered: Value = serde_json::from_str(rendered_arguments).unwrap();
assert_eq!(
rendered,
json!({"city": city}),
"tool arguments lost their call ID binding"
);
}
for (message, id, content) in [
(&api.messages[2], "weather_rome", "rainy"),
(&api.messages[3], "weather_paris", "sunny"),
] {
assert_eq!(message.role, ferrum_types::ApiMessageRole::Tool);
assert_eq!(message.tool_call_id.as_deref(), Some(id));
assert_eq!(message.content, content);
}
assert!(
!request.prompt.contains("<|assistant|>"),
"model-template tool prompt should not use generic fallback: {}",
request.prompt
);
assert!(
!request.prompt.contains("When a tool is needed"),
"model-template tool prompt should not inject fallback tool instructions: {}",
request.prompt
);
}
}
#[tokio::test]
async fn chat_omitted_output_budget_uses_auto_ceiling() {
let (router, engine) = router_with_capturing_llm();
let response = post_json(
router,
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "hello"}]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let request = engine.last_request();
assert_eq!(request.sampling_params.max_tokens, 4096);
assert_eq!(
request.metadata.get(DEFAULT_MAX_TOKENS_METADATA_KEY),
Some(&serde_json::json!(true))
);
}
#[tokio::test]
async fn chat_accepts_stop_string_and_max_completion_tokens() {
let (router, engine) = router_with_capturing_llm();
let response = post_json(
router,
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "hello"}],
"max_tokens": 99,
"max_completion_tokens": 3,
"stop": "<END>"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let request = engine.last_request();
let defaults = default_chat_sampling_params();
assert_eq!(request.sampling_params.max_tokens, 3);
assert!(!request
.metadata
.contains_key(DEFAULT_MAX_TOKENS_METADATA_KEY));
assert_eq!(request.sampling_params.temperature, defaults.temperature);
assert_eq!(
request.sampling_params.repetition_penalty,
defaults.repetition_penalty
);
assert_eq!(request.sampling_params.stop_sequences, vec!["<END>"]);
}
#[tokio::test]
async fn chat_maps_vllm_sampling_extensions_without_hidden_defaults() {
let (router, engine) = router_with_capturing_llm();
let response = post_json(
router,
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "hello"}],
"top_k": 20,
"min_p": 0.05,
"repetition_penalty": 1.25
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let request = engine.last_request();
assert_eq!(request.sampling_params.top_k, Some(20));
assert_eq!(request.sampling_params.min_p, Some(0.05));
assert_eq!(request.sampling_params.repetition_penalty, 1.25);
}
#[tokio::test]
async fn chat_normalizes_disabled_sampling_extensions_and_rejects_invalid_ranges() {
let (router, engine) = router_with_capturing_llm();
let response = post_json(
router,
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "hello"}],
"top_k": -1,
"min_p": 0.0,
"repetition_penalty": 1.0
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let request = engine.last_request();
assert_eq!(request.sampling_params.top_k, None);
assert_eq!(request.sampling_params.min_p, None);
assert_eq!(request.sampling_params.repetition_penalty, 1.0);
for (field, value) in [
("top_k", json!(-2)),
("min_p", json!(1.01)),
("repetition_penalty", json!(0.0)),
("presence_penalty", json!(2.01)),
("frequency_penalty", json!(-2.01)),
] {
let (router, _) = router_with_capturing_llm();
let response = post_json(
router,
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "hello"}],
(field): value
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST, "{field}");
let body = response_json(response).await;
assert_eq!(body["error"]["param"], field);
}
}
#[tokio::test]
async fn chat_request_forbids_initial_think_close_token() {
let engine = Arc::new(CapturingLlm::new());
let router = AxumServer::from_llm(engine.clone()).build_router();
let response = post_json(
router,
"/v1/chat/completions",
json!({
"model": "qwen3",
"messages": [{"role": "user", "content": "hello"}]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let request = engine.last_request();
assert_eq!(
request
.metadata
.get(INITIAL_FORBIDDEN_TOKEN_TEXTS_METADATA_KEY),
Some(&serde_json::json!([THINK_END_TAG]))
);
}
#[tokio::test]
async fn omitted_enable_thinking_preserves_model_template_default() {
let template = ModelChatTemplate::new(
"{% for message in messages %}{{ '<|im_start|>' ~ message.role ~ '\n' ~ message.content ~ '<|im_end|>\n' }}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% if enable_thinking is defined and enable_thinking is false %}{{ '<think>\n\n</think>\n\n' }}{% else %}{{ '<think>\n' }}{% endif %}{% endif %}",
"test-template",
);
let (router, engine) = router_with_capturing_llm_and_template(template);
let response = post_json(
router,
"/v1/chat/completions",
json!({
"model": "served-alias",
"messages": [{"role": "user", "content": "hello"}]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let request = engine.last_request();
assert!(request.prompt.ends_with("<|im_start|>assistant\n<think>\n"));
assert!(!request
.metadata
.contains_key(INITIAL_FORBIDDEN_TOKEN_TEXTS_METADATA_KEY));
}
#[tokio::test]
async fn server_thinking_default_applies_but_request_override_wins() {
let template = ModelChatTemplate::new(
"{% if add_generation_prompt %}<assistant>{% if enable_thinking is defined and enable_thinking is false %}<think>\n\n</think>\n\n{% else %}<think>\n{% endif %}{% endif %}",
"test-template",
);
let (router, engine) =
router_with_capturing_llm_and_template_default(template, Some(false));
let response = post_json(
router.clone(),
"/v1/chat/completions",
json!({
"model": "served-alias",
"messages": [{"role": "user", "content": "hello"}]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let request = engine.last_request();
assert_eq!(request.prompt, "<assistant><think>\n\n</think>\n\n");
assert_eq!(
request
.metadata
.get(INITIAL_FORBIDDEN_TOKEN_TEXTS_METADATA_KEY),
Some(&serde_json::json!([THINK_END_TAG, THINK_START_TAG]))
);
let response = post_json(
router,
"/v1/chat/completions",
json!({
"model": "served-alias",
"messages": [{"role": "user", "content": "hello"}],
"chat_template_kwargs": {"enable_thinking": true}
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let request = engine.last_request();
assert_eq!(request.prompt, "<assistant><think>\n");
assert!(!request
.metadata
.contains_key(INITIAL_FORBIDDEN_TOKEN_TEXTS_METADATA_KEY));
}
#[tokio::test]
async fn chat_template_enable_thinking_true_overrides_default() {
let template = ModelChatTemplate::new(
"{% if add_generation_prompt %}<assistant>{% if enable_thinking is defined and enable_thinking is false %}<think>\n\n</think>\n\n{% else %}<think>\n{% endif %}{% endif %}",
"test-template",
);
let (router, engine) = router_with_capturing_llm_and_template(template);
let response = post_json(
router,
"/v1/chat/completions",
json!({
"model": "served-alias",
"messages": [{"role": "user", "content": "hello"}],
"chat_template_kwargs": {"enable_thinking": true}
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let request = engine.last_request();
assert_eq!(request.prompt, "<assistant><think>\n");
assert!(!request
.metadata
.contains_key(INITIAL_FORBIDDEN_TOKEN_TEXTS_METADATA_KEY));
}
#[tokio::test]
async fn chat_template_enable_thinking_false_is_a_hard_override() {
let template = ModelChatTemplate::new(
"{% if add_generation_prompt %}<assistant>{% if enable_thinking is defined and enable_thinking is false %}<think>\n\n</think>\n\n{% else %}<think>\n{% endif %}{% endif %}",
"test-template",
);
let (router, engine) = router_with_capturing_llm_and_template(template);
let response = post_json(
router,
"/v1/chat/completions",
json!({
"model": "served-alias",
"messages": [{"role": "user", "content": "hello"}],
"chat_template_kwargs": {"enable_thinking": false}
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let request = engine.last_request();
assert_eq!(request.prompt, "<assistant><think>\n\n</think>\n\n");
assert_eq!(
request
.metadata
.get(INITIAL_FORBIDDEN_TOKEN_TEXTS_METADATA_KEY),
Some(&serde_json::json!([THINK_END_TAG, THINK_START_TAG]))
);
}
#[tokio::test]
async fn chat_template_reasoning_effort_is_typed_and_rendered() {
let template = ModelChatTemplate::new(
"{% if reasoning_effort is defined %}Reasoning: {{ reasoning_effort }}{% else %}Reasoning: model-default{% endif %}",
"test-template",
);
let (router, engine) = router_with_capturing_llm_and_template(template);
let response = post_json(
router.clone(),
"/v1/chat/completions",
json!({
"model": "served-alias",
"messages": [{"role": "user", "content": "hello"}],
"chat_template_kwargs": {"reasoning_effort": "low"}
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
assert_eq!(engine.last_request().prompt, "Reasoning: low");
let response = post_json(
router.clone(),
"/v1/chat/completions",
json!({
"model": "served-alias",
"messages": [{"role": "user", "content": "hello"}],
"chat_template_kwargs": {"reasoning_effort": "xhigh"}
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
assert_eq!(engine.last_request().prompt, "Reasoning: xhigh");
for invalid in [json!("extreme"), json!(1)] {
let response = post_json(
router.clone(),
"/v1/chat/completions",
json!({
"model": "served-alias",
"messages": [{"role": "user", "content": "hello"}],
"chat_template_kwargs": {"reasoning_effort": invalid}
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert!(body["error"]["message"]
.as_str()
.unwrap_or_default()
.contains("reasoning_effort"));
}
}
#[tokio::test]
async fn chat_template_enable_thinking_rejects_non_bool() {
let template = ModelChatTemplate::new(
"{% if add_generation_prompt %}<assistant>{% endif %}",
"test-template",
);
let (router, _) = router_with_capturing_llm_and_template(template);
let response = post_json(
router,
"/v1/chat/completions",
json!({
"model": "served-alias",
"messages": [{"role": "user", "content": "hello"}],
"chat_template_kwargs": {"enable_thinking": "false"}
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert!(body["error"]["message"]
.as_str()
.unwrap_or_default()
.contains("chat_template_kwargs.enable_thinking must be a boolean"));
}
#[tokio::test]
async fn stop_string_strips_chat_and_completion_suffixes() {
let chat = post_json(
router_with_stub("hello<END>"),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "hello"}],
"stop": "<END>"
}),
)
.await;
assert_eq!(chat.status(), AxumStatusCode::OK);
let chat_body = response_json(chat).await;
assert_eq!(chat_body["choices"][0]["message"]["content"], "hello");
let completion = post_json(
router_with_stub("done<END>"),
"/v1/completions",
json!({
"model": "stub-model",
"prompt": "complete",
"stop": "<END>"
}),
)
.await;
assert_eq!(completion.status(), AxumStatusCode::OK);
let completion_body = response_json(completion).await;
assert_eq!(completion_body["choices"][0]["text"], "done");
}
#[test]
fn started_in_think_parse_streams_reasoning_before_end_tag() {
// R1-distill templates open `<think>` inside the prompt, so the
// generated text never contains the start tag. Mid-think text must
// be reasoning, not content (this leaked as content deltas before).
let parsed = parse_reasoning_response_started_in_think("Okay, the user wants");
assert_eq!(parsed.reasoning.as_deref(), Some("Okay, the user wants"));
assert_eq!(parsed.content, "");
let parsed = parse_reasoning_response_started_in_think("thinking...</think>\nanswer");
assert_eq!(parsed.reasoning.as_deref(), Some("thinking..."));
assert_eq!(parsed.content, "answer");
// Model re-opening its own think block defers to the normal parse.
let parsed = parse_reasoning_response_started_in_think("<think>\nx\n</think>\n\nanswer");
assert_eq!(parsed.reasoning.as_deref(), Some("\nx\n"));
assert_eq!(parsed.content, "answer");
}
#[tokio::test]
async fn chat_response_splits_reasoning_from_content() {
let response = post_json(
router_with_stub("<think>\nreasoning\n</think>\n\nfinal answer"),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "hello"}]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
let message = &body["choices"][0]["message"];
assert_eq!(message["content"], "final answer");
assert_eq!(message["reasoning"], "\nreasoning\n");
assert!(message.get("reasoning_content").is_none());
}
#[tokio::test]
async fn streaming_chat_reasoning_prefix_chunks_do_not_panic_or_leak_content() {
let response = post_json(
router_with_stub_stream_chunks(&["<", "think", ">\nreason", "\n</think>\n\nfinal"]),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "think then answer"}],
"stream": true
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(body.contains("data: [DONE]"), "missing DONE: {body}");
assert!(
body.contains(r#""reasoning":"\nreason"#),
"stream should emit reasoning delta after full think prefix: {body}"
);
assert!(!body.contains("\"reasoning_content\":"));
assert!(
body.contains(r#""content":"final""#),
"stream should emit visible content after think close: {body}"
);
assert!(
!body.contains(r#""content":"<"#),
"partial think prefix must not leak as content: {body}"
);
}
#[tokio::test]
async fn route_rejects_unsupported_tool_and_function_selection() {
for (extra, param) in [
(
json!({
"tools": [{
"type": "function",
"function": {"name": "weather", "parameters": {"type": "object"}}
}],
"tool_choice": {
"type": "function",
"function": {"name": "calendar"}
}
}),
"tool_choice",
),
(
json!({
"functions": [{"name": "weather", "parameters": {"type": "object"}}],
"function_call": {"name": "calendar"}
}),
"function_call",
),
] {
let mut body = json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "hello"}]
});
body.as_object_mut()
.expect("object")
.extend(extra.as_object().expect("extra object").clone());
let response =
post_json(router_with_stub("unused"), "/v1/chat/completions", body).await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], param);
}
}
#[tokio::test]
async fn route_rejects_non_function_tools_with_openai_error_param() {
let response = post_json(
router_with_stub("unused"),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "hello"}],
"tools": [{
"type": "retrieval",
"function": {"name": "search", "parameters": {"type": "object"}}
}]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "tools");
}
#[tokio::test]
async fn route_rejects_tool_choice_required_without_tools() {
let response = post_json(
router_with_stub("unused"),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "hello"}],
"tool_choice": "required"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "tool_choice");
}
#[tokio::test]
async fn route_rejects_unknown_response_format_type_with_openai_error_param() {
let response = post_json(
router_with_stub("unused"),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "hello"}],
"response_format": {"type": "xml"}
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "response_format.type");
}
#[tokio::test]
async fn route_chat_engine_unavailable_maps_to_503() {
let response = post_json(
router_without_llm(),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "hello"}]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::SERVICE_UNAVAILABLE);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "service_unavailable_error");
assert_eq!(body["error"]["param"], Value::Null);
}
#[tokio::test]
async fn context_capacity_rejection_has_a_structured_code_on_openai_routes() {
for stream in [false, true] {
for (path, mut input) in [
(
"/v1/chat/completions",
json!({"messages":[{"role":"user","content":"hello"}],"max_tokens":100}),
),
(
"/v1/completions",
json!({"prompt":"hello","max_tokens":100}),
),
(
"/v1/responses",
json!({"input":"hello","max_output_tokens":100}),
),
] {
input["model"] = json!("failing-model");
input["stream"] = json!(stream);
let router = AxumServer::from_llm(Arc::new(FailingLlm::context_length_exceeded()))
.build_router();
let response = post_json(router, path, input).await;
assert_eq!(
response.status(),
AxumStatusCode::BAD_REQUEST,
"{path}, stream={stream}"
);
let body = response_json(response).await;
assert_eq!(body["error"]["code"], "context_length_exceeded", "{body}");
assert_eq!(body["error"]["type"], "invalid_request_error");
assert!(body["error"]["message"]
.as_str()
.unwrap()
.contains("500 input tokens + 100 output tokens"));
}
}
let ordinary =
server_error_from_ferrum_error(Error::request_validation("invalid parameter"))
.into_response();
assert_eq!(response_json(ordinary).await["error"]["code"], Value::Null);
}
#[tokio::test]
async fn route_chat_generation_failure_maps_to_500() {
let response = post_json(
router_with_failing_llm(),
"/v1/chat/completions",
json!({
"model": "failing-model",
"messages": [{"role": "user", "content": "hello"}]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::INTERNAL_SERVER_ERROR);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "internal_server_error");
assert!(body["error"]["message"]
.as_str()
.unwrap()
.contains("stub generation failed"));
}
#[tokio::test]
async fn route_chat_generation_failure_writes_replay_diagnostics() {
let root = unique_request_dump_dir("chat-sync-failure");
let profile = unique_profile_jsonl("chat-sync-failure");
let response = post_json(
router_with_failing_llm_request_dump_and_profile(root.clone(), profile.clone()),
"/v1/chat/completions",
json!({
"model": "failing-model",
"messages": [{"role": "user", "content": "hello"}]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::INTERNAL_SERVER_ERROR);
assert_chat_failure_replay_bundle(
&root,
"chat_completions_sync",
"internal",
"stub generation failed",
);
let event = read_profile_events(&profile)
.into_iter()
.find(|event| event["phase"] == "chat_completions_sync")
.expect("sync failure profile event");
assert_eq!(event["event_kind"], "timed_span");
assert_eq!(event["status"], "failure");
assert!(event["duration_us"].as_u64().is_some());
assert_eq!(event["attributes"]["terminal_failure_event"], true);
assert_eq!(event["error"]["kind"], "internal");
let _ = fs::remove_dir_all(root);
let _ = fs::remove_file(profile);
}
#[tokio::test]
async fn route_chat_resource_failure_writes_resource_replay_diagnostics() {
let root = unique_request_dump_dir("chat-resource-failure");
let response = post_json(
router_with_resource_exhausted_llm_and_request_dump_dir(root.clone()),
"/v1/chat/completions",
json!({
"model": "failing-model",
"messages": [{"role": "user", "content": "hello"}]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::SERVICE_UNAVAILABLE);
let bundle = only_replay_bundle(&root);
let bad_scan = read_json_file(bundle.join("bad_output_scan.json"));
assert_eq!(bad_scan["failure_kind"], "oom_admission");
let diagnostics = read_json_file(bundle.join("failure_diagnostics.json"));
assert_eq!(diagnostics["failure_kind"], "oom_admission");
assert_eq!(
diagnostics["first_failure_event"]["error_kind"],
"resource_exhausted"
);
assert_eq!(
diagnostics["capacity"]["resource_kind"],
"admission_capacity"
);
assert!(diagnostics["capacity"]["reason"]
.as_str()
.expect("capacity reason")
.contains("admission capacity exhausted"));
assert_eq!(
diagnostics["nearest_resource_event"]["resource_kind"],
"admission_capacity"
);
assert!(diagnostics["nearest_memory_snapshot"]["current_bytes"].is_number());
assert!(diagnostics["nearest_memory_snapshot"]["high_water_bytes"].is_number());
let _ = fs::remove_dir_all(root);
}
#[tokio::test]
async fn route_chat_sync_success_updates_replay_output_tokens() {
let root = unique_request_dump_dir("chat-sync-success-output");
let response = post_json(
router_with_stub_and_request_dump_dir("OK", root.clone()),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "hello"}]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["choices"][0]["message"]["content"], "OK");
assert_chat_success_replay_bundle(&root, &[11, 12], "stop", "OK");
let _ = fs::remove_dir_all(root);
}
#[tokio::test]
async fn route_chat_sync_success_writes_product_profile_event() {
let root = unique_request_dump_dir("chat-sync-profile");
let profile = unique_profile_jsonl("chat-sync-profile");
let response = post_json(
router_with_stub_request_dump_and_profile("OK", root.clone(), profile.clone()),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "hello"}]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let response_body = response_json(response).await;
let events = read_profile_events(&profile);
assert_eq!(events.len(), 2, "events: {events:#?}");
let event = events
.iter()
.find(|event| event["phase"] == "chat_completions_sync_complete")
.expect("sync completion profile event");
assert!(response_body["id"]
.as_str()
.is_some_and(|id| !id.is_empty()));
assert_eq!(response_body["id"], event["request_id"]);
assert_eq!(response_body["id"], event["correlation_id"]);
assert_eq!(
event["schema_version"],
OBSERVABILITY_PROFILE_SCHEMA_VERSION
);
assert_eq!(event["entrypoint"], "serve");
assert_eq!(event["event_kind"], "timed_span");
assert_eq!(event["status"], "ok");
assert_eq!(event["phase"], "chat_completions_sync_complete");
assert_eq!(event["attributes"]["actual_model_smoke"], true);
assert_eq!(event["attributes"]["profile_detail"], "latency");
assert_eq!(event["attributes"]["diagnostic_only"], false);
assert_eq!(event["attributes"]["stream"], false);
assert_eq!(event["attributes"]["output_token_count"], 2);
assert_eq!(event["attributes"]["prompt_token_count"], 7);
assert_eq!(event["attributes"]["completion_token_count"], 2);
assert_eq!(event["attributes"]["total_token_count"], 9);
assert_eq!(event["attributes"]["token_count_source"], "usage");
assert_eq!(event["attributes"]["finish_reason"], "stop");
assert_eq!(
event["attributes"]["engine_token_clock_source"],
"rust_std_instant"
);
assert_eq!(event["attributes"]["engine_token_commit_count"], 2);
assert_eq!(event["attributes"]["itl_interval_count"], 1);
assert_eq!(event["attributes"]["ttft_us"], 1_000);
assert_eq!(event["attributes"]["itl_us_avg"], 1_000);
assert!(event["attributes"]["http_first_sse_enqueue_us"].is_null());
assert!(event["duration_us"].as_u64().unwrap_or_default() > 0);
assert!(
event["attributes"]["e2e_duration_us"]
.as_u64()
.unwrap_or_default()
> 0
);
assert_eq!(
event["replay"]["bundle_dir"].as_str(),
Some(root.to_string_lossy().as_ref())
);
assert!(event["replay"]["command"]
.as_str()
.unwrap_or_default()
.contains("replay_body.json"));
let memory_event = events
.iter()
.find(|event| event["phase"] == "actual_serve_first_request_done")
.expect("first request memory profile event");
assert_eq!(memory_event["event_kind"], "memory");
assert_eq!(
memory_event["attributes"]["memory_stage"],
"first_request_done"
);
assert_eq!(
memory_event["attributes"]["memory_measurement"],
"process_rss"
);
assert!(memory_event["memory"]["current_bytes"]
.as_u64()
.is_some_and(|bytes| bytes > 0));
let _ = fs::remove_dir_all(root);
let _ = fs::remove_file(profile);
}
#[tokio::test]
async fn route_chat_profile_events_preserve_benchmark_correlation() {
let root = unique_request_dump_dir("chat-benchmark-correlation");
let profile = unique_profile_jsonl("chat-benchmark-correlation");
let correlation = BenchmarkRequestCorrelation::new(
"bench-123".to_string(),
"cell-1-closed-c8".to_string(),
2,
ferrum_bench_core::BenchmarkPhase::Measured,
17,
)
.unwrap();
let response = post_json_with_benchmark_correlation(
router_with_stub_request_dump_and_profile("OK", root.clone(), profile.clone()),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "hello"}]
}),
&correlation,
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let _ = response_json(response).await;
let events = read_profile_events(&profile);
assert_eq!(events.len(), 2, "events: {events:#?}");
for event in events {
assert_eq!(event["attributes"]["benchmark_run_id"], "bench-123");
assert_eq!(event["attributes"]["cell_id"], "cell-1-closed-c8");
assert_eq!(event["attributes"]["repeat_index"], 2);
assert_eq!(event["attributes"]["phase"], "measured");
assert_eq!(event["attributes"]["request_index"], 17);
}
let _ = fs::remove_dir_all(root);
let _ = fs::remove_file(profile);
}
#[tokio::test]
async fn route_chat_rejects_partial_benchmark_correlation_headers() {
let response = router_with_stub("OK")
.oneshot(
Request::builder()
.method("POST")
.uri("/v1/chat/completions")
.header(header::CONTENT_TYPE, "application/json")
.header(BENCHMARK_RUN_ID_HEADER, "bench-123")
.body(Body::from(
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "hello"}]
})
.to_string(),
))
.expect("request"),
)
.await
.expect("route response");
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
}
#[tokio::test]
async fn route_chat_sync_profile_jsonl_is_parseable_under_concurrent_requests() {
let root = unique_request_dump_dir("chat-sync-profile-concurrent");
let profile = unique_profile_jsonl("chat-sync-profile-concurrent");
let app = router_with_stub_request_dump_and_profile("OK", root.clone(), profile.clone());
let mut handles = Vec::new();
for request_index in 0..8 {
let app = app.clone();
handles.push(tokio::spawn(async move {
let response = post_json(
app,
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": format!("hello {request_index}")}]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["choices"][0]["message"]["content"], "OK");
}));
}
for handle in handles {
handle.await.expect("concurrent request task");
}
let raw = fs::read_to_string(&profile).expect("profile jsonl");
let mut completion_events = 0usize;
for (line_index, line) in raw
.lines()
.filter(|line| !line.trim().is_empty())
.enumerate()
{
let event: Value = serde_json::from_str(line).unwrap_or_else(|err| {
panic!(
"profile line {} invalid JSON: {err}: {line}",
line_index + 1
)
});
if event["phase"] == "chat_completions_sync_complete" {
completion_events += 1;
}
}
assert_eq!(completion_events, 8);
let _ = fs::remove_dir_all(root);
let _ = fs::remove_file(profile);
}
#[tokio::test]
async fn route_chat_stream_success_updates_replay_output_tokens() {
let root = unique_request_dump_dir("chat-stream-success-output");
let response = post_json(
router_with_stub_stream_chunks_and_request_dump_dir(&["O", "K"], root.clone()),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "hello"}],
"stream": true
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(body.contains("data: [DONE]"), "body: {body}");
assert_chat_success_replay_bundle(&root, &[11, 12], "stop", "OK");
let _ = fs::remove_dir_all(root);
}
#[tokio::test]
async fn route_chat_stream_success_writes_product_profile_event() {
let root = unique_request_dump_dir("chat-stream-profile");
let profile = unique_profile_jsonl("chat-stream-profile");
let response = post_json(
router_with_stub_stream_request_dump_and_profile(
&["O", "K"],
root.clone(),
profile.clone(),
),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "hello"}],
"stream": true
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(body.contains("data: [DONE]"), "body: {body}");
let events = read_profile_events(&profile);
assert_eq!(events.len(), 2, "events: {events:#?}");
let event = events
.iter()
.find(|event| event["phase"] == "chat_completions_stream_complete")
.expect("stream completion profile event");
let chunks = responses_sse_json_events(&body);
assert!(!chunks.is_empty());
for chunk in chunks {
assert!(chunk["id"].as_str().is_some_and(|id| !id.is_empty()));
assert_eq!(chunk["id"], event["request_id"]);
assert_eq!(chunk["id"], event["correlation_id"]);
}
assert_eq!(
event["schema_version"],
OBSERVABILITY_PROFILE_SCHEMA_VERSION
);
assert_eq!(event["entrypoint"], "serve");
assert_eq!(event["event_kind"], "timed_span");
assert_eq!(event["status"], "ok");
assert_eq!(event["phase"], "chat_completions_stream_complete");
assert_eq!(event["attributes"]["actual_model_smoke"], true);
assert_eq!(event["attributes"]["profile_detail"], "latency");
assert_eq!(event["attributes"]["diagnostic_only"], false);
assert_eq!(event["attributes"]["stream"], true);
assert_eq!(event["attributes"]["output_token_count"], 2);
assert_eq!(event["attributes"]["prompt_token_count"], 5);
assert_eq!(event["attributes"]["completion_token_count"], 2);
assert_eq!(event["attributes"]["total_token_count"], 7);
assert_eq!(event["attributes"]["token_count_source"], "usage");
assert_eq!(event["attributes"]["finish_reason"], "stop");
assert!(event["duration_us"].as_u64().unwrap_or_default() > 0);
assert!(
event["attributes"]["e2e_duration_us"]
.as_u64()
.unwrap_or_default()
> 0
);
assert!(event["attributes"]["ttft_us"].as_u64().is_some());
assert!(event["attributes"]["itl_us_avg"].as_u64().is_some());
assert_eq!(
event["attributes"]["engine_token_commit_nanos_since_request_start"],
json!([1_000_000, 2_000_000])
);
assert_eq!(event["attributes"]["itl_interval_count"], 1);
assert_eq!(event["attributes"]["itl_source"], "engine_token_commit");
assert!(event["attributes"]["engine_stream_first_chunk_received_us"]
.as_u64()
.is_some());
assert!(event["attributes"]["http_first_sse_enqueue_us"]
.as_u64()
.is_some());
assert!(event["attributes"]["http_stream_flush_unavailable_reason"]
.as_str()
.is_some());
assert_eq!(
event["replay"]["bundle_dir"].as_str(),
Some(root.to_string_lossy().as_ref())
);
let memory_event = events
.iter()
.find(|event| event["phase"] == "actual_serve_first_request_done")
.expect("first request memory profile event");
assert_eq!(memory_event["event_kind"], "memory");
assert_eq!(
memory_event["attributes"]["memory_stage"],
"first_request_done"
);
assert_eq!(
memory_event["attributes"]["memory_measurement"],
"process_rss"
);
assert!(memory_event["memory"]["current_bytes"]
.as_u64()
.is_some_and(|bytes| bytes > 0));
let _ = fs::remove_dir_all(root);
let _ = fs::remove_file(profile);
}
#[tokio::test]
async fn route_chat_stream_profile_retains_non_visible_terminal_token() {
let root = unique_request_dump_dir("chat-stream-profile-terminal-token");
let profile = unique_profile_jsonl("chat-stream-profile-terminal-token");
let llm = StubLlm {
stream_usage: Some(TokenUsage::new(5, 2)),
..StubLlm::with_stream_chunks(&["Paris"])
};
let app = AxumServer::from_state(
AppState::default()
.with_llm(Arc::new(llm))
.with_request_dump_dir(Some(root.clone()))
.with_profile_detail(ferrum_types::ObservabilityProfileDetail::Latency)
.with_profile_jsonl(Some(profile.clone())),
)
.build_router();
let response = post_json(
app,
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "hello"}],
"stream": true,
"stream_options": {"include_usage": true}
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(body.contains("\"completion_tokens\":2"), "body: {body}");
assert!(body.contains("data: [DONE]"), "body: {body}");
let events = read_profile_events(&profile);
let event = events
.iter()
.find(|event| event["phase"] == "chat_completions_stream_complete")
.expect("stream completion profile event");
assert_eq!(event["attributes"]["output_token_count"], 2);
assert_eq!(event["attributes"]["completion_token_count"], 2);
assert_eq!(event["attributes"]["engine_token_commit_count"], 2);
assert_chat_success_replay_bundle(&root, &[11, 12], "stop", "Paris");
let _ = fs::remove_dir_all(root);
let _ = fs::remove_file(profile);
}
#[tokio::test]
async fn route_chat_sync_bad_output_updates_replay_classifier() {
let root = unique_request_dump_dir("chat-sync-bad-output");
let response = post_json(
router_with_stub_and_request_dump_dir("<unk>", root.clone()),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "hello"}]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let _ = response_json(response).await;
let bundle = only_replay_bundle(&root);
let bad_scan = read_json_file(bundle.join("bad_output_scan.json"));
assert_eq!(bad_scan["bad_output"], true);
assert_eq!(bad_scan["reasons"], json!(["reserved_token"]));
assert_eq!(bad_scan["first_bad_text_span"]["reason"], "reserved_token");
let _ = fs::remove_dir_all(root);
}
#[tokio::test]
async fn route_chat_stream_generation_failure_emits_openai_error_event() {
let response = post_json(
router_with_failing_llm(),
"/v1/chat/completions",
json!({
"model": "failing-model",
"messages": [{"role": "user", "content": "hello"}],
"stream": true
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::INTERNAL_SERVER_ERROR);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "internal_server_error");
assert!(body["error"]["message"]
.as_str()
.unwrap_or_default()
.contains("stub stream failed"));
}
#[tokio::test]
async fn route_chat_stream_generation_failure_writes_replay_diagnostics() {
let root = unique_request_dump_dir("chat-stream-start-failure");
let response = post_json(
router_with_failing_llm_and_request_dump_dir(root.clone()),
"/v1/chat/completions",
json!({
"model": "failing-model",
"messages": [{"role": "user", "content": "hello"}],
"stream": true
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::INTERNAL_SERVER_ERROR);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "internal_server_error");
assert!(body["error"]["message"]
.as_str()
.unwrap_or_default()
.contains("stub stream failed"));
assert_chat_failure_replay_bundle(
&root,
"chat_completions_stream_start",
"internal",
"stub stream failed",
);
let _ = fs::remove_dir_all(root);
}
#[tokio::test]
async fn route_chat_stream_chunk_failure_emits_openai_error_event() {
let response = post_json(
router_with_stream_chunk_failing_llm(),
"/v1/chat/completions",
json!({
"model": "failing-model",
"messages": [{"role": "user", "content": "hello"}],
"stream": true
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert_openai_stream_error(&body, "stub stream chunk failed");
}
#[tokio::test]
async fn route_chat_stream_chunk_failure_writes_replay_diagnostics() {
let root = unique_request_dump_dir("chat-stream-chunk-failure");
let response = post_json(
router_with_stream_chunk_failing_llm_and_request_dump_dir(root.clone()),
"/v1/chat/completions",
json!({
"model": "failing-model",
"messages": [{"role": "user", "content": "hello"}],
"stream": true
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert_openai_stream_error(&body, "stub stream chunk failed");
assert_chat_failure_replay_bundle(
&root,
"chat_completions_stream_next",
"internal",
"stub stream chunk failed",
);
let _ = fs::remove_dir_all(root);
}
#[tokio::test]
async fn route_completions_engine_unavailable_maps_to_503() {
let response = post_json(
router_without_llm(),
"/v1/completions",
json!({
"model": "stub-model",
"prompt": "complete me"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::SERVICE_UNAVAILABLE);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "service_unavailable_error");
assert_eq!(body["error"]["param"], Value::Null);
}
#[tokio::test]
async fn route_embeddings_engine_unavailable_maps_to_503() {
let response = post_json(
router_without_llm(),
"/v1/embeddings",
json!({
"model": "embed-model",
"input": "hello"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::SERVICE_UNAVAILABLE);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "service_unavailable_error");
assert_eq!(body["error"]["param"], Value::Null);
}
#[tokio::test]
async fn route_embeddings_contract_uses_stub_engine() {
let response = post_json(
router_with_stub_embed(),
"/v1/embeddings",
json!({
"model": "stub-embed",
"input": ["hi", "world"],
"encoding_format": "float"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["object"], "list");
assert_eq!(body["model"], "stub-embed");
assert_eq!(body["usage"]["prompt_tokens"], 7);
assert_eq!(body["usage"]["total_tokens"], 7);
let data = body["data"].as_array().expect("embedding data");
assert_eq!(data.len(), 2, "body: {body}");
assert_eq!(data[0]["object"], "embedding");
assert_eq!(data[0]["index"], 0);
assert_eq!(data[0]["embedding"].as_array().unwrap().len(), 3);
assert_eq!(data[0]["embedding"][0].as_f64().unwrap(), 2.0);
assert_eq!(data[1]["index"], 1);
assert_eq!(data[1]["embedding"][0].as_f64().unwrap(), 5.0);
}
#[tokio::test]
async fn route_embeddings_public_alias_succeeds_and_unknown_alias_is_rejected() {
let registry = ServedModelRegistry::try_new(
"stub-embed",
ServedModelKind::Embedding,
vec!["public-embed".to_string()],
vec![],
)
.unwrap();
let server =
AxumServer::from_embed(Arc::new(StubEmbed::new())).with_served_model_registry(registry);
let accepted = post_json(
server.build_router(),
"/v1/embeddings",
json!({"model": "public-embed", "input": "hello"}),
)
.await;
assert_eq!(accepted.status(), AxumStatusCode::OK);
assert_eq!(response_json(accepted).await["model"], "public-embed");
let rejected = post_json(
server.build_router(),
"/v1/embeddings",
json!({"model": "stub-embed", "input": "hello"}),
)
.await;
assert_eq!(rejected.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(rejected).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "model");
}
#[tokio::test]
async fn route_embeddings_rejects_unsupported_encoding_format() {
let response = post_json(
router_with_stub_embed(),
"/v1/embeddings",
json!({
"model": "stub-embed",
"input": "hi",
"encoding_format": "base64"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "encoding_format");
}
#[tokio::test]
async fn route_embeddings_rejects_empty_input_with_field_param() {
let response = post_json(
router_with_stub_embed(),
"/v1/embeddings",
json!({
"model": "stub-embed",
"input": []
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "input");
}
#[tokio::test]
async fn route_embeddings_rejects_empty_item_with_field_param() {
let response = post_json(
router_with_stub_embed(),
"/v1/embeddings",
json!({
"model": "stub-embed",
"input": [{}]
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "input");
}
#[tokio::test]
async fn route_embeddings_invalid_json_maps_to_openai_error() {
let response = post_raw_json(router_with_stub_embed(), "/v1/embeddings", "{").await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], Value::Null);
assert!(body["error"]["message"]
.as_str()
.unwrap()
.contains("invalid embeddings request"));
}
#[tokio::test]
async fn route_transcriptions_engine_unavailable_maps_to_503() {
let boundary = "ferrum-test-boundary";
let body = concat!(
"--ferrum-test-boundary\r\n",
"Content-Disposition: form-data; name=\"file\"; filename=\"audio.wav\"\r\n",
"Content-Type: audio/wav\r\n",
"\r\n",
"RIFFtest\r\n",
"--ferrum-test-boundary--\r\n"
);
let response = post_multipart(
router_without_llm(),
"/v1/audio/transcriptions",
boundary,
body,
)
.await;
assert_eq!(response.status(), AxumStatusCode::SERVICE_UNAVAILABLE);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "service_unavailable_error");
assert_eq!(body["error"]["param"], Value::Null);
}
#[tokio::test]
async fn route_transcriptions_contract_uses_stub_engine() {
let boundary = "ferrum-test-boundary";
let body = concat!(
"--ferrum-test-boundary\r\n",
"Content-Disposition: form-data; name=\"file\"; filename=\"audio.wav\"\r\n",
"Content-Type: audio/wav\r\n",
"\r\n",
"RIFFtest\r\n",
"--ferrum-test-boundary\r\n",
"Content-Disposition: form-data; name=\"language\"\r\n",
"\r\n",
"en\r\n",
"--ferrum-test-boundary\r\n",
"Content-Disposition: form-data; name=\"response_format\"\r\n",
"\r\n",
"json\r\n",
"--ferrum-test-boundary--\r\n"
);
let response = post_multipart(
router_with_stub_transcribe(),
"/v1/audio/transcriptions",
boundary,
body,
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["text"], "bytes:8:en");
}
#[tokio::test]
async fn route_transcriptions_rejects_unsupported_response_format() {
let boundary = "ferrum-test-boundary";
let body = concat!(
"--ferrum-test-boundary\r\n",
"Content-Disposition: form-data; name=\"file\"; filename=\"audio.wav\"\r\n",
"Content-Type: audio/wav\r\n",
"\r\n",
"RIFFtest\r\n",
"--ferrum-test-boundary\r\n",
"Content-Disposition: form-data; name=\"response_format\"\r\n",
"\r\n",
"text\r\n",
"--ferrum-test-boundary--\r\n"
);
let response = post_multipart(
router_with_stub_transcribe(),
"/v1/audio/transcriptions",
boundary,
body,
)
.await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "response_format");
}
#[tokio::test]
async fn route_transcriptions_rejects_missing_file_with_field_param() {
let boundary = "ferrum-test-boundary";
let body = concat!(
"--ferrum-test-boundary\r\n",
"Content-Disposition: form-data; name=\"language\"\r\n",
"\r\n",
"en\r\n",
"--ferrum-test-boundary--\r\n"
);
let response = post_multipart(
router_with_stub_transcribe(),
"/v1/audio/transcriptions",
boundary,
body,
)
.await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "file");
}
#[tokio::test]
async fn route_transcriptions_invalid_multipart_maps_to_openai_error() {
let response = post_json(
router_with_stub_transcribe(),
"/v1/audio/transcriptions",
json!({}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], Value::Null);
assert!(body["error"]["message"]
.as_str()
.unwrap()
.contains("invalid transcriptions request"));
}
#[tokio::test]
async fn route_speech_engine_unavailable_maps_to_503() {
let response = post_json(
router_without_llm(),
"/v1/audio/speech",
json!({
"model": "tts-model",
"input": "hello",
"voice": "default"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::SERVICE_UNAVAILABLE);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "service_unavailable_error");
assert_eq!(body["error"]["param"], Value::Null);
}
#[tokio::test]
async fn route_speech_contract_uses_stub_engine() {
let response = post_json(
router_with_stub_tts(),
"/v1/audio/speech",
json!({
"model": "stub-tts",
"input": "hello",
"voice": "default",
"response_format": "wav",
"language": "english"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
assert_eq!(
response.headers().get(header::CONTENT_TYPE).unwrap(),
"audio/wav"
);
let body = response_bytes(response).await;
assert!(body.len() > 44, "WAV should include header and PCM data");
assert_eq!(&body[0..4], b"RIFF");
assert_eq!(&body[8..12], b"WAVE");
}
#[tokio::test]
async fn route_speech_streaming_contract_uses_stub_engine() {
let response = post_json(
router_with_stub_tts(),
"/v1/audio/speech",
json!({
"model": "stub-tts",
"input": "hello",
"voice": "default",
"response_format": "wav",
"stream": true
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
assert_eq!(
response.headers().get(header::CONTENT_TYPE).unwrap(),
"audio/wav"
);
assert_eq!(
response.headers().get(header::TRANSFER_ENCODING).unwrap(),
"chunked"
);
let body = response_bytes(response).await;
assert!(body.len() > 44, "streaming WAV should include audio bytes");
assert_eq!(&body[0..4], b"RIFF");
assert_eq!(&body[8..12], b"WAVE");
}
#[tokio::test]
async fn route_speech_pcm_response_format_returns_raw_pcm() {
let response = post_json(
router_with_stub_tts(),
"/v1/audio/speech",
json!({
"model": "stub-tts",
"input": "hello",
"voice": "default",
"response_format": "pcm"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
assert_eq!(
response.headers().get(header::CONTENT_TYPE).unwrap(),
"audio/pcm"
);
let body = response_bytes(response).await;
assert_eq!(body.len(), 6, "three f32 samples should encode as s16le");
assert_eq!(&body[0..2], &[0, 0]);
assert_ne!(&body[0..4], b"RIFF");
}
#[tokio::test]
async fn route_speech_rejects_unsupported_response_format() {
let response = post_json(
router_with_stub_tts(),
"/v1/audio/speech",
json!({
"model": "stub-tts",
"input": "hello",
"voice": "default",
"response_format": "mp3"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "response_format");
}
#[tokio::test]
async fn route_speech_invalid_json_maps_to_openai_error() {
let response = post_raw_json(router_with_stub_tts(), "/v1/audio/speech", "{").await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], Value::Null);
assert!(body["error"]["message"]
.as_str()
.unwrap()
.contains("invalid speech request"));
}
#[tokio::test]
async fn route_completions_generation_failure_maps_to_500() {
let response = post_json(
router_with_failing_llm(),
"/v1/completions",
json!({
"model": "failing-model",
"prompt": "complete me"
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::INTERNAL_SERVER_ERROR);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "internal_server_error");
assert!(body["error"]["message"]
.as_str()
.unwrap()
.contains("stub generation failed"));
}
#[tokio::test]
async fn route_completions_stream_start_failure_maps_to_500_before_sse() {
let response = post_json(
router_with_failing_llm(),
"/v1/completions",
json!({
"model": "failing-model",
"prompt": "complete me",
"stream": true
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::INTERNAL_SERVER_ERROR);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "internal_server_error");
assert!(body["error"]["message"]
.as_str()
.unwrap()
.contains("stub stream failed"));
}
#[tokio::test]
async fn route_completions_stream_chunk_failure_emits_openai_error_event() {
let response = post_json(
router_with_stream_chunk_failing_llm(),
"/v1/completions",
json!({
"model": "failing-model",
"prompt": "complete me",
"stream": true
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert_openai_stream_error(&body, "stub stream chunk failed");
}
#[tokio::test]
async fn route_completions_contract_uses_stub_engine() {
let response = post_json(
router_with_stub("done"),
"/v1/completions",
json!({
"model": "stub-model",
"prompt": "complete me",
"max_tokens": 8,
"temperature": 0.0
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["object"], "text_completion");
assert_eq!(body["choices"][0]["text"], "done");
assert_eq!(body["usage"]["prompt_tokens"], 7);
assert_eq!(body["usage"]["completion_tokens"], 2);
}
#[tokio::test]
async fn route_completions_public_alias_maps_to_internal_model() {
let engine = Arc::new(CapturingLlm::new());
let registry = ServedModelRegistry::try_new(
"qwen3",
ServedModelKind::Llm,
vec!["served-alias".to_string()],
vec![],
)
.unwrap();
let router = AxumServer::from_llm(engine.clone())
.with_served_model_registry(registry)
.build_router();
let response = post_json(
router,
"/v1/completions",
json!({"model": "served-alias", "prompt": "complete me"}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
assert_eq!(response_json(response).await["model"], "served-alias");
assert_eq!(engine.last_request().model_id, ModelId::new("qwen3"));
}
#[tokio::test]
async fn route_completions_streaming_contract_uses_stub_engine() {
let response = post_json(
router_with_stub("done"),
"/v1/completions",
json!({
"model": "stub-model",
"prompt": "complete me",
"max_tokens": 8,
"temperature": 0.0,
"stream": true
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(body.contains("data: [DONE]"), "missing DONE: {body}");
assert!(
body.contains("\"object\":\"text_completion\""),
"missing completion chunk: {body}"
);
assert!(body.contains("\"text\":\"done\""), "missing text: {body}");
assert!(
body.contains("\"choices\":[],\"usage\""),
"missing separate usage chunk: {body}"
);
assert!(
body.contains("\"prompt_tokens\":5"),
"stream usage should come from engine token usage: {body}"
);
assert!(
body.contains("\"completion_tokens\":1"),
"stream completion usage should come from engine token usage: {body}"
);
}
#[tokio::test]
async fn route_completions_stream_waits_for_separate_final_usage_at_max_tokens() {
let response = post_json(
router_with_stub_separate_final_stream_chunk(&["do", "ne"]),
"/v1/completions",
json!({
"model": "stub-model",
"prompt": "complete me",
"max_tokens": 2,
"temperature": 0.0,
"stream": true
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert_eq!(body.matches("data: [DONE]").count(), 1, "body: {body}");
assert!(
body.contains("\"text\":\"do\""),
"missing first chunk: {body}"
);
assert!(
body.contains("\"text\":\"ne\""),
"missing second chunk: {body}"
);
assert!(
body.contains("\"choices\":[],\"usage\""),
"missing separate usage chunk from final engine chunk: {body}"
);
assert!(
body.contains("\"prompt_tokens\":5"),
"stream usage should come from engine final usage: {body}"
);
}
#[tokio::test]
async fn route_completions_invalid_json_maps_to_openai_error() {
let response = post_raw_json(router_with_stub("unused"), "/v1/completions", "{").await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], Value::Null);
assert!(body["error"]["message"]
.as_str()
.unwrap()
.contains("invalid completions request"));
}
#[tokio::test]
async fn route_completions_rejects_unsupported_fields_explicitly() {
for (extra, param) in [
(json!({"n": 2}), "n"),
(json!({"logprobs": 3}), "logprobs"),
(json!({"logit_bias": {"42": 1.0}}), "logit_bias"),
] {
let mut body = json!({
"model": "stub-model",
"prompt": "complete me"
});
body.as_object_mut()
.expect("object")
.extend(extra.as_object().expect("extra object").clone());
let response = post_json(router_with_stub("unused"), "/v1/completions", body).await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], param);
}
}
#[tokio::test]
async fn streaming_completions_do_not_synthesize_whitespace_usage() {
let response = post_json(
router_with_stub_without_stream_usage("done"),
"/v1/completions",
json!({
"model": "stub-model",
"prompt": "one two three four",
"stream": true
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(body.contains("data: [DONE]"), "missing DONE: {body}");
assert!(
!body.contains("\"usage\":{\"prompt_tokens\""),
"server must not synthesize whitespace-count completion usage: {body}"
);
}
#[tokio::test]
async fn chat_rejects_n_not_one_with_openai_error_param() {
let request = chat_request(json!({"n": 2}));
let err = chat_completions_handler(
State(state_with_stub("unused")),
HeaderMap::new(),
Ok(Json(request)),
)
.await
.expect_err("n=2 should reject");
let (status, body) = error_json(err).await;
assert_eq!(status, AxumStatusCode::BAD_REQUEST);
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "n");
}
#[tokio::test]
async fn chat_rejects_logit_bias_and_logprobs_explicitly() {
for (extra, param) in [
(json!({"logit_bias": {"1": 100.0}}), "logit_bias"),
(json!({"logprobs": true}), "logprobs"),
(json!({"top_logprobs": 2}), "top_logprobs"),
] {
let request = chat_request(extra);
let err = chat_completions_handler(
State(state_with_stub("unused")),
HeaderMap::new(),
Ok(Json(request)),
)
.await
.expect_err("unsupported field should reject");
let (status, body) = error_json(err).await;
assert_eq!(status, AxumStatusCode::BAD_REQUEST);
assert_eq!(body["error"]["param"], param);
assert_eq!(body["error"]["type"], "invalid_request_error");
}
}
#[tokio::test]
async fn chat_stream_options_include_usage_controls_stream_usage() {
let request = chat_request(json!({
"stream": true,
"stream_options": {"include_usage": true}
}));
let response = chat_completions_handler(
State(state_with_stub("ok")),
HeaderMap::new(),
Ok(Json(request)),
)
.await
.expect("stream response");
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(body.contains("data: [DONE]"), "missing DONE: {body}");
assert!(
body.contains("\"usage\"") && body.contains("\"completion_tokens\":1"),
"include_usage=true should emit stream usage: {body}"
);
assert!(
body.contains("\"choices\":[],\"usage\""),
"include_usage=true should use a separate usage chunk: {body}"
);
assert!(
body.contains("\"prompt_tokens\":5"),
"stream usage should come from engine token usage: {body}"
);
let request = chat_request(json!({"stream": true}));
let response = chat_completions_handler(
State(state_with_stub("ok")),
HeaderMap::new(),
Ok(Json(request)),
)
.await
.expect("stream response");
let body = response_text(response).await;
assert!(
!body.contains("\"usage\":{\"prompt_tokens\""),
"stream usage should be omitted unless requested: {body}"
);
}
#[tokio::test]
async fn streaming_chat_does_not_synthesize_whitespace_usage() {
let response = post_json(
router_with_stub_without_stream_usage("ok"),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "one two three four"}],
"stream": true,
"stream_options": {"include_usage": true}
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(body.contains("data: [DONE]"), "missing DONE: {body}");
assert!(
!body.contains("\"usage\":{\"prompt_tokens\""),
"server must not synthesize whitespace-count usage when engine stream omits usage: {body}"
);
}
#[test]
fn tool_requests_and_tool_messages_parse_into_structured_api_request() {
let request: ChatCompletionsRequest = serde_json::from_value(json!({
"model": "qwen3",
"messages": [
{"role": "user", "content": "Use the weather tool."},
{
"role": "assistant",
"content": null,
"tool_calls": [{
"id": "call_1",
"type": "function",
"function": {"name": "weather", "arguments": "{\"city\":\"Paris\"}"}
}]
},
{"role": "tool", "tool_call_id": "call_1", "content": "sunny"}
],
"tools": [{
"type": "function",
"function": {
"name": "weather",
"description": "Get weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"]
}
}
}],
"tool_choice": "auto"
}))
.expect("tool request parses");
validate_chat_request(&request).expect("tool request validates");
let internal = convert_chat_request(&request).expect("convert");
assert!(internal.prompt.contains("\"tools\":[{"));
assert!(internal.prompt.contains("\"type\":\"function\""));
assert!(internal.prompt.contains("\"name\":\"weather\""));
assert!(internal.prompt.contains("<|im_start|>assistant\n{"));
assert!(internal.prompt.contains("\"tool_calls\":[{"));
assert!(internal.prompt.contains("\"id\":\"call_1\""));
assert!(internal
.prompt
.contains("<|im_start|>tool\nsunny<|im_end|>"));
assert_eq!(
internal.metadata["openai_tools"][0]["function"]["name"],
"weather"
);
assert_eq!(internal.metadata["openai_tool_choice"], "auto");
let Some(ferrum_types::ApiRequest::Chat(api)) = internal.api_request.as_ref() else {
panic!("expected structured chat api_request");
};
assert_eq!(api.messages.len(), 3);
assert_eq!(api.messages[2].role, ferrum_types::ApiMessageRole::Tool);
assert_eq!(api.messages[2].content, "sunny");
assert_eq!(api.messages[2].tool_call_id.as_deref(), Some("call_1"));
assert_eq!(api.tools[0].function.name, "weather");
assert_eq!(
api.tool_choice,
Some(ferrum_types::ApiToolChoice::Mode("auto".into()))
);
assert_eq!(
api.messages[1].tool_calls[0].function.arguments,
"{\"city\":\"Paris\"}"
);
}
#[test]
fn omitted_tool_choice_defaults_to_auto_when_tools_are_present() {
let request: ChatCompletionsRequest = serde_json::from_value(json!({
"model": "served-alias",
"messages": [{"role": "user", "content": "Use the weather tool."}],
"tools": [{
"type": "function",
"function": {
"name": "weather",
"description": "Get weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"]
}
}
}]
}))
.expect("tool request parses");
validate_chat_request(&request).expect("tool request validates");
let internal = convert_chat_request(&request).expect("convert");
assert!(internal.prompt.contains("\"tools\":[{"));
assert!(internal.prompt.contains("\"tool_choice\":\"auto\""));
assert_eq!(internal.metadata["openai_tool_choice"], "auto");
let initial_forbidden = internal.metadata[INITIAL_FORBIDDEN_TOKEN_TEXTS_METADATA_KEY]
.as_array()
.expect("initial forbidden token list");
assert_eq!(initial_forbidden, &[serde_json::json!(THINK_END_TAG)]);
assert_eq!(
internal.sampling_params.response_format,
ferrum_types::ResponseFormat::Text,
"auto tool choice must preserve native model selection instead of forcing arguments JSON",
);
let Some(ferrum_types::ApiRequest::Chat(api)) = internal.api_request.as_ref() else {
panic!("expected structured chat api_request");
};
assert_eq!(
api.tool_choice,
Some(ferrum_types::ApiToolChoice::Mode("auto".into()))
);
}
#[test]
fn omitted_tool_choice_uses_native_template_protocol_without_hard_schema() {
let request: ChatCompletionsRequest = serde_json::from_value(json!({
"model": "served-alias",
"messages": [{"role": "user", "content": "北京现在天气怎么样?用摄氏度。"}],
"tools": [{
"type": "function",
"function": {
"name": "get_weather",
"description": "查询指定城市的当前天气",
"parameters": {
"type": "object",
"properties": {
"city": {"type": "string"},
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}
},
"required": ["city"]
}
}
}]
}))
.expect("tool request parses");
let template = ModelChatTemplate::new(
"{% if tools %}<tools>{{ tools | tojson }}</tools>Use <tool_call><function=name><parameter=key>value</parameter></function></tool_call>{% endif %}{% for message in messages %}{{ message.content }}{% endfor %}",
"function-parameter-xml-template",
);
validate_chat_request(&request).expect("tool request validates");
let internal =
convert_chat_request_with_template_model(&request, "served-alias", Some(&template))
.expect("convert");
assert_eq!(internal.metadata["openai_tool_choice"], "auto");
assert_eq!(
internal.sampling_params.response_format,
ferrum_types::ResponseFormat::Text,
);
let Some(ferrum_types::ApiRequest::Chat(api)) = internal.api_request else {
panic!("expected chat API request");
};
assert_eq!(
api.tool_call_protocol,
ferrum_types::ApiToolCallProtocol::FunctionParameterXml,
);
}
#[test]
fn tool_schema_response_format_bounds_unconstrained_strings() {
let request: ChatCompletionsRequest = serde_json::from_value(json!({
"model": "served-alias",
"messages": [{"role": "user", "content": "Use the selected tool."}],
"tools": [{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {
"city": {"type": "string"},
"unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}
},
"required": ["city"]
}
}
}],
"tool_choice": {
"type": "function",
"function": {"name": "get_weather"}
}
}))
.expect("tool request parses");
validate_chat_request(&request).expect("tool request validates");
let internal = convert_chat_request(&request).expect("convert");
match internal.sampling_params.response_format {
ferrum_types::ResponseFormat::JsonSchema(ref schema) => {
let value: serde_json::Value =
serde_json::from_str(schema).expect("schema should be JSON");
assert_eq!(
value["properties"]["city"]["maxLength"],
DEFAULT_GUIDED_TOOL_ARGUMENT_STRING_MAX_LENGTH
);
assert_eq!(
value["properties"]["unit"]["enum"],
json!(["celsius", "fahrenheit"])
);
assert!(
value["properties"]["unit"]["maxLength"].is_null(),
"enum string should remain finite via enum instead of maxLength: {value}"
);
}
ref other => panic!("expected forced tool json schema, got {other:?}"),
}
}
#[test]
fn forced_native_tool_choice_preserves_xml_framing_with_a_tool_only_grammar() {
let template = ModelChatTemplate::new(
"{% if tools %}<tools>{{ tools | tojson }}</tools>Use <tool_call><function=name><parameter=key>value</parameter></function></tool_call>{% endif %}{% for message in messages %}{{ message.content }}{% endfor %}",
"native-tool-fixture",
);
for choice in [
json!({"type":"function","function":{"name":"calc"}}),
json!("required"),
] {
let request: ChatCompletionsRequest = serde_json::from_value(json!({
"model":"served-alias","messages":[{"role":"user","content":"Use the selected tool."}],
"tools":[
{"type":"function","function":{"name":"calc","parameters":{"type":"object","properties":{"expression":{"type":"string"}},"required":["expression"]}}},
{"type":"function","function":{"name":"lookup","parameters":{"type":"object"}}}
],
"tool_choice":choice
})).unwrap();
validate_chat_request(&request).unwrap();
let internal =
convert_chat_request_with_template_model(&request, "served-alias", Some(&template))
.unwrap();
assert!(internal.requires_structured_output());
assert_eq!(
internal.sampling_params.response_format,
ferrum_types::ResponseFormat::Text
);
assert!(internal.prompt.contains("<function=name>"));
let Some(ferrum_types::ApiRequest::Chat(chat)) = &internal.api_request else {
panic!("chat contract")
};
assert!(chat.requires_native_tool_call());
assert!(chat.allows_tool_name("calc"));
assert_eq!(chat.allows_tool_name("lookup"), choice == json!("required"));
assert_eq!(
internal.sampling_params.structured_output_start,
StructuredOutputStart::Immediate
);
}
}
#[test]
fn harmony_named_tool_choice_preserves_native_protocol_envelope() {
let request: ChatCompletionsRequest = serde_json::from_value(json!({
"model": "gpt-oss-20b-mxfp4",
"messages": [{
"role": "user",
"content": "Call get_weather exactly once with city set to Paris."
}],
"tools": [{
"type": "function",
"function": {
"name": "get_weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"],
"additionalProperties": false
}
}
}],
"tool_choice": {
"type": "function",
"function": {"name": "get_weather"}
}
}))
.expect("Harmony tool request parses");
let mut template = ModelChatTemplate::new(
"{% if tools %}<|start|>developer<|message|>{{ tools | tojson }}<|end|>{% endif %}{% for message in messages %}<|start|>{{ message.role }}<|message|>{{ message.content }}<|end|>{% endfor %}{% if add_generation_prompt %}<|start|>assistant{% endif %}",
"harmony-tool-template",
);
template.output_protocol = ModelOutputProtocol::HarmonyGptOss;
validate_chat_request(&request).expect("Harmony tool request validates");
let internal =
convert_chat_request_with_template_model(&request, "gpt-oss-20b", Some(&template))
.expect("convert Harmony tool request");
assert!(internal.prompt.ends_with("<|start|>assistant"));
assert_eq!(
internal.sampling_params.model_output_protocol,
ModelOutputProtocol::HarmonyGptOss
);
assert_eq!(
internal.sampling_params.response_format,
ferrum_types::ResponseFormat::Text,
"Harmony must generate its channel/message/call envelope before tool arguments"
);
assert_eq!(
internal.metadata[INITIAL_FORBIDDEN_TOKEN_TEXTS_METADATA_KEY],
json!([]),
"Harmony declares no think delimiter and must not receive the generic structured-call mask"
);
let Some(ferrum_types::ApiRequest::Chat(api)) = internal.api_request else {
panic!("expected chat API request");
};
assert_eq!(
api.tool_choice,
Some(ferrum_types::ApiToolChoice::Function {
tool_type: "function".to_string(),
function: ferrum_types::ApiToolChoiceFunction {
name: "get_weather".to_string(),
},
})
);
}
#[test]
fn required_tool_choice_uses_tool_schema_response_format_without_extra_prompt_instruction() {
let request: ChatCompletionsRequest = serde_json::from_value(json!({
"model": "served-alias",
"messages": [{"role": "user", "content": "Call capture_quality_marker."}],
"tools": [{
"type": "function",
"function": {
"name": "capture_quality_marker",
"description": "Record one marker.",
"parameters": {
"type": "object",
"properties": {
"marker": {"type": "string", "enum": ["ferrum0401"]},
"checksum": {"type": "string", "enum": ["S0004"]}
},
"required": ["marker", "checksum"]
}
}
}],
"tool_choice": "required"
}))
.expect("tool request parses");
validate_chat_request(&request).expect("tool request validates");
let internal = convert_chat_request(&request).expect("convert");
assert!(
!internal.prompt.contains(
"Output only a single JSON object containing the selected function arguments"
),
"{}",
internal.prompt
);
assert!(
internal.prompt.contains("\"tool_choice\":\"required\""),
"{}",
internal.prompt
);
assert_eq!(internal.metadata["openai_tool_choice"], "required");
match internal.sampling_params.response_format {
ferrum_types::ResponseFormat::JsonSchema(ref schema) => {
assert!(schema.contains(r#""enum":["ferrum0401"]"#), "{schema}");
assert!(schema.contains(r#""enum":["S0004"]"#), "{schema}");
}
ref other => panic!("expected forced tool json schema, got {other:?}"),
}
}
#[test]
fn required_tool_choice_suppresses_conflicting_response_format_instruction() {
let request: ChatCompletionsRequest =
serde_json::from_value(required_tool_with_strict_response_format_request(false))
.expect("request parses");
validate_chat_request(&request).expect("request validates");
let internal = convert_chat_request(&request).expect("convert");
assert!(
!internal.prompt.contains("response_format requires"),
"required tool output must not receive a conflicting content-schema instruction: {}",
internal.prompt
);
let ferrum_types::ResponseFormat::JsonSchema(schema) =
internal.sampling_params.response_format
else {
panic!("single required tool must use its argument schema");
};
let schema: Value = serde_json::from_str(&schema).expect("tool schema JSON");
assert!(schema["properties"].get("city").is_some(), "{schema}");
assert!(schema["properties"].get("answer").is_none(), "{schema}");
}
#[test]
fn required_multiple_tools_do_not_force_the_first_tool_schema() {
let request: ChatCompletionsRequest = serde_json::from_value(json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Use the appropriate tool."}],
"tools": [
{
"type": "function",
"function": {
"name": "weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"]
}
}
},
{
"type": "function",
"function": {
"name": "calendar",
"parameters": {
"type": "object",
"properties": {"date": {"type": "string"}},
"required": ["date"]
}
}
}
],
"tool_choice": "required"
}))
.expect("request parses");
validate_chat_request(&request).expect("request validates");
let internal = convert_chat_request(&request).expect("convert");
assert_eq!(
internal.sampling_params.response_format,
ferrum_types::ResponseFormat::Text,
"required permits either declared tool, so guided decoding cannot bind the first tool's arguments"
);
}
#[test]
fn omitted_single_unrelated_tool_keeps_text_response_format() {
let request: ChatCompletionsRequest = serde_json::from_value(json!({
"model": "served-alias",
"messages": [{"role": "user", "content": "讲一个短笑话。"}],
"tools": [{
"type": "function",
"function": {
"name": "get_weather",
"description": "查询指定城市的当前天气",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"]
}
}
}]
}))
.expect("tool request parses");
validate_chat_request(&request).expect("tool request validates");
let internal = convert_chat_request(&request).expect("convert");
assert_eq!(
internal.sampling_params.response_format,
ferrum_types::ResponseFormat::Text
);
}
#[test]
fn tool_choice_none_omits_tools_from_model_template_prompt() {
let request: ChatCompletionsRequest = serde_json::from_value(json!({
"model": "served-alias",
"messages": [
{"role": "user", "content": "Use the weather tool if needed."},
{
"role": "assistant",
"content": null,
"tool_calls": [{
"id": "call_1",
"type": "function",
"function": {"name": "weather", "arguments": "{\"city\":\"Paris\"}"}
}]
},
{"role": "tool", "tool_call_id": "call_1", "content": "{\"temp\":22}"}
],
"tools": [{
"type": "function",
"function": {"name": "weather", "parameters": {"type": "object"}}
}],
"tool_choice": "none"
}))
.expect("tool_choice none request parses");
let template = ModelChatTemplate::new(
"{% set tools_in_user_message = true %}{% if tools %}<tools>{% for tool in tools %}{{ tool.function.name }}{% endfor %}</tools>{% endif %}{% for message in messages %}[{{ message.role }}]{{ message.content }}{% endfor %}{% if add_generation_prompt %}[assistant]{% endif %}",
"tool-choice-none-template",
);
validate_chat_request(&request).expect("tool_choice none request validates");
let internal = convert_chat_request_with_template_model(
&request,
"served-template-model",
Some(&template),
)
.expect("convert");
assert!(
!internal.prompt.contains("<tools>"),
"tool_choice none must not expose tools to the model template: {}",
internal.prompt
);
assert!(internal.prompt.contains("[tool]"), "{}", internal.prompt);
assert_eq!(
internal.metadata["openai_tools"][0]["function"]["name"],
"weather"
);
assert_eq!(internal.metadata["openai_tool_choice"], "none");
let Some(ferrum_types::ApiRequest::Chat(api)) = internal.api_request.as_ref() else {
panic!("expected structured chat api_request");
};
assert_eq!(api.tools[0].function.name, "weather");
assert_eq!(
api.tool_choice,
Some(ferrum_types::ApiToolChoice::Mode("none".into()))
);
}
#[test]
fn specific_tool_choice_parses_into_structured_api_request() {
let request: ChatCompletionsRequest = serde_json::from_value(json!({
"model": "qwen3",
"messages": [{"role": "user", "content": "Use the selected tool."}],
"tools": [
{
"type": "function",
"function": {"name": "weather", "parameters": {"type": "object"}}
},
{
"type": "function",
"function": {"name": "calendar", "parameters": {"type": "object"}}
}
],
"tool_choice": {
"type": "function",
"function": {"name": "weather"}
}
}))
.expect("specific tool_choice request parses");
validate_chat_request(&request).expect("specific tool_choice validates");
let internal = convert_chat_request(&request).expect("convert");
assert!(internal.prompt.contains("\"tool_choice\":{"));
assert!(internal.prompt.contains("\"name\":\"weather\""));
let Some(ferrum_types::ApiRequest::Chat(api)) = internal.api_request.as_ref() else {
panic!("expected structured chat api_request");
};
assert_eq!(
api.tool_choice,
Some(ferrum_types::ApiToolChoice::Function {
tool_type: "function".to_string(),
function: ferrum_types::ApiToolChoiceFunction {
name: "weather".to_string()
},
})
);
let invalid: ChatCompletionsRequest = serde_json::from_value(json!({
"model": "qwen3",
"messages": [{"role": "user", "content": "Use the selected tool."}],
"tools": [{
"type": "function",
"function": {"name": "weather", "parameters": {"type": "object"}}
}],
"tool_choice": {
"type": "function",
"function": {"name": "calendar"}
}
}))
.expect("invalid specific tool_choice request parses");
let err = validate_chat_request(&invalid).expect_err("undeclared tool should reject");
match err {
ServerError::InvalidRequest { param, .. } => {
assert_eq!(param.as_deref(), Some("tool_choice"));
}
other => panic!("expected invalid_request_error for tool_choice, got {other:?}"),
}
}
#[test]
fn legacy_function_role_messages_parse_into_structured_api_request() {
let request: ChatCompletionsRequest = serde_json::from_value(json!({
"model": "mystery-model",
"messages": [
{"role": "user", "content": "Call weather."},
{
"role": "assistant",
"content": null,
"function_call": {"name": "weather", "arguments": "{\"city\":\"Paris\"}"}
},
{"role": "function", "name": "weather", "content": "{\"forecast\":\"sunny\"}"}
],
"functions": [{
"name": "weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"]
}
}],
"function_call": "auto"
}))
.expect("legacy function request parses");
validate_chat_request(&request).expect("legacy function request validates");
let internal = convert_chat_request(&request).expect("convert");
assert!(
internal
.prompt
.contains("<|function|>\n{\"forecast\":\"sunny\"}</s>"),
"legacy function role should be preserved in fallback template: {}",
internal.prompt
);
assert_eq!(
internal.metadata["openai_legacy_functions"][0]["name"],
"weather"
);
assert_eq!(internal.metadata["openai_legacy_function_call"], "auto");
let Some(ferrum_types::ApiRequest::Chat(api)) = internal.api_request.as_ref() else {
panic!("expected structured chat api_request");
};
assert_eq!(api.messages.len(), 3);
assert_eq!(api.messages[2].role, ferrum_types::ApiMessageRole::Function);
assert_eq!(api.messages[2].name.as_deref(), Some("weather"));
assert_eq!(
api.messages[1]
.function_call
.as_ref()
.map(|call| call.name.as_str()),
Some("weather")
);
assert_eq!(api.legacy_functions[0].name, "weather");
assert_eq!(
api.legacy_function_call,
Some(ferrum_types::ApiFunctionCallChoice::Mode("auto".into()))
);
}
#[test]
fn specific_legacy_function_call_parses_into_structured_api_request() {
let request: ChatCompletionsRequest = serde_json::from_value(json!({
"model": "mystery-model",
"messages": [{"role": "user", "content": "Use the selected function."}],
"functions": [
{"name": "weather", "parameters": {"type": "object"}},
{"name": "calendar", "parameters": {"type": "object"}}
],
"function_call": {"name": "weather"}
}))
.expect("specific function_call request parses");
validate_chat_request(&request).expect("specific function_call validates");
let internal = convert_chat_request(&request).expect("convert");
assert!(internal.prompt.contains("\"function_call\":{"));
assert!(internal.prompt.contains("\"name\":\"weather\""));
let Some(ferrum_types::ApiRequest::Chat(api)) = internal.api_request.as_ref() else {
panic!("expected structured chat api_request");
};
assert_eq!(
api.legacy_function_call,
Some(ferrum_types::ApiFunctionCallChoice::Function {
name: "weather".to_string(),
})
);
let invalid: ChatCompletionsRequest = serde_json::from_value(json!({
"model": "mystery-model",
"messages": [{"role": "user", "content": "Use the selected function."}],
"functions": [{"name": "weather", "parameters": {"type": "object"}}],
"function_call": {"name": "calendar"}
}))
.expect("invalid specific function_call request parses");
let err = validate_chat_request(&invalid).expect_err("undeclared function should reject");
match err {
ServerError::InvalidRequest { param, .. } => {
assert_eq!(param.as_deref(), Some("function_call"));
}
other => panic!("expected invalid_request_error for function_call, got {other:?}"),
}
}
#[test]
fn stream_text_delta_handles_unicode_boundaries() {
let mut sent_len = 0usize;
assert_eq!(stream_text_delta("你好", &mut sent_len), "你好");
assert_eq!(sent_len, "你好".len());
assert_eq!(stream_text_delta("你好世界", &mut sent_len), "世界");
assert_eq!(sent_len, "你好世界".len());
}
#[test]
fn stream_text_delta_recovers_from_non_boundary_offset() {
let mut sent_len = 1usize;
assert_eq!(stream_text_delta("你好", &mut sent_len), "");
assert_eq!(sent_len, "你好".len());
}
#[test]
fn assistant_tool_call_serializes_openai_shape() {
let message = ChatMessage {
role: MessageRole::Assistant,
content: String::new(),
reasoning: None,
name: None,
tool_calls: Some(vec![ChatToolCall {
index: None,
id: "call_1".to_string(),
tool_type: "function".to_string(),
function: ChatFunctionCall {
name: "weather".to_string(),
arguments: "{\"city\":\"Paris\"}".to_string(),
},
}]),
tool_call_id: None,
function_call: None,
};
let value = serde_json::to_value(message).expect("serialize");
assert_eq!(value["role"], "assistant");
assert_eq!(value["tool_calls"][0]["type"], "function");
assert_eq!(value["tool_calls"][0]["function"]["name"], "weather");
}
#[test]
fn unsupported_multimodal_content_is_not_silently_dropped() {
let err = serde_json::from_value::<ChatCompletionsRequest>(json!({
"model": "stub-model",
"messages": [{
"role": "user",
"content": [
{"type": "text", "text": "describe this"},
{"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}}
]
}]
}))
.expect_err("unsupported content part should fail parsing");
assert!(
err.to_string()
.contains("unsupported message content part type"),
"unexpected error: {err}"
);
}
#[tokio::test]
async fn completions_endpoint_uses_stub_engine() {
let request = CompletionsRequest {
model: "stub-model".to_string(),
prompt: CompletionPrompt::Text("complete me".to_string()),
max_tokens: Some(8),
temperature: Some(0.0),
top_p: None,
n: None,
stream: None,
stop: None,
logprobs: None,
logit_bias: None,
};
let response = completions_handler(State(state_with_stub("done")), Ok(Json(request)))
.await
.expect("completion response");
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(body["object"], "text_completion");
assert_eq!(body["choices"][0]["text"], "done");
assert_eq!(body["usage"]["prompt_tokens"], 7);
assert_eq!(body["usage"]["completion_tokens"], 2);
}
#[tokio::test]
async fn route_completions_rejects_non_string_prompt_with_field_param() {
for prompt in [
json!(["a", "b"]),
json!({"text": "complete me"}),
Value::Null,
] {
let response = post_json(
router_with_stub("unused"),
"/v1/completions",
json!({
"model": "stub-model",
"prompt": prompt
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "prompt");
}
let response = post_json(
router_with_stub("unused"),
"/v1/completions",
json!({"model": "stub-model"}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert_eq!(body["error"]["param"], "prompt");
}
#[tokio::test]
async fn stream_options_without_stream_is_invalid() {
let request = chat_request(json!({"stream_options": {"include_usage": true}}));
let err = chat_completions_handler(
State(state_with_stub("unused")),
HeaderMap::new(),
Ok(Json(request)),
)
.await
.expect_err("stream_options without stream should reject");
let (status, body) = error_json(err).await;
assert_eq!(status, AxumStatusCode::BAD_REQUEST);
assert_eq!(body["error"]["param"], "stream_options");
assert_eq!(body["error"]["type"], "invalid_request_error");
}
#[tokio::test]
async fn unknown_stream_option_is_rejected_instead_of_ignored() {
let response = post_json(
router_with_stub("unused"),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "hello"}],
"stream": true,
"stream_options": {"continuous_usage_stats": true}
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::BAD_REQUEST);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "invalid_request_error");
assert!(
body["error"]["message"]
.as_str()
.unwrap_or_default()
.contains("invalid chat completions request"),
"body: {body}"
);
}
#[tokio::test]
async fn json_object_rejects_markdown_fence_instead_of_repairing() {
let request = chat_request(json!({
"response_format": {"type": "json_object"}
}));
let err = chat_completions_handler(
State(state_with_stub("```json\n{\"answer\":\"yes\"}\n```")),
HeaderMap::new(),
Ok(Json(request)),
)
.await
.expect_err("fenced json_object must fail");
let (status, body) = error_json(err).await;
assert_eq!(status, AxumStatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(body["error"]["type"], "internal_server_error");
assert!(body["error"]["message"]
.as_str()
.unwrap_or_default()
.contains("response_format.json_object: invalid JSON"));
}
#[tokio::test]
async fn streaming_json_object_buffers_thinking_and_emits_clean_json_content() {
let response = post_json(
router_with_stub_stream_chunks(&[
"<think>\n好的，我需要输出 JSON。",
"\n</think>\n\n",
"{\"name\":\"李四\",\"age\":30}",
]),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "输出JSON(name,age)：李四,30岁"}],
"stream": true,
"response_format": {"type": "json_object"}
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(body.contains("data: [DONE]"), "missing DONE: {body}");
assert!(
body.contains(r#""content":"{\"name\":\"李四\",\"age\":30}""#),
"stream should emit clean JSON content: {body}"
);
assert!(
body.contains(r#""reasoning":"\n好的，我需要输出 JSON。\n""#),
"stream should keep thinking in reasoning field: {body}"
);
assert!(
!body.contains(r#""content":"<think"#)
&& !body.contains(r#""content":"好的"#)
&& !body.contains(r#""content":"我需要"#),
"thinking text must not leak as streamed content: {body}"
);
}
fn prompt_opened_literal_json_template() -> ModelChatTemplate {
let template = ModelChatTemplate::new(
"{% for message in messages %}[{{ message.role }}]{{ message.content }}{% endfor %}{% if add_generation_prompt %}<assistant><think>{% endif %}",
"prompt-opened-text-test",
);
assert_eq!(template.output_protocol, ModelOutputProtocol::Text);
assert_eq!(
template.reasoning_protocol,
ModelReasoningProtocol::PromptOpened
);
let request = chat_request(json!({"response_format": {"type": "json_object"}}));
let internal =
convert_chat_request_with_template_model(&request, "stub-model", Some(&template))
.expect("convert prompt-opened Text request");
assert!(internal.prompt.ends_with("<think>"));
template
}
#[tokio::test]
async fn json_object_preserves_literal_think_tags_after_prompt_opened_reasoning_sync() {
let response = post_json(
router_with_stub_and_template(
"reason</think>\n{\"text\":\"<think>literal</think>\"}",
prompt_opened_literal_json_template(),
),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Return a JSON object."}],
"response_format": {"type": "json_object"}
}),
)
.await;
let status = response.status();
let body = response_json(response).await;
assert_eq!(status, AxumStatusCode::OK, "{body}");
assert!(body.get("error").is_none(), "{body}");
let message = &body["choices"][0]["message"];
assert_eq!(message["content"], r#"{"text":"<think>literal</think>"}"#);
assert_eq!(message["reasoning"], "reason");
assert_eq!(body["choices"][0]["finish_reason"], "stop");
}
#[tokio::test]
async fn json_object_preserves_literal_think_tags_after_prompt_opened_reasoning_sse() {
for chunks in [
vec!["reason</think>\n{\"text\":\"<think>literal</think>\"}"],
vec![
"reason</thi",
"nk>\n{\"text\":\"<thi",
"nk>literal</thi",
"nk>\"}",
],
] {
let router = AxumServer::from_llm(Arc::new(StubLlm::with_stream_chunks(&chunks)))
.with_prompt_template(Some(prompt_opened_literal_json_template()))
.build_router();
let response = post_json(
router,
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Return a JSON object."}],
"stream": true,
"stream_options": {"include_usage": true},
"response_format": {"type": "json_object"}
}),
)
.await;
let status = response.status();
let body = response_text(response).await;
assert_eq!(status, AxumStatusCode::OK, "{body}");
let normalized = body.replace("\r\n", "\n");
assert_eq!(normalized.matches("data: [DONE]").count(), 1, "{body}");
assert!(normalized.ends_with("data: [DONE]\n\n"), "{body}");
let events = responses_sse_json_events(&body);
assert!(
events.iter().all(|event| event.get("error").is_none()),
"{body}"
);
let content: String = events
.iter()
.filter_map(|event| event["choices"][0]["delta"]["content"].as_str())
.collect();
let reasoning: String = events
.iter()
.filter_map(|event| event["choices"][0]["delta"]["reasoning"].as_str())
.collect();
assert_eq!(content, r#"{"text":"<think>literal</think>"}"#);
assert_eq!(reasoning, "reason");
assert_eq!(
serde_json::from_str::<Value>(&content).expect("intact JSON body"),
json!({"text": "<think>literal</think>"})
);
let terminals: Vec<_> = events
.iter()
.enumerate()
.filter(|(_, event)| !event["choices"][0]["finish_reason"].is_null())
.collect();
assert_eq!(terminals.len(), 1, "{body}");
let (terminal_index, terminal) = terminals[0];
assert_eq!(terminal["choices"][0]["finish_reason"], "stop");
for event in &events[terminal_index..] {
for field in ["content", "reasoning", "reasoning_content"] {
assert!(
event["choices"][0]["delta"][field]
.as_str()
.unwrap_or_default()
.is_empty(),
"payload after terminal: {event}"
);
}
}
let usages: Vec<_> = events
.iter()
.enumerate()
.filter(|(_, event)| !event["usage"].is_null())
.collect();
assert_eq!(usages.len(), 1, "{body}");
let (usage_index, usage) = usages[0];
assert!(terminal_index < usage_index, "{body}");
assert_eq!(usage_index, events.len() - 1, "usage must be last: {body}");
assert_eq!(usage["choices"], json!([]));
}
}
#[tokio::test]
async fn json_object_rejects_non_json_model_output() {
let request = chat_request(json!({
"response_format": {"type": "json_object"}
}));
let err = chat_completions_handler(
State(state_with_stub("not json")),
HeaderMap::new(),
Ok(Json(request)),
)
.await
.expect_err("invalid json_object must fail");
let (status, body) = error_json(err).await;
assert_eq!(status, AxumStatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(body["error"]["type"], "internal_server_error");
assert!(body["error"]["message"]
.as_str()
.unwrap_or_default()
.contains("response_format.json_object"));
}
#[test]
fn one_of_strict_json_schema_reaches_hard_decoder() {
let request = chat_request(json!({
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "unsupported",
"strict": true,
"schema": {"oneOf": [{"type": "string"}, {"type": "integer"}]}
}
}
}));
validate_chat_request(&request).expect("oneOf strict schema should validate");
let internal = convert_chat_request(&request).expect("convert oneOf strict schema");
let ferrum_types::ResponseFormat::JsonSchema(schema) =
internal.sampling_params.response_format
else {
panic!("strict schema did not reach hard decoder");
};
assert_eq!(
serde_json::from_str::<serde_json::Value>(&schema).unwrap()["oneOf"],
json!([{"type": "string"}, {"type": "integer"}])
);
let schema = serde_json::from_str::<serde_json::Value>(&schema).unwrap();
validate_json_text_against_schema(&schema, r#""answer""#)
.expect("oneOf string branch should pass final validation");
validate_json_text_against_schema(&schema, "7")
.expect("oneOf integer branch should pass final validation");
assert!(validate_json_text_against_schema(&schema, "true").is_err());
}
#[tokio::test]
async fn missing_json_schema_schema_rejects_with_field_param() {
let request = chat_request(json!({
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "missing_schema",
"strict": true
}
}
}));
let err = chat_completions_handler(
State(state_with_stub("unused")),
HeaderMap::new(),
Ok(Json(request)),
)
.await
.expect_err("missing strict schema should reject");
let (status, body) = error_json(err).await;
assert_eq!(status, AxumStatusCode::BAD_REQUEST);
assert_eq!(body["error"]["param"], "response_format.json_schema");
assert_eq!(body["error"]["type"], "invalid_request_error");
assert!(body["error"]["message"]
.as_str()
.unwrap()
.contains("schema is required"));
}
#[test]
fn non_strict_json_schema_is_preserved_but_not_hard_masked() {
let request = chat_request(json!({
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "best_effort",
"strict": false,
"schema": {"oneOf": [{"type": "string"}, {"type": "integer"}]}
}
}
}));
validate_chat_request(&request).expect("non-strict schema should not boundary reject");
let internal = convert_chat_request(&request).expect("convert non-strict schema");
assert!(
internal
.prompt
.contains("response_format requires a single valid JSON value"),
"response_format instruction should reach the model prompt: {}",
internal.prompt
);
assert!(
internal.prompt.contains("\"oneOf\""),
"schema should reach the model prompt: {}",
internal.prompt
);
assert_eq!(
internal.sampling_params.response_format,
ferrum_types::ResponseFormat::Text,
"non-strict json_schema must stay best-effort instead of enabling hard guided decode"
);
let Some(ferrum_types::ApiRequest::Chat(api)) = internal.api_request.as_ref() else {
panic!("expected structured chat api_request");
};
assert_eq!(
api.response_format
.as_ref()
.and_then(|format| format.json_schema.as_ref())
.and_then(|schema| schema.strict),
Some(false)
);
}
#[test]
fn json_object_response_format_instruction_reaches_model_prompt() {
let request = chat_request(json!({
"response_format": {"type": "json_object"}
}));
let internal = convert_chat_request(&request).expect("convert json_object");
assert!(
internal
.prompt
.contains("response_format requires a single valid JSON object"),
"response_format instruction should reach the model prompt: {}",
internal.prompt
);
assert!(
internal.prompt.contains("Output only JSON"),
"JSON-only instruction should reach the model prompt: {}",
internal.prompt
);
assert_eq!(
internal.sampling_params.response_format,
ferrum_types::ResponseFormat::JsonObject,
"json_object must reach the tokenizer-aware hard decoder"
);
assert_eq!(
internal.sampling_params.structured_output_start,
StructuredOutputStart::Immediate
);
}
fn harmony_json_template() -> ModelChatTemplate {
let mut template = ModelChatTemplate::new(
"{% for message in messages %}<|start|>{{ message.role }}<|message|>{{ message.content }}<|end|>{% endfor %}{% if add_generation_prompt %}<|start|>assistant{% endif %}",
"harmony-structured-template",
);
template.output_protocol = ModelOutputProtocol::HarmonyGptOss;
template
}
#[test]
fn harmony_structured_format_activates_at_final_payload() {
let template = harmony_json_template();
for (response_format, constrained) in [
(json!({"type": "json_object"}), true),
(
json!({
"type": "json_schema",
"json_schema": {
"name": "answer",
"strict": true,
"schema": {
"type": "object",
"properties": {"answer": {"type": "integer"}},
"required": ["answer"],
"additionalProperties": false
}
}
}),
true,
),
(
json!({
"type": "json_schema",
"json_schema": {
"name": "best_effort",
"strict": false,
"schema": {"type": "object"}
}
}),
false,
),
(json!({"type": "text"}), false),
] {
let request = chat_request(json!({"response_format": response_format}));
let internal =
convert_chat_request_with_template_model(&request, "stub-model", Some(&template))
.unwrap();
assert_eq!(
internal.sampling_params.structured_output_start,
if constrained {
StructuredOutputStart::HarmonyFinal
} else {
StructuredOutputStart::Immediate
}
);
assert_eq!(
internal.sampling_params.response_completion_boundary,
ResponseCompletionBoundary::Immediate,
"Harmony framing must not be gated on a Text reasoning delimiter"
);
internal.sampling_params.validate().unwrap();
}
}
fn harmony_json_request(stream: bool) -> Value {
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Return an answer object."}],
"stream": stream,
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "answer",
"strict": true,
"schema": {
"type": "object",
"properties": {"answer": {"type": "integer"}},
"required": ["answer"],
"additionalProperties": false
}
}
}
})
}
#[tokio::test]
async fn harmony_strict_json_routes_validate_final_payload_in_sync_and_sse() {
for (chunks, finish_reason, reasoning) in [
(
vec![
"<|channel|>fi",
"nal<|message|>{\"answer\":",
"42}<|return|>",
],
FinishReason::EOS,
"",
),
(
vec![
"<|channel|>analysis<|message|>Compute.",
"<|end|><|start|>assistant<|channel|>fi",
"nal<|message|>{\"answer\":42}<|return|>",
],
FinishReason::EOS,
"Compute.",
),
(
vec!["<|channel|>final<|message|>{\"answer\":", "42}"],
FinishReason::Length,
"",
),
(
vec!["<|channel|>final<|message|>{\"answer\":", "42}"],
FinishReason::Stop,
"",
),
] {
for stream in [false, true] {
let engine = StubLlm {
finish_reason,
..StubLlm::with_stream_chunks(&chunks)
};
let router = AxumServer::from_llm(Arc::new(engine))
.with_prompt_template(Some(harmony_json_template()))
.build_router();
let mut request = harmony_json_request(stream);
if finish_reason == FinishReason::Stop {
// An explicit stop at the model's terminal removes that
// marker while preserving the already complete JSON value.
request["stop"] = json!(["<|return|>"]);
}
let response = post_json(router, "/v1/chat/completions", request).await;
assert_eq!(response.status(), AxumStatusCode::OK);
if stream {
let body = response_text(response).await;
assert!(body.contains("data: [DONE]"));
let events = responses_sse_json_events(&body);
assert!(events.iter().all(|event| event.get("error").is_none()));
let content: String = events
.iter()
.filter_map(|event| event["choices"][0]["delta"]["content"].as_str())
.collect();
let actual_reasoning: String = events
.iter()
.filter_map(|event| event["choices"][0]["delta"]["reasoning"].as_str())
.collect();
assert_eq!(
serde_json::from_str::<Value>(&content).unwrap(),
json!({"answer": 42})
);
assert_eq!(actual_reasoning, reasoning);
} else {
let body = response_json(response).await;
let message = &body["choices"][0]["message"];
assert_eq!(message["content"], "{\"answer\":42}");
assert_eq!(message["reasoning"].as_str().unwrap_or(""), reasoning);
}
}
}
}
#[tokio::test]
async fn harmony_strict_json_routes_reject_bad_framing_and_payload_without_sse_leaks() {
for output in [
"{\"answer\":42}",
"<|channel|>final<|message|>{\"answer\":42}",
"<|channel|>final<|message|>{\"answer\":42}<|call|>",
"<|channel|>analysis<|message|>Compute.<|end|>\
<|start|>assistant<|channel|>final<|message|>{\"answer\":\"wrong\"}<|return|>",
] {
for stream in [false, true] {
let response = post_json(
router_with_stub_and_template(output, harmony_json_template()),
"/v1/chat/completions",
harmony_json_request(stream),
)
.await;
if stream {
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(body.contains("data: [DONE]"));
let events = responses_sse_json_events(&body);
assert!(events.iter().any(|event| event.get("error").is_some()));
for event in events {
for field in ["content", "reasoning"] {
assert!(event["choices"][0]["delta"][field]
.as_str()
.unwrap_or("")
.is_empty());
}
}
} else {
assert_eq!(response.status(), AxumStatusCode::INTERNAL_SERVER_ERROR);
let body = response_json(response).await;
assert_eq!(body["error"]["type"], "internal_server_error");
assert!(body.get("choices").is_none());
}
}
}
}
#[test]
fn json_object_thinking_template_activates_after_typed_end_delimiter() {
let request = chat_request(json!({
"response_format": {"type": "json_object"}
}));
let template = ModelChatTemplate::new(
"{% for message in messages %}[{{ message.role }}]{{ message.content }}{% endfor %}{% if add_generation_prompt %}<assistant><think>\n{% endif %}",
"thinking-test-template",
);
assert_eq!(
template.reasoning_protocol,
ModelReasoningProtocol::PromptOpened
);
let internal =
convert_chat_request_with_template_model(&request, "stub-model", Some(&template))
.expect("convert thinking json_object");
assert!(internal.prompt.ends_with("<assistant><think>\n"));
assert!(internal
.prompt
.contains("Output only JSON, with no markdown fences, no explanation, no chain-of-thought, and no extra text"));
assert!(
!internal.prompt.contains(THINK_END_TAG),
"the instruction must not echo the typed end delimiter: {}",
internal.prompt
);
assert_eq!(
internal.sampling_params.structured_output_start,
StructuredOutputStart::AfterDelimiter(THINK_END_TAG.to_string())
);
assert_eq!(
internal.sampling_params.response_completion_boundary,
ResponseCompletionBoundary::AfterDelimiterAndPayload {
delimiter: THINK_END_TAG.to_string(),
alternate_envelope: None,
}
);
}
#[test]
fn json_object_model_generated_thinking_activates_after_typed_end_delimiter() {
let request = chat_request(json!({
"response_format": {"type": "json_object"},
"chat_template_kwargs": {"enable_thinking": true}
}));
let template = ModelChatTemplate::new(
"{% for message in messages %}[{{ message.role }}]{{ message.content }}{% endfor %}{% if add_generation_prompt %}<assistant>{% if enable_thinking is defined and enable_thinking is false %}<think>\n\n</think>\n\n{% endif %}{% endif %}",
"qwen3-model-generated-thinking-template",
);
let internal =
convert_chat_request_with_template_model(&request, "stub-model", Some(&template))
.expect("convert model-generated thinking json_object");
assert!(!has_unclosed_thinking_block(&internal.prompt));
assert!(internal.prompt.ends_with("<assistant>"));
assert!(internal
.prompt
.contains("Output only JSON, with no markdown fences, no explanation, no chain-of-thought, and no extra text"));
assert!(
!internal.prompt.contains(THINK_START_TAG) && !internal.prompt.contains(THINK_END_TAG),
"the instruction must not teach the model the typed reasoning delimiter: {}",
internal.prompt
);
assert_eq!(
internal.sampling_params.structured_output_start,
StructuredOutputStart::AfterDelimiter(THINK_END_TAG.to_string())
);
assert_eq!(
internal.sampling_params.response_completion_boundary,
ResponseCompletionBoundary::AfterDelimiterAndPayload {
delimiter: THINK_END_TAG.to_string(),
alternate_envelope: None,
}
);
}
#[test]
fn strict_schema_model_generated_thinking_does_not_echo_typed_delimiter() {
let request = chat_request(json!({
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "reasoning_result",
"strict": true,
"schema": {
"type": "object",
"properties": {
"result": {"type": "string", "const": "G00-c21-schema-OK"}
},
"required": ["result"],
"additionalProperties": false
}
}
},
"chat_template_kwargs": {"enable_thinking": true}
}));
let template = ModelChatTemplate::new(
"{% for message in messages %}[{{ message.role }}]{{ message.content }}{% endfor %}{% if add_generation_prompt %}<assistant>{% if enable_thinking is defined and enable_thinking is false %}<think>\n\n</think>\n\n{% endif %}{% endif %}",
"qwen3-model-generated-thinking-template",
);
let internal =
convert_chat_request_with_template_model(&request, "stub-model", Some(&template))
.expect("convert model-generated thinking strict schema");
assert!(!has_unclosed_thinking_block(&internal.prompt));
assert!(internal.prompt.ends_with("<assistant>"));
assert!(internal.prompt.contains("G00-c21-schema-OK"));
assert!(
!internal.prompt.contains(THINK_START_TAG) && !internal.prompt.contains(THINK_END_TAG),
"the instruction must not teach the model the typed reasoning delimiter: {}",
internal.prompt
);
assert_eq!(
internal.sampling_params.structured_output_start,
StructuredOutputStart::AfterDelimiter(THINK_END_TAG.to_string())
);
assert_eq!(
internal.sampling_params.response_completion_boundary,
ResponseCompletionBoundary::AfterDelimiterAndPayload {
delimiter: THINK_END_TAG.to_string(),
alternate_envelope: None,
}
);
}
#[test]
fn json_object_model_generated_thinking_hard_off_starts_immediately() {
let request = chat_request(json!({
"response_format": {"type": "json_object"},
"chat_template_kwargs": {"enable_thinking": false}
}));
let template = ModelChatTemplate::new(
"{% for message in messages %}[{{ message.role }}]{{ message.content }}{% endfor %}{% if add_generation_prompt %}<assistant>{% if enable_thinking is defined and enable_thinking is false %}<think>\n\n</think>\n\n{% endif %}{% endif %}",
"qwen3-model-generated-thinking-template",
);
let internal =
convert_chat_request_with_template_model(&request, "stub-model", Some(&template))
.expect("convert disabled model-generated thinking json_object");
assert_eq!(
internal.sampling_params.structured_output_start,
StructuredOutputStart::Immediate
);
assert_eq!(
internal.sampling_params.response_completion_boundary,
ResponseCompletionBoundary::Immediate
);
assert!(internal.prompt.contains("no chain-of-thought"));
}
#[test]
fn response_completion_contract_is_set_on_text_thinking_template() {
let request = chat_request(json!({}));
let template = ModelChatTemplate::new(
"{% if add_generation_prompt %}<assistant><think>\n{% endif %}",
"thinking-test-template",
);
let internal =
convert_chat_request_with_template_model(&request, "stub-model", Some(&template))
.expect("convert thinking text request");
assert_eq!(
internal.sampling_params.structured_output_start,
StructuredOutputStart::Immediate
);
assert_eq!(
internal.sampling_params.response_completion_boundary,
ResponseCompletionBoundary::AfterDelimiterAndPayload {
delimiter: THINK_END_TAG.to_string(),
alternate_envelope: None,
}
);
}
#[test]
fn thinking_tool_request_compiles_typed_envelope_into_completion_contract() {
let request = chat_request(json!({
"tools": [{
"type": "function",
"function": {
"name": "weather",
"parameters": {
"type": "object",
"properties": {"city": {"type": "string"}},
"required": ["city"]
}
}
}]
}));
let template = ModelChatTemplate::new(
"{% if tools %}<tool_call><function=name><parameter=key>value</parameter></function></tool_call>{% endif %}{% if add_generation_prompt %}<assistant><think>\n{% endif %}",
"thinking-tool-template",
);
let internal =
convert_chat_request_with_template_model(&request, "stub-model", Some(&template))
.expect("convert thinking tool request");
assert_eq!(
internal.sampling_params.response_completion_boundary,
ResponseCompletionBoundary::AfterDelimiterAndPayload {
delimiter: THINK_END_TAG.to_string(),
alternate_envelope: Some(ferrum_types::ResponseCompletionEnvelope {
open_token_text: "<tool_call>".to_string(),
close_token_text: "</tool_call>".to_string(),
max_envelopes: 32,
}),
}
);
}
#[test]
fn strict_json_schema_response_format_uses_guided_sampling_mode() {
let request = chat_request(json!({
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "answer",
"strict": true,
"schema": {
"type": "object",
"properties": {"answer": {"type": "string"}},
"required": ["answer"]
}
}
}
}));
let internal = convert_chat_request(&request).expect("convert strict json_schema");
assert!(
internal
.prompt
.contains("response_format requires a single valid JSON value"),
"response_format instruction should reach the model prompt: {}",
internal.prompt
);
let ferrum_types::ResponseFormat::JsonSchema(schema) =
internal.sampling_params.response_format
else {
panic!(
"strict json_schema must reach guided decoding, got {:?}",
internal.sampling_params.response_format
);
};
let schema: serde_json::Value = serde_json::from_str(&schema).unwrap();
assert_eq!(schema["type"], "object");
assert_eq!(schema["properties"]["answer"]["type"], "string");
assert_eq!(schema["required"], json!(["answer"]));
}
#[tokio::test]
async fn strict_json_schema_validates_non_streaming_response() {
let request = chat_request(json!({
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "answer",
"strict": true,
"schema": {
"type": "object",
"properties": {"answer": {"type": "string"}},
"required": ["answer"]
}
}
}
}));
let response = chat_completions_handler(
State(state_with_stub("{\"answer\":\"yes\"}")),
HeaderMap::new(),
Ok(Json(request)),
)
.await
.expect("strict response");
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(
body["choices"][0]["message"]["content"],
"{\"answer\":\"yes\"}"
);
}
#[tokio::test]
async fn strict_json_schema_validates_non_streaming_response_after_reasoning_block() {
let request = chat_request(json!({
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "answer",
"strict": true,
"schema": {
"type": "object",
"properties": {"answer": {"type": "string"}},
"required": ["answer"]
}
}
}
}));
let response = chat_completions_handler(
State(state_with_stub(
"<think>\nreasoning\n</think>\n\n{\"answer\":\"yes\"}",
)),
HeaderMap::new(),
Ok(Json(request)),
)
.await
.expect("strict response with reasoning");
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_json(response).await;
assert_eq!(
body["choices"][0]["message"]["content"],
"{\"answer\":\"yes\"}"
);
assert_eq!(body["choices"][0]["message"]["reasoning"], "\nreasoning\n");
}
#[tokio::test]
async fn strict_json_schema_validates_streaming_final_response() {
let response = post_json(
router_with_stub("{\"answer\":\"yes\"}"),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Return an answer object."}],
"stream": true,
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "answer",
"strict": true,
"schema": {
"type": "object",
"properties": {"answer": {"type": "string"}},
"required": ["answer"]
}
}
}
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(body.contains("data: [DONE]"), "missing DONE: {body}");
assert!(
body.contains("\\\"answer\\\":\\\"yes\\\""),
"strict streaming content missing: {body}"
);
assert!(
!body.contains("\"error\""),
"valid strict streaming response should not emit error: {body}"
);
}
#[tokio::test]
async fn strict_json_schema_validates_streaming_final_response_after_reasoning_block() {
let response = post_json(
router_with_stub_stream_chunks(&[
"<think>\nreasoning",
"\n</think>\n\n",
"{\"answer\":\"yes\"}",
]),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Return an answer object."}],
"stream": true,
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "answer",
"strict": true,
"schema": {
"type": "object",
"properties": {"answer": {"type": "string"}},
"required": ["answer"]
}
}
}
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(body.contains("data: [DONE]"), "missing DONE: {body}");
assert!(
body.contains("\\\"answer\\\":\\\"yes\\\""),
"strict streaming content missing: {body}"
);
assert!(
body.contains(r#""reasoning":"\nreasoning\n""#),
"strict streaming should keep reasoning separate: {body}"
);
assert!(
!body.contains("\"error\""),
"valid strict streaming response should not emit error: {body}"
);
}
#[tokio::test]
async fn strict_json_schema_invalid_streaming_output_emits_error_event() {
let response = post_json(
router_with_stub("not json"),
"/v1/chat/completions",
json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Return an answer object."}],
"stream": true,
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "answer",
"strict": true,
"schema": {
"type": "object",
"properties": {"answer": {"type": "string"}},
"required": ["answer"]
}
}
}
}),
)
.await;
assert_eq!(response.status(), AxumStatusCode::OK);
let body = response_text(response).await;
assert!(body.contains("data: [DONE]"), "missing DONE: {body}");
assert!(
body.contains("\"type\":\"internal_server_error\""),
"strict streaming validation failure should emit OpenAI error: {body}"
);
assert!(
body.contains("\"param\":\"response_format.json_schema\""),
"strict streaming validation error should identify schema param: {body}"
);
assert!(
body.contains("invalid JSON"),
"strict streaming validation should report invalid JSON: {body}"
);
assert!(
!body.contains("not json"),
"strict streaming must not emit invalid partial deltas before validation failure: {body}"
);
}
#[tokio::test]
async fn route_strict_json_schema_supported_schema_passes_100_runs() {
let request_body = json!({
"model": "stub-model",
"messages": [{"role": "user", "content": "Return an answer object."}],
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "answer",
"strict": true,
"schema": {
"type": "object",
"properties": {"answer": {"type": "string"}},
"required": ["answer"]
}
}
}
});
let router = router_with_stub("{\"answer\":\"yes\"}");
for run in 0..100 {
let response =
post_json(router.clone(), "/v1/chat/completions", request_body.clone()).await;
assert_eq!(
response.status(),
AxumStatusCode::OK,
"strict schema run {run} returned non-200"
);
let body = response_json(response).await;
let content = body["choices"][0]["message"]["content"]
.as_str()
.unwrap_or("");
assert_eq!(
content, "{\"answer\":\"yes\"}",
"strict schema run {run} returned unexpected content"
);
let parsed: serde_json::Value =
serde_json::from_str(content).expect("strict content JSON");
assert_eq!(parsed["answer"], "yes");
}
}
#[test]
fn cache_metrics_use_engine_real_kv_snapshot_when_available() {
let cache = CacheRuntimeState::default();
let policy = CachePolicy {
prefix_cache_enabled: true,
session_cache_mode: "memory".to_string(),
session_cache_max_entries: 128,
session_cache_max_tokens: 4096,
};
cache.record_prefix_prompt("alpha beta gamma", &policy);
cache.record_prefix_prompt("alpha beta delta", &policy);
let engine_snapshot = json!({
"position": "real-kv-reuse",
"source": "llama-family-paged-block-prefix-cache",
"enabled": true,
"hits": 7,
"misses": 3,
"evictions": 1,
"saved_prefill_tokens": 64,
"entries": 5,
"bytes": 8192,
"block_size": 16,
"kv_dtype": "fp16",
"selected_pipeline_mode": "batch",
"selected_stage_bridge": "host",
"stage_count": 2,
});
let health = cache.health_json(&policy, Some(&engine_snapshot));
let prefix = &health["prefix_cache"];
assert_eq!(prefix["position"], "real-kv-reuse");
assert_eq!(prefix["source"], "llama-family-paged-block-prefix-cache");
assert_eq!(prefix["hits"], 7);
assert_eq!(prefix["misses"], 3);
assert_eq!(prefix["evictions"], 1);
assert_eq!(prefix["saved_prefill_tokens"], 64);
assert_eq!(prefix["entries"], 5);
assert_eq!(prefix["bytes"], 8192);
assert_eq!(prefix["block_size"], 16);
assert_eq!(prefix["kv_dtype"], "fp16");
assert_eq!(prefix["selected_pipeline_mode"], "batch");
assert_eq!(prefix["selected_stage_bridge"], "host");
assert_eq!(prefix["stage_count"], 2);
let metrics = cache.prometheus_metrics(Some(&engine_snapshot));
assert!(metrics.contains("ferrum_prefix_cache_hits_total 7\n"));
assert!(metrics.contains("ferrum_prefix_cache_misses_total 3\n"));
assert!(metrics.contains("ferrum_prefix_cache_saved_prefill_tokens_total 64\n"));
assert!(metrics.contains("ferrum_prefix_cache_entries 5\n"));
assert!(metrics.contains("ferrum_prefix_cache_bytes 8192\n"));
}
#[tokio::test]
async fn strict_json_schema_invalid_model_output_fails_before_response() {
let request = chat_request(json!({
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "answer",
"strict": true,
"schema": {
"type": "object",
"properties": {"answer": {"type": "string"}},
"required": ["answer"]
}
}
}
}));
let err = chat_completions_handler(
State(state_with_stub("not json")),
HeaderMap::new(),
Ok(Json(request)),
)
.await
.expect_err("invalid strict response should fail");
let (status, body) = error_json(err).await;
assert_eq!(status, AxumStatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(body["error"]["type"], "internal_server_error");
assert!(body["error"]["message"]
.as_str()
.unwrap()
.contains("json_schema.strict"));
}
#[tokio::test]
async fn strict_json_schema_does_not_rely_on_markdown_fence_stripping() {
let request = chat_request(json!({
"response_format": {
"type": "json_schema",
"json_schema": {
"name": "answer",
"strict": true,
"schema": {
"type": "object",
"properties": {"answer": {"type": "string"}},
"required": ["answer"]
}
}
}
}));
let err = chat_completions_handler(
State(state_with_stub("```json\n{\"answer\":\"yes\"}\n```")),
HeaderMap::new(),
Ok(Json(request)),
)
.await
.expect_err("strict schema should fail fenced JSON instead of repairing it");
let (status, body) = error_json(err).await;
assert_eq!(status, AxumStatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(body["error"]["type"], "internal_server_error");
assert!(body["error"]["message"]
.as_str()
.unwrap()
.contains("json_schema.strict: invalid JSON"));
}
}