use serde::{Deserialize, Serialize};
use crate::entities::sampling::{ReasoningEffort, SamplingConfig};
use crate::shared::api::contract::{ApiMessage, ApiRole, ChatRequest};
use crate::shared::api::effort;
use crate::shared::api::error::EngineError;
#[derive(Debug, Serialize)]
pub struct ChatCompletionRequest {
#[serde(skip_serializing_if = "Option::is_none")]
pub model: Option<String>,
pub messages: Vec<WireMessage>,
pub stream: bool,
#[serde(skip_serializing_if = "Option::is_none")]
pub stream_options: Option<StreamOptions>,
#[serde(skip_serializing_if = "Option::is_none")]
pub temperature: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub dynatemp_range: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub dynatemp_exponent: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub max_tokens: Option<usize>,
#[serde(skip_serializing_if = "Option::is_none")]
pub top_k: Option<i64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub top_p: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub min_p: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub top_n_sigma: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub typical_p: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub adaptive_target: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub adaptive_decay: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub frequency_penalty: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub presence_penalty: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub repeat_penalty: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub repetition_penalty: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub repeat_last_n: Option<i64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub dry_multiplier: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub dry_base: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub dry_allowed_length: Option<i64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub dry_penalty_last_n: Option<i64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub dry_sequence_breakers: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub xtc_probability: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub xtc_threshold: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub mirostat: Option<i64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub mirostat_tau: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub mirostat_eta: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
pub seed: Option<i64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub samplers: Option<Vec<String>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub thinking: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub reasoning_effort: Option<&'static str>,
#[serde(skip_serializing_if = "Option::is_none")]
pub reasoning_budget: Option<i64>,
#[serde(skip_serializing_if = "Option::is_none")]
pub reasoning: Option<WireReasoning>,
#[serde(skip_serializing_if = "Option::is_none")]
pub chat_template_kwargs: Option<serde_json::Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub continue_final_message: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub add_generation_prompt: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub tools: Option<Vec<WireTool>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub tool_choice: Option<&'static str>,
}
#[derive(Debug, Serialize)]
pub struct StreamOptions {
pub include_usage: bool,
}
#[derive(Debug, Serialize)]
pub struct WireMessage {
pub role: &'static str,
#[serde(skip_serializing_if = "Option::is_none")]
pub content: Option<WireContent>,
#[serde(skip_serializing_if = "Option::is_none")]
pub tool_call_id: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub tool_calls: Option<Vec<WireToolCall>>,
}
#[derive(Debug, Serialize)]
#[serde(untagged)]
pub enum WireContent {
Text(String),
Parts(Vec<serde_json::Value>),
}
fn wire_content(m: &ApiMessage) -> WireContent {
if m.images.is_empty() {
return WireContent::Text(m.content.clone());
}
let text_part = |text: &str| serde_json::json!({ "type": "text", "text": text });
let text_leads = m.role == ApiRole::Tool;
let mut parts = Vec::with_capacity(m.images.len() * 2 + 1);
if text_leads && !m.content.is_empty() {
parts.push(text_part(&m.content));
}
for image in &m.images {
if let Some(label) = &image.label {
parts.push(text_part(label));
}
parts.push(serde_json::json!({
"type": "image_url",
"image_url": { "url": format!("data:{};base64,{}", image.mime, image.data) },
}));
}
if !text_leads && !m.content.is_empty() {
parts.push(text_part(&m.content));
}
WireContent::Parts(parts)
}
pub fn carries_tool_images(messages: &[ApiMessage]) -> bool {
messages
.iter()
.any(|m| m.role == ApiRole::Tool && !m.images.is_empty())
}
pub fn rehome_tool_images(messages: &[ApiMessage]) -> Option<Vec<ApiMessage>> {
if !carries_tool_images(messages) {
return None;
}
let mut out = Vec::with_capacity(messages.len() + 1);
let mut moved: Vec<crate::shared::api::ApiImage> = Vec::new();
for m in messages {
if m.role != ApiRole::Tool && !moved.is_empty() {
out.push(ApiMessage::user("").with_images(std::mem::take(&mut moved)));
}
let mut m = m.clone();
if m.role == ApiRole::Tool {
moved.append(&mut m.images);
}
out.push(m);
}
if !moved.is_empty() {
out.push(ApiMessage::user("").with_images(moved));
}
Some(out)
}
#[derive(Debug, Serialize)]
pub struct WireTool {
#[serde(rename = "type")]
pub kind: &'static str,
pub function: WireFunction,
}
#[derive(Debug, Serialize)]
pub struct WireFunction {
pub name: String,
pub description: String,
pub parameters: serde_json::Value,
}
fn wire_tools(tools: &[crate::shared::api::contract::ToolSchema]) -> Vec<WireTool> {
tools
.iter()
.map(|t| WireTool {
kind: "function",
function: WireFunction {
name: t.name.clone(),
description: t.description.clone(),
parameters: t.parameters.clone(),
},
})
.collect()
}
pub fn tools_json(tools: &[crate::shared::api::contract::ToolSchema]) -> String {
serde_json::to_string(&wire_tools(tools)).unwrap_or_default()
}
#[derive(Debug, Serialize)]
pub struct WireToolCall {
pub id: String,
#[serde(rename = "type")]
pub kind: &'static str,
pub function: WireFunctionCall,
}
#[derive(Debug, Serialize)]
pub struct WireFunctionCall {
pub name: String,
pub arguments: String,
}
pub fn build_chat_request(
req: &ChatRequest,
stream: bool,
model: Option<&str>,
omit_effort_none: bool,
) -> ChatCompletionRequest {
let mut messages = Vec::with_capacity(req.messages.len() + 1);
if let Some(system) = &req.system {
messages.push(WireMessage {
role: "system",
content: Some(WireContent::Text(system.clone())),
tool_call_id: None,
tool_calls: None,
});
}
for m in &req.messages {
let tool_calls = if m.tool_calls.is_empty() {
None
} else {
Some(
m.tool_calls
.iter()
.map(|tc| WireToolCall {
id: tc.id.clone(),
kind: "function",
function: WireFunctionCall {
name: tc.name.clone(),
arguments: tc.arguments.clone(),
},
})
.collect(),
)
};
messages.push(WireMessage {
role: m.role.as_wire(),
content: Some(wire_content(m)),
tool_call_id: m.tool_call_id.clone(),
tool_calls,
});
}
let tools = if req.tools.is_empty() {
None
} else {
Some(wire_tools(&req.tools))
};
let tool_choice = tools.as_ref().map(|_| "auto");
let s = &req.sampling;
let chat_template_kwargs = (s.reasoning_budget == Some(0) || req.continue_final)
.then(|| serde_json::json!({ "enable_thinking": false }));
let non_empty = |v: &Option<Vec<String>>| v.clone().filter(|x| !x.is_empty());
ChatCompletionRequest {
model: model.map(str::to_string),
messages,
stream,
stream_options: stream.then_some(StreamOptions {
include_usage: true,
}),
temperature: s.temperature,
dynatemp_range: s.dynatemp_range,
dynatemp_exponent: s.dynatemp_exponent,
max_tokens: s.max_tokens,
top_k: s.top_k,
top_p: s.top_p,
min_p: s.min_p,
top_n_sigma: s.top_n_sigma,
typical_p: s.typical_p,
adaptive_target: s.adaptive_target,
adaptive_decay: s.adaptive_decay,
frequency_penalty: s.frequency_penalty,
presence_penalty: s.presence_penalty,
repeat_penalty: s.repeat_penalty,
repetition_penalty: None,
repeat_last_n: s.repeat_last_n,
dry_multiplier: s.dry_multiplier,
dry_base: s.dry_base,
dry_allowed_length: s.dry_allowed_length,
dry_penalty_last_n: s.dry_penalty_last_n,
dry_sequence_breakers: non_empty(&s.dry_sequence_breakers),
xtc_probability: s.xtc_probability,
xtc_threshold: s.xtc_threshold,
mirostat: s.mirostat,
mirostat_tau: s.mirostat_tau,
mirostat_eta: s.mirostat_eta,
seed: s.seed,
samplers: non_empty(&s.samplers),
thinking: s.thinking,
reasoning_effort: s
.reasoning_effort
.filter(|r| !(omit_effort_none && *r == ReasoningEffort::None))
.map(|r| r.as_wire()),
reasoning_budget: s.reasoning_budget,
reasoning: None,
chat_template_kwargs,
continue_final_message: req.continue_final.then_some(true),
add_generation_prompt: req.continue_final.then_some(false),
tools,
tool_choice,
}
}
impl ChatCompletionRequest {
pub fn for_gateway(mut self) -> Self {
self.repetition_penalty = self.repeat_penalty.take();
self.repeat_last_n = None;
self.dynatemp_range = None;
self.dynatemp_exponent = None;
self.top_n_sigma = None;
self.typical_p = None;
self.adaptive_target = None;
self.adaptive_decay = None;
self.dry_multiplier = None;
self.dry_base = None;
self.dry_allowed_length = None;
self.dry_penalty_last_n = None;
self.dry_sequence_breakers = None;
self.xtc_probability = None;
self.xtc_threshold = None;
self.mirostat = None;
self.mirostat_tau = None;
self.mirostat_eta = None;
self.samplers = None;
self.thinking = None;
self.reasoning_budget = None;
self.chat_template_kwargs = None;
self.continue_final_message = None;
self.add_generation_prompt = None;
self
}
}
impl effort::CarriesEffort for ChatCompletionRequest {
fn effort(&self) -> Option<&'static str> {
self.reasoning_effort
}
fn respell_effort(&mut self, instead: Option<&'static str>) {
self.reasoning_effort = instead;
}
}
pub fn asks_reasoning_off(sampling: &SamplingConfig) -> bool {
match sampling.reasoning_effort {
Some(effort) => effort == ReasoningEffort::None,
None => sampling.reasoning_budget == Some(0) || sampling.thinking == Some(false),
}
}
pub fn effort_wish(sampling: &SamplingConfig) -> Option<&'static str> {
if asks_reasoning_off(sampling) {
return Some(effort::NONE);
}
sampling.reasoning_effort.map(ReasoningEffort::as_wire)
}
pub fn xai_effort(wish: &'static str, listed: Option<&[&'static str]>) -> Option<&'static str> {
match listed {
Some(listed) if listed.contains(&wish) => Some(wish),
Some(listed) => effort::nearest(wish, listed),
None if wish == effort::NONE => None,
None if wish == ReasoningEffort::Max.as_wire() => Some(ReasoningEffort::XHigh.as_wire()),
None => Some(wish),
}
}
pub fn refuses_effort_parameter(err: &EngineError) -> bool {
if err.status != Some(400) {
return false;
}
let message = err.message.to_lowercase();
message.contains("does not support parameter")
&& (message.contains("reasoningeffort") || message.contains("reasoning_effort"))
}
pub fn gateway_muted_effort(
sampling: &SamplingConfig,
entry: Option<&ModelEntry>,
off_refused: bool,
) -> Option<&'static str> {
if !asks_reasoning_off(sampling) {
return None;
}
let mandatory = off_refused || entry.and_then(ModelEntry::reasoning_mandatory) == Some(true);
if !mandatory {
return None;
}
entry?.lowest_effort()
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub struct WireReasoning {
pub enabled: bool,
}
pub fn gateway_reasoning(
sampling: &SamplingConfig,
entry: &ModelEntry,
off_refused: bool,
) -> Option<WireReasoning> {
if sampling.reasoning_effort.is_some() || !entry.lists_parameter("reasoning") {
return None;
}
let on = match (sampling.reasoning_budget, sampling.thinking) {
(Some(0), _) => false,
(_, Some(on)) => on,
(_, None) => return None,
};
if on {
return Some(WireReasoning { enabled: true });
}
(entry.reasoning_mandatory() == Some(false) && !off_refused)
.then_some(WireReasoning { enabled: false })
}
#[derive(Debug, Deserialize)]
pub struct ChatCompletionChunk {
#[serde(default)]
pub choices: Vec<ChatChoiceChunk>,
#[serde(default)]
pub provider: Option<String>,
#[serde(default)]
pub usage: Option<Usage>,
#[serde(default)]
pub timings: Option<Timings>,
}
#[derive(Debug, Default, Deserialize)]
pub struct Timings {
#[serde(default)]
pub prompt_n: u32,
#[serde(default)]
pub prompt_ms: f64,
}
#[derive(Debug, Deserialize)]
struct StreamErrorEnvelope {
error: StreamErrorBody,
}
#[derive(Debug, Default, Deserialize)]
struct StreamErrorBody {
#[serde(default)]
message: String,
#[serde(default, rename = "type")]
name: String,
#[serde(default)]
code: Option<serde_json::Value>,
}
pub struct StreamError {
pub message: String,
pub name: String,
pub transient: bool,
}
pub fn parse_stream_error(data: &str) -> Option<StreamError> {
let env: StreamErrorEnvelope = serde_json::from_str(data).ok()?;
let body = env.error;
if body.name.trim().is_empty() && body.message.trim().is_empty() {
return None;
}
let status = body.code.as_ref().and_then(|c| c.as_u64()).and_then(|c| {
u16::try_from(c).ok().filter(|s| (100..=599).contains(s))
});
let transient = crate::shared::api::error::stream_error_transient(&body.name, status);
Some(StreamError {
message: crate::shared::api::error::stream_error_text(&body.name, &body.message),
name: body.name,
transient,
})
}
#[derive(Debug, Default, Deserialize)]
pub struct Usage {
#[serde(default)]
pub prompt_tokens: u32,
#[serde(default)]
pub completion_tokens: u32,
#[serde(default)]
pub completion_tokens_details: CompletionTokensDetails,
#[serde(default)]
pub cost: Option<f64>,
}
impl Usage {
pub fn cost_nanos(&self) -> Option<u64> {
let cost = self.cost.filter(|c| c.is_finite() && *c >= 0.0)?;
Some((cost * 1e9).round() as u64)
}
}
#[derive(Debug, Default, Deserialize)]
pub struct CompletionTokensDetails {
#[serde(default)]
pub reasoning_tokens: u32,
}
#[derive(Debug, Deserialize)]
pub struct ChatChoiceChunk {
#[serde(default)]
pub delta: Delta,
#[serde(default)]
pub finish_reason: Option<String>,
}
#[derive(Debug, Default, Deserialize)]
pub struct Delta {
#[serde(default)]
pub content: Option<String>,
#[serde(default)]
pub reasoning_content: Option<String>,
#[serde(default)]
pub reasoning: Option<String>,
#[serde(default)]
pub tool_calls: Option<Vec<DeltaToolCall>>,
}
impl Delta {
pub fn thoughts(&mut self) -> Option<String> {
self.reasoning_content
.take()
.or_else(|| self.reasoning.take())
}
}
#[derive(Debug, Deserialize)]
pub struct DeltaToolCall {
#[serde(default)]
pub index: usize,
#[serde(default)]
pub id: Option<String>,
#[serde(default)]
pub function: Option<DeltaFunction>,
}
#[derive(Debug, Default, Deserialize)]
pub struct DeltaFunction {
#[serde(default)]
pub name: Option<String>,
#[serde(default)]
pub arguments: Option<String>,
}
#[derive(Debug, Deserialize)]
pub struct ModelList {
#[serde(default)]
pub data: Vec<ModelEntry>,
}
#[derive(Debug, Deserialize)]
pub struct ModelEnvelope {
pub data: ModelEntry,
}
impl From<ModelEnvelope> for ModelEntry {
fn from(envelope: ModelEnvelope) -> Self {
envelope.data
}
}
#[derive(Debug, Clone, Deserialize)]
pub struct ModelEntry {
#[serde(default)]
pub id: String,
#[serde(default)]
pub context_length: Option<u32>,
#[serde(default)]
pub supported_parameters: Option<Vec<String>>,
#[serde(default)]
pub reasoning: Option<serde_json::Value>,
#[serde(default)]
pub architecture: Option<serde_json::Value>,
#[serde(default)]
pub capabilities: Option<serde_json::Value>,
}
impl ModelEntry {
pub fn takes_images(&self) -> Option<bool> {
let listed = self
.architecture
.as_ref()?
.get("input_modalities")?
.as_array()?
.iter()
.map(serde_json::Value::as_str)
.collect::<Option<Vec<_>>>()?;
(!listed.is_empty()).then(|| listed.contains(&"image"))
}
pub fn lists_parameter(&self, name: &str) -> bool {
self.supported_parameters
.as_ref()
.is_some_and(|p| p.iter().any(|x| x == name))
}
pub fn reasoning_mandatory(&self) -> Option<bool> {
self.reasoning.as_ref()?.get("mandatory")?.as_bool()
}
pub fn lowest_effort(&self) -> Option<&'static str> {
let listed = self
.reasoning
.as_ref()?
.get("supported_efforts")?
.as_array()?;
effort::LADDER
.into_iter()
.find(|effort| listed.iter().any(|l| l.as_str() == Some(effort)))
}
pub fn listed_efforts(&self) -> Option<Vec<&'static str>> {
let listed = self
.capabilities
.as_ref()?
.get("reasoning_effort")?
.as_array()?;
let known: Vec<&'static str> = std::iter::once(effort::NONE)
.chain(effort::LADDER)
.filter(|value| listed.iter().any(|l| l.as_str() == Some(value)))
.collect();
(!known.is_empty()).then_some(known)
}
}
#[derive(Debug, Serialize)]
pub struct EmbeddingRequest {
#[serde(skip_serializing_if = "Option::is_none")]
pub model: Option<String>,
pub input: Vec<String>,
}
#[derive(Debug, Deserialize)]
pub struct EmbeddingResponse {
#[serde(default)]
pub data: Vec<EmbeddingData>,
}
#[derive(Debug, Deserialize)]
pub struct EmbeddingData {
pub embedding: Vec<f32>,
}
#[cfg(test)]
mod tests {
use super::*;
use crate::entities::sampling::{ReasoningEffort, SamplingConfig};
use crate::shared::api::contract::ApiMessage;
#[test]
fn omits_stop_and_none_fields() {
let req = ChatRequest {
continue_final: false,
system: Some("sys".into()),
messages: vec![ApiMessage::user("hi")],
sampling: SamplingConfig::default(),
tools: vec![],
};
let body = build_chat_request(&req, true, None, false);
let json = serde_json::to_value(&body).unwrap();
assert!(json.get("stop").is_none(), "stop must never be sent");
assert!(json.get("temperature").is_none());
assert_eq!(json["stream"], true);
assert_eq!(json["stream_options"]["include_usage"], true);
assert_eq!(json["messages"][0]["role"], "system");
assert_eq!(json["messages"][1]["role"], "user");
assert_eq!(json["messages"][1]["content"], "hi");
}
fn image(mime: &str, data: &str, label: Option<&str>) -> crate::shared::api::ApiImage {
crate::shared::api::ApiImage {
mime: mime.to_string(),
data: std::sync::Arc::from(data),
label: label.map(str::to_string),
}
}
#[test]
fn a_text_only_request_is_unchanged_by_the_image_support() {
let req = ChatRequest {
continue_final: false,
system: Some("be brief".into()),
messages: vec![
ApiMessage::user("hi"),
ApiMessage::assistant("hello"),
ApiMessage::tool("call-1", "42"),
],
sampling: SamplingConfig::default(),
tools: vec![],
};
let json = serde_json::to_value(build_chat_request(&req, false, None, false)).unwrap();
for i in 0..4 {
assert!(
json["messages"][i]["content"].is_string(),
"message {i} must serialize its content as a string, got {:?}",
json["messages"][i]["content"]
);
}
assert_eq!(json["messages"][0]["content"], "be brief");
assert_eq!(json["messages"][1]["content"], "hi");
assert_eq!(json["messages"][3]["content"], "42");
}
#[test]
fn images_become_content_parts_ahead_of_the_text() {
let req = ChatRequest {
continue_final: false,
system: None,
messages: vec![ApiMessage::user("what is this?").with_images(vec![image(
"image/png",
"QUJD",
Some("Image #1 — \"a.png\":"),
)])],
sampling: SamplingConfig::default(),
tools: vec![],
};
let json = serde_json::to_value(build_chat_request(&req, false, None, false)).unwrap();
let parts = json["messages"][0]["content"].as_array().unwrap();
assert_eq!(parts.len(), 3);
assert_eq!(parts[0]["type"], "text");
assert_eq!(parts[0]["text"], "Image #1 — \"a.png\":");
assert_eq!(parts[1]["type"], "image_url");
assert_eq!(
parts[1]["image_url"]["url"], "data:image/png;base64,QUJD",
"the payload must be a data URI, which is what llama.cpp and xAI both accept"
);
assert_eq!(parts[2]["type"], "text");
assert_eq!(parts[2]["text"], "what is this?");
}
#[test]
fn several_images_keep_their_order_and_a_labelless_one_emits_no_text_part() {
let req = ChatRequest {
continue_final: false,
system: None,
messages: vec![ApiMessage::user("").with_images(vec![
image("image/png", "AAA", None),
image("image/jpeg", "BBB", None),
])],
sampling: SamplingConfig::default(),
tools: vec![],
};
let json = serde_json::to_value(build_chat_request(&req, false, None, false)).unwrap();
let parts = json["messages"][0]["content"].as_array().unwrap();
assert_eq!(parts.len(), 2);
assert_eq!(parts[0]["image_url"]["url"], "data:image/png;base64,AAA");
assert_eq!(parts[1]["image_url"]["url"], "data:image/jpeg;base64,BBB");
}
#[test]
fn a_tool_result_without_images_is_unchanged() {
let req = ChatRequest {
continue_final: false,
system: None,
messages: vec![ApiMessage::tool("call-1", "42")],
sampling: SamplingConfig::default(),
tools: vec![],
};
let json = serde_json::to_value(build_chat_request(&req, false, None, false)).unwrap();
assert_eq!(json["messages"][0]["role"], "tool");
assert!(
json["messages"][0]["content"].is_string(),
"a tool result with no images must stay a string, got {:?}",
json["messages"][0]["content"]
);
assert_eq!(json["messages"][0]["content"], "42");
}
#[test]
fn a_tool_result_image_follows_the_result_text() {
let req = ChatRequest {
continue_final: false,
system: None,
messages: vec![
ApiMessage::tool("call-1", "screenshot taken").with_images(vec![image(
"image/png",
"QUJD",
Some("Image #1 — \"shot.png\":"),
)]),
],
sampling: SamplingConfig::default(),
tools: vec![],
};
let json = serde_json::to_value(build_chat_request(&req, false, None, false)).unwrap();
assert_eq!(json["messages"][0]["role"], "tool");
assert_eq!(json["messages"][0]["tool_call_id"], "call-1");
let parts = json["messages"][0]["content"].as_array().unwrap();
assert_eq!(parts.len(), 3);
assert_eq!(parts[0]["type"], "text");
assert_eq!(parts[0]["text"], "screenshot taken");
assert_eq!(parts[1]["type"], "text");
assert_eq!(parts[1]["text"], "Image #1 — \"shot.png\":");
assert_eq!(parts[2]["type"], "image_url");
assert_eq!(
parts[2]["image_url"]["url"], "data:image/png;base64,QUJD",
"a tool result carries the payload as the same data URI a user image does"
);
}
#[test]
fn an_image_only_tool_result_carries_no_empty_text_part() {
let req = ChatRequest {
continue_final: false,
system: None,
messages: vec![ApiMessage::tool("call-1", "").with_images(vec![image(
"image/jpeg",
"QQ==",
None,
)])],
sampling: SamplingConfig::default(),
tools: vec![],
};
let json = serde_json::to_value(build_chat_request(&req, false, None, false)).unwrap();
let parts = json["messages"][0]["content"].as_array().unwrap();
assert_eq!(parts.len(), 1);
assert_eq!(parts[0]["image_url"]["url"], "data:image/jpeg;base64,QQ==");
}
#[test]
fn a_rounds_tool_images_are_re_homed_after_the_run() {
let call = |id: &str| crate::shared::api::ApiToolCall {
id: id.into(),
name: "screenshot".into(),
arguments: "{}".into(),
thought_signature: None,
};
let messages = vec![
ApiMessage::user("look"),
ApiMessage::assistant_tool_calls("", vec![call("a"), call("b")]),
ApiMessage::tool("a", "first").with_images(vec![image("image/png", "AAA", Some("#1"))]),
ApiMessage::tool("b", "second").with_images(vec![image(
"image/png",
"BBB",
Some("#2"),
)]),
ApiMessage::assistant_tool_calls("", vec![call("c")]),
ApiMessage::tool("c", "third").with_images(vec![image(
"image/jpeg",
"CCC",
Some("#3"),
)]),
];
let out = rehome_tool_images(&messages).expect("tool results carry images");
let roles: Vec<ApiRole> = out.iter().map(|m| m.role).collect();
use ApiRole::{Assistant, Tool, User};
assert_eq!(
roles,
[User, Assistant, Tool, Tool, User, Assistant, Tool, User],
"one user message after each run of tool results"
);
assert!(
out.iter()
.filter(|m| m.role == Tool)
.all(|m| m.images.is_empty()),
"every tool result goes out text-only"
);
assert_eq!(out[2].content, "first");
assert_eq!(out[3].content, "second");
let labels = |m: &ApiMessage| -> Vec<Option<String>> {
m.images.iter().map(|i| i.label.clone()).collect()
};
assert_eq!(labels(&out[4]), [Some("#1".into()), Some("#2".into())]);
assert!(out[4].content.is_empty());
assert_eq!(labels(&out[7]), [Some("#3".into())]);
let req = ChatRequest {
continue_final: false,
system: None,
messages: out,
sampling: SamplingConfig::default(),
tools: vec![],
};
let json = serde_json::to_value(build_chat_request(&req, true, None, false)).unwrap();
assert!(json["messages"][2]["content"].is_string());
assert!(json["messages"][3]["content"].is_string());
let parts = json["messages"][4]["content"].as_array().unwrap();
assert_eq!(parts.len(), 4, "label, image, label, image: {parts:?}");
assert_eq!(parts[1]["image_url"]["url"], "data:image/png;base64,AAA");
assert_eq!(parts[3]["image_url"]["url"], "data:image/png;base64,BBB");
}
#[test]
fn a_conversation_without_tool_images_is_not_re_homed() {
let messages = vec![
ApiMessage::user("what is this?").with_images(vec![image("image/png", "AAA", None)]),
ApiMessage::tool("a", "42"),
];
assert!(!carries_tool_images(&messages));
assert!(rehome_tool_images(&messages).is_none());
}
#[test]
fn maps_supported_sampling_fields() {
let req = ChatRequest {
continue_final: false,
system: None,
messages: vec![ApiMessage::user("hi")],
sampling: SamplingConfig {
temperature: Some(0.8),
dynatemp_range: Some(0.4),
dynatemp_exponent: Some(1.0),
top_k: Some(40),
top_p: Some(0.95),
min_p: Some(0.03),
top_n_sigma: Some(1.5),
adaptive_target: Some(0.1),
adaptive_decay: Some(0.9),
frequency_penalty: Some(0.1),
presence_penalty: Some(0.2),
repeat_penalty: Some(1.0),
dry_multiplier: Some(0.8),
dry_base: Some(1.75),
dry_allowed_length: Some(2),
xtc_probability: Some(0.3),
xtc_threshold: Some(0.15),
seed: Some(-1),
max_tokens: Some(256),
thinking: Some(true),
reasoning_effort: Some(ReasoningEffort::High),
reasoning_budget: Some(0),
..Default::default()
},
tools: vec![],
};
let json = serde_json::to_value(build_chat_request(&req, false, None, false)).unwrap();
let approx = |v: &serde_json::Value, want: f64| (v.as_f64().unwrap() - want).abs() < 1e-6;
assert!(approx(&json["temperature"], 0.8));
assert!(approx(&json["dynatemp_range"], 0.4));
assert!(approx(&json["dynatemp_exponent"], 1.0));
assert_eq!(json["top_k"], 40);
assert!(approx(&json["top_p"], 0.95));
assert!(approx(&json["min_p"], 0.03));
assert!(approx(&json["top_n_sigma"], 1.5));
assert!(approx(&json["adaptive_target"], 0.1));
assert!(approx(&json["adaptive_decay"], 0.9));
assert!(approx(&json["frequency_penalty"], 0.1));
assert!(approx(&json["presence_penalty"], 0.2));
assert!(approx(&json["repeat_penalty"], 1.0));
assert!(approx(&json["dry_multiplier"], 0.8));
assert!(approx(&json["dry_base"], 1.75));
assert_eq!(json["dry_allowed_length"], 2);
assert!(approx(&json["xtc_probability"], 0.3));
assert!(approx(&json["xtc_threshold"], 0.15));
assert_eq!(json["seed"], -1);
assert_eq!(json["max_tokens"], 256);
assert_eq!(json["thinking"], true);
assert_eq!(json["reasoning_effort"], "high");
assert_eq!(json["reasoning_budget"], 0);
assert_eq!(json["chat_template_kwargs"]["enable_thinking"], false);
assert_eq!(json["stream"], false);
assert!(json.get("stream_options").is_none());
assert!(json.get("typical_p").is_none());
assert!(json.get("mirostat").is_none());
}
#[test]
fn list_fields_sent_as_arrays_and_empty_omitted() {
let req = ChatRequest {
continue_final: false,
system: None,
messages: vec![ApiMessage::user("hi")],
sampling: SamplingConfig {
dry_sequence_breakers: Some(vec!["\n".into(), ":".into()]),
samplers: Some(vec!["penalties".into(), "temperature".into()]),
..Default::default()
},
tools: vec![],
};
let json = serde_json::to_value(build_chat_request(&req, false, None, false)).unwrap();
assert_eq!(json["dry_sequence_breakers"][0], "\n");
assert_eq!(json["dry_sequence_breakers"][1], ":");
assert_eq!(json["samplers"][0], "penalties");
assert_eq!(json["samplers"][1], "temperature");
let req_empty = ChatRequest {
continue_final: false,
system: None,
messages: vec![ApiMessage::user("hi")],
sampling: SamplingConfig {
dry_sequence_breakers: Some(vec![]),
samplers: Some(vec![]),
..Default::default()
},
tools: vec![],
};
let json_empty =
serde_json::to_value(build_chat_request(&req_empty, false, None, false)).unwrap();
assert!(json_empty.get("dry_sequence_breakers").is_none());
assert!(json_empty.get("samplers").is_none());
}
#[test]
fn model_is_sent_when_some_and_omitted_when_none() {
let req = ChatRequest {
continue_final: false,
system: None,
messages: vec![ApiMessage::user("hi")],
sampling: SamplingConfig::default(),
tools: vec![],
};
let with_model =
serde_json::to_value(build_chat_request(&req, true, Some("some-model"), false))
.unwrap();
assert_eq!(with_model["model"], "some-model");
let no_model = serde_json::to_value(build_chat_request(&req, true, None, false)).unwrap();
assert!(no_model.get("model").is_none());
}
#[test]
fn effort_none_is_omitted_only_when_asked() {
let req = |e: ReasoningEffort| ChatRequest {
continue_final: false,
system: None,
messages: vec![ApiMessage::user("hi")],
sampling: SamplingConfig {
reasoning_effort: Some(e),
..Default::default()
},
tools: vec![],
};
let json = |e: ReasoningEffort, omit: bool| {
serde_json::to_value(build_chat_request(&req(e), true, None, omit)).unwrap()
};
assert_eq!(
json(ReasoningEffort::None, false)["reasoning_effort"],
"none"
);
assert!(
json(ReasoningEffort::None, true)
.get("reasoning_effort")
.is_none()
);
assert_eq!(json(ReasoningEffort::Low, true)["reasoning_effort"], "low");
assert_eq!(
json(ReasoningEffort::XHigh, true)["reasoning_effort"],
"xhigh"
);
}
#[test]
fn parses_streaming_chunk() {
let raw = r#"{"choices":[{"delta":{"content":"hello","reasoning_content":"hmm"},"finish_reason":null}]}"#;
let chunk: ChatCompletionChunk = serde_json::from_str(raw).unwrap();
let c = &chunk.choices[0];
assert_eq!(c.delta.content.as_deref(), Some("hello"));
assert_eq!(c.delta.reasoning_content.as_deref(), Some("hmm"));
assert!(c.finish_reason.is_none());
}
#[test]
fn parses_usage_chunk() {
let raw = r#"{"choices":[],"usage":{"prompt_tokens":42,"completion_tokens":7,"total_tokens":49}}"#;
let chunk: ChatCompletionChunk = serde_json::from_str(raw).unwrap();
assert!(chunk.choices.is_empty());
let u = chunk.usage.unwrap();
assert_eq!(u.prompt_tokens, 42);
assert_eq!(u.completion_tokens, 7);
}
#[test]
fn chunk_without_usage_is_none() {
let raw = r#"{"choices":[{"delta":{"content":"hi"},"finish_reason":null}]}"#;
let chunk: ChatCompletionChunk = serde_json::from_str(raw).unwrap();
assert!(chunk.usage.is_none());
}
#[test]
fn parses_finish_chunk() {
let raw = r#"{"choices":[{"delta":{},"finish_reason":"tool_calls"}]}"#;
let chunk: ChatCompletionChunk = serde_json::from_str(raw).unwrap();
assert_eq!(
chunk.choices[0].finish_reason.as_deref(),
Some("tool_calls")
);
}
#[test]
fn parses_tool_call_delta() {
let raw = r#"{"choices":[{"delta":{"tool_calls":[{"index":0,"id":"c1","type":"function","function":{"name":"note_save","arguments":"{\"x\":1}"}}]},"finish_reason":null}]}"#;
let chunk: ChatCompletionChunk = serde_json::from_str(raw).unwrap();
let tc = chunk.choices[0].delta.tool_calls.as_ref().unwrap();
assert_eq!(tc[0].index, 0);
assert_eq!(tc[0].id.as_deref(), Some("c1"));
let f = tc[0].function.as_ref().unwrap();
assert_eq!(f.name.as_deref(), Some("note_save"));
assert_eq!(f.arguments.as_deref(), Some("{\"x\":1}"));
}
#[test]
fn builds_tools_and_tool_choice() {
use crate::shared::api::contract::ToolSchema;
let req = ChatRequest {
continue_final: false,
system: None,
messages: vec![ApiMessage::user("hi")],
sampling: SamplingConfig::default(),
tools: vec![ToolSchema {
name: "note_save".into(),
description: "Сохранить заметку".into(),
parameters: serde_json::json!({"type":"object"}),
}],
};
let json = serde_json::to_value(build_chat_request(&req, true, None, false)).unwrap();
assert_eq!(json["tool_choice"], "auto");
assert_eq!(json["tools"][0]["type"], "function");
assert_eq!(json["tools"][0]["function"]["name"], "note_save");
}
#[test]
fn serializes_assistant_tool_calls_in_history() {
use crate::shared::api::contract::ApiToolCall;
let req = ChatRequest {
continue_final: false,
system: None,
messages: vec![
ApiMessage::assistant_tool_calls(
"",
vec![ApiToolCall {
thought_signature: None,
id: "c1".into(),
name: "f".into(),
arguments: "{}".into(),
}],
),
ApiMessage::tool("c1", "result"),
],
sampling: SamplingConfig::default(),
tools: vec![],
};
let json = serde_json::to_value(build_chat_request(&req, true, None, false)).unwrap();
assert_eq!(json["messages"][0]["tool_calls"][0]["id"], "c1");
assert_eq!(
json["messages"][0]["tool_calls"][0]["function"]["name"],
"f"
);
assert_eq!(json["messages"][1]["role"], "tool");
assert_eq!(json["messages"][1]["tool_call_id"], "c1");
assert!(json.get("tool_choice").is_none());
}
fn a_body_with_everything_set() -> ChatCompletionRequest {
use crate::shared::api::contract::{ApiToolCall, ToolSchema};
let sampling: SamplingConfig = serde_json::from_str(
r#"{"temperature":0.7,"dynatemp_range":0.5,"dynatemp_exponent":1.0,"top_k":40,
"top_p":0.9,"min_p":0.05,"top_n_sigma":1.0,"typical_p":0.9,
"adaptive_target":0.5,"adaptive_decay":0.9,"frequency_penalty":0.1,
"presence_penalty":0.2,"repeat_penalty":1.15,"repeat_last_n":64,
"dry_multiplier":0.8,"dry_base":1.75,"dry_allowed_length":2,
"dry_penalty_last_n":64,"dry_sequence_breakers":["\n"],
"xtc_probability":0.5,"xtc_threshold":0.1,"mirostat":2,"mirostat_tau":5.0,
"mirostat_eta":0.1,"max_tokens":512,"seed":7,"samplers":["top_k"],
"thinking":true,"reasoning_effort":"low","reasoning_budget":0}"#,
)
.unwrap();
let call = ApiToolCall {
thought_signature: None,
id: "c1".into(),
name: "note_save".into(),
arguments: "{}".into(),
};
let req = ChatRequest {
continue_final: true,
system: Some("sys".into()),
messages: vec![
ApiMessage::user("q"),
ApiMessage::assistant_tool_calls("", vec![call]),
ApiMessage::tool("c1", "saved"),
ApiMessage::assistant("part"),
],
sampling,
tools: vec![ToolSchema {
name: "note_save".into(),
description: "Save a note".into(),
parameters: serde_json::json!({"type":"object"}),
}],
};
let mut body = build_chat_request(&req, true, Some("anthropic/claude-haiku-4.5"), false);
body.reasoning = Some(WireReasoning { enabled: true });
body
}
const LEFT_OUT_FOR_THE_GATEWAY: &str = "add_generation_prompt adaptive_decay \
adaptive_target chat_template_kwargs continue_final_message dry_allowed_length \
dry_base dry_multiplier dry_penalty_last_n dry_sequence_breakers dynatemp_exponent \
dynatemp_range mirostat mirostat_eta mirostat_tau reasoning_budget repeat_last_n \
repeat_penalty samplers thinking top_n_sigma typical_p xtc_probability xtc_threshold";
const READ_BY_THE_GATEWAY: &str = "frequency_penalty max_tokens messages min_p model \
presence_penalty reasoning reasoning_effort seed stream stream_options temperature \
tool_choice tools top_k top_p";
#[test]
fn the_gateway_dialect_renames_one_field_and_leaves_out_llama_cpps_own() {
type Body = serde_json::Map<String, serde_json::Value>;
fn sorted(mut keys: Vec<&str>) -> Vec<&str> {
keys.sort_unstable();
keys
}
fn only_in<'a>(a: &'a Body, b: &Body) -> Vec<&'a str> {
let keys = a.keys().filter(|k| !b.contains_key(*k));
sorted(keys.map(String::as_str).collect())
}
let before = serde_json::to_value(a_body_with_everything_set()).unwrap();
let after = serde_json::to_value(a_body_with_everything_set().for_gateway()).unwrap();
let (before, after) = (before.as_object().unwrap(), after.as_object().unwrap());
assert_eq!(
only_in(before, after),
sorted(LEFT_OUT_FOR_THE_GATEWAY.split_whitespace().collect())
);
assert_eq!(only_in(after, before), ["repetition_penalty"]);
assert_eq!(after["repetition_penalty"], before["repeat_penalty"]);
let stayed = after.keys().filter(|k| before.get(*k) == after.get(*k));
assert_eq!(
sorted(stayed.map(String::as_str).collect()),
sorted(READ_BY_THE_GATEWAY.split_whitespace().collect()),
"present on both sides with the same value"
);
}
const COSTS: &str = r#"
{"prompt_tokens":17,"completion_tokens":4,"cost":0.000037} => 37000 | a measured turn on Haiku 4.5
{"cost":0.000065} => 65000 | rounded, not cut to 64999
{"cost":0} => 0 | a free model costs nothing, and says so
{"cost":12.5} => 12500000000 | dollars, not fractions of a cent
{"cost":0.0000000004} => 0 | less than half a billionth is nothing
{"cost":-0.000001} => - | a negative figure is not a cost
{"cost":null} => - | null
{"prompt_tokens":17,"completion_tokens":4} => - | every server that is not a gateway
"#;
#[test]
fn a_cost_is_stored_in_whole_billionths_of_a_dollar() {
let mut wrong = Vec::new();
for row in COSTS.lines().filter(|l| !l.trim().is_empty()) {
let (columns, why) = row.split_once('|').unwrap();
let (usage, nanos) = columns.split_once("=>").unwrap();
let usage: Usage = serde_json::from_str(usage.trim()).unwrap();
let got = usage.cost_nanos();
if got != nanos.trim().parse().ok() {
wrong.push(format!("{}: {got:?}", why.trim()));
}
}
assert!(wrong.is_empty(), "{wrong:#?}");
for not_a_number in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] {
let usage = Usage {
cost: Some(not_a_number),
..Default::default()
};
assert_eq!(usage.cost_nanos(), None, "{not_a_number}");
}
}
#[test]
fn the_gateways_answer_about_one_model_is_an_entry_under_data() {
let one: ModelEnvelope = serde_json::from_str(
r#"{"data":{"id":"google/gemini-3.5-flash","name":"Google: Gemini 3.5 Flash",
"context_length":1048576,
"architecture":{"input_modalities":["text","image","video"],"output_modalities":["text"]},
"supported_parameters":["reasoning","max_tokens","temperature","tools"],
"reasoning":{"mandatory":true,"supported_efforts":["high","medium","low","minimal"]},
"pricing":{"prompt":"0.0000003","completion":"0.0000025"}}}"#,
)
.expect("the gateway's own shape");
let entry = one.data;
assert_eq!(entry.id, "google/gemini-3.5-flash");
assert_eq!(entry.context_length, Some(1_048_576));
assert_eq!(entry.takes_images(), Some(true));
assert!(entry.lists_parameter("tools") && !entry.lists_parameter("top_k"));
assert_eq!(entry.reasoning_mandatory(), Some(true));
assert_eq!(entry.lowest_effort(), Some("minimal"));
for not_an_entry in [
r#"{"data":[{"id":"google/gemini-3.5-flash"}]}"#,
r#"{"error":{"message":"nope/not-a-model is not a valid model ID","code":400}}"#,
"{}",
] {
assert!(
serde_json::from_str::<ModelEnvelope>(not_an_entry).is_err(),
"{not_an_entry}"
);
}
}
}
#[cfg(test)]
mod stream_error_tests {
use super::*;
#[test]
fn a_llama_cpp_server_error_is_transient() {
let data = r#"{"error":{"code":500,"message":"failed to decode","type":"server_error"}}"#;
let e = parse_stream_error(data).expect("an error envelope must be recognized");
assert!(e.transient);
assert!(e.message.contains("failed to decode"));
assert_eq!(e.name, "server_error");
}
#[test]
fn a_context_overflow_is_not_transient() {
let data = r#"{"error":{"code":400,"message":"the request exceeds the available context size","type":"exceed_context_size_error"}}"#;
let e = parse_stream_error(data).unwrap();
assert!(!e.transient);
assert!(crate::features::compaction::is_context_overflow(&e.message));
}
#[test]
fn a_string_code_does_not_read_as_a_status() {
let data = r#"{"error":{"code":"context_length_exceeded","message":"too long","type":"invalid_request_error"}}"#;
let e = parse_stream_error(data).unwrap();
assert!(!e.transient, "a 4xx-class name must not be retried");
}
#[test]
fn a_rate_limit_name_is_transient_without_any_code() {
let data = r#"{"error":{"message":"slow down","type":"rate_limit_exceeded"}}"#;
assert!(parse_stream_error(data).unwrap().transient);
}
#[test]
fn non_errors_are_not_mistaken_for_errors() {
for data in [
r#"{"choices":[{"delta":{"content":"hi"},"index":0}]}"#,
r#"{"usage":{"prompt_tokens":1,"completion_tokens":2}}"#,
"not json at all",
r#"{"error":{}}"#,
r#"{"error":{"message":" ","type":""}}"#,
] {
assert!(parse_stream_error(data).is_none(), "{data}");
}
}
}
#[cfg(test)]
mod continuation_tests {
use super::*;
use crate::shared::api::contract::ApiMessage;
#[test]
fn continuation_adds_its_fields_and_absence_changes_nothing() {
let mut req = ChatRequest {
system: None,
messages: vec![ApiMessage::user("q"), ApiMessage::assistant("part")],
sampling: Default::default(),
tools: vec![],
continue_final: false,
};
let off = serde_json::to_value(build_chat_request(&req, true, None, false)).unwrap();
assert!(off.get("continue_final_message").is_none());
assert!(off.get("add_generation_prompt").is_none());
assert!(off.get("chat_template_kwargs").is_none());
req.continue_final = true;
let on = serde_json::to_value(build_chat_request(&req, true, None, false)).unwrap();
assert_eq!(on["continue_final_message"], true);
assert_eq!(on["add_generation_prompt"], false);
assert_eq!(on["chat_template_kwargs"]["enable_thinking"], false);
let (mut a, mut b) = (off, on);
for k in [
"continue_final_message",
"add_generation_prompt",
"chat_template_kwargs",
] {
a.as_object_mut().unwrap().remove(k);
b.as_object_mut().unwrap().remove(k);
}
assert_eq!(a, b, "the flag must touch nothing but its three fields");
}
#[test]
fn parses_timings_beside_usage() {
let raw = r#"{"choices":[],"usage":{"prompt_tokens":1250,"completion_tokens":7,"total_tokens":1257},"timings":{"cache_n":50,"prompt_n":1200,"prompt_ms":13333.4,"prompt_per_second":90.0,"predicted_n":7,"predicted_ms":700.0}}"#;
let chunk: ChatCompletionChunk = serde_json::from_str(raw).unwrap();
let t = chunk.timings.expect("timings");
assert_eq!(t.prompt_n, 1200);
assert!((t.prompt_ms - 13333.4).abs() < 0.01);
let plain = r#"{"choices":[],"usage":{"prompt_tokens":42,"completion_tokens":7,"total_tokens":49}}"#;
let chunk: ChatCompletionChunk = serde_json::from_str(plain).unwrap();
assert!(chunk.timings.is_none());
}
}
#[cfg(test)]
mod gateway_reasoning_tests {
use super::*;
const SWITCH_CASES: &str = "
on - - optional no on | the switch on
off - - optional no off | the switch off
- - - optional no - | nothing asked
on low - optional no - | an effort already reaches the gateway
off none 0 optional no - | the silent turns' shape
on - 0 optional no off | a zero budget is off whatever thinking says
- - 0 optional no off | ...and with no switch set at all
off - - mandatory no - | a model that must reason is never asked off
off - - unstated no - | an unstated flag is silence
on - - unstated no on | ...while on needs no flag
on - - unlisted no - | a catalogue that does not list reasoning
off - - optional yes - | a refused off is not asked again
on - - optional yes on | ...and on is untouched by that refusal
";
#[test]
fn the_thinking_switch_is_spelled_for_a_gateway_only_where_it_is_read() {
let entry = |kind: &str| -> ModelEntry {
serde_json::from_str(match kind {
"optional" => r#"{"id":"m","supported_parameters":["reasoning"],"reasoning":{"mandatory":false}}"#,
"mandatory" => r#"{"id":"m","supported_parameters":["reasoning"],"reasoning":{"mandatory":true}}"#,
"unstated" => r#"{"id":"m","supported_parameters":["reasoning"]}"#,
_ => r#"{"id":"m","supported_parameters":["temperature"],"reasoning":{"mandatory":false}}"#,
})
.unwrap()
};
let switch = |v: &str| match v {
"on" => Some(true),
"off" => Some(false),
_ => None,
};
let rows = SWITCH_CASES.lines().filter(|l| !l.trim().is_empty());
for row in rows {
let (columns, why) = row.split_once('|').unwrap();
let c: Vec<&str> = columns.split_whitespace().collect();
let sampling = SamplingConfig {
thinking: switch(c[0]),
reasoning_effort: match c[1] {
"low" => Some(ReasoningEffort::Low),
"none" => Some(ReasoningEffort::None),
_ => None,
},
reasoning_budget: c[2].parse().ok(),
..Default::default()
};
assert_eq!(
gateway_reasoning(&sampling, &entry(c[3]), c[4] == "yes"),
switch(c[5]).map(|enabled| WireReasoning { enabled }),
"{}",
why.trim()
);
}
}
#[test]
fn an_odd_reasoning_key_leaves_the_catalogue_readable() {
let list: ModelList = serde_json::from_str(
r#"{"data":[{"id":"a","context_length":8192,"reasoning":true},
{"id":"b","reasoning":{"mandatory":"no"}}]}"#,
)
.unwrap();
assert_eq!(list.data[0].context_length, Some(8192));
assert_eq!(list.data[0].reasoning_mandatory(), None);
assert_eq!(list.data[1].reasoning_mandatory(), None);
}
#[test]
fn input_modalities_say_whether_the_model_takes_images() {
let list: ModelList = serde_json::from_str(
r#"{"data":[
{"id":"deepseek/deepseek-r1","architecture":{"modality":"text->text",
"input_modalities":["text"],"output_modalities":["text"]}},
{"id":"google/gemma-4-31b-it","architecture":{"modality":"text+image+video->text",
"input_modalities":["image","text","video"]}},
{"id":"llama.cpp-shaped"},
{"id":"no-list","architecture":{"modality":"text+image->text"}},
{"id":"empty","architecture":{"input_modalities":[]}},
{"id":"not-strings","architecture":{"input_modalities":["text",{"image":true}]}},
{"id":"not-a-list","architecture":{"input_modalities":"text+image"}},
{"id":"odd","architecture":"text->text","context_length":4096}
]}"#,
)
.unwrap();
let answers: Vec<_> = list.data.iter().map(ModelEntry::takes_images).collect();
assert_eq!(
answers,
[Some(false), Some(true), None, None, None, None, None, None]
);
assert_eq!(
list.data[7].context_length,
Some(4096),
"an odd key must not cost the entry"
);
}
fn sampling(thinking: &str, effort: &str, budget: &str) -> SamplingConfig {
SamplingConfig {
thinking: match thinking {
"on" => Some(true),
"off" => Some(false),
_ => None,
},
reasoning_effort: serde_json::from_value(serde_json::json!(effort)).ok(),
reasoning_budget: budget.parse().ok(),
..Default::default()
}
}
const ASKED_OFF: &str = "
- - - no | nothing set
off - - yes | the switch off
on - - no | the switch on
- none - yes | an effort of none
- low - no | an effort chosen
- - 0 yes | a zero budget
- - -1 no | an unlimited budget
- - 512 no | a budget to think within
on - 0 yes | a zero budget is off whatever the switch says
off none 0 yes | the silent turns' shape
on none - yes | none outranks the switch
off high - no | a chosen effort outranks the switch
- high 0 no | ...and the budget
";
#[test]
fn a_request_says_do_not_reason_in_one_of_three_ways() {
let mut wrong = Vec::new();
for row in ASKED_OFF.lines().filter(|l| !l.trim().is_empty()) {
let (columns, why) = row.split_once('|').unwrap();
let c: Vec<&str> = columns.split_whitespace().collect();
if asks_reasoning_off(&sampling(c[0], c[1], c[2])) != (c[3] == "yes") {
wrong.push(why.trim());
}
}
assert!(wrong.is_empty(), "{wrong:#?}");
}
fn entry(kind: &str) -> Option<ModelEntry> {
let reasoning = match kind {
"gemini" => {
r#"{"mandatory":true,"supported_efforts":["high","medium","low","minimal"]}"#
}
"sonnet" => {
r#"{"mandatory":true,"supported_efforts":["max","xhigh","high","medium","low"]}"#
}
"r1" => r#"{"mandatory":true}"#,
"haiku" => r#"{"mandatory":false}"#,
"optional" => r#"{"mandatory":false,"supported_efforts":["high","low"]}"#,
"unstated" => r#"{"supported_efforts":["high","low"]}"#,
"silent" => "null",
_ => return None,
};
serde_json::from_str(&format!(r#"{{"id":"m","reasoning":{reasoning}}}"#)).ok()
}
const MUTED_EFFORT: &str = "
off none 0 gemini no minimal | the silent turns' shape, on a model that must reason
off - - gemini no minimal | the switch alone
- - 0 gemini no minimal | a zero budget alone
- none - sonnet no low | the lowest listed, whatever the list opens with
on - - gemini no - | a turn that may reason is sent as built
- high - gemini no - | an effort somebody chose is nobody's to replace
- - - gemini no - | nothing asked
off none 0 r1 no - | must reason and lists no efforts: nothing lower to ask for
off none 0 haiku no - | may stop, so is asked to stop
off none 0 optional no - | ...even where it lists efforts
off none 0 unstated no - | an unstated flag is silence
off none 0 optional yes low | a refusal corrects an entry that said it may stop
off none 0 unstated yes low | ...and one that did not say
off none 0 silent yes - | refused, with no list to pick from
off none 0 absent yes - | refused, with no entry at all
off none 0 absent no - | no entry, nothing claimed
on - - optional yes - | a refusal mutes no turn that did not ask
";
#[test]
fn a_muted_turn_is_given_the_lowest_effort_only_where_stopping_is_refused() {
let mut wrong = Vec::new();
for row in MUTED_EFFORT.lines().filter(|l| !l.trim().is_empty()) {
let (columns, why) = row.split_once('|').unwrap();
let c: Vec<&str> = columns.split_whitespace().collect();
let got = gateway_muted_effort(
&sampling(c[0], c[1], c[2]),
entry(c[3]).as_ref(),
c[4] == "yes",
);
if got != Some(c[5]).filter(|effort| *effort != "-") {
wrong.push(format!("{}: {got:?}", why.trim()));
}
}
assert!(wrong.is_empty(), "{wrong:#?}");
}
const LOWEST_EFFORT: &str = r#"
{"supported_efforts":["high","medium","low","minimal"]} => minimal | Gemini 3.5 Flash, highest first
{"supported_efforts":["minimal","low","medium","high"]} => minimal | the same list, lowest first
{"supported_efforts":["xhigh","high","medium","low","minimal","none"]} => minimal | GPT-5.5: none is not an effort
{"supported_efforts":["max","xhigh","high","medium","low"]} => low | Claude Sonnet 5.5
{"supported_efforts":["max","high","xhigh"]} => high | high is below xhigh, and both below max
{"supported_efforts":["max"]} => max | one effort is the lowest there is
{"supported_efforts":["none"]} => - | none alone is nothing to ask for
{"supported_efforts":["turbo","deep"]} => - | words with no order here
{"supported_efforts":["Low","HIGH"]} => - | another spelling is another word
{"supported_efforts":[3,"medium",null]} => medium | what is not a word is passed over
{"supported_efforts":[]} => - | an empty list
{"supported_efforts":"low"} => - | not a list
{"mandatory":true} => - | no list
"low" => - | not an object
"#;
#[test]
fn the_lowest_effort_is_read_in_this_clients_order() {
let mut wrong = Vec::new();
for row in LOWEST_EFFORT.lines().filter(|l| !l.trim().is_empty()) {
let (columns, why) = row.split_once('|').unwrap();
let (reasoning, lowest) = columns.split_once("=>").unwrap();
let entry: ModelEntry =
serde_json::from_str(&format!(r#"{{"id":"m","reasoning":{reasoning}}}"#)).unwrap();
let got = entry.lowest_effort();
if got != Some(lowest.trim()).filter(|effort| *effort != "-") {
wrong.push(format!("{}: {got:?}", why.trim()));
}
}
assert!(wrong.is_empty(), "{wrong:#?}");
let silent: ModelEntry = serde_json::from_str(r#"{"id":"m"}"#).unwrap();
assert_eq!(silent.lowest_effort(), None, "no reasoning key at all");
}
}
#[cfg(test)]
mod xai_effort_tests {
use super::*;
fn listed(kind: &str) -> Option<Vec<&'static str>> {
let capabilities = match kind {
"4.3" => {
r#"{"reasoning_effort":["none","low","medium","high","xhigh"],"default_reasoning_effort":"low"}"#
}
"4.7" => {
r#"{"reasoning_effort":["low","medium","high","xhigh"],"default_reasoning_effort":"high"}"#
}
_ => return None,
};
let entry: ModelEntry =
serde_json::from_str(&format!(r#"{{"id":"m","capabilities":{capabilities}}}"#))
.unwrap();
entry.listed_efforts()
}
const ASKED: &str = "
4.3 none none | listed: a muted turn does not reason at all
4.3 low low | listed, as it is
4.3 xhigh xhigh | the top of the list
4.3 minimal low | not listed: the nearest depth, never the switch beside it
4.3 max xhigh | above the list
4.7 none low | a model that cannot stop: its lowest depth, not its default
4.7 minimal low | not listed
4.7 medium medium | listed
4.7 max xhigh | above the list
- none - | no list: three models of four refuse the value
- max xhigh | no list: all four refuse it
- minimal minimal | no list: every other depth goes as chosen
- xhigh xhigh | ...the highest xAI has among them
";
#[test]
fn an_effort_is_asked_as_the_models_list_has_it() {
let mut wrong = Vec::new();
for row in ASKED.lines().filter(|l| !l.trim().is_empty()) {
let (columns, why) = row.split_once('|').unwrap();
let c: Vec<&str> = columns.split_whitespace().collect();
let wish = std::iter::once(effort::NONE)
.chain(effort::LADDER)
.find(|value| *value == c[1])
.unwrap();
let got = xai_effort(wish, listed(c[0]).as_deref());
if got != Some(c[2]).filter(|asked| *asked != "-") {
wrong.push(format!("{}: {got:?}", why.trim()));
}
}
assert!(wrong.is_empty(), "{wrong:#?}");
}
const LISTED: &str = r#"
{"reasoning_effort":["none","low","medium","high","xhigh"]} => none low medium high xhigh | grok-4.3
{"reasoning_effort":["xhigh","low","high","medium"]} => low medium high xhigh | the scale's order, not the list's
{"reasoning_effort":["low","turbo",7,null]} => low | what is not a word of the scale is passed over
{"reasoning_effort":["turbo"]} => - | words with no place on the scale
{"reasoning_effort":[]} => - | an empty list
{"reasoning_effort":"low"} => - | not a list
{"default_reasoning_effort":"high"} => - | no list
null => - | no capabilities
"#;
#[test]
fn the_models_efforts_are_read_from_its_capabilities() {
let mut wrong = Vec::new();
for row in LISTED.lines().filter(|l| !l.trim().is_empty()) {
let (columns, why) = row.split_once('|').unwrap();
let (capabilities, efforts) = columns.split_once("=>").unwrap();
let entry: ModelEntry =
serde_json::from_str(&format!(r#"{{"id":"m","capabilities":{capabilities}}}"#))
.unwrap();
let expected: Vec<&str> = efforts.split_whitespace().filter(|e| *e != "-").collect();
let got = entry.listed_efforts().unwrap_or_default();
if got != expected {
wrong.push(format!("{}: {got:?}", why.trim()));
}
}
assert!(wrong.is_empty(), "{wrong:#?}");
let silent: ModelEntry =
serde_json::from_str(r#"{"id":"grok-build-0.1","context_length":256000}"#).unwrap();
assert_eq!(silent.listed_efforts(), None);
}
#[test]
fn a_requests_wish_is_none_a_depth_or_nothing() {
let wish = |thinking, effort, budget| {
effort_wish(&SamplingConfig {
thinking,
reasoning_effort: effort,
reasoning_budget: budget,
..Default::default()
})
};
assert_eq!(wish(None, Some(ReasoningEffort::None), None), Some("none"));
assert_eq!(wish(Some(false), None, None), Some("none"));
assert_eq!(wish(None, None, Some(0)), Some("none"));
assert_eq!(wish(None, Some(ReasoningEffort::Max), None), Some("max"));
assert_eq!(
wish(Some(false), Some(ReasoningEffort::High), Some(0)),
Some("high")
);
assert_eq!(wish(Some(true), None, None), None);
assert_eq!(wish(None, None, None), None);
}
#[test]
fn only_the_refusal_of_the_parameter_itself_is_read_as_one() {
let refusal = |status: u16, body: &str| {
EngineError::status(
crate::shared::api::error::SUBJECT_ENGINE,
reqwest::StatusCode::from_u16(status).unwrap(),
&reqwest::header::HeaderMap::new(),
body,
)
};
const PARAMETER: &str = r#"{"code":"invalid-argument","error":"Model grok-build-0.1 does not support parameter reasoningEffort."}"#;
assert!(refuses_effort_parameter(&refusal(400, PARAMETER)));
assert!(!refuses_effort_parameter(&refusal(503, PARAMETER)));
for other in [
r#"{"code":"invalid-argument","error":"Model grok-4.5 does not support parameter presencePenalty."}"#,
r#"{"code":"invalid-argument","error":"This model does not support `reasoning_effort` value `none`."}"#,
r#"{"code":"invalid-argument","error":"Invalid reasoning effort."}"#,
] {
assert!(!refuses_effort_parameter(&refusal(400, other)), "{other}");
}
}
#[test]
fn a_chat_body_carries_its_effort() {
use effort::CarriesEffort;
let req = ChatRequest {
continue_final: false,
system: None,
messages: vec![ApiMessage::user("hi")],
sampling: SamplingConfig {
reasoning_effort: Some(ReasoningEffort::High),
..Default::default()
},
tools: vec![],
};
let mut body = build_chat_request(&req, true, Some("m"), false);
assert_eq!(body.effort(), Some("high"));
body.respell_effort(None);
assert_eq!(body.effort(), None);
let json = serde_json::to_value(&body).unwrap();
assert!(json.get("reasoning_effort").is_none(), "{json}");
}
}