ferrum_server/
openai.rs

1//! OpenAI API compatibility types
2//!
3//! This module defines types that match the OpenAI API specification
4//! for chat completions, completions, and model management.
5
6use serde::{de, Deserialize, Serialize};
7use std::collections::HashMap;
8
9/// Chat completions request (OpenAI compatible)
10#[derive(Debug, Clone, Serialize, Deserialize)]
11pub struct ChatCompletionsRequest {
12    /// Model to use for completion
13    pub model: String,
14
15    /// List of messages
16    pub messages: Vec<ChatMessage>,
17
18    /// Maximum number of tokens to generate
19    #[serde(skip_serializing_if = "Option::is_none")]
20    pub max_tokens: Option<u32>,
21
22    /// Newer OpenAI chat field replacing `max_tokens` for completion budget.
23    /// When both are supplied, Ferrum uses this value.
24    #[serde(skip_serializing_if = "Option::is_none")]
25    pub max_completion_tokens: Option<u32>,
26
27    /// Temperature for sampling
28    #[serde(skip_serializing_if = "Option::is_none")]
29    pub temperature: Option<f32>,
30
31    /// Top-p for nucleus sampling
32    #[serde(skip_serializing_if = "Option::is_none")]
33    pub top_p: Option<f32>,
34
35    /// Number of completions to generate
36    #[serde(skip_serializing_if = "Option::is_none")]
37    pub n: Option<u32>,
38
39    /// Whether to stream responses
40    #[serde(skip_serializing_if = "Option::is_none")]
41    pub stream: Option<bool>,
42
43    /// vLLM-compatible extension for benchmark/throughput workloads.
44    /// When true, Ferrum ignores model EOS tokens and stops only on the
45    /// requested token budget or explicit user stop sequences.
46    #[serde(skip_serializing_if = "Option::is_none")]
47    pub ignore_eos: Option<bool>,
48
49    /// Stop sequences
50    #[serde(default, deserialize_with = "deserialize_stop_sequences")]
51    #[serde(skip_serializing_if = "Option::is_none")]
52    pub stop: Option<Vec<String>>,
53
54    /// Presence penalty
55    #[serde(skip_serializing_if = "Option::is_none")]
56    pub presence_penalty: Option<f32>,
57
58    /// Frequency penalty
59    #[serde(skip_serializing_if = "Option::is_none")]
60    pub frequency_penalty: Option<f32>,
61
62    /// Logit bias
63    #[serde(skip_serializing_if = "Option::is_none")]
64    pub logit_bias: Option<HashMap<String, f32>>,
65
66    /// Return log probabilities. Ferrum rejects this until implemented so
67    /// clients get an explicit OpenAI-style error instead of silent ignore.
68    #[serde(skip_serializing_if = "Option::is_none")]
69    pub logprobs: Option<bool>,
70
71    /// Number of top log probabilities to return.
72    #[serde(skip_serializing_if = "Option::is_none")]
73    pub top_logprobs: Option<u32>,
74
75    /// User identifier
76    #[serde(skip_serializing_if = "Option::is_none")]
77    pub user: Option<String>,
78
79    /// Random seed
80    #[serde(skip_serializing_if = "Option::is_none")]
81    pub seed: Option<u64>,
82
83    /// Response format constraint (e.g., `{"type": "json_object"}`)
84    #[serde(skip_serializing_if = "Option::is_none")]
85    pub response_format: Option<OpenAiResponseFormat>,
86
87    /// OpenAI tool definitions. Function tools are parsed, carried through
88    /// structured request data, and can shape model-emitted tool-call JSON.
89    /// Tool execution itself stays caller-owned.
90    #[serde(skip_serializing_if = "Option::is_none")]
91    pub tools: Option<Vec<ChatTool>>,
92
93    /// OpenAI tool selection policy.
94    #[serde(skip_serializing_if = "Option::is_none")]
95    pub tool_choice: Option<ToolChoice>,
96
97    /// Streaming response options.
98    #[serde(skip_serializing_if = "Option::is_none")]
99    pub stream_options: Option<StreamOptions>,
100
101    /// Legacy OpenAI functions compatibility.
102    #[serde(skip_serializing_if = "Option::is_none")]
103    pub functions: Option<Vec<ChatFunction>>,
104
105    /// Legacy OpenAI function-call selector.
106    #[serde(skip_serializing_if = "Option::is_none")]
107    pub function_call: Option<FunctionCallChoice>,
108
109    /// Ferrum extension metadata. Used for opt-in product features such as
110    /// `metadata.ferrum_session_id` when callers prefer body metadata over
111    /// the `X-Ferrum-Session` header.
112    #[serde(default, skip_serializing_if = "Option::is_none")]
113    pub metadata: Option<HashMap<String, serde_json::Value>>,
114
115    /// vLLM-compatible chat-template variables. Ferrum forwards supported
116    /// values to the model-provided chat template; templates that do not read
117    /// a variable are unaffected.
118    #[serde(default, skip_serializing_if = "Option::is_none")]
119    pub chat_template_kwargs: Option<HashMap<String, serde_json::Value>>,
120}
121
122/// OpenAI streaming options.
123#[derive(Debug, Clone, Serialize, Deserialize)]
124pub struct StreamOptions {
125    #[serde(default, skip_serializing_if = "Option::is_none")]
126    pub include_usage: Option<bool>,
127}
128
129/// Tool definition in OpenAI chat-completion requests.
130#[derive(Debug, Clone, Serialize, Deserialize)]
131pub struct ChatTool {
132    #[serde(rename = "type")]
133    pub tool_type: String,
134    pub function: ChatFunction,
135}
136
137/// Function schema for `tools[].function` and legacy `functions[]`.
138#[derive(Debug, Clone, Serialize, Deserialize)]
139pub struct ChatFunction {
140    pub name: String,
141    #[serde(default, skip_serializing_if = "Option::is_none")]
142    pub description: Option<String>,
143    #[serde(default, skip_serializing_if = "Option::is_none")]
144    pub parameters: Option<serde_json::Value>,
145    #[serde(default, skip_serializing_if = "Option::is_none")]
146    pub strict: Option<bool>,
147}
148
149/// OpenAI `tool_choice` accepts either a simple mode string or a specific
150/// function-tool selector object.
151#[derive(Debug, Clone, Serialize, Deserialize)]
152#[serde(untagged)]
153pub enum ToolChoice {
154    Mode(String),
155    Function {
156        #[serde(rename = "type")]
157        tool_type: String,
158        function: ToolChoiceFunction,
159    },
160}
161
162#[derive(Debug, Clone, Serialize, Deserialize)]
163pub struct ToolChoiceFunction {
164    pub name: String,
165}
166
167/// Legacy `function_call` accepts a simple mode string or a named function.
168#[derive(Debug, Clone, Serialize, Deserialize)]
169#[serde(untagged)]
170pub enum FunctionCallChoice {
171    Mode(String),
172    Function { name: String },
173}
174
175/// OpenAI-compatible response format specifier.
176///
177/// Mirrors OpenAI's `response_format` field on `/v1/chat/completions`:
178///   - `{"type": "text"}`         — default, no constraint
179///   - `{"type": "json_object"}`  — output must be valid JSON
180///   - `{"type": "json_schema", "json_schema": {"name":..., "strict":true,
181///      "schema": {...}}}` — output must conform to the inline JSON Schema
182#[derive(Debug, Clone, Serialize, Deserialize)]
183pub struct OpenAiResponseFormat {
184    #[serde(rename = "type")]
185    pub format_type: String,
186    /// Present only when `format_type == "json_schema"`.
187    #[serde(default, skip_serializing_if = "Option::is_none")]
188    pub json_schema: Option<OpenAiJsonSchema>,
189}
190
191#[derive(Debug, Clone, Serialize, Deserialize)]
192pub struct OpenAiJsonSchema {
193    /// Optional name for the schema (ignored internally, kept for round-trip).
194    #[serde(default, skip_serializing_if = "Option::is_none")]
195    pub name: Option<String>,
196    /// The actual JSON Schema. Stored as raw JSON value so callers can pass
197    /// any valid schema object; we re-serialise when forwarding to the
198    /// guided-decoding pipeline. Optional at deserialization time so the
199    /// HTTP layer can return an OpenAI-shaped `param` error for missing
200    /// schemas instead of Axum's generic JSON rejection.
201    #[serde(default, skip_serializing_if = "Option::is_none")]
202    pub schema: Option<serde_json::Value>,
203    /// OpenAI's `strict` flag. When true, Ferrum rejects schemas outside the
204    /// currently supported guided-decoding subset instead of silently falling
205    /// back to best-effort JSON.
206    #[serde(default, skip_serializing_if = "Option::is_none")]
207    pub strict: Option<bool>,
208}
209
210/// Chat message
211#[derive(Debug, Clone, Serialize, Deserialize)]
212pub struct ChatMessage {
213    /// Message role
214    pub role: MessageRole,
215
216    /// Message content. Accepts either a plain string or the OpenAI
217    /// "typed parts" array form (`[{"type":"text","text":"..."}]`)
218    /// — both shapes deserialize into a single String. Non-text parts
219    /// fail deserialization so multimodal input is rejected instead of
220    /// silently dropped.
221    #[serde(default)]
222    #[serde(deserialize_with = "deserialize_message_content")]
223    pub content: String,
224
225    /// vLLM-compatible parsed reasoning text. When Ferrum parses
226    /// `<think>...</think>`, `content` contains only the final visible
227    /// answer and this field contains the reasoning block text.
228    #[serde(default, skip_serializing_if = "Option::is_none")]
229    pub reasoning: Option<String>,
230
231    /// Message name (for function calls)
232    #[serde(skip_serializing_if = "Option::is_none")]
233    pub name: Option<String>,
234
235    /// Assistant tool calls.
236    #[serde(skip_serializing_if = "Option::is_none")]
237    pub tool_calls: Option<Vec<ChatToolCall>>,
238
239    /// Tool response correlation id.
240    #[serde(skip_serializing_if = "Option::is_none")]
241    pub tool_call_id: Option<String>,
242
243    /// Legacy assistant function call.
244    #[serde(skip_serializing_if = "Option::is_none")]
245    pub function_call: Option<ChatFunctionCall>,
246}
247
248/// Assistant tool call in OpenAI responses and historical conversation input.
249#[derive(Debug, Clone, Serialize, Deserialize)]
250pub struct ChatToolCall {
251    #[serde(skip_serializing_if = "Option::is_none")]
252    pub index: Option<u32>,
253    pub id: String,
254    #[serde(rename = "type")]
255    pub tool_type: String,
256    pub function: ChatFunctionCall,
257}
258
259/// Function call payload. OpenAI serializes arguments as a JSON string.
260#[derive(Debug, Clone, Serialize, Deserialize)]
261pub struct ChatFunctionCall {
262    pub name: String,
263    pub arguments: String,
264}
265
266/// Deserialize chat message content from either a plain string or the
267/// OpenAI typed-parts array form. Real OpenAI clients (and `vllm bench
268/// serve`'s openai-chat backend) send `content` as
269/// `[{"type":"text","text":"..."}]` even for plain text; refusing that
270/// breaks every standard client.
271fn deserialize_message_content<'de, D>(deserializer: D) -> Result<String, D::Error>
272where
273    D: serde::Deserializer<'de>,
274{
275    let value = serde_json::Value::deserialize(deserializer)?;
276    match value {
277        serde_json::Value::Null => Ok(String::new()),
278        serde_json::Value::String(s) => Ok(s),
279        serde_json::Value::Array(parts) => {
280            let mut text_parts = Vec::with_capacity(parts.len());
281            for part in parts {
282                let ty = part
283                    .get("type")
284                    .and_then(|v| v.as_str())
285                    .ok_or_else(|| de::Error::custom("message content part missing type"))?;
286                if ty != "text" {
287                    return Err(de::Error::custom(format!(
288                        "unsupported message content part type `{ty}`"
289                    )));
290                }
291                if let Some(text) = part.get("text").and_then(|v| v.as_str()) {
292                    text_parts.push(text.to_string());
293                }
294            }
295            Ok(text_parts.join("\n"))
296        }
297        _ => Err(de::Error::custom(
298            "message content must be a string, null, or an array of text parts",
299        )),
300    }
301}
302
303fn deserialize_stop_sequences<'de, D>(deserializer: D) -> Result<Option<Vec<String>>, D::Error>
304where
305    D: serde::Deserializer<'de>,
306{
307    let value = Option::<serde_json::Value>::deserialize(deserializer)?;
308    match value {
309        None | Some(serde_json::Value::Null) => Ok(None),
310        Some(serde_json::Value::String(stop)) => Ok(Some(vec![stop])),
311        Some(serde_json::Value::Array(values)) => {
312            let mut stops = Vec::with_capacity(values.len());
313            for value in values {
314                match value {
315                    serde_json::Value::String(stop) => stops.push(stop),
316                    _ => {
317                        return Err(de::Error::custom(
318                            "stop must be a string or an array of strings",
319                        ))
320                    }
321                }
322            }
323            Ok(Some(stops))
324        }
325        _ => Err(de::Error::custom(
326            "stop must be a string or an array of strings",
327        )),
328    }
329}
330
331/// Message roles
332#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
333#[serde(rename_all = "lowercase")]
334pub enum MessageRole {
335    System,
336    User,
337    Assistant,
338    Function,
339    Tool,
340}
341
342/// Chat completions response
343#[derive(Debug, Clone, Serialize, Deserialize)]
344pub struct ChatCompletionsResponse {
345    /// Response ID
346    pub id: String,
347
348    /// Object type
349    pub object: String,
350
351    /// Creation timestamp
352    pub created: u64,
353
354    /// Model used
355    pub model: String,
356
357    /// Choices array
358    pub choices: Vec<ChatChoice>,
359
360    /// Token usage information
361    #[serde(skip_serializing_if = "Option::is_none")]
362    pub usage: Option<Usage>,
363}
364
365/// Chat choice
366#[derive(Debug, Clone, Serialize, Deserialize)]
367pub struct ChatChoice {
368    /// Choice index
369    pub index: u32,
370
371    /// Message content
372    #[serde(skip_serializing_if = "Option::is_none")]
373    pub message: Option<ChatMessage>,
374
375    /// Delta for streaming
376    #[serde(skip_serializing_if = "Option::is_none")]
377    pub delta: Option<ChatMessage>,
378
379    /// Finish reason
380    #[serde(skip_serializing_if = "Option::is_none")]
381    pub finish_reason: Option<String>,
382}
383
384/// Legacy completions request
385#[derive(Debug, Clone, Serialize, Deserialize)]
386pub struct CompletionsRequest {
387    /// Model to use
388    pub model: String,
389
390    /// Prompt text. OpenAI's legacy completions endpoint also accepts prompt
391    /// arrays, but Ferrum currently supports only a single string and rejects
392    /// other shapes with `param=prompt`.
393    #[serde(default)]
394    pub prompt: CompletionPrompt,
395
396    /// Maximum tokens
397    #[serde(skip_serializing_if = "Option::is_none")]
398    pub max_tokens: Option<u32>,
399
400    /// Temperature
401    #[serde(skip_serializing_if = "Option::is_none")]
402    pub temperature: Option<f32>,
403
404    /// Top-p
405    #[serde(skip_serializing_if = "Option::is_none")]
406    pub top_p: Option<f32>,
407
408    /// Number of completions to generate. Ferrum currently supports only
409    /// `n=1` and rejects larger values explicitly.
410    #[serde(skip_serializing_if = "Option::is_none")]
411    pub n: Option<u32>,
412
413    /// Stream responses
414    #[serde(skip_serializing_if = "Option::is_none")]
415    pub stream: Option<bool>,
416
417    /// Stop sequences
418    #[serde(default, deserialize_with = "deserialize_stop_sequences")]
419    #[serde(skip_serializing_if = "Option::is_none")]
420    pub stop: Option<Vec<String>>,
421
422    /// Legacy completions log probabilities. Explicitly rejected until
423    /// implemented so clients don't mistake a silent ignore for support.
424    #[serde(skip_serializing_if = "Option::is_none")]
425    pub logprobs: Option<u32>,
426
427    /// Logit bias.
428    #[serde(skip_serializing_if = "Option::is_none")]
429    pub logit_bias: Option<HashMap<String, f32>>,
430}
431
432/// Legacy completions prompt. Kept as a parsed enum so the HTTP layer can
433/// return an OpenAI-shaped field error instead of a generic JSON rejection.
434#[derive(Debug, Clone, Serialize, Deserialize)]
435#[serde(untagged)]
436pub enum CompletionPrompt {
437    Text(String),
438    Unsupported(serde_json::Value),
439}
440
441impl Default for CompletionPrompt {
442    fn default() -> Self {
443        Self::Unsupported(serde_json::Value::Null)
444    }
445}
446
447impl CompletionPrompt {
448    pub fn as_text(&self) -> Option<&str> {
449        match self {
450            Self::Text(text) => Some(text),
451            Self::Unsupported(_) => None,
452        }
453    }
454}
455
456/// Completions response
457#[derive(Debug, Clone, Serialize, Deserialize)]
458pub struct CompletionsResponse {
459    pub id: String,
460    pub object: String,
461    pub created: u64,
462    pub model: String,
463    pub choices: Vec<CompletionChoice>,
464    pub usage: Option<Usage>,
465}
466
467/// Completion choice
468#[derive(Debug, Clone, Serialize, Deserialize)]
469pub struct CompletionChoice {
470    pub text: String,
471    pub index: u32,
472    pub finish_reason: Option<String>,
473}
474
475/// Token usage information
476#[derive(Debug, Clone, Serialize, Deserialize)]
477pub struct Usage {
478    pub prompt_tokens: u32,
479    pub completion_tokens: u32,
480    pub total_tokens: u32,
481}
482
483/// Model list response
484#[derive(Debug, Clone, Serialize, Deserialize)]
485pub struct ModelListResponse {
486    pub object: String,
487    pub data: Vec<ModelInfo>,
488}
489
490/// Model information
491#[derive(Debug, Clone, Serialize, Deserialize)]
492pub struct ModelInfo {
493    pub id: String,
494    pub object: String,
495    pub created: u64,
496    pub owned_by: String,
497    pub permission: Vec<ModelPermission>,
498    pub root: Option<String>,
499    pub parent: Option<String>,
500}
501
502/// Model permission
503#[derive(Debug, Clone, Serialize, Deserialize)]
504pub struct ModelPermission {
505    pub id: String,
506    pub object: String,
507    pub created: u64,
508    pub allow_create_engine: bool,
509    pub allow_sampling: bool,
510    pub allow_logprobs: bool,
511    pub allow_search_indices: bool,
512    pub allow_view: bool,
513    pub allow_fine_tuning: bool,
514    pub organization: String,
515    pub group: Option<String>,
516    pub is_blocking: bool,
517}
518
519// ======================== Embeddings API ========================
520
521/// Embeddings request (OpenAI-compatible, extended for images)
522#[derive(Debug, Clone, Serialize, Deserialize)]
523pub struct EmbeddingsRequest {
524    /// Model identifier
525    pub model: String,
526
527    /// Input to embed — text string, array of strings, or objects with text/image fields
528    pub input: EmbeddingInput,
529
530    /// Encoding format: "float" (default) or "base64"
531    #[serde(skip_serializing_if = "Option::is_none")]
532    pub encoding_format: Option<String>,
533}
534
535/// Polymorphic embedding input.
536/// Supports: single string, array of strings, single object, array of objects.
537#[derive(Debug, Clone, Serialize, Deserialize)]
538#[serde(untagged)]
539pub enum EmbeddingInput {
540    /// Single text string (OpenAI standard)
541    Single(String),
542    /// Batch of text strings (OpenAI standard)
543    Batch(Vec<String>),
544    /// Single multimodal item (Jina-style extension)
545    SingleObject(EmbeddingItem),
546    /// Batch of multimodal items
547    BatchObjects(Vec<EmbeddingItem>),
548}
549
550/// A single embedding input item — text or image.
551#[derive(Debug, Clone, Serialize, Deserialize)]
552pub struct EmbeddingItem {
553    /// Text to embed
554    #[serde(skip_serializing_if = "Option::is_none")]
555    pub text: Option<String>,
556    /// Image: file path or base64 data URI
557    #[serde(skip_serializing_if = "Option::is_none")]
558    pub image: Option<String>,
559}
560
561/// Embeddings response (OpenAI-compatible)
562#[derive(Debug, Clone, Serialize, Deserialize)]
563pub struct EmbeddingsResponse {
564    pub object: String,
565    pub data: Vec<EmbeddingData>,
566    pub model: String,
567    pub usage: EmbeddingUsage,
568}
569
570/// Single embedding result
571#[derive(Debug, Clone, Serialize, Deserialize)]
572pub struct EmbeddingData {
573    pub object: String,
574    pub embedding: Vec<f32>,
575    pub index: usize,
576}
577
578/// Token usage for embeddings
579#[derive(Debug, Clone, Serialize, Deserialize)]
580pub struct EmbeddingUsage {
581    pub prompt_tokens: u32,
582    pub total_tokens: u32,
583}
584
585// ======================== Audio Transcription API ========================
586
587/// Transcription response (OpenAI-compatible)
588#[derive(Debug, Clone, Serialize, Deserialize)]
589pub struct TranscriptionResponse {
590    pub text: String,
591}
592
593// ======================== Error types ========================
594
595/// OpenAI API error
596#[derive(Debug, Clone, Serialize, Deserialize)]
597pub struct OpenAiError {
598    pub error: OpenAiErrorDetail,
599}
600
601/// OpenAI error detail
602#[derive(Debug, Clone, Serialize, Deserialize)]
603pub struct OpenAiErrorDetail {
604    pub message: String,
605    #[serde(rename = "type")]
606    pub error_type: String,
607    pub param: Option<String>,
608    pub code: Option<String>,
609}
610
611/// OpenAI error types
612#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
613pub enum OpenAiErrorType {
614    InvalidRequestError,
615    AuthenticationError,
616    PermissionError,
617    NotFoundError,
618    RateLimitError,
619    InternalServerError,
620    ServiceUnavailableError,
621}
622
623/// Server-sent event for streaming
624#[derive(Debug, Clone)]
625pub struct SseEvent {
626    pub event: Option<String>,
627    pub data: String,
628    pub id: Option<String>,
629    pub retry: Option<u32>,
630}
631
632impl SseEvent {
633    pub fn data(data: String) -> Self {
634        Self {
635            event: None,
636            data,
637            id: None,
638            retry: None,
639        }
640    }
641
642    pub fn json(value: &serde_json::Value) -> Result<Self, serde_json::Error> {
643        Ok(Self::data(serde_json::to_string(value)?))
644    }
645
646    pub fn to_string(&self) -> String {
647        let mut result = String::new();
648
649        if let Some(event) = &self.event {
650            result.push_str(&format!("event: {}\n", event));
651        }
652
653        if let Some(id) = &self.id {
654            result.push_str(&format!("id: {}\n", id));
655        }
656
657        if let Some(retry) = self.retry {
658            result.push_str(&format!("retry: {}\n", retry));
659        }
660
661        result.push_str(&format!("data: {}\n\n", self.data));
662        result
663    }
664}
665
666/// TTS speech request (OpenAI compatible /v1/audio/speech)
667#[derive(Debug, Clone, Serialize, Deserialize)]
668pub struct SpeechRequest {
669    /// Model name (e.g., "qwen3-tts", "tts-1")
670    #[serde(default = "default_tts_model")]
671    pub model: String,
672
673    /// Text to synthesize
674    pub input: String,
675
676    /// Voice preset (ignored for now — uses default speaker)
677    #[serde(default = "default_voice")]
678    pub voice: String,
679
680    /// Response format: "wav", "pcm" (default: "wav")
681    #[serde(default = "default_audio_format")]
682    pub response_format: String,
683
684    /// Language hint: "auto", "chinese", "english"
685    #[serde(default = "default_language")]
686    pub language: String,
687
688    /// Enable streaming (chunked transfer)
689    #[serde(default)]
690    pub stream: bool,
691}
692
693fn default_tts_model() -> String {
694    "qwen3-tts".to_string()
695}
696fn default_voice() -> String {
697    "default".to_string()
698}
699fn default_audio_format() -> String {
700    "wav".to_string()
701}
702fn default_language() -> String {
703    "auto".to_string()
704}
ferrum_server/openai.rs

ferrum_server/
openai.rs
