ferrum_server/
openai.rs

1//! OpenAI API compatibility types
2//!
3//! This module defines types that match the OpenAI API specification
4//! for chat completions, completions, and model management.
5
6use serde::{de, Deserialize, Serialize};
7use std::collections::HashMap;
8
9/// Chat completions request (OpenAI compatible)
10#[derive(Debug, Clone, Serialize, Deserialize)]
11pub struct ChatCompletionsRequest {
12    /// Model to use for completion
13    pub model: String,
14
15    /// List of messages
16    pub messages: Vec<ChatMessage>,
17
18    /// Maximum number of tokens to generate
19    #[serde(skip_serializing_if = "Option::is_none")]
20    pub max_tokens: Option<u32>,
21
22    /// Newer OpenAI chat field replacing `max_tokens` for completion budget.
23    /// When both are supplied, Ferrum uses this value.
24    #[serde(skip_serializing_if = "Option::is_none")]
25    pub max_completion_tokens: Option<u32>,
26
27    /// Temperature for sampling
28    #[serde(skip_serializing_if = "Option::is_none")]
29    pub temperature: Option<f32>,
30
31    /// Top-p for nucleus sampling
32    #[serde(skip_serializing_if = "Option::is_none")]
33    pub top_p: Option<f32>,
34
35    /// Number of completions to generate
36    #[serde(skip_serializing_if = "Option::is_none")]
37    pub n: Option<u32>,
38
39    /// Whether to stream responses
40    #[serde(skip_serializing_if = "Option::is_none")]
41    pub stream: Option<bool>,
42
43    /// vLLM-compatible extension for benchmark/throughput workloads.
44    /// When true, Ferrum ignores model EOS tokens and stops only on the
45    /// requested token budget or explicit user stop sequences.
46    #[serde(skip_serializing_if = "Option::is_none")]
47    pub ignore_eos: Option<bool>,
48
49    /// Stop sequences
50    #[serde(default, deserialize_with = "deserialize_stop_sequences")]
51    #[serde(skip_serializing_if = "Option::is_none")]
52    pub stop: Option<Vec<String>>,
53
54    /// Presence penalty
55    #[serde(skip_serializing_if = "Option::is_none")]
56    pub presence_penalty: Option<f32>,
57
58    /// Frequency penalty
59    #[serde(skip_serializing_if = "Option::is_none")]
60    pub frequency_penalty: Option<f32>,
61
62    /// Logit bias
63    #[serde(skip_serializing_if = "Option::is_none")]
64    pub logit_bias: Option<HashMap<String, f32>>,
65
66    /// Return log probabilities. Ferrum rejects this until implemented so
67    /// clients get an explicit OpenAI-style error instead of silent ignore.
68    #[serde(skip_serializing_if = "Option::is_none")]
69    pub logprobs: Option<bool>,
70
71    /// Number of top log probabilities to return.
72    #[serde(skip_serializing_if = "Option::is_none")]
73    pub top_logprobs: Option<u32>,
74
75    /// User identifier
76    #[serde(skip_serializing_if = "Option::is_none")]
77    pub user: Option<String>,
78
79    /// Random seed
80    #[serde(skip_serializing_if = "Option::is_none")]
81    pub seed: Option<u64>,
82
83    /// Response format constraint (e.g., `{"type": "json_object"}`)
84    #[serde(skip_serializing_if = "Option::is_none")]
85    pub response_format: Option<OpenAiResponseFormat>,
86
87    /// OpenAI tool definitions. Function tools are parsed, carried through
88    /// structured request data, and can shape model-emitted tool-call JSON.
89    /// Tool execution itself stays caller-owned.
90    #[serde(skip_serializing_if = "Option::is_none")]
91    pub tools: Option<Vec<ChatTool>>,
92
93    /// OpenAI tool selection policy.
94    #[serde(skip_serializing_if = "Option::is_none")]
95    pub tool_choice: Option<ToolChoice>,
96
97    /// Streaming response options.
98    #[serde(skip_serializing_if = "Option::is_none")]
99    pub stream_options: Option<StreamOptions>,
100
101    /// Legacy OpenAI functions compatibility.
102    #[serde(skip_serializing_if = "Option::is_none")]
103    pub functions: Option<Vec<ChatFunction>>,
104
105    /// Legacy OpenAI function-call selector.
106    #[serde(skip_serializing_if = "Option::is_none")]
107    pub function_call: Option<FunctionCallChoice>,
108}
109
110/// OpenAI streaming options.
111#[derive(Debug, Clone, Serialize, Deserialize)]
112pub struct StreamOptions {
113    #[serde(default, skip_serializing_if = "Option::is_none")]
114    pub include_usage: Option<bool>,
115}
116
117/// Tool definition in OpenAI chat-completion requests.
118#[derive(Debug, Clone, Serialize, Deserialize)]
119pub struct ChatTool {
120    #[serde(rename = "type")]
121    pub tool_type: String,
122    pub function: ChatFunction,
123}
124
125/// Function schema for `tools[].function` and legacy `functions[]`.
126#[derive(Debug, Clone, Serialize, Deserialize)]
127pub struct ChatFunction {
128    pub name: String,
129    #[serde(default, skip_serializing_if = "Option::is_none")]
130    pub description: Option<String>,
131    #[serde(default, skip_serializing_if = "Option::is_none")]
132    pub parameters: Option<serde_json::Value>,
133    #[serde(default, skip_serializing_if = "Option::is_none")]
134    pub strict: Option<bool>,
135}
136
137/// OpenAI `tool_choice` accepts either a simple mode string or a specific
138/// function-tool selector object.
139#[derive(Debug, Clone, Serialize, Deserialize)]
140#[serde(untagged)]
141pub enum ToolChoice {
142    Mode(String),
143    Function {
144        #[serde(rename = "type")]
145        tool_type: String,
146        function: ToolChoiceFunction,
147    },
148}
149
150#[derive(Debug, Clone, Serialize, Deserialize)]
151pub struct ToolChoiceFunction {
152    pub name: String,
153}
154
155/// Legacy `function_call` accepts a simple mode string or a named function.
156#[derive(Debug, Clone, Serialize, Deserialize)]
157#[serde(untagged)]
158pub enum FunctionCallChoice {
159    Mode(String),
160    Function { name: String },
161}
162
163/// OpenAI-compatible response format specifier.
164///
165/// Mirrors OpenAI's `response_format` field on `/v1/chat/completions`:
166///   - `{"type": "text"}`         — default, no constraint
167///   - `{"type": "json_object"}`  — output must be valid JSON
168///   - `{"type": "json_schema", "json_schema": {"name":..., "strict":true,
169///      "schema": {...}}}` — output must conform to the inline JSON Schema
170#[derive(Debug, Clone, Serialize, Deserialize)]
171pub struct OpenAiResponseFormat {
172    #[serde(rename = "type")]
173    pub format_type: String,
174    /// Present only when `format_type == "json_schema"`.
175    #[serde(default, skip_serializing_if = "Option::is_none")]
176    pub json_schema: Option<OpenAiJsonSchema>,
177}
178
179#[derive(Debug, Clone, Serialize, Deserialize)]
180pub struct OpenAiJsonSchema {
181    /// Optional name for the schema (ignored internally, kept for round-trip).
182    #[serde(default, skip_serializing_if = "Option::is_none")]
183    pub name: Option<String>,
184    /// The actual JSON Schema. Stored as raw JSON value so callers can pass
185    /// any valid schema object; we re-serialise when forwarding to the
186    /// guided-decoding pipeline. Optional at deserialization time so the
187    /// HTTP layer can return an OpenAI-shaped `param` error for missing
188    /// schemas instead of Axum's generic JSON rejection.
189    #[serde(default, skip_serializing_if = "Option::is_none")]
190    pub schema: Option<serde_json::Value>,
191    /// OpenAI's `strict` flag. When true, Ferrum rejects schemas outside the
192    /// currently supported guided-decoding subset instead of silently falling
193    /// back to best-effort JSON.
194    #[serde(default, skip_serializing_if = "Option::is_none")]
195    pub strict: Option<bool>,
196}
197
198/// Chat message
199#[derive(Debug, Clone, Serialize, Deserialize)]
200pub struct ChatMessage {
201    /// Message role
202    pub role: MessageRole,
203
204    /// Message content. Accepts either a plain string or the OpenAI
205    /// "typed parts" array form (`[{"type":"text","text":"..."}]`)
206    /// — both shapes deserialize into a single String. Non-text parts
207    /// fail deserialization so multimodal input is rejected instead of
208    /// silently dropped.
209    #[serde(default)]
210    #[serde(deserialize_with = "deserialize_message_content")]
211    pub content: String,
212
213    /// vLLM-compatible parsed reasoning text. When Ferrum parses
214    /// `<think>...</think>`, `content` contains only the final visible
215    /// answer and this field contains the reasoning block text.
216    #[serde(default, skip_serializing_if = "Option::is_none")]
217    pub reasoning: Option<String>,
218
219    /// Message name (for function calls)
220    #[serde(skip_serializing_if = "Option::is_none")]
221    pub name: Option<String>,
222
223    /// Assistant tool calls.
224    #[serde(skip_serializing_if = "Option::is_none")]
225    pub tool_calls: Option<Vec<ChatToolCall>>,
226
227    /// Tool response correlation id.
228    #[serde(skip_serializing_if = "Option::is_none")]
229    pub tool_call_id: Option<String>,
230
231    /// Legacy assistant function call.
232    #[serde(skip_serializing_if = "Option::is_none")]
233    pub function_call: Option<ChatFunctionCall>,
234}
235
236/// Assistant tool call in OpenAI responses and historical conversation input.
237#[derive(Debug, Clone, Serialize, Deserialize)]
238pub struct ChatToolCall {
239    #[serde(skip_serializing_if = "Option::is_none")]
240    pub index: Option<u32>,
241    pub id: String,
242    #[serde(rename = "type")]
243    pub tool_type: String,
244    pub function: ChatFunctionCall,
245}
246
247/// Function call payload. OpenAI serializes arguments as a JSON string.
248#[derive(Debug, Clone, Serialize, Deserialize)]
249pub struct ChatFunctionCall {
250    pub name: String,
251    pub arguments: String,
252}
253
254/// Deserialize chat message content from either a plain string or the
255/// OpenAI typed-parts array form. Real OpenAI clients (and `vllm bench
256/// serve`'s openai-chat backend) send `content` as
257/// `[{"type":"text","text":"..."}]` even for plain text; refusing that
258/// breaks every standard client.
259fn deserialize_message_content<'de, D>(deserializer: D) -> Result<String, D::Error>
260where
261    D: serde::Deserializer<'de>,
262{
263    let value = serde_json::Value::deserialize(deserializer)?;
264    match value {
265        serde_json::Value::Null => Ok(String::new()),
266        serde_json::Value::String(s) => Ok(s),
267        serde_json::Value::Array(parts) => {
268            let mut text_parts = Vec::with_capacity(parts.len());
269            for part in parts {
270                let ty = part
271                    .get("type")
272                    .and_then(|v| v.as_str())
273                    .ok_or_else(|| de::Error::custom("message content part missing type"))?;
274                if ty != "text" {
275                    return Err(de::Error::custom(format!(
276                        "unsupported message content part type `{ty}`"
277                    )));
278                }
279                if let Some(text) = part.get("text").and_then(|v| v.as_str()) {
280                    text_parts.push(text.to_string());
281                }
282            }
283            Ok(text_parts.join("\n"))
284        }
285        _ => Err(de::Error::custom(
286            "message content must be a string, null, or an array of text parts",
287        )),
288    }
289}
290
291fn deserialize_stop_sequences<'de, D>(deserializer: D) -> Result<Option<Vec<String>>, D::Error>
292where
293    D: serde::Deserializer<'de>,
294{
295    let value = Option::<serde_json::Value>::deserialize(deserializer)?;
296    match value {
297        None | Some(serde_json::Value::Null) => Ok(None),
298        Some(serde_json::Value::String(stop)) => Ok(Some(vec![stop])),
299        Some(serde_json::Value::Array(values)) => {
300            let mut stops = Vec::with_capacity(values.len());
301            for value in values {
302                match value {
303                    serde_json::Value::String(stop) => stops.push(stop),
304                    _ => {
305                        return Err(de::Error::custom(
306                            "stop must be a string or an array of strings",
307                        ))
308                    }
309                }
310            }
311            Ok(Some(stops))
312        }
313        _ => Err(de::Error::custom(
314            "stop must be a string or an array of strings",
315        )),
316    }
317}
318
319/// Message roles
320#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
321#[serde(rename_all = "lowercase")]
322pub enum MessageRole {
323    System,
324    User,
325    Assistant,
326    Function,
327    Tool,
328}
329
330/// Chat completions response
331#[derive(Debug, Clone, Serialize, Deserialize)]
332pub struct ChatCompletionsResponse {
333    /// Response ID
334    pub id: String,
335
336    /// Object type
337    pub object: String,
338
339    /// Creation timestamp
340    pub created: u64,
341
342    /// Model used
343    pub model: String,
344
345    /// Choices array
346    pub choices: Vec<ChatChoice>,
347
348    /// Token usage information
349    #[serde(skip_serializing_if = "Option::is_none")]
350    pub usage: Option<Usage>,
351}
352
353/// Chat choice
354#[derive(Debug, Clone, Serialize, Deserialize)]
355pub struct ChatChoice {
356    /// Choice index
357    pub index: u32,
358
359    /// Message content
360    #[serde(skip_serializing_if = "Option::is_none")]
361    pub message: Option<ChatMessage>,
362
363    /// Delta for streaming
364    #[serde(skip_serializing_if = "Option::is_none")]
365    pub delta: Option<ChatMessage>,
366
367    /// Finish reason
368    #[serde(skip_serializing_if = "Option::is_none")]
369    pub finish_reason: Option<String>,
370}
371
372/// Legacy completions request
373#[derive(Debug, Clone, Serialize, Deserialize)]
374pub struct CompletionsRequest {
375    /// Model to use
376    pub model: String,
377
378    /// Prompt text. OpenAI's legacy completions endpoint also accepts prompt
379    /// arrays, but Ferrum currently supports only a single string and rejects
380    /// other shapes with `param=prompt`.
381    #[serde(default)]
382    pub prompt: CompletionPrompt,
383
384    /// Maximum tokens
385    #[serde(skip_serializing_if = "Option::is_none")]
386    pub max_tokens: Option<u32>,
387
388    /// Temperature
389    #[serde(skip_serializing_if = "Option::is_none")]
390    pub temperature: Option<f32>,
391
392    /// Top-p
393    #[serde(skip_serializing_if = "Option::is_none")]
394    pub top_p: Option<f32>,
395
396    /// Number of completions to generate. Ferrum currently supports only
397    /// `n=1` and rejects larger values explicitly.
398    #[serde(skip_serializing_if = "Option::is_none")]
399    pub n: Option<u32>,
400
401    /// Stream responses
402    #[serde(skip_serializing_if = "Option::is_none")]
403    pub stream: Option<bool>,
404
405    /// Stop sequences
406    #[serde(default, deserialize_with = "deserialize_stop_sequences")]
407    #[serde(skip_serializing_if = "Option::is_none")]
408    pub stop: Option<Vec<String>>,
409
410    /// Legacy completions log probabilities. Explicitly rejected until
411    /// implemented so clients don't mistake a silent ignore for support.
412    #[serde(skip_serializing_if = "Option::is_none")]
413    pub logprobs: Option<u32>,
414
415    /// Logit bias.
416    #[serde(skip_serializing_if = "Option::is_none")]
417    pub logit_bias: Option<HashMap<String, f32>>,
418}
419
420/// Legacy completions prompt. Kept as a parsed enum so the HTTP layer can
421/// return an OpenAI-shaped field error instead of a generic JSON rejection.
422#[derive(Debug, Clone, Serialize, Deserialize)]
423#[serde(untagged)]
424pub enum CompletionPrompt {
425    Text(String),
426    Unsupported(serde_json::Value),
427}
428
429impl Default for CompletionPrompt {
430    fn default() -> Self {
431        Self::Unsupported(serde_json::Value::Null)
432    }
433}
434
435impl CompletionPrompt {
436    pub fn as_text(&self) -> Option<&str> {
437        match self {
438            Self::Text(text) => Some(text),
439            Self::Unsupported(_) => None,
440        }
441    }
442}
443
444/// Completions response
445#[derive(Debug, Clone, Serialize, Deserialize)]
446pub struct CompletionsResponse {
447    pub id: String,
448    pub object: String,
449    pub created: u64,
450    pub model: String,
451    pub choices: Vec<CompletionChoice>,
452    pub usage: Option<Usage>,
453}
454
455/// Completion choice
456#[derive(Debug, Clone, Serialize, Deserialize)]
457pub struct CompletionChoice {
458    pub text: String,
459    pub index: u32,
460    pub finish_reason: Option<String>,
461}
462
463/// Token usage information
464#[derive(Debug, Clone, Serialize, Deserialize)]
465pub struct Usage {
466    pub prompt_tokens: u32,
467    pub completion_tokens: u32,
468    pub total_tokens: u32,
469}
470
471/// Model list response
472#[derive(Debug, Clone, Serialize, Deserialize)]
473pub struct ModelListResponse {
474    pub object: String,
475    pub data: Vec<ModelInfo>,
476}
477
478/// Model information
479#[derive(Debug, Clone, Serialize, Deserialize)]
480pub struct ModelInfo {
481    pub id: String,
482    pub object: String,
483    pub created: u64,
484    pub owned_by: String,
485    pub permission: Vec<ModelPermission>,
486    pub root: Option<String>,
487    pub parent: Option<String>,
488}
489
490/// Model permission
491#[derive(Debug, Clone, Serialize, Deserialize)]
492pub struct ModelPermission {
493    pub id: String,
494    pub object: String,
495    pub created: u64,
496    pub allow_create_engine: bool,
497    pub allow_sampling: bool,
498    pub allow_logprobs: bool,
499    pub allow_search_indices: bool,
500    pub allow_view: bool,
501    pub allow_fine_tuning: bool,
502    pub organization: String,
503    pub group: Option<String>,
504    pub is_blocking: bool,
505}
506
507// ======================== Embeddings API ========================
508
509/// Embeddings request (OpenAI-compatible, extended for images)
510#[derive(Debug, Clone, Serialize, Deserialize)]
511pub struct EmbeddingsRequest {
512    /// Model identifier
513    pub model: String,
514
515    /// Input to embed — text string, array of strings, or objects with text/image fields
516    pub input: EmbeddingInput,
517
518    /// Encoding format: "float" (default) or "base64"
519    #[serde(skip_serializing_if = "Option::is_none")]
520    pub encoding_format: Option<String>,
521}
522
523/// Polymorphic embedding input.
524/// Supports: single string, array of strings, single object, array of objects.
525#[derive(Debug, Clone, Serialize, Deserialize)]
526#[serde(untagged)]
527pub enum EmbeddingInput {
528    /// Single text string (OpenAI standard)
529    Single(String),
530    /// Batch of text strings (OpenAI standard)
531    Batch(Vec<String>),
532    /// Single multimodal item (Jina-style extension)
533    SingleObject(EmbeddingItem),
534    /// Batch of multimodal items
535    BatchObjects(Vec<EmbeddingItem>),
536}
537
538/// A single embedding input item — text or image.
539#[derive(Debug, Clone, Serialize, Deserialize)]
540pub struct EmbeddingItem {
541    /// Text to embed
542    #[serde(skip_serializing_if = "Option::is_none")]
543    pub text: Option<String>,
544    /// Image: file path or base64 data URI
545    #[serde(skip_serializing_if = "Option::is_none")]
546    pub image: Option<String>,
547}
548
549/// Embeddings response (OpenAI-compatible)
550#[derive(Debug, Clone, Serialize, Deserialize)]
551pub struct EmbeddingsResponse {
552    pub object: String,
553    pub data: Vec<EmbeddingData>,
554    pub model: String,
555    pub usage: EmbeddingUsage,
556}
557
558/// Single embedding result
559#[derive(Debug, Clone, Serialize, Deserialize)]
560pub struct EmbeddingData {
561    pub object: String,
562    pub embedding: Vec<f32>,
563    pub index: usize,
564}
565
566/// Token usage for embeddings
567#[derive(Debug, Clone, Serialize, Deserialize)]
568pub struct EmbeddingUsage {
569    pub prompt_tokens: u32,
570    pub total_tokens: u32,
571}
572
573// ======================== Audio Transcription API ========================
574
575/// Transcription response (OpenAI-compatible)
576#[derive(Debug, Clone, Serialize, Deserialize)]
577pub struct TranscriptionResponse {
578    pub text: String,
579}
580
581// ======================== Error types ========================
582
583/// OpenAI API error
584#[derive(Debug, Clone, Serialize, Deserialize)]
585pub struct OpenAiError {
586    pub error: OpenAiErrorDetail,
587}
588
589/// OpenAI error detail
590#[derive(Debug, Clone, Serialize, Deserialize)]
591pub struct OpenAiErrorDetail {
592    pub message: String,
593    #[serde(rename = "type")]
594    pub error_type: String,
595    pub param: Option<String>,
596    pub code: Option<String>,
597}
598
599/// OpenAI error types
600#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
601pub enum OpenAiErrorType {
602    InvalidRequestError,
603    AuthenticationError,
604    PermissionError,
605    NotFoundError,
606    RateLimitError,
607    InternalServerError,
608    ServiceUnavailableError,
609}
610
611/// Server-sent event for streaming
612#[derive(Debug, Clone)]
613pub struct SseEvent {
614    pub event: Option<String>,
615    pub data: String,
616    pub id: Option<String>,
617    pub retry: Option<u32>,
618}
619
620impl SseEvent {
621    pub fn data(data: String) -> Self {
622        Self {
623            event: None,
624            data,
625            id: None,
626            retry: None,
627        }
628    }
629
630    pub fn json(value: &serde_json::Value) -> Result<Self, serde_json::Error> {
631        Ok(Self::data(serde_json::to_string(value)?))
632    }
633
634    pub fn to_string(&self) -> String {
635        let mut result = String::new();
636
637        if let Some(event) = &self.event {
638            result.push_str(&format!("event: {}\n", event));
639        }
640
641        if let Some(id) = &self.id {
642            result.push_str(&format!("id: {}\n", id));
643        }
644
645        if let Some(retry) = self.retry {
646            result.push_str(&format!("retry: {}\n", retry));
647        }
648
649        result.push_str(&format!("data: {}\n\n", self.data));
650        result
651    }
652}
653
654/// TTS speech request (OpenAI compatible /v1/audio/speech)
655#[derive(Debug, Clone, Serialize, Deserialize)]
656pub struct SpeechRequest {
657    /// Model name (e.g., "qwen3-tts", "tts-1")
658    #[serde(default = "default_tts_model")]
659    pub model: String,
660
661    /// Text to synthesize
662    pub input: String,
663
664    /// Voice preset (ignored for now — uses default speaker)
665    #[serde(default = "default_voice")]
666    pub voice: String,
667
668    /// Response format: "wav", "pcm" (default: "wav")
669    #[serde(default = "default_audio_format")]
670    pub response_format: String,
671
672    /// Language hint: "auto", "chinese", "english"
673    #[serde(default = "default_language")]
674    pub language: String,
675
676    /// Enable streaming (chunked transfer)
677    #[serde(default)]
678    pub stream: bool,
679}
680
681fn default_tts_model() -> String {
682    "qwen3-tts".to_string()
683}
684fn default_voice() -> String {
685    "default".to_string()
686}
687fn default_audio_format() -> String {
688    "wav".to_string()
689}
690fn default_language() -> String {
691    "auto".to_string()
692}
ferrum_server/openai.rs

ferrum_server/
openai.rs
