ferrin_spec/
realtime_model.rs

1//! Realtime (WebSocket) model interface.
2//!
3//! A realtime model does not own the connection. It issues client secrets,
4//! describes how to open the WebSocket, and translates between provider wire
5//! events and the standardized [`RealtimeServerEvent`] /
6//! [`RealtimeClientEvent`] sets. The session loop lives in the core.
7
8use std::future::Future;
9
10use bytes::Bytes;
11use serde::Deserialize;
12use serde::Serialize;
13use url::Url;
14
15use crate::error::NoSuchModelError;
16use crate::error::ProviderError;
17use crate::json::JsonObject;
18use crate::json::JsonValue;
19use crate::shared::AudioFormat;
20use crate::shared::ModelId;
21use crate::shared::ProviderId;
22use crate::shared::base64_bytes;
23
24/// A realtime model.
25pub trait RealtimeModel: Send + Sync + 'static {
26    /// Provider identifier.
27    fn provider(&self) -> &ProviderId;
28
29    /// Model identifier.
30    fn model_id(&self) -> &ModelId;
31
32    /// Creates a short-lived client secret for opening a session.
33    fn do_create_client_secret(
34        &self,
35        options: ClientSecretOptions,
36    ) -> impl Future<Output = Result<ClientSecret, ProviderError>> + Send;
37
38    /// Returns the WebSocket URL and sub-protocols for a token.
39    fn websocket_config(&self, token: &str, url: &Url) -> WebSocketConfig;
40
41    /// Converts a raw server message into zero or more standardized events.
42    ///
43    /// # Errors
44    ///
45    /// Returns an error when the message cannot be interpreted at all;
46    /// unknown but well-formed messages should map to
47    /// [`RealtimeServerEvent::Custom`].
48    fn parse_server_event(&self, raw: JsonValue)
49    -> Result<Vec<RealtimeServerEvent>, ProviderError>;
50
51    /// Converts a standardized client event into the provider wire format.
52    fn serialize_client_event(
53        &self,
54        event: RealtimeClientEvent,
55    ) -> impl Future<Output = Result<JsonValue, ProviderError>> + Send;
56
57    /// Converts a session configuration into the provider wire format.
58    ///
59    /// # Errors
60    ///
61    /// Returns an error when the configuration cannot be expressed.
62    fn build_session_config(
63        &self,
64        config: &RealtimeSessionConfig,
65    ) -> Result<JsonValue, ProviderError>;
66
67    /// Returns the reply to send when `raw` is a provider health-check ping.
68    fn health_check_response(&self, raw: &JsonValue) -> Option<JsonValue> {
69        let _ = raw;
70        None
71    }
72}
73
74/// Creates realtime models and session tokens for a provider.
75pub trait RealtimeFactory: Send + Sync + 'static {
76    /// Provider identifier.
77    fn provider(&self) -> &ProviderId;
78
79    /// Returns the realtime model with `model_id`.
80    ///
81    /// # Errors
82    ///
83    /// Returns [`NoSuchModelError`] when the model is unknown.
84    fn model(&self, model_id: &str) -> Result<crate::dynamic::RealtimeModelRef, NoSuchModelError>;
85
86    /// Creates a session token for `options.model`.
87    fn get_token(
88        &self,
89        options: GetTokenOptions,
90    ) -> impl Future<Output = Result<ClientSecret, ProviderError>> + Send;
91}
92
93/// Options for creating a client secret.
94#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
95pub struct ClientSecretOptions {
96    /// Requested lifetime in seconds.
97    #[serde(default, skip_serializing_if = "Option::is_none")]
98    pub expires_after_seconds: Option<u64>,
99    /// Initial session configuration.
100    #[serde(default, skip_serializing_if = "Option::is_none")]
101    pub session_config: Option<RealtimeSessionConfig>,
102}
103
104/// Options for [`RealtimeFactory::get_token`].
105#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
106pub struct GetTokenOptions {
107    /// Model to open the session with.
108    pub model: ModelId,
109    /// Requested lifetime in seconds.
110    #[serde(default, skip_serializing_if = "Option::is_none")]
111    pub expires_after_seconds: Option<u64>,
112    /// Initial session configuration.
113    #[serde(default, skip_serializing_if = "Option::is_none")]
114    pub session_config: Option<RealtimeSessionConfig>,
115}
116
117/// A client secret for opening a session.
118#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
119pub struct ClientSecret {
120    /// The token; never logged.
121    pub token: String,
122    /// WebSocket URL to connect to.
123    pub url: Url,
124    /// Expiry as a Unix timestamp in seconds.
125    #[serde(default, skip_serializing_if = "Option::is_none")]
126    pub expires_at: Option<u64>,
127}
128
129impl std::fmt::Debug for ClientSecret {
130    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
131        f.debug_struct("ClientSecret")
132            .field("token", &"***")
133            .field("url", &self.url)
134            .field("expires_at", &self.expires_at)
135            .finish()
136    }
137}
138
139/// WebSocket connection parameters; URL and protocols are redacted in debug output.
140#[derive(Clone, PartialEq, Eq)]
141pub struct WebSocketConfig {
142    /// URL to connect to.
143    pub url: Url,
144    /// Sub-protocols to request.
145    pub protocols: Vec<String>,
146}
147
148impl std::fmt::Debug for WebSocketConfig {
149    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
150        f.debug_struct("WebSocketConfig")
151            .field("url", &"***")
152            .field("protocols", &"***")
153            .finish()
154    }
155}
156
157/// Output modality.
158#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
159#[serde(rename_all = "lowercase")]
160#[non_exhaustive]
161pub enum Modality {
162    /// Text.
163    Text,
164    /// Audio.
165    Audio,
166}
167
168/// Transcription settings for input or output audio.
169#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
170pub struct TranscriptionConfig {
171    /// Transcription model.
172    #[serde(default, skip_serializing_if = "Option::is_none")]
173    pub model: Option<String>,
174    /// Language hint.
175    #[serde(default, skip_serializing_if = "Option::is_none")]
176    pub language: Option<String>,
177    /// Prompt hint.
178    #[serde(default, skip_serializing_if = "Option::is_none")]
179    pub prompt: Option<String>,
180}
181
182/// Turn detection mode.
183#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
184#[serde(rename_all = "kebab-case")]
185#[non_exhaustive]
186pub enum TurnDetectionKind {
187    /// Server-side voice activity detection.
188    ServerVad,
189    /// Semantic voice activity detection.
190    SemanticVad,
191    /// No automatic turn detection.
192    Disabled,
193}
194
195/// Turn detection settings.
196#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
197pub struct TurnDetection {
198    /// Mode.
199    #[serde(rename = "type")]
200    pub kind: TurnDetectionKind,
201    /// Activation threshold.
202    #[serde(default, skip_serializing_if = "Option::is_none")]
203    pub threshold: Option<f64>,
204    /// Silence duration in milliseconds that ends a turn.
205    #[serde(default, skip_serializing_if = "Option::is_none")]
206    pub silence_duration_ms: Option<u64>,
207    /// Audio kept before detected speech, in milliseconds.
208    #[serde(default, skip_serializing_if = "Option::is_none")]
209    pub prefix_padding_ms: Option<u64>,
210}
211
212/// A function tool exposed in a realtime session.
213#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
214pub struct RealtimeToolDefinition {
215    /// Tool name.
216    pub name: String,
217    /// Description.
218    #[serde(default, skip_serializing_if = "Option::is_none")]
219    pub description: Option<String>,
220    /// JSON Schema of the parameters.
221    pub parameters: JsonValue,
222}
223
224/// Session configuration.
225#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
226pub struct RealtimeSessionConfig {
227    /// System instructions.
228    #[serde(default, skip_serializing_if = "Option::is_none")]
229    pub instructions: Option<String>,
230    /// Voice.
231    #[serde(default, skip_serializing_if = "Option::is_none")]
232    pub voice: Option<String>,
233    /// Output modalities.
234    #[serde(default, skip_serializing_if = "Option::is_none")]
235    pub output_modalities: Option<Vec<Modality>>,
236    /// Input audio format.
237    #[serde(default, skip_serializing_if = "Option::is_none")]
238    pub input_audio_format: Option<AudioFormat>,
239    /// Input transcription settings.
240    #[serde(default, skip_serializing_if = "Option::is_none")]
241    pub input_audio_transcription: Option<TranscriptionConfig>,
242    /// Output transcription settings.
243    #[serde(default, skip_serializing_if = "Option::is_none")]
244    pub output_audio_transcription: Option<TranscriptionConfig>,
245    /// Output audio format.
246    #[serde(default, skip_serializing_if = "Option::is_none")]
247    pub output_audio_format: Option<AudioFormat>,
248    /// Turn detection.
249    #[serde(default, skip_serializing_if = "Option::is_none")]
250    pub turn_detection: Option<TurnDetection>,
251    /// Tools.
252    #[serde(default, skip_serializing_if = "Vec::is_empty")]
253    pub tools: Vec<RealtimeToolDefinition>,
254    /// Provider-specific options.
255    #[serde(default, skip_serializing_if = "Option::is_none")]
256    pub provider_options: Option<JsonObject>,
257}
258
259/// Role of a conversation item created by the client.
260#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
261#[serde(rename_all = "lowercase")]
262#[non_exhaustive]
263pub enum ConversationRole {
264    /// The user.
265    User,
266}
267
268/// A conversation item created by the client, tagged by `type`.
269#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
270#[serde(tag = "type", rename_all = "kebab-case")]
271#[non_exhaustive]
272pub enum ConversationItem {
273    /// A text message.
274    TextMessage {
275        /// Role.
276        role: ConversationRole,
277        /// Text.
278        text: String,
279    },
280    /// An audio message.
281    AudioMessage {
282        /// Role.
283        role: ConversationRole,
284        /// Audio bytes.
285        #[serde(with = "base64_bytes")]
286        audio: Bytes,
287    },
288    /// Output of a function call.
289    FunctionCallOutput {
290        /// Call id.
291        call_id: String,
292        /// Function name.
293        #[serde(default, skip_serializing_if = "Option::is_none")]
294        name: Option<String>,
295        /// Output text.
296        output: String,
297    },
298}
299
300/// Options of a `response-create` event.
301#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
302pub struct ResponseCreateOptions {
303    /// Modalities.
304    #[serde(default, skip_serializing_if = "Option::is_none")]
305    pub modalities: Option<Vec<String>>,
306    /// Instructions for this response.
307    #[serde(default, skip_serializing_if = "Option::is_none")]
308    pub instructions: Option<String>,
309    /// Metadata.
310    #[serde(default, skip_serializing_if = "Option::is_none")]
311    pub metadata: Option<JsonObject>,
312}
313
314/// Events sent by the client, tagged by `type`.
315#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
316#[serde(tag = "type", rename_all = "kebab-case")]
317#[non_exhaustive]
318pub enum RealtimeClientEvent {
319    /// Update the session configuration.
320    SessionUpdate {
321        /// New configuration.
322        config: Box<RealtimeSessionConfig>,
323    },
324    /// Append input audio.
325    InputAudioAppend {
326        /// Audio bytes.
327        #[serde(with = "base64_bytes")]
328        audio: Bytes,
329    },
330    /// Commit the input audio buffer.
331    InputAudioCommit,
332    /// Clear the input audio buffer.
333    InputAudioClear,
334    /// Create a conversation item.
335    ConversationItemCreate {
336        /// The item.
337        item: ConversationItem,
338    },
339    /// Truncate a conversation item.
340    ConversationItemTruncate {
341        /// Item id.
342        item_id: String,
343        /// Content index.
344        content_index: u32,
345        /// Audio end in milliseconds.
346        audio_end_ms: u64,
347    },
348    /// Request a response.
349    ResponseCreate {
350        /// Options.
351        #[serde(default, skip_serializing_if = "Option::is_none")]
352        options: Option<ResponseCreateOptions>,
353    },
354    /// Cancel the in-progress response.
355    ResponseCancel,
356}
357
358/// Events received from the server, tagged by `type`.
359///
360/// Every variant carries the raw provider message in `raw`.
361#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
362#[serde(tag = "type", rename_all = "kebab-case")]
363#[non_exhaustive]
364pub enum RealtimeServerEvent {
365    /// Session created.
366    SessionCreated {
367        /// Session id.
368        #[serde(default, skip_serializing_if = "Option::is_none")]
369        session_id: Option<String>,
370        /// Raw message.
371        raw: JsonValue,
372    },
373    /// Session updated.
374    SessionUpdated {
375        /// Raw message.
376        raw: JsonValue,
377    },
378    /// Speech started.
379    SpeechStarted {
380        /// Item id.
381        #[serde(default, skip_serializing_if = "Option::is_none")]
382        item_id: Option<String>,
383        /// Raw message.
384        raw: JsonValue,
385    },
386    /// Speech stopped.
387    SpeechStopped {
388        /// Item id.
389        #[serde(default, skip_serializing_if = "Option::is_none")]
390        item_id: Option<String>,
391        /// Raw message.
392        raw: JsonValue,
393    },
394    /// Input audio committed.
395    AudioCommitted {
396        /// Item id.
397        #[serde(default, skip_serializing_if = "Option::is_none")]
398        item_id: Option<String>,
399        /// Previous item id.
400        #[serde(default, skip_serializing_if = "Option::is_none")]
401        previous_item_id: Option<String>,
402        /// Raw message.
403        raw: JsonValue,
404    },
405    /// Conversation item added.
406    ConversationItemAdded {
407        /// Item id.
408        item_id: String,
409        /// The item.
410        item: JsonValue,
411        /// Raw message.
412        raw: JsonValue,
413    },
414    /// Input transcription completed.
415    InputTranscriptionCompleted {
416        /// Item id.
417        item_id: String,
418        /// Transcript.
419        transcript: String,
420        /// Raw message.
421        raw: JsonValue,
422    },
423    /// Response created.
424    ResponseCreated {
425        /// Response id.
426        response_id: String,
427        /// Raw message.
428        raw: JsonValue,
429    },
430    /// Response done.
431    ResponseDone {
432        /// Response id.
433        response_id: String,
434        /// Status.
435        status: String,
436        /// Raw message.
437        raw: JsonValue,
438    },
439    /// Output item added.
440    OutputItemAdded {
441        /// Response id.
442        response_id: String,
443        /// Item id.
444        item_id: String,
445        /// Raw message.
446        raw: JsonValue,
447    },
448    /// Output item done.
449    OutputItemDone {
450        /// Response id.
451        response_id: String,
452        /// Item id.
453        item_id: String,
454        /// Raw message.
455        raw: JsonValue,
456    },
457    /// Content part added.
458    ContentPartAdded {
459        /// Response id.
460        response_id: String,
461        /// Item id.
462        item_id: String,
463        /// Raw message.
464        raw: JsonValue,
465    },
466    /// Content part done.
467    ContentPartDone {
468        /// Response id.
469        response_id: String,
470        /// Item id.
471        item_id: String,
472        /// Raw message.
473        raw: JsonValue,
474    },
475    /// Audio increment.
476    AudioDelta {
477        /// Response id.
478        response_id: String,
479        /// Item id.
480        item_id: String,
481        /// Audio bytes.
482        #[serde(with = "base64_bytes")]
483        delta: Bytes,
484        /// Raw message.
485        raw: JsonValue,
486    },
487    /// Audio done.
488    AudioDone {
489        /// Response id.
490        response_id: String,
491        /// Item id.
492        item_id: String,
493        /// Raw message.
494        raw: JsonValue,
495    },
496    /// Audio transcript increment.
497    AudioTranscriptDelta {
498        /// Response id.
499        response_id: String,
500        /// Item id.
501        item_id: String,
502        /// Text.
503        delta: String,
504        /// Raw message.
505        raw: JsonValue,
506    },
507    /// Audio transcript done.
508    AudioTranscriptDone {
509        /// Response id.
510        response_id: String,
511        /// Item id.
512        item_id: String,
513        /// Full transcript.
514        #[serde(default, skip_serializing_if = "Option::is_none")]
515        transcript: Option<String>,
516        /// Raw message.
517        raw: JsonValue,
518    },
519    /// Text increment.
520    TextDelta {
521        /// Response id.
522        response_id: String,
523        /// Item id.
524        item_id: String,
525        /// Text.
526        delta: String,
527        /// Raw message.
528        raw: JsonValue,
529    },
530    /// Text done.
531    TextDone {
532        /// Response id.
533        response_id: String,
534        /// Item id.
535        item_id: String,
536        /// Full text.
537        #[serde(default, skip_serializing_if = "Option::is_none")]
538        text: Option<String>,
539        /// Raw message.
540        raw: JsonValue,
541    },
542    /// Function call arguments increment.
543    FunctionCallArgumentsDelta {
544        /// Response id.
545        response_id: String,
546        /// Item id.
547        item_id: String,
548        /// Call id.
549        call_id: String,
550        /// Arguments text.
551        delta: String,
552        /// Raw message.
553        raw: JsonValue,
554    },
555    /// Function call arguments done.
556    FunctionCallArgumentsDone {
557        /// Response id.
558        response_id: String,
559        /// Item id.
560        item_id: String,
561        /// Call id.
562        call_id: String,
563        /// Function name.
564        name: String,
565        /// Complete arguments JSON text.
566        arguments: String,
567        /// Raw message.
568        raw: JsonValue,
569    },
570    /// Error.
571    Error {
572        /// Message.
573        message: String,
574        /// Code.
575        #[serde(default, skip_serializing_if = "Option::is_none")]
576        code: Option<String>,
577        /// Raw message.
578        raw: JsonValue,
579    },
580    /// A provider event without a standardized mapping.
581    Custom {
582        /// Provider event type.
583        raw_type: String,
584        /// Raw message.
585        raw: JsonValue,
586    },
587}
ferrin_spec/realtime_model.rs

ferrin_spec/
realtime_model.rs
