ferrin_spec/
realtime_model.rs

1//! Realtime (WebSocket) model interface.
2//!
3//! A realtime model does not own the connection. It issues client secrets,
4//! describes how to open the WebSocket, and translates between provider wire
5//! events and the standardized [`RealtimeServerEvent`] /
6//! [`RealtimeClientEvent`] sets. The session loop lives in the core.
7
8use std::future::Future;
9
10use bytes::Bytes;
11use serde::Deserialize;
12use serde::Serialize;
13use url::Url;
14
15use crate::error::NoSuchModelError;
16use crate::error::ProviderError;
17use crate::json::JsonObject;
18use crate::json::JsonValue;
19use crate::shared::AudioFormat;
20use crate::shared::ModelId;
21use crate::shared::ProviderId;
22use crate::shared::base64_bytes;
23
24/// A realtime model.
25pub trait RealtimeModel: Send + Sync + 'static {
26    /// Provider identifier.
27    fn provider(&self) -> &ProviderId;
28
29    /// Model identifier.
30    fn model_id(&self) -> &ModelId;
31
32    /// Creates a short-lived client secret for opening a session.
33    fn do_create_client_secret(
34        &self,
35        options: ClientSecretOptions,
36    ) -> impl Future<Output = Result<ClientSecret, ProviderError>> + Send;
37
38    /// Returns the WebSocket URL and sub-protocols for a token.
39    fn websocket_config(&self, token: &str, url: &Url) -> WebSocketConfig;
40
41    /// Converts a raw server message into zero or more standardized events.
42    ///
43    /// # Errors
44    ///
45    /// Returns an error when the message cannot be interpreted at all;
46    /// unknown but well-formed messages should map to
47    /// [`RealtimeServerEvent::Custom`].
48    fn parse_server_event(&self, raw: JsonValue)
49    -> Result<Vec<RealtimeServerEvent>, ProviderError>;
50
51    /// Converts a standardized client event into the provider wire format.
52    fn serialize_client_event(
53        &self,
54        event: RealtimeClientEvent,
55    ) -> impl Future<Output = Result<JsonValue, ProviderError>> + Send;
56
57    /// Converts a session configuration into the provider wire format.
58    ///
59    /// # Errors
60    ///
61    /// Returns an error when the configuration cannot be expressed.
62    fn build_session_config(
63        &self,
64        config: &RealtimeSessionConfig,
65    ) -> Result<JsonValue, ProviderError>;
66
67    /// Returns the reply to send when `raw` is a provider health-check ping.
68    fn health_check_response(&self, raw: &JsonValue) -> Option<JsonValue> {
69        let _ = raw;
70        None
71    }
72}
73
74/// Creates realtime models and session tokens for a provider.
75pub trait RealtimeFactory: Send + Sync + 'static {
76    /// Provider identifier.
77    fn provider(&self) -> &ProviderId;
78
79    /// Returns the realtime model with `model_id`.
80    ///
81    /// # Errors
82    ///
83    /// Returns [`NoSuchModelError`] when the model is unknown.
84    fn model(&self, model_id: &str) -> Result<crate::dynamic::RealtimeModelRef, NoSuchModelError>;
85
86    /// Creates a session token for `options.model`.
87    fn get_token(
88        &self,
89        options: GetTokenOptions,
90    ) -> impl Future<Output = Result<ClientSecret, ProviderError>> + Send;
91}
92
93/// Options for creating a client secret.
94#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
95pub struct ClientSecretOptions {
96    /// Requested lifetime in seconds.
97    #[serde(default, skip_serializing_if = "Option::is_none")]
98    pub expires_after_seconds: Option<u64>,
99    /// Initial session configuration.
100    #[serde(default, skip_serializing_if = "Option::is_none")]
101    pub session_config: Option<RealtimeSessionConfig>,
102}
103
104/// Options for [`RealtimeFactory::get_token`].
105#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
106pub struct GetTokenOptions {
107    /// Model to open the session with.
108    pub model: ModelId,
109    /// Requested lifetime in seconds.
110    #[serde(default, skip_serializing_if = "Option::is_none")]
111    pub expires_after_seconds: Option<u64>,
112    /// Initial session configuration.
113    #[serde(default, skip_serializing_if = "Option::is_none")]
114    pub session_config: Option<RealtimeSessionConfig>,
115}
116
117/// A client secret for opening a session.
118#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
119pub struct ClientSecret {
120    /// The token; never logged.
121    pub token: String,
122    /// WebSocket URL to connect to.
123    pub url: Url,
124    /// Expiry as a Unix timestamp in seconds.
125    #[serde(default, skip_serializing_if = "Option::is_none")]
126    pub expires_at: Option<u64>,
127}
128
129impl std::fmt::Debug for ClientSecret {
130    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
131        f.debug_struct("ClientSecret")
132            .field("token", &"***")
133            .field("url", &self.url)
134            .field("expires_at", &self.expires_at)
135            .finish()
136    }
137}
138
139/// WebSocket connection parameters.
140#[derive(Debug, Clone, PartialEq, Eq)]
141pub struct WebSocketConfig {
142    /// URL to connect to.
143    pub url: Url,
144    /// Sub-protocols to request.
145    pub protocols: Vec<String>,
146}
147
148/// Output modality.
149#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
150#[serde(rename_all = "lowercase")]
151#[non_exhaustive]
152pub enum Modality {
153    /// Text.
154    Text,
155    /// Audio.
156    Audio,
157}
158
159/// Transcription settings for input or output audio.
160#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
161pub struct TranscriptionConfig {
162    /// Transcription model.
163    #[serde(default, skip_serializing_if = "Option::is_none")]
164    pub model: Option<String>,
165    /// Language hint.
166    #[serde(default, skip_serializing_if = "Option::is_none")]
167    pub language: Option<String>,
168    /// Prompt hint.
169    #[serde(default, skip_serializing_if = "Option::is_none")]
170    pub prompt: Option<String>,
171}
172
173/// Turn detection mode.
174#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
175#[serde(rename_all = "kebab-case")]
176#[non_exhaustive]
177pub enum TurnDetectionKind {
178    /// Server-side voice activity detection.
179    ServerVad,
180    /// Semantic voice activity detection.
181    SemanticVad,
182    /// No automatic turn detection.
183    Disabled,
184}
185
186/// Turn detection settings.
187#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
188pub struct TurnDetection {
189    /// Mode.
190    #[serde(rename = "type")]
191    pub kind: TurnDetectionKind,
192    /// Activation threshold.
193    #[serde(default, skip_serializing_if = "Option::is_none")]
194    pub threshold: Option<f64>,
195    /// Silence duration in milliseconds that ends a turn.
196    #[serde(default, skip_serializing_if = "Option::is_none")]
197    pub silence_duration_ms: Option<u64>,
198    /// Audio kept before detected speech, in milliseconds.
199    #[serde(default, skip_serializing_if = "Option::is_none")]
200    pub prefix_padding_ms: Option<u64>,
201}
202
203/// A function tool exposed in a realtime session.
204#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
205pub struct RealtimeToolDefinition {
206    /// Tool name.
207    pub name: String,
208    /// Description.
209    #[serde(default, skip_serializing_if = "Option::is_none")]
210    pub description: Option<String>,
211    /// JSON Schema of the parameters.
212    pub parameters: JsonValue,
213}
214
215/// Session configuration.
216#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
217pub struct RealtimeSessionConfig {
218    /// System instructions.
219    #[serde(default, skip_serializing_if = "Option::is_none")]
220    pub instructions: Option<String>,
221    /// Voice.
222    #[serde(default, skip_serializing_if = "Option::is_none")]
223    pub voice: Option<String>,
224    /// Output modalities.
225    #[serde(default, skip_serializing_if = "Option::is_none")]
226    pub output_modalities: Option<Vec<Modality>>,
227    /// Input audio format.
228    #[serde(default, skip_serializing_if = "Option::is_none")]
229    pub input_audio_format: Option<AudioFormat>,
230    /// Input transcription settings.
231    #[serde(default, skip_serializing_if = "Option::is_none")]
232    pub input_audio_transcription: Option<TranscriptionConfig>,
233    /// Output transcription settings.
234    #[serde(default, skip_serializing_if = "Option::is_none")]
235    pub output_audio_transcription: Option<TranscriptionConfig>,
236    /// Output audio format.
237    #[serde(default, skip_serializing_if = "Option::is_none")]
238    pub output_audio_format: Option<AudioFormat>,
239    /// Turn detection.
240    #[serde(default, skip_serializing_if = "Option::is_none")]
241    pub turn_detection: Option<TurnDetection>,
242    /// Tools.
243    #[serde(default, skip_serializing_if = "Vec::is_empty")]
244    pub tools: Vec<RealtimeToolDefinition>,
245    /// Provider-specific options.
246    #[serde(default, skip_serializing_if = "Option::is_none")]
247    pub provider_options: Option<JsonObject>,
248}
249
250/// Role of a conversation item created by the client.
251#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
252#[serde(rename_all = "lowercase")]
253#[non_exhaustive]
254pub enum ConversationRole {
255    /// The user.
256    User,
257}
258
259/// A conversation item created by the client, tagged by `type`.
260#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
261#[serde(tag = "type", rename_all = "kebab-case")]
262#[non_exhaustive]
263pub enum ConversationItem {
264    /// A text message.
265    TextMessage {
266        /// Role.
267        role: ConversationRole,
268        /// Text.
269        text: String,
270    },
271    /// An audio message.
272    AudioMessage {
273        /// Role.
274        role: ConversationRole,
275        /// Audio bytes.
276        #[serde(with = "base64_bytes")]
277        audio: Bytes,
278    },
279    /// Output of a function call.
280    FunctionCallOutput {
281        /// Call id.
282        call_id: String,
283        /// Function name.
284        #[serde(default, skip_serializing_if = "Option::is_none")]
285        name: Option<String>,
286        /// Output text.
287        output: String,
288    },
289}
290
291/// Options of a `response-create` event.
292#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
293pub struct ResponseCreateOptions {
294    /// Modalities.
295    #[serde(default, skip_serializing_if = "Option::is_none")]
296    pub modalities: Option<Vec<String>>,
297    /// Instructions for this response.
298    #[serde(default, skip_serializing_if = "Option::is_none")]
299    pub instructions: Option<String>,
300    /// Metadata.
301    #[serde(default, skip_serializing_if = "Option::is_none")]
302    pub metadata: Option<JsonObject>,
303}
304
305/// Events sent by the client, tagged by `type`.
306#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
307#[serde(tag = "type", rename_all = "kebab-case")]
308#[non_exhaustive]
309pub enum RealtimeClientEvent {
310    /// Update the session configuration.
311    SessionUpdate {
312        /// New configuration.
313        config: Box<RealtimeSessionConfig>,
314    },
315    /// Append input audio.
316    InputAudioAppend {
317        /// Audio bytes.
318        #[serde(with = "base64_bytes")]
319        audio: Bytes,
320    },
321    /// Commit the input audio buffer.
322    InputAudioCommit,
323    /// Clear the input audio buffer.
324    InputAudioClear,
325    /// Create a conversation item.
326    ConversationItemCreate {
327        /// The item.
328        item: ConversationItem,
329    },
330    /// Truncate a conversation item.
331    ConversationItemTruncate {
332        /// Item id.
333        item_id: String,
334        /// Content index.
335        content_index: u32,
336        /// Audio end in milliseconds.
337        audio_end_ms: u64,
338    },
339    /// Request a response.
340    ResponseCreate {
341        /// Options.
342        #[serde(default, skip_serializing_if = "Option::is_none")]
343        options: Option<ResponseCreateOptions>,
344    },
345    /// Cancel the in-progress response.
346    ResponseCancel,
347}
348
349/// Events received from the server, tagged by `type`.
350///
351/// Every variant carries the raw provider message in `raw`.
352#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
353#[serde(tag = "type", rename_all = "kebab-case")]
354#[non_exhaustive]
355pub enum RealtimeServerEvent {
356    /// Session created.
357    SessionCreated {
358        /// Session id.
359        #[serde(default, skip_serializing_if = "Option::is_none")]
360        session_id: Option<String>,
361        /// Raw message.
362        raw: JsonValue,
363    },
364    /// Session updated.
365    SessionUpdated {
366        /// Raw message.
367        raw: JsonValue,
368    },
369    /// Speech started.
370    SpeechStarted {
371        /// Item id.
372        #[serde(default, skip_serializing_if = "Option::is_none")]
373        item_id: Option<String>,
374        /// Raw message.
375        raw: JsonValue,
376    },
377    /// Speech stopped.
378    SpeechStopped {
379        /// Item id.
380        #[serde(default, skip_serializing_if = "Option::is_none")]
381        item_id: Option<String>,
382        /// Raw message.
383        raw: JsonValue,
384    },
385    /// Input audio committed.
386    AudioCommitted {
387        /// Item id.
388        #[serde(default, skip_serializing_if = "Option::is_none")]
389        item_id: Option<String>,
390        /// Previous item id.
391        #[serde(default, skip_serializing_if = "Option::is_none")]
392        previous_item_id: Option<String>,
393        /// Raw message.
394        raw: JsonValue,
395    },
396    /// Conversation item added.
397    ConversationItemAdded {
398        /// Item id.
399        item_id: String,
400        /// The item.
401        item: JsonValue,
402        /// Raw message.
403        raw: JsonValue,
404    },
405    /// Input transcription completed.
406    InputTranscriptionCompleted {
407        /// Item id.
408        item_id: String,
409        /// Transcript.
410        transcript: String,
411        /// Raw message.
412        raw: JsonValue,
413    },
414    /// Response created.
415    ResponseCreated {
416        /// Response id.
417        response_id: String,
418        /// Raw message.
419        raw: JsonValue,
420    },
421    /// Response done.
422    ResponseDone {
423        /// Response id.
424        response_id: String,
425        /// Status.
426        status: String,
427        /// Raw message.
428        raw: JsonValue,
429    },
430    /// Output item added.
431    OutputItemAdded {
432        /// Response id.
433        response_id: String,
434        /// Item id.
435        item_id: String,
436        /// Raw message.
437        raw: JsonValue,
438    },
439    /// Output item done.
440    OutputItemDone {
441        /// Response id.
442        response_id: String,
443        /// Item id.
444        item_id: String,
445        /// Raw message.
446        raw: JsonValue,
447    },
448    /// Content part added.
449    ContentPartAdded {
450        /// Response id.
451        response_id: String,
452        /// Item id.
453        item_id: String,
454        /// Raw message.
455        raw: JsonValue,
456    },
457    /// Content part done.
458    ContentPartDone {
459        /// Response id.
460        response_id: String,
461        /// Item id.
462        item_id: String,
463        /// Raw message.
464        raw: JsonValue,
465    },
466    /// Audio increment.
467    AudioDelta {
468        /// Response id.
469        response_id: String,
470        /// Item id.
471        item_id: String,
472        /// Audio bytes.
473        #[serde(with = "base64_bytes")]
474        delta: Bytes,
475        /// Raw message.
476        raw: JsonValue,
477    },
478    /// Audio done.
479    AudioDone {
480        /// Response id.
481        response_id: String,
482        /// Item id.
483        item_id: String,
484        /// Raw message.
485        raw: JsonValue,
486    },
487    /// Audio transcript increment.
488    AudioTranscriptDelta {
489        /// Response id.
490        response_id: String,
491        /// Item id.
492        item_id: String,
493        /// Text.
494        delta: String,
495        /// Raw message.
496        raw: JsonValue,
497    },
498    /// Audio transcript done.
499    AudioTranscriptDone {
500        /// Response id.
501        response_id: String,
502        /// Item id.
503        item_id: String,
504        /// Full transcript.
505        #[serde(default, skip_serializing_if = "Option::is_none")]
506        transcript: Option<String>,
507        /// Raw message.
508        raw: JsonValue,
509    },
510    /// Text increment.
511    TextDelta {
512        /// Response id.
513        response_id: String,
514        /// Item id.
515        item_id: String,
516        /// Text.
517        delta: String,
518        /// Raw message.
519        raw: JsonValue,
520    },
521    /// Text done.
522    TextDone {
523        /// Response id.
524        response_id: String,
525        /// Item id.
526        item_id: String,
527        /// Full text.
528        #[serde(default, skip_serializing_if = "Option::is_none")]
529        text: Option<String>,
530        /// Raw message.
531        raw: JsonValue,
532    },
533    /// Function call arguments increment.
534    FunctionCallArgumentsDelta {
535        /// Response id.
536        response_id: String,
537        /// Item id.
538        item_id: String,
539        /// Call id.
540        call_id: String,
541        /// Arguments text.
542        delta: String,
543        /// Raw message.
544        raw: JsonValue,
545    },
546    /// Function call arguments done.
547    FunctionCallArgumentsDone {
548        /// Response id.
549        response_id: String,
550        /// Item id.
551        item_id: String,
552        /// Call id.
553        call_id: String,
554        /// Function name.
555        name: String,
556        /// Complete arguments JSON text.
557        arguments: String,
558        /// Raw message.
559        raw: JsonValue,
560    },
561    /// Error.
562    Error {
563        /// Message.
564        message: String,
565        /// Code.
566        #[serde(default, skip_serializing_if = "Option::is_none")]
567        code: Option<String>,
568        /// Raw message.
569        raw: JsonValue,
570    },
571    /// A provider event without a standardized mapping.
572    Custom {
573        /// Provider event type.
574        raw_type: String,
575        /// Raw message.
576        raw: JsonValue,
577    },
578}
ferrin_spec/realtime_model.rs

ferrin_spec/
realtime_model.rs
