openai_tools/realtime/events/
server.rs

1//! Server-to-client events for the Realtime API.
2
3use serde::{Deserialize, Serialize};
4
5use crate::realtime::audio::{AudioFormat, Voice};
6use crate::realtime::conversation::ItemStatus;
7use crate::realtime::session::{MaxTokens, Modality, RealtimeTool, ToolChoice};
8use crate::realtime::vad::TurnDetection;
9
10/// Server events received from the OpenAI Realtime API.
11#[derive(Debug, Clone, Deserialize)]
12#[serde(tag = "type")]
13#[non_exhaustive]
14pub enum ServerEvent {
15    // ==================== Session Events ====================
16    /// Session was created (first event after connection).
17    #[serde(rename = "session.created")]
18    SessionCreated(SessionCreatedEvent),
19
20    /// Session configuration was updated.
21    #[serde(rename = "session.updated")]
22    SessionUpdated(SessionUpdatedEvent),
23
24    // ==================== Conversation Events ====================
25    /// Conversation was created.
26    #[serde(rename = "conversation.created")]
27    ConversationCreated(ConversationCreatedEvent),
28
29    /// Conversation item was created.
30    #[serde(rename = "conversation.item.created")]
31    ConversationItemCreated(ConversationItemCreatedEvent),
32
33    /// Conversation item was retrieved.
34    #[serde(rename = "conversation.item.retrieved")]
35    ConversationItemRetrieved(ConversationItemRetrievedEvent),
36
37    /// Conversation item was deleted.
38    #[serde(rename = "conversation.item.deleted")]
39    ConversationItemDeleted(ConversationItemDeletedEvent),
40
41    /// Conversation item was truncated.
42    #[serde(rename = "conversation.item.truncated")]
43    ConversationItemTruncated(ConversationItemTruncatedEvent),
44
45    /// Input audio transcription completed.
46    #[serde(rename = "conversation.item.input_audio_transcription.completed")]
47    InputAudioTranscriptionCompleted(InputAudioTranscriptionCompletedEvent),
48
49    /// Input audio transcription failed.
50    #[serde(rename = "conversation.item.input_audio_transcription.failed")]
51    InputAudioTranscriptionFailed(InputAudioTranscriptionFailedEvent),
52
53    // ==================== Input Audio Buffer Events ====================
54    /// Input audio buffer was committed.
55    #[serde(rename = "input_audio_buffer.committed")]
56    InputAudioBufferCommitted(InputAudioBufferCommittedEvent),
57
58    /// Input audio buffer was cleared.
59    #[serde(rename = "input_audio_buffer.cleared")]
60    InputAudioBufferCleared(InputAudioBufferClearedEvent),
61
62    /// Speech started in input audio.
63    #[serde(rename = "input_audio_buffer.speech_started")]
64    InputAudioBufferSpeechStarted(SpeechStartedEvent),
65
66    /// Speech stopped in input audio.
67    #[serde(rename = "input_audio_buffer.speech_stopped")]
68    InputAudioBufferSpeechStopped(SpeechStoppedEvent),
69
70    // ==================== Output Audio Buffer Events (WebRTC) ====================
71    /// Output audio buffer playback started.
72    #[serde(rename = "output_audio_buffer.started")]
73    OutputAudioBufferStarted(OutputAudioBufferEvent),
74
75    /// Output audio buffer playback stopped.
76    #[serde(rename = "output_audio_buffer.stopped")]
77    OutputAudioBufferStopped(OutputAudioBufferStoppedEvent),
78
79    /// Output audio buffer was cleared.
80    #[serde(rename = "output_audio_buffer.cleared")]
81    OutputAudioBufferCleared(OutputAudioBufferEvent),
82
83    // ==================== Response Events ====================
84    /// Response was created.
85    #[serde(rename = "response.created")]
86    ResponseCreated(ResponseCreatedEvent),
87
88    /// Response generation completed.
89    #[serde(rename = "response.done")]
90    ResponseDone(ResponseDoneEvent),
91
92    /// Output item was added to response.
93    #[serde(rename = "response.output_item.added")]
94    ResponseOutputItemAdded(ResponseOutputItemEvent),
95
96    /// Output item completed.
97    #[serde(rename = "response.output_item.done")]
98    ResponseOutputItemDone(ResponseOutputItemEvent),
99
100    /// Content part was added.
101    #[serde(rename = "response.content_part.added")]
102    ResponseContentPartAdded(ResponseContentPartEvent),
103
104    /// Content part completed.
105    #[serde(rename = "response.content_part.done")]
106    ResponseContentPartDone(ResponseContentPartEvent),
107
108    /// Text delta received.
109    #[serde(rename = "response.text.delta")]
110    ResponseTextDelta(ResponseTextDeltaEvent),
111
112    /// Text output completed.
113    #[serde(rename = "response.text.done")]
114    ResponseTextDone(ResponseTextDoneEvent),
115
116    /// Audio delta received.
117    #[serde(rename = "response.audio.delta")]
118    ResponseAudioDelta(ResponseAudioDeltaEvent),
119
120    /// Audio output completed.
121    #[serde(rename = "response.audio.done")]
122    ResponseAudioDone(ResponseAudioDoneEvent),
123
124    /// Audio transcript delta received.
125    #[serde(rename = "response.audio_transcript.delta")]
126    ResponseAudioTranscriptDelta(ResponseAudioTranscriptDeltaEvent),
127
128    /// Audio transcript completed.
129    #[serde(rename = "response.audio_transcript.done")]
130    ResponseAudioTranscriptDone(ResponseAudioTranscriptDoneEvent),
131
132    /// Function call arguments delta.
133    #[serde(rename = "response.function_call_arguments.delta")]
134    ResponseFunctionCallArgumentsDelta(ResponseFunctionCallArgumentsDeltaEvent),
135
136    /// Function call arguments completed.
137    #[serde(rename = "response.function_call_arguments.done")]
138    ResponseFunctionCallArgumentsDone(ResponseFunctionCallArgumentsDoneEvent),
139
140    // ==================== Rate Limits ====================
141    /// Rate limits updated.
142    #[serde(rename = "rate_limits.updated")]
143    RateLimitsUpdated(RateLimitsUpdatedEvent),
144
145    // ==================== Error ====================
146    /// Error occurred.
147    #[serde(rename = "error")]
148    Error(ErrorEvent),
149
150    // ==================== Unknown ====================
151    /// An event type this version of the library does not know about.
152    ///
153    /// OpenAI adds realtime events regularly. Without this arm a single
154    /// unrecognised `type` would fail deserialization and tear down the
155    /// session, so unknown events are surfaced here instead.
156    ///
157    /// Note that the payload is not preserved: `serde`'s `other` attribute is
158    /// only allowed on a unit variant.
159    #[serde(other)]
160    Unknown,
161}
162
163// ==================== Session Event Types ====================
164
165/// Session created event payload.
166#[derive(Debug, Clone, Deserialize)]
167pub struct SessionCreatedEvent {
168    pub event_id: String,
169    pub session: SessionInfo,
170}
171
172/// Session updated event payload.
173#[derive(Debug, Clone, Deserialize)]
174pub struct SessionUpdatedEvent {
175    pub event_id: String,
176    pub session: SessionInfo,
177}
178
179/// Session information.
180#[derive(Debug, Clone, Deserialize)]
181pub struct SessionInfo {
182    pub id: String,
183    pub object: String,
184    pub model: String,
185    #[serde(default)]
186    pub modalities: Vec<Modality>,
187    #[serde(default)]
188    pub instructions: String,
189    pub voice: Option<Voice>,
190    pub input_audio_format: Option<AudioFormat>,
191    pub output_audio_format: Option<AudioFormat>,
192    pub turn_detection: Option<TurnDetection>,
193    #[serde(default)]
194    pub tools: Vec<RealtimeTool>,
195    pub tool_choice: Option<ToolChoice>,
196    pub temperature: Option<f32>,
197    pub max_response_output_tokens: Option<MaxTokens>,
198}
199
200// ==================== Conversation Event Types ====================
201
202/// Conversation created event.
203#[derive(Debug, Clone, Deserialize)]
204pub struct ConversationCreatedEvent {
205    pub event_id: String,
206    pub conversation: ConversationInfo,
207}
208
209/// Conversation information.
210#[derive(Debug, Clone, Deserialize)]
211pub struct ConversationInfo {
212    pub id: String,
213    pub object: String,
214}
215
216/// Conversation item created event.
217#[derive(Debug, Clone, Deserialize)]
218pub struct ConversationItemCreatedEvent {
219    pub event_id: String,
220    #[serde(default)]
221    pub previous_item_id: Option<String>,
222    pub item: ResponseItem,
223}
224
225/// Conversation item retrieved event.
226#[derive(Debug, Clone, Deserialize)]
227pub struct ConversationItemRetrievedEvent {
228    pub event_id: String,
229    pub item: ResponseItem,
230}
231
232/// Conversation item deleted event.
233#[derive(Debug, Clone, Deserialize)]
234pub struct ConversationItemDeletedEvent {
235    pub event_id: String,
236    pub item_id: String,
237}
238
239/// Conversation item truncated event.
240#[derive(Debug, Clone, Deserialize)]
241pub struct ConversationItemTruncatedEvent {
242    pub event_id: String,
243    pub item_id: String,
244    pub content_index: u32,
245    pub audio_end_ms: u32,
246}
247
248/// Input audio transcription completed event.
249#[derive(Debug, Clone, Deserialize)]
250pub struct InputAudioTranscriptionCompletedEvent {
251    pub event_id: String,
252    pub item_id: String,
253    pub content_index: u32,
254    pub transcript: String,
255}
256
257/// Input audio transcription failed event.
258#[derive(Debug, Clone, Deserialize)]
259pub struct InputAudioTranscriptionFailedEvent {
260    pub event_id: String,
261    pub item_id: String,
262    pub content_index: u32,
263    pub error: RealtimeError,
264}
265
266// ==================== Input Audio Buffer Event Types ====================
267
268/// Input audio buffer committed event.
269#[derive(Debug, Clone, Deserialize)]
270pub struct InputAudioBufferCommittedEvent {
271    pub event_id: String,
272    #[serde(default)]
273    pub previous_item_id: Option<String>,
274    pub item_id: String,
275}
276
277/// Input audio buffer cleared event.
278#[derive(Debug, Clone, Deserialize)]
279pub struct InputAudioBufferClearedEvent {
280    pub event_id: String,
281}
282
283/// Speech started event.
284#[derive(Debug, Clone, Deserialize)]
285pub struct SpeechStartedEvent {
286    pub event_id: String,
287    pub audio_start_ms: u32,
288    pub item_id: String,
289}
290
291/// Speech stopped event.
292#[derive(Debug, Clone, Deserialize)]
293pub struct SpeechStoppedEvent {
294    pub event_id: String,
295    pub audio_end_ms: u32,
296    #[serde(default)]
297    pub item_id: Option<String>,
298}
299
300// ==================== Output Audio Buffer Event Types ====================
301
302/// Output audio buffer event (started/cleared).
303#[derive(Debug, Clone, Deserialize)]
304pub struct OutputAudioBufferEvent {
305    pub event_id: String,
306    pub response_id: String,
307}
308
309/// Output audio buffer stopped event.
310#[derive(Debug, Clone, Deserialize)]
311pub struct OutputAudioBufferStoppedEvent {
312    pub event_id: String,
313    pub response_id: String,
314    pub audio_end_ms: u32,
315    pub item_id: String,
316}
317
318// ==================== Response Event Types ====================
319
320/// Response created event.
321#[derive(Debug, Clone, Deserialize)]
322pub struct ResponseCreatedEvent {
323    pub event_id: String,
324    pub response: ResponseInfo,
325}
326
327/// Response done event.
328#[derive(Debug, Clone, Deserialize)]
329pub struct ResponseDoneEvent {
330    pub event_id: String,
331    pub response: ResponseInfo,
332}
333
334/// Response information.
335#[derive(Debug, Clone, Deserialize)]
336pub struct ResponseInfo {
337    pub id: String,
338    pub object: String,
339    pub status: ResponseStatus,
340    #[serde(default)]
341    pub status_details: Option<serde_json::Value>,
342    #[serde(default)]
343    pub output: Vec<ResponseItem>,
344    #[serde(default)]
345    pub usage: Option<RealtimeUsage>,
346}
347
348/// Response status.
349#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
350#[serde(rename_all = "snake_case")]
351#[non_exhaustive]
352pub enum ResponseStatus {
353    InProgress,
354    Completed,
355    Cancelled,
356    Incomplete,
357    Failed,
358    /// A status this version of the library does not know about.
359    ///
360    /// OpenAI can add lifecycle states at any time; capturing the raw value
361    /// keeps the surrounding response parseable.
362    #[serde(untagged)]
363    Other(String),
364}
365
366/// Response item (output item in a response).
367#[derive(Debug, Clone, Deserialize)]
368pub struct ResponseItem {
369    pub id: String,
370    pub object: String,
371    #[serde(rename = "type")]
372    pub item_type: String,
373    #[serde(default)]
374    pub role: Option<String>,
375    #[serde(default)]
376    pub content: Vec<ResponseContentPart>,
377    #[serde(default)]
378    pub status: Option<ItemStatus>,
379    // Function call fields
380    #[serde(default)]
381    pub call_id: Option<String>,
382    #[serde(default)]
383    pub name: Option<String>,
384    #[serde(default)]
385    pub arguments: Option<String>,
386    #[serde(default)]
387    pub output: Option<String>,
388}
389
390/// Response content part.
391#[derive(Debug, Clone, Deserialize)]
392pub struct ResponseContentPart {
393    #[serde(rename = "type")]
394    pub content_type: String,
395    #[serde(default)]
396    pub text: Option<String>,
397    #[serde(default)]
398    pub audio: Option<String>,
399    #[serde(default)]
400    pub transcript: Option<String>,
401}
402
403/// Response output item event.
404#[derive(Debug, Clone, Deserialize)]
405pub struct ResponseOutputItemEvent {
406    pub event_id: String,
407    pub response_id: String,
408    pub output_index: u32,
409    pub item: ResponseItem,
410}
411
412/// Response content part event.
413#[derive(Debug, Clone, Deserialize)]
414pub struct ResponseContentPartEvent {
415    pub event_id: String,
416    pub response_id: String,
417    pub item_id: String,
418    pub output_index: u32,
419    pub content_index: u32,
420    pub part: ResponseContentPart,
421}
422
423/// Response text delta event.
424#[derive(Debug, Clone, Deserialize)]
425pub struct ResponseTextDeltaEvent {
426    pub event_id: String,
427    pub response_id: String,
428    pub item_id: String,
429    pub output_index: u32,
430    pub content_index: u32,
431    pub delta: String,
432}
433
434/// Response text done event.
435#[derive(Debug, Clone, Deserialize)]
436pub struct ResponseTextDoneEvent {
437    pub event_id: String,
438    pub response_id: String,
439    pub item_id: String,
440    pub output_index: u32,
441    pub content_index: u32,
442    pub text: String,
443}
444
445/// Response audio delta event.
446#[derive(Debug, Clone, Deserialize)]
447pub struct ResponseAudioDeltaEvent {
448    pub event_id: String,
449    pub response_id: String,
450    pub item_id: String,
451    pub output_index: u32,
452    pub content_index: u32,
453    /// Base64-encoded audio chunk.
454    pub delta: String,
455}
456
457/// Response audio done event.
458#[derive(Debug, Clone, Deserialize)]
459pub struct ResponseAudioDoneEvent {
460    pub event_id: String,
461    pub response_id: String,
462    pub item_id: String,
463    pub output_index: u32,
464    pub content_index: u32,
465}
466
467/// Response audio transcript delta event.
468#[derive(Debug, Clone, Deserialize)]
469pub struct ResponseAudioTranscriptDeltaEvent {
470    pub event_id: String,
471    pub response_id: String,
472    pub item_id: String,
473    pub output_index: u32,
474    pub content_index: u32,
475    pub delta: String,
476}
477
478/// Response audio transcript done event.
479#[derive(Debug, Clone, Deserialize)]
480pub struct ResponseAudioTranscriptDoneEvent {
481    pub event_id: String,
482    pub response_id: String,
483    pub item_id: String,
484    pub output_index: u32,
485    pub content_index: u32,
486    pub transcript: String,
487}
488
489/// Response function call arguments delta event.
490#[derive(Debug, Clone, Deserialize)]
491pub struct ResponseFunctionCallArgumentsDeltaEvent {
492    pub event_id: String,
493    pub response_id: String,
494    pub item_id: String,
495    pub output_index: u32,
496    pub call_id: String,
497    pub delta: String,
498}
499
500/// Response function call arguments done event.
501#[derive(Debug, Clone, Deserialize)]
502pub struct ResponseFunctionCallArgumentsDoneEvent {
503    pub event_id: String,
504    pub response_id: String,
505    pub item_id: String,
506    pub output_index: u32,
507    pub call_id: String,
508    pub name: String,
509    pub arguments: String,
510}
511
512// ==================== Rate Limits ====================
513
514/// Rate limits updated event.
515#[derive(Debug, Clone, Deserialize)]
516pub struct RateLimitsUpdatedEvent {
517    pub event_id: String,
518    pub rate_limits: Vec<RateLimit>,
519}
520
521/// Rate limit information.
522#[derive(Debug, Clone, Deserialize)]
523pub struct RateLimit {
524    pub name: String,
525    pub limit: u32,
526    pub remaining: u32,
527    pub reset_seconds: f32,
528}
529
530// ==================== Error ====================
531
532/// Error event.
533#[derive(Debug, Clone, Deserialize)]
534pub struct ErrorEvent {
535    pub event_id: String,
536    pub error: RealtimeError,
537}
538
539/// Realtime API error.
540#[derive(Debug, Clone, Deserialize)]
541pub struct RealtimeError {
542    #[serde(rename = "type")]
543    pub error_type: Option<String>,
544    pub code: Option<String>,
545    pub message: String,
546    #[serde(default)]
547    pub param: Option<String>,
548    #[serde(default)]
549    pub event_id: Option<String>,
550}
551
552// ==================== Usage ====================
553
554/// Token usage information.
555#[derive(Debug, Clone, Default, Deserialize)]
556pub struct RealtimeUsage {
557    pub total_tokens: u32,
558    pub input_tokens: u32,
559    pub output_tokens: u32,
560    #[serde(default)]
561    pub input_token_details: Option<InputTokenDetails>,
562    #[serde(default)]
563    pub output_token_details: Option<OutputTokenDetails>,
564}
565
566/// Input token details.
567#[derive(Debug, Clone, Default, Deserialize)]
568pub struct InputTokenDetails {
569    #[serde(default)]
570    pub cached_tokens: u32,
571    #[serde(default)]
572    pub text_tokens: u32,
573    #[serde(default)]
574    pub audio_tokens: u32,
575}
576
577/// Output token details.
578#[derive(Debug, Clone, Default, Deserialize)]
579pub struct OutputTokenDetails {
580    #[serde(default)]
581    pub text_tokens: u32,
582    #[serde(default)]
583    pub audio_tokens: u32,
584}
585
586impl ServerEvent {
587    /// Check if this is an error event.
588    pub fn is_error(&self) -> bool {
589        matches!(self, Self::Error(_))
590    }
591
592    /// Get the event ID if available.
593    pub fn event_id(&self) -> Option<&str> {
594        match self {
595            Self::SessionCreated(e) => Some(&e.event_id),
596            Self::SessionUpdated(e) => Some(&e.event_id),
597            Self::ConversationCreated(e) => Some(&e.event_id),
598            Self::ConversationItemCreated(e) => Some(&e.event_id),
599            Self::ConversationItemRetrieved(e) => Some(&e.event_id),
600            Self::ConversationItemDeleted(e) => Some(&e.event_id),
601            Self::ConversationItemTruncated(e) => Some(&e.event_id),
602            Self::InputAudioTranscriptionCompleted(e) => Some(&e.event_id),
603            Self::InputAudioTranscriptionFailed(e) => Some(&e.event_id),
604            Self::InputAudioBufferCommitted(e) => Some(&e.event_id),
605            Self::InputAudioBufferCleared(e) => Some(&e.event_id),
606            Self::InputAudioBufferSpeechStarted(e) => Some(&e.event_id),
607            Self::InputAudioBufferSpeechStopped(e) => Some(&e.event_id),
608            Self::OutputAudioBufferStarted(e) => Some(&e.event_id),
609            Self::OutputAudioBufferStopped(e) => Some(&e.event_id),
610            Self::OutputAudioBufferCleared(e) => Some(&e.event_id),
611            Self::ResponseCreated(e) => Some(&e.event_id),
612            Self::ResponseDone(e) => Some(&e.event_id),
613            Self::ResponseOutputItemAdded(e) => Some(&e.event_id),
614            Self::ResponseOutputItemDone(e) => Some(&e.event_id),
615            Self::ResponseContentPartAdded(e) => Some(&e.event_id),
616            Self::ResponseContentPartDone(e) => Some(&e.event_id),
617            Self::ResponseTextDelta(e) => Some(&e.event_id),
618            Self::ResponseTextDone(e) => Some(&e.event_id),
619            Self::ResponseAudioDelta(e) => Some(&e.event_id),
620            Self::ResponseAudioDone(e) => Some(&e.event_id),
621            Self::ResponseAudioTranscriptDelta(e) => Some(&e.event_id),
622            Self::ResponseAudioTranscriptDone(e) => Some(&e.event_id),
623            Self::ResponseFunctionCallArgumentsDelta(e) => Some(&e.event_id),
624            Self::ResponseFunctionCallArgumentsDone(e) => Some(&e.event_id),
625            Self::RateLimitsUpdated(e) => Some(&e.event_id),
626            Self::Error(e) => Some(&e.event_id),
627            // The payload of an unrecognised event is not retained.
628            Self::Unknown => None,
629        }
630    }
631}
openai_tools/realtime/events/server.rs

openai_tools/realtime/events/
server.rs
