1use serde::{Deserialize, Serialize};
8
9use super::protocol::{ItemContent, RealtimeConversationItem, RealtimeResponse, SessionConfig};
10
11#[derive(Debug, Clone, Deserialize)]
13pub struct RealtimeConversation {
14 pub id: String,
16 pub object: String,
18}
19
20#[derive(Debug, Clone, Deserialize)]
22pub struct ServerErrorBody {
23 #[serde(rename = "type")]
25 pub type_: String,
26 #[serde(default)]
28 pub code: Option<String>,
29 pub message: String,
31}
32
33#[derive(Debug, Clone, Deserialize)]
39pub struct RealtimeSessionInfo {
40 #[serde(default)]
42 pub id: Option<String>,
43 #[serde(default)]
45 pub object: Option<String>,
46 #[serde(default)]
48 pub model: Option<String>,
49 #[serde(default)]
51 pub modalities: Vec<String>,
52 #[serde(default)]
55 pub voice: Option<String>,
56 #[serde(default)]
58 pub input_audio_format: Option<String>,
59 #[serde(default)]
61 pub output_audio_format: Option<String>,
62 #[serde(default)]
64 pub temperature: Option<f64>,
65 #[serde(flatten)]
67 pub extra: serde_json::Map<String, serde_json::Value>,
68}
69
70#[derive(Debug, Clone, Deserialize)]
72pub struct RealtimeRateLimit {
73 pub name: String,
75 pub limit: u64,
77 pub remaining: u64,
79 pub reset_seconds: f64,
81}
82
83#[derive(Debug, Clone, Serialize)]
89#[serde(tag = "type")]
90pub enum ClientEvent {
91 #[serde(rename = "session.update")]
93 SessionUpdate {
94 #[serde(skip_serializing_if = "Option::is_none")]
96 event_id: Option<String>,
97 session: SessionConfig,
99 },
100
101 #[serde(rename = "input_audio_buffer.append")]
103 InputAudioBufferAppend {
104 audio: String,
106 #[serde(skip_serializing_if = "Option::is_none")]
108 client_timestamp: Option<i64>,
109 },
110
111 #[serde(rename = "input_audio_buffer.append_video_frame")]
113 InputAudioBufferAppendVideoFrame {
114 video_frame: String,
116 #[serde(skip_serializing_if = "Option::is_none")]
118 client_timestamp: Option<i64>,
119 },
120
121 #[serde(rename = "input_audio_buffer.commit")]
123 InputAudioBufferCommit {
124 #[serde(skip_serializing_if = "Option::is_none")]
126 client_timestamp: Option<i64>,
127 },
128
129 #[serde(rename = "input_audio_buffer.clear")]
131 InputAudioBufferClear,
132
133 #[serde(rename = "conversation.item.create")]
136 ConversationItemCreate {
137 #[serde(skip_serializing_if = "Option::is_none")]
139 event_id: Option<String>,
140 item: RealtimeConversationItem,
142 },
143
144 #[serde(rename = "conversation.item.delete")]
146 ConversationItemDelete {
147 #[serde(skip_serializing_if = "Option::is_none")]
149 event_id: Option<String>,
150 #[serde(skip_serializing_if = "Option::is_none")]
152 client_timestamp: Option<i64>,
153 item_id: String,
155 },
156
157 #[serde(rename = "conversation.item.retrieve")]
159 ConversationItemRetrieve {
160 #[serde(skip_serializing_if = "Option::is_none")]
162 event_id: Option<String>,
163 #[serde(skip_serializing_if = "Option::is_none")]
165 client_timestamp: Option<i64>,
166 item_id: String,
168 },
169
170 #[serde(rename = "response.create")]
172 ResponseCreate {
173 #[serde(skip_serializing_if = "Option::is_none")]
175 client_timestamp: Option<i64>,
176 },
177
178 #[serde(rename = "response.cancel")]
180 ResponseCancel {
181 #[serde(skip_serializing_if = "Option::is_none")]
183 client_timestamp: Option<i64>,
184 },
185}
186
187#[non_exhaustive]
198#[derive(Debug, Clone, Deserialize)]
199#[serde(tag = "type")]
200pub enum ServerEvent {
201 #[serde(rename = "error")]
203 Error {
204 error: ServerErrorBody,
206 },
207
208 #[serde(rename = "session.created")]
210 SessionCreated {
211 session: RealtimeSessionInfo,
213 },
214
215 #[serde(rename = "session.updated")]
217 SessionUpdated {
218 session: SessionConfig,
220 },
221
222 #[serde(rename = "conversation.created")]
224 ConversationCreated {
225 conversation: RealtimeConversation,
227 },
228
229 #[serde(rename = "conversation.item.created")]
231 ConversationItemCreated {
232 item: RealtimeConversationItem,
234 },
235
236 #[serde(rename = "conversation.item.deleted")]
238 ConversationItemDeleted {
239 item_id: String,
241 },
242
243 #[serde(rename = "conversation.item.retrieved")]
245 ConversationItemRetrieved {
246 item: RealtimeConversationItem,
248 },
249
250 #[serde(rename = "conversation.item.input_audio_transcription.completed")]
252 InputAudioTranscriptionCompleted {
253 item_id: String,
255 #[serde(default)]
257 content_index: Option<u64>,
258 transcript: String,
260 },
261
262 #[serde(rename = "conversation.item.input_audio_transcription.failed")]
264 InputAudioTranscriptionFailed {
265 item_id: String,
267 #[serde(default)]
269 content_index: Option<u64>,
270 error: ServerErrorBody,
272 },
273
274 #[serde(rename = "input_audio_buffer.committed")]
276 InputAudioBufferCommitted {
277 item_id: String,
279 },
280
281 #[serde(rename = "input_audio_buffer.cleared")]
283 InputAudioBufferCleared,
284
285 #[serde(rename = "input_audio_buffer.speech_started")]
287 InputAudioBufferSpeechStarted {
288 #[serde(default)]
290 audio_start_ms: Option<u64>,
291 #[serde(default)]
293 item_id: Option<String>,
294 },
295
296 #[serde(rename = "input_audio_buffer.speech_stopped")]
298 InputAudioBufferSpeechStopped {
299 #[serde(default)]
301 audio_end_ms: Option<u64>,
302 #[serde(default)]
304 item_id: Option<String>,
305 },
306
307 #[serde(rename = "response.output_item.added")]
309 ResponseOutputItemAdded {
310 response_id: String,
312 output_index: u64,
314 item: RealtimeConversationItem,
316 },
317
318 #[serde(rename = "response.output_item.done")]
320 ResponseOutputItemDone {
321 response_id: String,
323 output_index: u64,
325 item: RealtimeConversationItem,
327 },
328
329 #[serde(rename = "response.content_part.added")]
331 ResponseContentPartAdded {
332 response_id: String,
334 item_id: String,
336 output_index: u64,
338 content_index: u64,
340 part: ItemContent,
342 },
343
344 #[serde(rename = "response.content_part.done")]
346 ResponseContentPartDone {
347 response_id: String,
349 item_id: String,
351 output_index: u64,
353 content_index: u64,
355 part: ItemContent,
357 },
358
359 #[serde(rename = "response.created")]
361 ResponseCreated {
362 response: RealtimeResponse,
364 },
365
366 #[serde(rename = "response.done")]
368 ResponseDone {
369 response: RealtimeResponse,
371 },
372
373 #[serde(rename = "response.cancelled")]
375 ResponseCancelled {
376 response: RealtimeResponse,
378 },
379
380 #[serde(rename = "response.text.delta")]
382 ResponseTextDelta {
383 response_id: String,
385 item_id: String,
387 #[serde(default)]
389 output_index: Option<u64>,
390 #[serde(default)]
392 content_index: Option<u64>,
393 delta: String,
395 },
396
397 #[serde(rename = "response.text.done")]
399 ResponseTextDone {
400 response_id: String,
402 item_id: String,
404 #[serde(default)]
406 output_index: Option<u64>,
407 #[serde(default)]
409 content_index: Option<u64>,
410 #[serde(default)]
412 text: Option<String>,
413 },
414
415 #[serde(rename = "response.audio.delta")]
417 ResponseAudioDelta {
418 response_id: String,
420 item_id: String,
422 #[serde(default)]
424 output_index: Option<u64>,
425 #[serde(default)]
427 content_index: Option<u64>,
428 delta: String,
430 },
431
432 #[serde(rename = "response.audio.done")]
434 ResponseAudioDone {
435 response_id: String,
437 item_id: String,
439 #[serde(default)]
441 output_index: Option<u64>,
442 #[serde(default)]
444 content_index: Option<u64>,
445 },
446
447 #[serde(rename = "response.audio_transcript.delta")]
449 ResponseAudioTranscriptDelta {
450 response_id: String,
452 item_id: String,
454 #[serde(default)]
456 output_index: Option<u64>,
457 #[serde(default)]
459 content_index: Option<u64>,
460 delta: String,
462 },
463
464 #[serde(rename = "response.audio_transcript.done")]
466 ResponseAudioTranscriptDone {
467 response_id: String,
469 item_id: String,
471 #[serde(default)]
473 output_index: Option<u64>,
474 #[serde(default)]
476 content_index: Option<u64>,
477 transcript: String,
479 },
480
481 #[serde(rename = "response.function_call_arguments.done")]
483 ResponseFunctionCallArgumentsDone {
484 response_id: String,
486 #[serde(default)]
488 output_index: Option<u64>,
489 name: String,
491 arguments: String,
493 },
494
495 #[serde(rename = "response.function_call.simple_browser")]
497 ResponseFunctionCallSimpleBrowser {
498 name: String,
500 #[serde(default)]
503 session: Option<serde_json::Value>,
504 },
505
506 #[serde(rename = "rate_limits.updated")]
508 RateLimitsUpdated {
509 rate_limits: Vec<RealtimeRateLimit>,
511 },
512
513 #[serde(rename = "heartbeat")]
515 Heartbeat,
516
517 #[serde(other)]
523 Unknown,
524}
525
526#[cfg(test)]
527mod tests {
528 use super::*;
529 use crate::realtime::{InputAudioNoiseReduction, NoiseReductionType, RealtimeVoice};
530
531 #[test]
532 fn session_update_serializes_to_official_shape() {
533 let session = SessionConfig {
534 model: Some("glm-realtime-flash".to_string()),
535 voice: Some(RealtimeVoice::FemaleTianmei),
536 temperature: Some(0.7),
537 max_response_output_tokens: Some(512),
538 input_audio_noise_reduction: Some(InputAudioNoiseReduction::new(
539 NoiseReductionType::NearField,
540 )),
541 ..SessionConfig::default()
542 };
543 let ev = ClientEvent::SessionUpdate {
544 event_id: Some("evt_1".into()),
545 session,
546 };
547 let json = serde_json::to_value(&ev).unwrap();
548 assert_eq!(json["type"], "session.update");
549 assert_eq!(json["event_id"], "evt_1");
550 assert_eq!(json["session"]["input_audio_format"], "wav");
551 assert_eq!(json["session"]["output_audio_format"], "pcm");
552 assert_eq!(json["session"]["turn_detection"]["type"], "client_vad");
553 assert_eq!(json["session"]["model"], "glm-realtime-flash");
554 assert_eq!(
555 json["session"]["modalities"],
556 serde_json::json!(["text", "audio"])
557 );
558 assert_eq!(json["session"]["temperature"], 0.7);
559 assert_eq!(json["session"]["max_response_output_tokens"], "512");
560 assert_eq!(json["session"]["voice"], "female-tianmei");
561 assert_eq!(
562 json["session"]["input_audio_noise_reduction"]["type"],
563 "near_field"
564 );
565 assert_eq!(json["session"]["beta_fields"]["chat_mode"], "audio");
566 assert_eq!(
567 json["session"]["turn_detection"],
568 serde_json::json!({ "type": "client_vad" })
569 );
570 }
571
572 #[test]
573 fn session_updated_accepts_server_infinite_token_default() {
574 let event: ServerEvent = serde_json::from_str(
575 r#"{"type":"session.updated","session":{"input_audio_format":"wav","output_audio_format":"pcm","turn_detection":{"type":"server_vad","create_response":true},"max_response_output_tokens":"inf","beta_fields":{"chat_mode":"audio"}}}"#,
576 )
577 .unwrap();
578 assert!(matches!(
579 event,
580 ServerEvent::SessionUpdated { session }
581 if session.max_response_output_tokens.is_none()
582 && session.turn_detection.create_response == Some(true)
583 ));
584 }
585
586 #[test]
587 fn audio_append_round_trips() {
588 let ev = ClientEvent::InputAudioBufferAppend {
589 audio: "UklGRiQ".into(),
590 client_timestamp: Some(1731999464667),
591 };
592 let json = serde_json::to_string(&ev).unwrap();
593 assert!(json.contains("\"type\":\"input_audio_buffer.append\""));
594 assert!(json.contains("\"audio\":\"UklGRiQ\""));
595 assert!(json.contains("\"client_timestamp\":1731999464667"));
596 }
597
598 #[test]
599 fn conversation_history_commands_use_official_shapes() {
600 let delete = ClientEvent::ConversationItemDelete {
601 event_id: Some("evt_delete".into()),
602 client_timestamp: Some(42),
603 item_id: "item_1".into(),
604 };
605 let json = serde_json::to_value(delete).unwrap();
606 assert_eq!(json["type"], "conversation.item.delete");
607 assert_eq!(json["item_id"], "item_1");
608 assert_eq!(json["client_timestamp"], 42);
609
610 let retrieve = ClientEvent::ConversationItemRetrieve {
611 event_id: None,
612 client_timestamp: None,
613 item_id: "item_2".into(),
614 };
615 let json = serde_json::to_value(retrieve).unwrap();
616 assert_eq!(json["type"], "conversation.item.retrieve");
617 assert_eq!(json["item_id"], "item_2");
618 assert!(json.get("event_id").is_none());
619 }
620
621 #[test]
622 fn server_audio_delta_parses_official_example() {
623 let raw = r#"{"event_id":"event89","type":"response.audio.delta","client_timestamp":1737454096061,"response_id":"respbc50304acdea479b8bd55efd5346dbdf","item_id":"item_1","output_index":0,"content_index":0,"delta":"+w6hBu39"}"#;
625 let ev: ServerEvent = serde_json::from_str(raw).unwrap();
626 match ev {
627 ServerEvent::ResponseAudioDelta {
628 response_id, delta, ..
629 } => {
630 assert_eq!(response_id, "respbc50304acdea479b8bd55efd5346dbdf");
631 assert_eq!(delta, "+w6hBu39");
632 },
633 _ => panic!("wrong variant"),
634 }
635 }
636
637 #[test]
638 fn server_response_done_parses_official_example() {
639 let raw = r#"{"event_id":"eventb94","type":"response.done","client_timestamp":1739001415611,"response":{"id":"respee64945eafb44facac88cea6f9de86f5","object":"realtime.response","status":"completed","usage":{"total_tokens":7,"input_tokens":4,"output_tokens":3,"input_token_details":{"text_tokens":4,"audio_tokens":0},"output_token_details":{"text_tokens":3,"audio_tokens":0}}}}"#;
640 let ev: ServerEvent = serde_json::from_str(raw).unwrap();
641 match ev {
642 ServerEvent::ResponseDone { response } => {
643 assert_eq!(response.status, "completed");
644 let usage = response.usage.unwrap();
645 assert_eq!(usage.total_tokens, 7);
646 assert_eq!(usage.output_tokens, 3);
647 },
648 _ => panic!("wrong variant"),
649 }
650 }
651
652 #[test]
653 fn server_text_events_preserve_consumable_text() {
654 let delta: ServerEvent = serde_json::from_str(
655 r#"{"type":"response.text.delta","response_id":"resp_1","item_id":"item_1","output_index":0,"content_index":1,"delta":"你好"}"#,
656 )
657 .unwrap();
658 match delta {
659 ServerEvent::ResponseTextDelta {
660 response_id,
661 item_id,
662 output_index,
663 content_index,
664 delta,
665 } => {
666 assert_eq!(response_id, "resp_1");
667 assert_eq!(item_id, "item_1");
668 assert_eq!(output_index, Some(0));
669 assert_eq!(content_index, Some(1));
670 assert_eq!(delta, "你好");
671 },
672 _ => panic!("wrong variant"),
673 }
674
675 let done: ServerEvent = serde_json::from_str(
676 r#"{"type":"response.text.done","response_id":"resp_1","item_id":"item_1","text":"你好!"}"#,
677 )
678 .unwrap();
679 assert!(matches!(
680 done,
681 ServerEvent::ResponseTextDone {
682 text: Some(text),
683 ..
684 } if text == "你好!"
685 ));
686 }
687
688 #[test]
689 fn server_cancelled_event_preserves_final_response() {
690 let event: ServerEvent = serde_json::from_str(
691 r#"{"type":"response.cancelled","response":{"id":"resp_1","object":"realtime.response","status":"cancelled"}}"#,
692 )
693 .unwrap();
694 assert!(matches!(
695 event,
696 ServerEvent::ResponseCancelled { response }
697 if response.id == "resp_1" && response.status == "cancelled"
698 ));
699 }
700
701 #[test]
702 fn server_events_preserve_required_correlation_fields() {
703 let conversation: ServerEvent = serde_json::from_str(
704 r#"{"type":"conversation.created","conversation":{"id":"conv_1","object":"realtime.conversation"}}"#,
705 )
706 .unwrap();
707 assert!(matches!(
708 conversation,
709 ServerEvent::ConversationCreated { conversation }
710 if conversation.id == "conv_1"
711 && conversation.object == "realtime.conversation"
712 ));
713
714 let transcript: ServerEvent = serde_json::from_str(
715 r#"{"type":"response.audio_transcript.delta","response_id":"resp_1","item_id":"item_1","output_index":2,"content_index":3,"delta":"hello"}"#,
716 )
717 .unwrap();
718 assert!(matches!(
719 transcript,
720 ServerEvent::ResponseAudioTranscriptDelta {
721 response_id,
722 item_id,
723 output_index: Some(2),
724 content_index: Some(3),
725 delta,
726 } if response_id == "resp_1" && item_id == "item_1" && delta == "hello"
727 ));
728
729 let function_call: ServerEvent = serde_json::from_str(
730 r#"{"type":"response.function_call_arguments.done","response_id":"resp_1","output_index":1,"name":"weather","arguments":"{}"}"#,
731 )
732 .unwrap();
733 assert!(matches!(
734 function_call,
735 ServerEvent::ResponseFunctionCallArgumentsDone {
736 output_index: Some(1),
737 name,
738 ..
739 } if name == "weather"
740 ));
741
742 let browser: ServerEvent = serde_json::from_str(
743 r#"{"type":"response.function_call.simple_browser","name":"simple_browser","session":{"beta_fields":{"simple_browser":{"description":"searching"}}}}"#,
744 )
745 .unwrap();
746 assert!(matches!(
747 browser,
748 ServerEvent::ResponseFunctionCallSimpleBrowser {
749 name,
750 session: Some(_),
751 } if name == "simple_browser"
752 ));
753
754 assert!(
755 serde_json::from_str::<ServerEvent>(
756 r#"{"type":"response.audio.delta","response_id":"resp_1","delta":"AA=="}"#
757 )
758 .is_err()
759 );
760 }
761
762 #[test]
763 fn server_error_parses() {
764 let raw = r#"{"event_id":"event_890","type":"error","error":{"type":"invalid_request_error","code":"invalid_event","message":"The 'type' field is missing."}}"#;
765 let ev: ServerEvent = serde_json::from_str(raw).unwrap();
766 match ev {
767 ServerEvent::Error { error } => {
768 assert_eq!(error.code.as_deref(), Some("invalid_event"));
769 },
770 _ => panic!("wrong variant"),
771 }
772 }
773
774 #[test]
775 fn guide_only_conversation_and_lifecycle_events_parse() {
776 let created: ServerEvent = serde_json::from_str(
777 r#"{"type":"session.created","session":{"id":"sess_1","object":"realtime.session","voice":"default","future_field":true}}"#,
778 )
779 .unwrap();
780 assert!(matches!(
781 created,
782 ServerEvent::SessionCreated { session }
783 if session.id.as_deref() == Some("sess_1")
784 && session.extra.contains_key("future_field")
785 ));
786
787 let deleted: ServerEvent =
788 serde_json::from_str(r#"{"type":"conversation.item.deleted","item_id":"item_1"}"#)
789 .unwrap();
790 assert!(matches!(
791 deleted,
792 ServerEvent::ConversationItemDeleted { item_id } if item_id == "item_1"
793 ));
794
795 let item_done: ServerEvent = serde_json::from_str(
796 r#"{"type":"response.output_item.done","response_id":"resp_1","output_index":0,"item":{"id":"item_1","type":"message","object":"realtime.item","status":"completed","role":"assistant","content":[{}]}}"#,
797 )
798 .unwrap();
799 assert!(matches!(
800 item_done,
801 ServerEvent::ResponseOutputItemDone {
802 output_index: 0,
803 item,
804 ..
805 } if item.content.len() == 1 && item.content[0].type_.is_none()
806 ));
807
808 let limits: ServerEvent = serde_json::from_str(
809 r#"{"type":"rate_limits.updated","rate_limits":[{"name":"requests","limit":5,"remaining":4,"reset_seconds":1.0}]}"#,
810 )
811 .unwrap();
812 assert!(matches!(
813 limits,
814 ServerEvent::RateLimitsUpdated { rate_limits }
815 if rate_limits.len() == 1 && rate_limits[0].remaining == 4
816 ));
817 }
818
819 #[test]
820 fn unknown_events_are_forward_compatible_but_known_shapes_are_strict() {
821 let event: ServerEvent =
822 serde_json::from_str(r#"{"type":"future.event","payload":"ignored"}"#).unwrap();
823 assert!(matches!(event, ServerEvent::Unknown));
824
825 assert!(
826 serde_json::from_str::<ServerEvent>(
827 r#"{"type":"response.text.delta","response_id":"resp_1","delta":"missing item"}"#
828 )
829 .is_err()
830 );
831 assert!(
832 serde_json::from_str::<ServerEvent>(
833 r#"{"type":"error","error":{"message":"missing error type"}}"#
834 )
835 .is_err()
836 );
837 }
838}