zai_rs/realtime/
events.rs

1//! Typed realtime events, mapped 1:1 to the official GLM-Realtime protocol
2//! client/server event names.
3//!
4//! See <https://github.com/MetaGLM/glm-realtime-sdk/blob/main/GLM-Realtime-doc-for-llm.md>.
5
6use serde::{Deserialize, Serialize};
7
8use super::protocol::{RealtimeConversationItem, RealtimeResponse, SessionConfig};
9
10/// Body of a server `error` event.
11#[derive(Debug, Clone, Deserialize)]
12pub struct ServerErrorBody {
13    /// Error type, e.g. `"invalid_request_error"`, `"server_error"`.
14    #[serde(rename = "type", default)]
15    pub type_: Option<String>,
16    /// Machine-readable error code (string per the GLM protocol).
17    #[serde(default)]
18    pub code: Option<String>,
19    /// Human-readable message.
20    #[serde(default)]
21    pub message: Option<String>,
22}
23
24// ---------------------------------------------------------------------------
25// Client events (sent client → server). Serialize-only.
26// ---------------------------------------------------------------------------
27
28/// A client event, tagged by `type` to match the official event names.
29#[derive(Debug, Clone, Serialize)]
30#[serde(tag = "type")]
31pub enum ClientEvent {
32    /// `session.update` — set the session defaults (formats, VAD, tools, …).
33    #[serde(rename = "session.update")]
34    SessionUpdate {
35        #[serde(skip_serializing_if = "Option::is_none")]
36        event_id: Option<String>,
37        session: SessionConfig,
38    },
39
40    /// `input_audio_buffer.append` — upload base64 WAV audio.
41    #[serde(rename = "input_audio_buffer.append")]
42    InputAudioBufferAppend {
43        audio: String,
44        #[serde(skip_serializing_if = "Option::is_none")]
45        client_timestamp: Option<i64>,
46    },
47
48    /// `input_audio_buffer.append_video_frame` — upload a base64 JPEG frame.
49    #[serde(rename = "input_audio_buffer.append_video_frame")]
50    InputAudioBufferAppendVideoFrame {
51        video_frame: String,
52        #[serde(skip_serializing_if = "Option::is_none")]
53        client_timestamp: Option<i64>,
54    },
55
56    /// `input_audio_buffer.commit` — commit buffered audio for inference.
57    #[serde(rename = "input_audio_buffer.commit")]
58    InputAudioBufferCommit {
59        #[serde(skip_serializing_if = "Option::is_none")]
60        client_timestamp: Option<i64>,
61    },
62
63    /// `input_audio_buffer.clear` — clear the buffer.
64    #[serde(rename = "input_audio_buffer.clear")]
65    InputAudioBufferClear,
66
67    /// `conversation.item.create` — inject a text message or function-call
68    /// output into the conversation history.
69    #[serde(rename = "conversation.item.create")]
70    ConversationItemCreate {
71        #[serde(skip_serializing_if = "Option::is_none")]
72        event_id: Option<String>,
73        item: RealtimeConversationItem,
74    },
75
76    /// `response.create` — trigger model inference.
77    #[serde(rename = "response.create")]
78    ResponseCreate {
79        #[serde(skip_serializing_if = "Option::is_none")]
80        client_timestamp: Option<i64>,
81    },
82
83    /// `response.cancel` — cancel the in-flight response (interruption).
84    #[serde(rename = "response.cancel")]
85    ResponseCancel {
86        #[serde(skip_serializing_if = "Option::is_none")]
87        client_timestamp: Option<i64>,
88    },
89}
90
91// ---------------------------------------------------------------------------
92// Server events (received server → client). Deserialize-only.
93// ---------------------------------------------------------------------------
94
95/// A server event, tagged by `type`. Unknown/extra fields are ignored.
96#[derive(Debug, Clone, Deserialize)]
97#[serde(tag = "type")]
98pub enum ServerEvent {
99    /// Server-side error (most are recoverable; the session stays open).
100    #[serde(rename = "error")]
101    Error { error: ServerErrorBody },
102
103    /// `session.created` — session established.
104    #[serde(rename = "session.created")]
105    SessionCreated,
106
107    /// `session.updated` — confirms a `session.update`.
108    #[serde(rename = "session.updated")]
109    SessionUpdated,
110
111    /// `conversation.created` — one per session.
112    #[serde(rename = "conversation.created")]
113    ConversationCreated,
114
115    /// `conversation.item.created`.
116    #[serde(rename = "conversation.item.created")]
117    ConversationItemCreated { item: RealtimeConversationItem },
118
119    /// `conversation.item.input_audio_transcription.completed`.
120    #[serde(rename = "conversation.item.input_audio_transcription.completed")]
121    InputAudioTranscriptionCompleted { item_id: String, transcript: String },
122
123    /// `conversation.item.input_audio_transcription.failed`.
124    #[serde(rename = "conversation.item.input_audio_transcription.failed")]
125    InputAudioTranscriptionFailed {
126        item_id: String,
127        error: ServerErrorBody,
128    },
129
130    /// `input_audio_buffer.committed`.
131    #[serde(rename = "input_audio_buffer.committed")]
132    InputAudioBufferCommitted {
133        #[serde(default)]
134        item_id: Option<String>,
135    },
136
137    /// `input_audio_buffer.cleared`.
138    #[serde(rename = "input_audio_buffer.cleared")]
139    InputAudioBufferCleared,
140
141    /// `input_audio_buffer.speech_started` (server-VAD only).
142    #[serde(rename = "input_audio_buffer.speech_started")]
143    InputAudioBufferSpeechStarted,
144
145    /// `input_audio_buffer.speech_stopped` (server-VAD only).
146    #[serde(rename = "input_audio_buffer.speech_stopped")]
147    InputAudioBufferSpeechStopped,
148
149    /// `response.created`.
150    #[serde(rename = "response.created")]
151    ResponseCreated { response: RealtimeResponse },
152
153    /// `response.done` — final state + usage. Always emitted.
154    #[serde(rename = "response.done")]
155    ResponseDone { response: RealtimeResponse },
156
157    /// `response.audio.delta` — base64 audio chunk (mp3 or pcm).
158    #[serde(rename = "response.audio.delta")]
159    ResponseAudioDelta {
160        response_id: String,
161        #[serde(default)]
162        item_id: Option<String>,
163        delta: String,
164    },
165
166    /// `response.audio.done`.
167    #[serde(rename = "response.audio.done")]
168    ResponseAudioDone {
169        response_id: String,
170        #[serde(default)]
171        item_id: Option<String>,
172    },
173
174    /// `response.audio_transcript.delta` — incremental transcript text.
175    #[serde(rename = "response.audio_transcript.delta")]
176    ResponseAudioTranscriptDelta { response_id: String, delta: String },
177
178    /// `response.audio_transcript.done` — final transcript.
179    #[serde(rename = "response.audio_transcript.done")]
180    ResponseAudioTranscriptDone {
181        response_id: String,
182        transcript: String,
183    },
184
185    /// `response.function_call_arguments.done` — completed tool call.
186    #[serde(rename = "response.function_call_arguments.done")]
187    ResponseFunctionCallArgumentsDone {
188        response_id: String,
189        name: String,
190        arguments: String,
191    },
192
193    /// `response.function_call.simple_browser` — video link triggered search.
194    #[serde(rename = "response.function_call.simple_browser")]
195    ResponseFunctionCallSimpleBrowser,
196
197    /// `heartbeat` — keepalive (every ~30s).
198    #[serde(rename = "heartbeat")]
199    Heartbeat,
200}
201
202#[cfg(test)]
203mod tests {
204    use super::*;
205
206    #[test]
207    fn session_update_serializes_to_official_shape() {
208        let ev = ClientEvent::SessionUpdate {
209            event_id: Some("evt_1".into()),
210            session: SessionConfig::default(),
211        };
212        let json = serde_json::to_value(&ev).unwrap();
213        assert_eq!(json["type"], "session.update");
214        assert_eq!(json["event_id"], "evt_1");
215        assert_eq!(json["session"]["input_audio_format"], "wav");
216        assert_eq!(json["session"]["output_audio_format"], "pcm");
217        assert_eq!(json["session"]["turn_detection"]["type"], "client_vad");
218    }
219
220    #[test]
221    fn audio_append_round_trips() {
222        let ev = ClientEvent::InputAudioBufferAppend {
223            audio: "UklGRiQ".into(),
224            client_timestamp: Some(1731999464667),
225        };
226        let json = serde_json::to_string(&ev).unwrap();
227        assert!(json.contains("\"type\":\"input_audio_buffer.append\""));
228        assert!(json.contains("\"audio\":\"UklGRiQ\""));
229        assert!(json.contains("\"client_timestamp\":1731999464667"));
230    }
231
232    #[test]
233    fn server_audio_delta_parses_official_example() {
234        // Trimmed shape of the official response.audio.delta example.
235        let raw = r#"{"event_id":"event89","type":"response.audio.delta","client_timestamp":1737454096061,"response_id":"respbc50304acdea479b8bd55efd5346dbdf","output_index":0,"content_index":0,"delta":"+w6hBu39"}"#;
236        let ev: ServerEvent = serde_json::from_str(raw).unwrap();
237        match ev {
238            ServerEvent::ResponseAudioDelta {
239                response_id, delta, ..
240            } => {
241                assert_eq!(response_id, "respbc50304acdea479b8bd55efd5346dbdf");
242                assert_eq!(delta, "+w6hBu39");
243            },
244            _ => panic!("wrong variant"),
245        }
246    }
247
248    #[test]
249    fn server_response_done_parses_official_example() {
250        let raw = r#"{"event_id":"eventb94","type":"response.done","client_timestamp":1739001415611,"response":{"id":"respee64945eafb44facac88cea6f9de86f5","object":"realtime.response","status":"completed","usage":{"total_tokens":7,"input_tokens":4,"output_tokens":3,"input_token_details":{"text_tokens":4,"audio_tokens":0},"output_token_details":{"text_tokens":3,"audio_tokens":0}}}}"#;
251        let ev: ServerEvent = serde_json::from_str(raw).unwrap();
252        match ev {
253            ServerEvent::ResponseDone { response } => {
254                assert_eq!(response.status.as_deref(), Some("completed"));
255                let usage = response.usage.unwrap();
256                assert_eq!(usage.total_tokens, 7);
257                assert_eq!(usage.output_tokens, 3);
258            },
259            _ => panic!("wrong variant"),
260        }
261    }
262
263    #[test]
264    fn server_error_parses() {
265        let raw = r#"{"event_id":"event_890","type":"error","error":{"type":"invalid_request_error","code":"invalid_event","message":"The 'type' field is missing."}}"#;
266        let ev: ServerEvent = serde_json::from_str(raw).unwrap();
267        match ev {
268            ServerEvent::Error { error } => {
269                assert_eq!(error.code.as_deref(), Some("invalid_event"));
270            },
271            _ => panic!("wrong variant"),
272        }
273    }
274}
zai_rs/realtime/events.rs

zai_rs/realtime/
events.rs
