zai_rs/realtime/
protocol.rs

1//! Data structures from the official GLM-Realtime protocol.
2//!
3//! Mirrors `RealtimeConversationItem`, `RealtimeResponse`, the `session.update`
4//! payload, and supporting enums. Wire shapes are pinned to the repository's
5//! `spec/upstream/asyncapi-2026-07-11.json` snapshot.
6
7use serde::{Deserialize, Serialize};
8
9use super::audio::{InputAudioFormat, OutputAudioFormat};
10
11/// VAD (voice-activity-detection) mode. `ClientVad` (default) lets the client
12/// decide when to commit audio; `ServerVad` has the server detect speech and
13/// auto-commit (and supports interruption handling).
14#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
15#[serde(rename_all = "snake_case")]
16pub enum TurnDetectionType {
17    /// Client-driven VAD (client uploads + commits audio manually).
18    #[default]
19    ClientVad,
20    /// Server-driven VAD (server detects speech, auto-commits, handles
21    /// interruption via `response.cancel`).
22    ServerVad,
23}
24
25/// `turn_detection` object inside `session.update`.
26///
27/// The tuning fields apply to server-side VAD. Leave them unset for
28/// client-driven VAD, where the caller explicitly commits audio and creates a
29/// response.
30#[derive(Debug, Clone, Serialize, Deserialize)]
31pub struct TurnDetection {
32    /// VAD strategy.
33    #[serde(rename = "type")]
34    pub type_: TurnDetectionType,
35    /// Automatically create a response when server VAD detects the end of a
36    /// turn.
37    #[serde(skip_serializing_if = "Option::is_none")]
38    pub create_response: Option<bool>,
39    /// Interrupt an in-progress response when server VAD detects new speech.
40    #[serde(skip_serializing_if = "Option::is_none")]
41    pub interrupt_response: Option<bool>,
42    /// Audio retained before detected speech, in milliseconds.
43    #[serde(skip_serializing_if = "Option::is_none")]
44    pub prefix_padding_ms: Option<u32>,
45    /// Silence required to end a turn, in milliseconds.
46    #[serde(skip_serializing_if = "Option::is_none")]
47    pub silence_duration_ms: Option<u32>,
48    /// VAD activation threshold in the inclusive range `0.0..=1.0`.
49    #[serde(skip_serializing_if = "Option::is_none")]
50    pub threshold: Option<f64>,
51}
52
53impl TurnDetection {
54    /// Create a `turn_detection` object with the given VAD strategy.
55    pub fn new(type_: TurnDetectionType) -> Self {
56        Self {
57            type_,
58            create_response: None,
59            interrupt_response: None,
60            prefix_padding_ms: None,
61            silence_duration_ms: None,
62            threshold: None,
63        }
64    }
65}
66
67/// Conversation mode under `beta_fields.chat_mode`.
68#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
69#[serde(rename_all = "snake_case")]
70pub enum ChatMode {
71    /// Passive video: video frames are sent alongside audio.
72    VideoPassive,
73    /// Audio-only conversation (default).
74    #[default]
75    Audio,
76}
77
78/// Output modality requested from the model.
79#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
80#[serde(rename_all = "snake_case")]
81pub enum RealtimeModality {
82    /// Generate text output.
83    Text,
84    /// Generate audio output.
85    Audio,
86}
87
88/// Built-in voice used for audio output.
89#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
90pub enum RealtimeVoice {
91    /// Default Tongtong voice.
92    #[default]
93    #[serde(rename = "tongtong")]
94    Tongtong,
95    /// General-purpose male voice.
96    #[serde(rename = "xiaochen")]
97    Xiaochen,
98    /// Tianmei female voice.
99    #[serde(rename = "female-tianmei")]
100    FemaleTianmei,
101    /// Young male voice.
102    #[serde(rename = "male-qn-daxuesheng")]
103    MaleQnDaxuesheng,
104    /// Professional male voice.
105    #[serde(rename = "male-qn-jingying")]
106    MaleQnJingying,
107    /// Lovely-girl voice.
108    #[serde(rename = "lovely_girl")]
109    LovelyGirl,
110    /// Young female voice.
111    #[serde(rename = "female-shaonv")]
112    FemaleShaonv,
113}
114
115/// Microphone placement used by input-audio noise reduction.
116#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
117#[serde(rename_all = "snake_case")]
118pub enum NoiseReductionType {
119    /// A microphone close to the speaker.
120    NearField,
121    /// A microphone farther from the speaker.
122    FarField,
123}
124
125/// `input_audio_noise_reduction` inside `session.update`.
126#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
127pub struct InputAudioNoiseReduction {
128    /// Noise-reduction profile.
129    #[serde(rename = "type")]
130    pub type_: NoiseReductionType,
131}
132
133impl InputAudioNoiseReduction {
134    /// Select a noise-reduction profile.
135    pub fn new(type_: NoiseReductionType) -> Self {
136        Self { type_ }
137    }
138}
139
140/// Optional server-generated greeting configuration.
141#[derive(Debug, Clone, Default, Serialize, Deserialize)]
142pub struct GreetingConfig {
143    /// Whether the greeting is enabled.
144    #[serde(skip_serializing_if = "Option::is_none")]
145    pub enable: Option<bool>,
146    /// Greeting text supplied to the model.
147    #[serde(skip_serializing_if = "Option::is_none")]
148    pub content: Option<String>,
149}
150
151/// `beta_fields` object inside `session.update`.
152#[derive(Debug, Clone, Serialize, Deserialize)]
153pub struct BetaFields {
154    /// Conversation mode.
155    #[serde(skip_serializing_if = "Option::is_none")]
156    pub chat_mode: Option<ChatMode>,
157    /// TTS source, e.g. `"e2e"`.
158    #[serde(skip_serializing_if = "Option::is_none")]
159    pub tts_source: Option<String>,
160    /// Enable the server-side built-in web search (audio mode only).
161    #[serde(skip_serializing_if = "Option::is_none")]
162    pub auto_search: Option<bool>,
163}
164
165impl Default for BetaFields {
166    fn default() -> Self {
167        Self {
168            chat_mode: Some(ChatMode::Audio),
169            tts_source: None,
170            auto_search: None,
171        }
172    }
173}
174
175/// A function tool advertised to the model via `session.update.tools`.
176#[derive(Debug, Clone, Serialize, Deserialize)]
177pub struct RealtimeTool {
178    /// Always `"function"`.
179    #[serde(rename = "type")]
180    pub type_: String,
181    /// Function name.
182    pub name: String,
183    /// Human-readable description the model uses to decide when to call.
184    pub description: String,
185    /// JSON Schema describing accepted parameters.
186    #[serde(default = "empty_json_object")]
187    pub parameters: serde_json::Value,
188}
189
190fn empty_json_object() -> serde_json::Value {
191    serde_json::Value::Object(serde_json::Map::new())
192}
193
194impl RealtimeTool {
195    /// Build a function tool from a name + description + JSON-Schema
196    /// parameters.
197    pub fn function(
198        name: impl Into<String>,
199        description: impl Into<String>,
200        parameters: serde_json::Value,
201    ) -> Self {
202        Self {
203            type_: "function".to_string(),
204            name: name.into(),
205            description: description.into(),
206            parameters,
207        }
208    }
209}
210
211/// The inner `session` object carried by the `session.update` client event.
212#[derive(Debug, Clone, Serialize, Deserialize)]
213pub struct SessionConfig {
214    /// Model id selected for this session. [`SessionBuilder`](super::SessionBuilder)
215    /// fills this from the type-safe model passed to
216    /// [`RealtimeClient::session`](super::RealtimeClient::session).
217    #[serde(skip_serializing_if = "Option::is_none")]
218    pub model: Option<String>,
219    /// Input audio format (`wav`, `pcm16`, or `pcm24`).
220    pub input_audio_format: InputAudioFormat,
221    /// Output audio format. The current protocol accepts only `pcm`.
222    pub output_audio_format: OutputAudioFormat,
223    /// System instructions guiding the model.
224    #[serde(skip_serializing_if = "Option::is_none")]
225    pub instructions: Option<String>,
226    /// Output modalities. The protocol default is text plus audio.
227    #[serde(default = "default_modalities")]
228    pub modalities: Vec<RealtimeModality>,
229    /// Voice used when audio output is requested.
230    #[serde(skip_serializing_if = "Option::is_none")]
231    pub voice: Option<RealtimeVoice>,
232    /// Sampling temperature in the inclusive range `0.0..=1.0`.
233    #[serde(skip_serializing_if = "Option::is_none")]
234    pub temperature: Option<f64>,
235    /// Maximum number of response text tokens (`1..=1024`). `None` uses the
236    /// server default (`"inf"`, currently equivalent to 1024).
237    ///
238    /// The upstream schema represents this numeric limit as a JSON string, so
239    /// custom serde preserves that exact wire shape while callers use a `u16`.
240    #[serde(
241        default,
242        skip_serializing_if = "Option::is_none",
243        with = "optional_u16_string"
244    )]
245    pub max_response_output_tokens: Option<u16>,
246    /// VAD strategy.
247    pub turn_detection: TurnDetection,
248    /// Optional microphone noise-reduction profile.
249    #[serde(skip_serializing_if = "Option::is_none")]
250    pub input_audio_noise_reduction: Option<InputAudioNoiseReduction>,
251    /// Beta / mode toggles.
252    #[serde(default)]
253    pub beta_fields: BetaFields,
254    /// Optional greeting generated when the session starts.
255    #[serde(skip_serializing_if = "Option::is_none")]
256    pub greeting_config: Option<GreetingConfig>,
257    /// Function tools advertised to the model.
258    #[serde(default, skip_serializing_if = "Vec::is_empty")]
259    pub tools: Vec<RealtimeTool>,
260}
261
262impl Default for SessionConfig {
263    fn default() -> Self {
264        Self {
265            model: None,
266            input_audio_format: InputAudioFormat::default(),
267            output_audio_format: OutputAudioFormat::default(),
268            instructions: None,
269            modalities: default_modalities(),
270            voice: None,
271            temperature: None,
272            max_response_output_tokens: None,
273            turn_detection: TurnDetection::new(TurnDetectionType::default()),
274            input_audio_noise_reduction: None,
275            beta_fields: BetaFields::default(),
276            greeting_config: None,
277            tools: Vec::new(),
278        }
279    }
280}
281
282fn default_modalities() -> Vec<RealtimeModality> {
283    vec![RealtimeModality::Text, RealtimeModality::Audio]
284}
285
286mod optional_u16_string {
287    use serde::{Deserialize as _, Deserializer, Serializer, de::Error as _};
288
289    pub(super) fn serialize<S>(value: &Option<u16>, serializer: S) -> Result<S::Ok, S::Error>
290    where
291        S: Serializer,
292    {
293        match value {
294            Some(value) => serializer.serialize_some(&value.to_string()),
295            None => serializer.serialize_none(),
296        }
297    }
298
299    pub(super) fn deserialize<'de, D>(deserializer: D) -> Result<Option<u16>, D::Error>
300    where
301        D: Deserializer<'de>,
302    {
303        let value = Option::<String>::deserialize(deserializer)?;
304        value
305            .map(|value| {
306                if value == "inf" {
307                    return Ok(None);
308                }
309                value
310                    .parse::<u16>()
311                    .map_err(|_| {
312                        D::Error::custom(
313                            "max_response_output_tokens must be \"inf\" or a decimal u16 string",
314                        )
315                    })
316                    .map(Some)
317            })
318            .transpose()
319            .map(Option::flatten)
320    }
321}
322
323/// `item.type` discriminator inside [`RealtimeConversationItem`].
324#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
325#[serde(rename_all = "snake_case")]
326pub enum ItemType {
327    /// A chat message (with `role` + `content`).
328    Message,
329    /// A function/tool call emitted by the model.
330    FunctionCall,
331    /// The caller's reply to a function/tool call.
332    FunctionCallOutput,
333}
334
335/// One content part within a conversation item message.
336#[derive(Debug, Clone, Serialize, Deserialize)]
337pub struct ItemContent {
338    /// Content type: `input_audio`, `input_text`, `text`.
339    #[serde(rename = "type", skip_serializing_if = "Option::is_none")]
340    pub type_: Option<String>,
341    /// Text content (`input_text` / `text`).
342    #[serde(skip_serializing_if = "Option::is_none")]
343    pub text: Option<String>,
344    /// Base64 audio (`input_audio`).
345    #[serde(skip_serializing_if = "Option::is_none")]
346    pub audio: Option<String>,
347    /// Audio transcript.
348    #[serde(skip_serializing_if = "Option::is_none")]
349    pub transcript: Option<String>,
350}
351
352/// `RealtimeConversationItem`: a message, function call, or function-call
353/// output inserted via `conversation.item.create`.
354#[derive(Debug, Clone, Serialize, Deserialize)]
355pub struct RealtimeConversationItem {
356    /// Item id (client- or server-generated).
357    #[serde(skip_serializing_if = "Option::is_none")]
358    pub id: Option<String>,
359    /// Item kind.
360    #[serde(rename = "type")]
361    pub type_: ItemType,
362    /// Always `"realtime.item"`.
363    pub object: String,
364    /// `completed` / `incomplete`.
365    #[serde(skip_serializing_if = "Option::is_none")]
366    pub status: Option<String>,
367    /// Sender role (`message` only): `user` / `assistant` / `system`.
368    #[serde(skip_serializing_if = "Option::is_none")]
369    pub role: Option<String>,
370    /// Message content parts (`message` only).
371    #[serde(default, skip_serializing_if = "Vec::is_empty")]
372    pub content: Vec<ItemContent>,
373    /// Function name (`function_call` only).
374    #[serde(skip_serializing_if = "Option::is_none")]
375    pub name: Option<String>,
376    /// Function arguments (`function_call` only).
377    #[serde(skip_serializing_if = "Option::is_none")]
378    pub arguments: Option<String>,
379    /// Function output (`function_call_output` only).
380    #[serde(skip_serializing_if = "Option::is_none")]
381    pub output: Option<String>,
382}
383
384impl RealtimeConversationItem {
385    /// A user text message: `conversation.item.create` for textual input.
386    pub fn user_text(text: impl Into<String>) -> Self {
387        Self {
388            id: None,
389            type_: ItemType::Message,
390            object: "realtime.item".to_string(),
391            status: Some("completed".to_string()),
392            role: Some("user".to_string()),
393            content: vec![ItemContent {
394                type_: Some("input_text".to_string()),
395                text: Some(text.into()),
396                audio: None,
397                transcript: None,
398            }],
399            name: None,
400            arguments: None,
401            output: None,
402        }
403    }
404
405    /// A function-call output to feed back to the model.
406    pub fn function_output(call_name: impl Into<String>, output: impl Into<String>) -> Self {
407        Self {
408            id: None,
409            type_: ItemType::FunctionCallOutput,
410            object: "realtime.item".to_string(),
411            status: None,
412            role: None,
413            content: Vec::new(),
414            name: Some(call_name.into()),
415            arguments: None,
416            output: Some(output.into()),
417        }
418    }
419}
420
421/// Token-usage detail breakdown.
422#[derive(Debug, Clone, Default, Serialize, Deserialize)]
423pub struct TokenDetails {
424    /// Number of text tokens.
425    #[serde(skip_serializing_if = "Option::is_none")]
426    pub text_tokens: Option<u64>,
427    /// Number of audio tokens.
428    #[serde(skip_serializing_if = "Option::is_none")]
429    pub audio_tokens: Option<u64>,
430    /// Number of cached tokens.
431    #[serde(skip_serializing_if = "Option::is_none")]
432    pub cached_tokens: Option<u64>,
433}
434
435/// `RealtimeResponse.usage`.
436#[derive(Debug, Clone, Default, Serialize, Deserialize)]
437pub struct RealtimeUsage {
438    /// Total tokens for this response.
439    #[serde(default)]
440    pub total_tokens: u64,
441    /// Input tokens.
442    #[serde(default)]
443    pub input_tokens: u64,
444    /// Output tokens.
445    #[serde(default)]
446    pub output_tokens: u64,
447    /// Input-token breakdown.
448    #[serde(skip_serializing_if = "Option::is_none")]
449    pub input_token_details: Option<TokenDetails>,
450    /// Output-token breakdown.
451    #[serde(skip_serializing_if = "Option::is_none")]
452    pub output_token_details: Option<TokenDetails>,
453}
454
455/// `RealtimeResponse`: emitted by `response.created` / `response.done`.
456#[derive(Debug, Clone, Serialize, Deserialize)]
457pub struct RealtimeResponse {
458    /// Response id.
459    pub id: String,
460    /// Always `"realtime.response"`.
461    pub object: String,
462    /// Response status (`in_progress`, `completed`, `cancelled`, `failed`, or
463    /// `incomplete`).
464    pub status: String,
465    /// Token usage (present on `response.done`).
466    #[serde(skip_serializing_if = "Option::is_none")]
467    pub usage: Option<RealtimeUsage>,
468}
zai_rs/realtime/protocol.rs

zai_rs/realtime/
protocol.rs
