zai_rs/realtime/
protocol.rs

1//! Data structures from the official GLM-Realtime protocol.
2//!
3//! Mirrors `RealtimeConversationItem`, `RealtimeResponse`, the `session.update`
4//! payload, and supporting enums. See:
5//! <https://github.com/MetaGLM/glm-realtime-sdk/blob/main/GLM-Realtime-doc-for-llm.md>
6
7use serde::{Deserialize, Serialize};
8
9use super::audio::{InputAudioFormat, OutputAudioFormat};
10
11/// VAD (voice-activity-detection) mode. `ClientVad` (default) lets the client
12/// decide when to commit audio; `ServerVad` has the server detect speech and
13/// auto-commit (and supports interruption handling).
14#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
15#[serde(rename_all = "snake_case")]
16pub enum TurnDetectionType {
17    /// Client-driven VAD (client uploads + commits audio manually).
18    #[default]
19    ClientVad,
20    /// Server-driven VAD (server detects speech, auto-commits, handles
21    /// interruption via `response.cancel`).
22    ServerVad,
23}
24
25/// `turn_detection` object inside `session.update`.
26#[derive(Debug, Clone, Serialize, Deserialize)]
27pub struct TurnDetection {
28    /// VAD strategy.
29    #[serde(rename = "type")]
30    pub type_: TurnDetectionType,
31}
32
33impl TurnDetection {
34    pub fn new(type_: TurnDetectionType) -> Self {
35        Self { type_ }
36    }
37}
38
39/// Conversation mode under `beta_fields.chat_mode`.
40#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
41#[serde(rename_all = "snake_case")]
42pub enum ChatMode {
43    /// Passive video: video frames are sent alongside audio.
44    VideoPassive,
45    /// Audio-only conversation (default).
46    #[default]
47    Audio,
48}
49
50/// `beta_fields` object inside `session.update`.
51#[derive(Debug, Clone, Default, Serialize, Deserialize)]
52pub struct BetaFields {
53    /// Conversation mode.
54    #[serde(skip_serializing_if = "Option::is_none")]
55    pub chat_mode: Option<ChatMode>,
56    /// TTS source, e.g. `"e2e"`.
57    #[serde(skip_serializing_if = "Option::is_none")]
58    pub tts_source: Option<String>,
59    /// Enable the server-side built-in web search (audio mode only).
60    #[serde(skip_serializing_if = "Option::is_none")]
61    pub auto_search: Option<bool>,
62}
63
64/// A function tool advertised to the model via `session.update.tools`.
65#[derive(Debug, Clone, Serialize, Deserialize)]
66pub struct RealtimeTool {
67    /// Always `"function"`.
68    #[serde(rename = "type")]
69    pub type_: String,
70    /// Function name.
71    pub name: String,
72    /// Human-readable description the model uses to decide when to call.
73    pub description: String,
74    /// JSON Schema describing accepted parameters.
75    pub parameters: serde_json::Value,
76}
77
78impl RealtimeTool {
79    /// Build a function tool from a name + description + JSON-Schema
80    /// parameters.
81    pub fn function(
82        name: impl Into<String>,
83        description: impl Into<String>,
84        parameters: serde_json::Value,
85    ) -> Self {
86        Self {
87            type_: "function".to_string(),
88            name: name.into(),
89            description: description.into(),
90            parameters,
91        }
92    }
93}
94
95/// The inner `session` object carried by the `session.update` client event.
96#[derive(Debug, Clone, Serialize, Deserialize)]
97pub struct SessionConfig {
98    /// Input audio format (`wav` / `wav48`).
99    pub input_audio_format: InputAudioFormat,
100    /// Output audio format (`pcm` / `mp3`).
101    pub output_audio_format: OutputAudioFormat,
102    /// System instructions guiding the model.
103    #[serde(skip_serializing_if = "Option::is_none")]
104    pub instructions: Option<String>,
105    /// VAD strategy.
106    pub turn_detection: TurnDetection,
107    /// Beta / mode toggles.
108    pub beta_fields: BetaFields,
109    /// Function tools (ServerVAD requires re-sending `turn_detection`).
110    #[serde(default, skip_serializing_if = "Vec::is_empty")]
111    pub tools: Vec<RealtimeTool>,
112}
113
114impl Default for SessionConfig {
115    fn default() -> Self {
116        Self {
117            input_audio_format: InputAudioFormat::default(),
118            output_audio_format: OutputAudioFormat::default(),
119            instructions: None,
120            turn_detection: TurnDetection::new(TurnDetectionType::default()),
121            beta_fields: BetaFields::default(),
122            tools: Vec::new(),
123        }
124    }
125}
126
127/// `item.type` discriminator inside [`RealtimeConversationItem`].
128#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
129#[serde(rename_all = "snake_case")]
130pub enum ItemType {
131    Message,
132    FunctionCall,
133    FunctionCallOutput,
134}
135
136/// One content part within a conversation item message.
137#[derive(Debug, Clone, Serialize, Deserialize)]
138pub struct ItemContent {
139    /// Content type: `input_audio`, `input_text`, `text`.
140    #[serde(rename = "type")]
141    pub type_: String,
142    /// Text content (`input_text` / `text`).
143    #[serde(skip_serializing_if = "Option::is_none")]
144    pub text: Option<String>,
145    /// Base64 audio (`input_audio`).
146    #[serde(skip_serializing_if = "Option::is_none")]
147    pub audio: Option<String>,
148    /// Audio transcript.
149    #[serde(skip_serializing_if = "Option::is_none")]
150    pub transcript: Option<String>,
151}
152
153/// `RealtimeConversationItem`: a message, function call, or function-call
154/// output inserted via `conversation.item.create`.
155#[derive(Debug, Clone, Serialize, Deserialize)]
156pub struct RealtimeConversationItem {
157    /// Item id (client- or server-generated).
158    #[serde(skip_serializing_if = "Option::is_none")]
159    pub id: Option<String>,
160    /// Item kind.
161    #[serde(rename = "type")]
162    pub type_: ItemType,
163    /// Always `"realtime.item"`.
164    pub object: String,
165    /// `completed` / `incomplete`.
166    #[serde(skip_serializing_if = "Option::is_none")]
167    pub status: Option<String>,
168    /// Sender role (`message` only): `user` / `assistant` / `system`.
169    #[serde(skip_serializing_if = "Option::is_none")]
170    pub role: Option<String>,
171    /// Message content parts (`message` only).
172    #[serde(default, skip_serializing_if = "Vec::is_empty")]
173    pub content: Vec<ItemContent>,
174    /// Function name (`function_call` only).
175    #[serde(skip_serializing_if = "Option::is_none")]
176    pub name: Option<String>,
177    /// Function arguments (`function_call` only).
178    #[serde(skip_serializing_if = "Option::is_none")]
179    pub arguments: Option<String>,
180    /// Function output (`function_call_output` only).
181    #[serde(skip_serializing_if = "Option::is_none")]
182    pub output: Option<String>,
183}
184
185impl RealtimeConversationItem {
186    /// A user text message: `conversation.item.create` for textual input.
187    pub fn user_text(text: impl Into<String>) -> Self {
188        Self {
189            id: None,
190            type_: ItemType::Message,
191            object: "realtime.item".to_string(),
192            status: Some("completed".to_string()),
193            role: Some("user".to_string()),
194            content: vec![ItemContent {
195                type_: "input_text".to_string(),
196                text: Some(text.into()),
197                audio: None,
198                transcript: None,
199            }],
200            name: None,
201            arguments: None,
202            output: None,
203        }
204    }
205
206    /// A function-call output to feed back to the model.
207    pub fn function_output(call_name: impl Into<String>, output: impl Into<String>) -> Self {
208        Self {
209            id: None,
210            type_: ItemType::FunctionCallOutput,
211            object: "realtime.item".to_string(),
212            status: None,
213            role: None,
214            content: Vec::new(),
215            name: Some(call_name.into()),
216            arguments: None,
217            output: Some(output.into()),
218        }
219    }
220}
221
222/// Token-usage detail breakdown.
223#[derive(Debug, Clone, Default, Serialize, Deserialize)]
224pub struct TokenDetails {
225    #[serde(skip_serializing_if = "Option::is_none")]
226    pub text_tokens: Option<u64>,
227    #[serde(skip_serializing_if = "Option::is_none")]
228    pub audio_tokens: Option<u64>,
229    #[serde(skip_serializing_if = "Option::is_none")]
230    pub cached_tokens: Option<u64>,
231}
232
233/// `RealtimeResponse.usage`.
234#[derive(Debug, Clone, Default, Serialize, Deserialize)]
235pub struct RealtimeUsage {
236    #[serde(default)]
237    pub total_tokens: u64,
238    #[serde(default)]
239    pub input_tokens: u64,
240    #[serde(default)]
241    pub output_tokens: u64,
242    #[serde(skip_serializing_if = "Option::is_none")]
243    pub input_token_details: Option<TokenDetails>,
244    #[serde(skip_serializing_if = "Option::is_none")]
245    pub output_token_details: Option<TokenDetails>,
246}
247
248/// `RealtimeResponse`: emitted by `response.created` / `response.done`.
249#[derive(Debug, Clone, Serialize, Deserialize)]
250pub struct RealtimeResponse {
251    #[serde(skip_serializing_if = "Option::is_none")]
252    pub id: Option<String>,
253    /// Always `"realtime.response"`.
254    pub object: String,
255    #[serde(skip_serializing_if = "Option::is_none")]
256    pub status: Option<String>,
257    #[serde(skip_serializing_if = "Option::is_none")]
258    pub usage: Option<RealtimeUsage>,
259}
zai_rs/realtime/protocol.rs

zai_rs/realtime/
protocol.rs
