adk_realtime/
config.rs

1//! Configuration types for realtime sessions.
2
3use crate::audio::AudioEncoding;
4use serde::{Deserialize, Serialize};
5use serde_json::Value;
6use std::ops::{Deref, DerefMut};
7
8/// Controls how the realtime session handles user interruptions during agent
9/// audio output.
10///
11/// When set to [`Automatic`](InterruptionDetection::Automatic), the session
12/// uses voice activity detection to detect user speech onset and immediately
13/// cancels the current agent audio output, enabling natural conversational
14/// turn-taking.
15///
16/// When set to [`Manual`](InterruptionDetection::Manual) (the default), the
17/// session relies on explicit API calls (e.g. `response.cancel`) to signal
18/// that the user is interrupting. This gives the application full control
19/// over interruption behavior.
20#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
21#[serde(rename_all = "snake_case")]
22pub enum InterruptionDetection {
23    /// Rely on explicit API calls to signal interruptions.
24    ///
25    /// The application is responsible for detecting user speech and calling
26    /// the appropriate cancellation method on the session. No automatic
27    /// voice activity detection is performed for interruption purposes.
28    #[default]
29    Manual,
30    /// Detect user speech onset and cancel the current agent audio output.
31    ///
32    /// The session monitors incoming audio for voice activity. When user
33    /// speech is detected while the agent is producing audio, the agent's
34    /// audio output is automatically cancelled, allowing the user to
35    /// take the conversational turn.
36    Automatic,
37}
38
39/// Voice Activity Detection mode.
40#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)]
41#[serde(rename_all = "snake_case")]
42pub enum VadMode {
43    /// Server-side VAD (default for most providers).
44    #[default]
45    ServerVad,
46    /// Semantic VAD (OpenAI-specific).
47    SemanticVad,
48    /// No automatic VAD - manual turn management.
49    ///
50    /// **The application becomes responsible for turn detection.** On Gemini
51    /// Live this sends
52    /// `setup.realtimeInputConfig.automaticActivityDetection.disabled = true`,
53    /// after which the server performs no turn detection at all and the client
54    /// must send `activityStart` / `activityEnd` itself, via the
55    /// `ActivitySignaller` that a Gemini session offers only in this mode.
56    /// Selecting this and never signalling leaves a session that never takes
57    /// a turn.
58    None,
59}
60
61/// VAD configuration options.
62#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
63pub struct VadConfig {
64    /// VAD mode to use.
65    #[serde(rename = "type")]
66    pub mode: VadMode,
67    /// Silence duration (ms) before considering speech ended.
68    #[serde(skip_serializing_if = "Option::is_none")]
69    pub silence_duration_ms: Option<u32>,
70    /// Detection threshold (0.0 - 1.0).
71    #[serde(skip_serializing_if = "Option::is_none")]
72    pub threshold: Option<f32>,
73    /// Prefix padding (ms) to include before detected speech.
74    #[serde(skip_serializing_if = "Option::is_none")]
75    pub prefix_padding_ms: Option<u32>,
76    /// Whether to interrupt the model when user starts speaking.
77    #[serde(skip_serializing_if = "Option::is_none")]
78    pub interrupt_response: Option<bool>,
79    /// Eagerness of turn detection (OpenAI-specific).
80    #[serde(skip_serializing_if = "Option::is_none")]
81    pub eagerness: Option<String>,
82}
83
84impl Default for VadConfig {
85    fn default() -> Self {
86        Self {
87            mode: VadMode::ServerVad,
88            silence_duration_ms: Some(500),
89            threshold: None,
90            prefix_padding_ms: None,
91            interrupt_response: Some(true),
92            eagerness: None,
93        }
94    }
95}
96
97impl VadConfig {
98    /// Create a server VAD config with default settings.
99    pub fn server_vad() -> Self {
100        Self::default()
101    }
102
103    /// Create a semantic VAD config (OpenAI).
104    pub fn semantic_vad() -> Self {
105        Self { mode: VadMode::SemanticVad, ..Default::default() }
106    }
107
108    /// Create a config with VAD disabled.
109    pub fn disabled() -> Self {
110        Self { mode: VadMode::None, ..Default::default() }
111    }
112
113    /// Set silence duration threshold.
114    pub fn with_silence_duration(mut self, ms: u32) -> Self {
115        self.silence_duration_ms = Some(ms);
116        self
117    }
118
119    /// Set whether to interrupt on user speech.
120    pub fn with_interrupt(mut self, interrupt: bool) -> Self {
121        self.interrupt_response = Some(interrupt);
122        self
123    }
124}
125
126/// Tool/function definition for realtime sessions.
127#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
128pub struct ToolDefinition {
129    /// Tool name.
130    pub name: String,
131    /// Tool description.
132    #[serde(skip_serializing_if = "Option::is_none")]
133    pub description: Option<String>,
134    /// JSON Schema for parameters.
135    #[serde(skip_serializing_if = "Option::is_none")]
136    pub parameters: Option<Value>,
137}
138
139impl ToolDefinition {
140    /// Create a new tool definition.
141    pub fn new(name: impl Into<String>) -> Self {
142        Self { name: name.into(), description: None, parameters: None }
143    }
144
145    /// Set the tool description.
146    pub fn with_description(mut self, desc: impl Into<String>) -> Self {
147        self.description = Some(desc.into());
148        self
149    }
150
151    /// Set the parameters schema.
152    pub fn with_parameters(mut self, schema: Value) -> Self {
153        self.parameters = Some(schema);
154        self
155    }
156}
157
158/// Configuration for a realtime session.
159#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
160pub struct RealtimeConfig {
161    /// Model to use (provider-specific).
162    #[serde(skip_serializing_if = "Option::is_none")]
163    pub model: Option<String>,
164
165    /// System instruction for the agent.
166    #[serde(skip_serializing_if = "Option::is_none")]
167    pub instruction: Option<String>,
168
169    /// Voice to use for audio output.
170    #[serde(skip_serializing_if = "Option::is_none")]
171    pub voice: Option<String>,
172
173    /// Output modalities: ["text"], ["audio"], or ["text", "audio"].
174    #[serde(skip_serializing_if = "Option::is_none")]
175    pub modalities: Option<Vec<String>>,
176
177    /// Input audio format.
178    #[serde(skip_serializing_if = "Option::is_none")]
179    pub input_audio_format: Option<AudioEncoding>,
180
181    /// Output audio format.
182    #[serde(skip_serializing_if = "Option::is_none")]
183    pub output_audio_format: Option<AudioEncoding>,
184
185    /// VAD configuration.
186    #[serde(skip_serializing_if = "Option::is_none")]
187    pub turn_detection: Option<VadConfig>,
188
189    /// Available tools/functions.
190    #[serde(skip_serializing_if = "Option::is_none")]
191    pub tools: Option<Vec<ToolDefinition>>,
192
193    /// Tool selection mode: "auto", "none", "required".
194    #[serde(skip_serializing_if = "Option::is_none")]
195    pub tool_choice: Option<String>,
196
197    /// Whether to include input audio transcription.
198    #[serde(skip_serializing_if = "Option::is_none")]
199    pub input_audio_transcription: Option<TranscriptionConfig>,
200
201    /// Temperature for response generation.
202    #[serde(skip_serializing_if = "Option::is_none")]
203    pub temperature: Option<f32>,
204
205    /// Maximum output tokens.
206    #[serde(skip_serializing_if = "Option::is_none")]
207    pub max_response_output_tokens: Option<u32>,
208
209    /// Cached content resource name (e.g. `cachedContents/1234`).
210    #[serde(skip_serializing_if = "Option::is_none")]
211    pub cached_content: Option<String>,
212
213    /// Interruption detection mode for voice sessions.
214    ///
215    /// Controls whether the session automatically detects user speech to
216    /// cancel agent audio output, or relies on explicit API calls.
217    /// Defaults to [`Manual`](InterruptionDetection::Manual) when `None`.
218    #[serde(skip_serializing_if = "Option::is_none")]
219    pub interruption_detection: Option<InterruptionDetection>,
220
221    /// Enable emotion-aware ("affective") dialog, where the model adapts its
222    /// tone to the user's emotional state. Gemini Live native-audio models only
223    /// (requires the v1alpha endpoint); ignored by other providers.
224    #[serde(skip_serializing_if = "Option::is_none")]
225    pub affective_dialog: Option<bool>,
226
227    /// Provider-specific options.
228    #[serde(skip_serializing_if = "Option::is_none")]
229    pub extra: Option<Value>,
230}
231
232/// A delta payload for safely updating an active realtime session.
233///
234/// Wraps `RealtimeConfig` to prevent struct duplication. Since all fields are
235/// `Option<T>` and skip serialization if `None`, omitting fields preserves
236/// the server's active state.
237///
238/// **⚠️ WARNING:** You must construct a *fresh* configuration containing **only**
239/// the fields to modify. Wrapping your original startup config will resend
240/// immutable fields (like `model`), causing the provider to reject the update.
241///
242/// This is the idiomatic mechanism for dynamic Finite State Machine (FSM) state
243/// transitions, allowing seamless "persona shifts" or tool swaps without
244/// dropping the audio connection.
245///
246/// # Example
247///
248/// ```rust
249/// use adk_realtime::config::{SessionUpdateConfig, RealtimeConfig};
250///
251/// // Update *only* the instruction mid-session.
252/// let delta = SessionUpdateConfig(
253///     RealtimeConfig::default().with_instruction("You are now a travel agent.")
254/// );
255/// ```
256#[derive(Debug, Clone, Default, Serialize, Deserialize)]
257#[serde(transparent)]
258pub struct SessionUpdateConfig(pub RealtimeConfig);
259
260impl Deref for SessionUpdateConfig {
261    type Target = RealtimeConfig;
262
263    fn deref(&self) -> &Self::Target {
264        &self.0
265    }
266}
267
268impl DerefMut for SessionUpdateConfig {
269    fn deref_mut(&mut self) -> &mut Self::Target {
270        &mut self.0
271    }
272}
273
274impl From<RealtimeConfig> for SessionUpdateConfig {
275    fn from(config: RealtimeConfig) -> Self {
276        Self(config)
277    }
278}
279
280/// Transcription configuration.
281#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
282pub struct TranscriptionConfig {
283    /// Transcription model to use.
284    pub model: String,
285}
286
287impl TranscriptionConfig {
288    /// Use whisper-1 for transcription.
289    pub fn whisper() -> Self {
290        Self { model: "whisper-1".to_string() }
291    }
292}
293
294impl RealtimeConfig {
295    /// Create a new empty configuration.
296    pub fn new() -> Self {
297        Self::default()
298    }
299
300    /// Create a builder for RealtimeConfig.
301    pub fn builder() -> RealtimeConfigBuilder {
302        RealtimeConfigBuilder::new()
303    }
304
305    /// Set the model.
306    pub fn with_model(mut self, model: impl Into<String>) -> Self {
307        self.model = Some(model.into());
308        self
309    }
310
311    /// Set the system instruction.
312    pub fn with_instruction(mut self, instruction: impl Into<String>) -> Self {
313        self.instruction = Some(instruction.into());
314        self
315    }
316
317    /// Set the voice.
318    pub fn with_voice(mut self, voice: impl Into<String>) -> Self {
319        self.voice = Some(voice.into());
320        self
321    }
322
323    /// Set output modalities.
324    pub fn with_modalities(mut self, modalities: Vec<String>) -> Self {
325        self.modalities = Some(modalities);
326        self
327    }
328
329    /// Enable text and audio output.
330    pub fn with_text_and_audio(mut self) -> Self {
331        self.modalities = Some(vec!["text".to_string(), "audio".to_string()]);
332        self
333    }
334
335    /// Enable audio-only output.
336    pub fn with_audio_only(mut self) -> Self {
337        self.modalities = Some(vec!["audio".to_string()]);
338        self
339    }
340
341    /// Set VAD configuration.
342    pub fn with_vad(mut self, vad: VadConfig) -> Self {
343        self.turn_detection = Some(vad);
344        self
345    }
346
347    /// Enable server-side VAD with default settings.
348    pub fn with_server_vad(self) -> Self {
349        self.with_vad(VadConfig::server_vad())
350    }
351
352    /// Disable automatic VAD, taking ownership of turn detection.
353    ///
354    /// See [`VadMode::None`] — on Gemini Live this is also what unlocks client
355    /// `activityStart` / `activityEnd` signalling, which the protocol permits
356    /// only while server-side detection is disabled.
357    pub fn without_vad(mut self) -> Self {
358        self.turn_detection = Some(VadConfig::disabled());
359        self
360    }
361
362    /// Add a tool definition.
363    pub fn with_tool(mut self, tool: ToolDefinition) -> Self {
364        self.tools.get_or_insert_with(Vec::new).push(tool);
365        self
366    }
367
368    /// Set multiple tools.
369    pub fn with_tools(mut self, tools: Vec<ToolDefinition>) -> Self {
370        self.tools = Some(tools);
371        self
372    }
373
374    /// Enable input audio transcription.
375    pub fn with_transcription(mut self) -> Self {
376        self.input_audio_transcription = Some(TranscriptionConfig::whisper());
377        self
378    }
379
380    /// Set temperature.
381    pub fn with_temperature(mut self, temp: f32) -> Self {
382        self.temperature = Some(temp);
383        self
384    }
385
386    /// Enable emotion-aware ("affective") dialog. Only honored by Gemini Live
387    /// native-audio models (v1alpha); a no-op for other providers/models.
388    pub fn with_affective_dialog(mut self, enabled: bool) -> Self {
389        self.affective_dialog = Some(enabled);
390        self
391    }
392
393    /// Set cached content resource.
394    pub fn with_cached_content(mut self, content: impl Into<String>) -> Self {
395        self.cached_content = Some(content.into());
396        self
397    }
398
399    /// Set the interruption detection mode.
400    ///
401    /// See [`InterruptionDetection`] for details on each variant.
402    pub fn with_interruption_detection(mut self, mode: InterruptionDetection) -> Self {
403        self.interruption_detection = Some(mode);
404        self
405    }
406
407    /// Enable automatic interruption detection.
408    ///
409    /// The session will detect user speech onset and cancel the current
410    /// agent audio output automatically.
411    pub fn with_automatic_interruption(self) -> Self {
412        self.with_interruption_detection(InterruptionDetection::Automatic)
413    }
414}
415
416/// Builder for RealtimeConfig.
417#[derive(Debug, Clone, Default)]
418pub struct RealtimeConfigBuilder {
419    config: RealtimeConfig,
420}
421
422impl RealtimeConfigBuilder {
423    /// Create a new builder.
424    pub fn new() -> Self {
425        Self::default()
426    }
427
428    /// Set the model.
429    pub fn model(mut self, model: impl Into<String>) -> Self {
430        self.config.model = Some(model.into());
431        self
432    }
433
434    /// Set the system instruction.
435    pub fn instruction(mut self, instruction: impl Into<String>) -> Self {
436        self.config.instruction = Some(instruction.into());
437        self
438    }
439
440    /// Set the voice.
441    pub fn voice(mut self, voice: impl Into<String>) -> Self {
442        self.config.voice = Some(voice.into());
443        self
444    }
445
446    /// Enable VAD.
447    pub fn vad_enabled(mut self, enabled: bool) -> Self {
448        if enabled {
449            self.config.turn_detection = Some(VadConfig::server_vad());
450        } else {
451            self.config.turn_detection = Some(VadConfig::disabled());
452        }
453        self
454    }
455
456    /// Set VAD configuration.
457    pub fn vad(mut self, vad: VadConfig) -> Self {
458        self.config.turn_detection = Some(vad);
459        self
460    }
461
462    /// Add a tool.
463    pub fn tool(mut self, tool: ToolDefinition) -> Self {
464        self.config.tools.get_or_insert_with(Vec::new).push(tool);
465        self
466    }
467
468    /// Set temperature.
469    pub fn temperature(mut self, temp: f32) -> Self {
470        self.config.temperature = Some(temp);
471        self
472    }
473
474    /// Set cached content resource.
475    pub fn cached_content(mut self, content: impl Into<String>) -> Self {
476        self.config.cached_content = Some(content.into());
477        self
478    }
479
480    /// Set the interruption detection mode.
481    pub fn interruption_detection(mut self, mode: InterruptionDetection) -> Self {
482        self.config.interruption_detection = Some(mode);
483        self
484    }
485
486    /// Build the configuration.
487    pub fn build(self) -> RealtimeConfig {
488        self.config
489    }
490}
adk_realtime/config.rs

adk_realtime/
config.rs
