Skip to main content

adk_realtime/
agent.rs

1//! RealtimeAgent - an Agent implementation for real-time voice interactions.
2//!
3//! This module provides `RealtimeAgent`, which implements the `adk_core::Agent` trait
4//! and provides the same callback/tool/instruction features as `LlmAgent`, but uses
5//! real-time bidirectional audio streaming instead of text-based LLM calls.
6//!
7//! # Architecture
8//!
9//! ```text
10//!                     ┌─────────────────────────────────────────┐
11//!                     │              Agent Trait                │
12//!                     │  (name, description, run, sub_agents)   │
13//!                     └────────────────┬────────────────────────┘
14//!                                      │
15//!              ┌───────────────────────┼───────────────────────┐
16//!              │                       │                       │
17//!     ┌────────▼────────┐    ┌─────────▼─────────┐   ┌─────────▼─────────┐
18//!     │    LlmAgent     │    │  RealtimeAgent    │   │  SequentialAgent  │
19//!     │  (text-based)   │    │  (voice-based)    │   │   (workflow)      │
20//!     └─────────────────┘    └───────────────────┘   └───────────────────┘
21//! ```
22//!
23//! # Shared Features with LlmAgent
24//!
25//! - **Tools**: Function tools that can be called during conversation
26//! - **Callbacks**: before_agent, after_agent, before_tool, after_tool
27//! - **Instructions**: Static or dynamic instruction providers
28//! - **Sub-agents**: Agent handoff/transfer support
29//! - **Context**: Full access to InvocationContext (session, memory, artifacts)
30//!
31//! # Example
32//!
33//! ```rust,ignore
34//! use adk_realtime::RealtimeAgent;
35//! use adk_realtime::openai::OpenAIRealtimeModel;
36//!
37//! let model = OpenAIRealtimeModel::new(api_key, "gpt-realtime-2.1");
38//!
39//! let agent = RealtimeAgent::builder("voice_assistant")
40//!     .model(Box::new(model))
41//!     .instruction("You are a helpful voice assistant.")
42//!     .voice("alloy")
43//!     .tool(Arc::new(weather_tool))
44//!     .before_agent_callback(|ctx| async move {
45//!         println!("Starting voice session for user: {}", ctx.user_id());
46//!         Ok(None)
47//!     })
48//!     .build()?;
49//!
50//! // Run through standard ADK runner
51//! let runner = Runner::new(agent);
52//! runner.run(session, user_content).await?;
53//! ```
54
55use crate::config::{RealtimeConfig, ToolDefinition, VadConfig, VadMode};
56use crate::events::{ServerEvent, ToolResponse};
57use adk_core::{
58    AdkError, AfterAgentCallback, AfterToolCallback, Agent, AgentInteractionMode,
59    BeforeAgentCallback, BeforeToolCallback, CallbackContext, Content, Event, EventActions,
60    EventStream, GlobalInstructionProvider, InstructionProvider, InvocationContext, MemoryEntry,
61    Part, ReadonlyContext, Result, Tool, ToolCallbackContext, ToolContext, Toolset,
62};
63use async_stream::stream;
64use async_trait::async_trait;
65
66use std::sync::{Arc, Mutex};
67
68const MAX_BUFFERED_PLAYBACK_AUDIO_BYTES: usize = 16 * 1024 * 1024;
69
70/// Shared realtime model type (thread-safe for async usage).
71pub type BoxedRealtimeModel = Arc<dyn crate::model::RealtimeModel>;
72
73/// A real-time voice agent that implements the ADK Agent trait.
74///
75/// `RealtimeAgent` provides bidirectional audio streaming while maintaining
76/// compatibility with the standard ADK agent ecosystem. It supports the same
77/// callbacks, tools, and instruction patterns as `LlmAgent`.
78pub struct RealtimeAgent {
79    name: String,
80    description: String,
81    model: BoxedRealtimeModel,
82
83    // Instructions (same as LlmAgent)
84    instruction: Option<String>,
85    instruction_provider: Option<Arc<InstructionProvider>>,
86    global_instruction: Option<String>,
87    global_instruction_provider: Option<Arc<GlobalInstructionProvider>>,
88
89    // Voice-specific settings
90    voice: Option<String>,
91    vad_config: Option<VadConfig>,
92    modalities: Vec<String>,
93
94    // Tools (same as LlmAgent)
95    tools: Vec<Arc<dyn Tool>>,
96    toolsets: Vec<Arc<dyn Toolset>>,
97    sub_agents: Vec<Arc<dyn Agent>>,
98
99    // Callbacks (same as LlmAgent)
100    before_callbacks: Arc<Vec<BeforeAgentCallback>>,
101    after_callbacks: Arc<Vec<AfterAgentCallback>>,
102    before_tool_callbacks: Arc<Vec<BeforeToolCallback>>,
103    after_tool_callbacks: Arc<Vec<AfterToolCallback>>,
104
105    // Realtime-specific callbacks
106    on_audio: Option<AudioCallback>,
107    on_transcript: Option<TranscriptCallback>,
108    on_speech_started: Option<SpeechCallback>,
109    on_speech_stopped: Option<SpeechCallback>,
110
111    // Video avatar configuration
112    #[cfg(feature = "video-avatar")]
113    avatar_config: Option<crate::avatar::AvatarConfig>,
114
115    // Video avatar provider instance
116    #[cfg(feature = "video-avatar")]
117    avatar_provider: Option<std::sync::Arc<dyn crate::avatar::AvatarProvider>>,
118}
119
120/// Callback for audio output events (receives raw PCM bytes).
121pub type AudioCallback = Arc<
122    dyn Fn(&[u8], &str) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>>
123        + Send
124        + Sync,
125>;
126
127/// Callback for transcript events.
128pub type TranscriptCallback = Arc<
129    dyn Fn(&str, &str) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>>
130        + Send
131        + Sync,
132>;
133
134/// Callback for speech detection events.
135pub type SpeechCallback = Arc<
136    dyn Fn(u64) -> std::pin::Pin<Box<dyn std::future::Future<Output = ()> + Send>> + Send + Sync,
137>;
138
139impl std::fmt::Debug for RealtimeAgent {
140    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
141        f.debug_struct("RealtimeAgent")
142            .field("name", &self.name)
143            .field("description", &self.description)
144            .field("model", &self.model.model_id())
145            .field("voice", &self.voice)
146            .field("tools_count", &self.tools.len())
147            .field("toolsets_count", &self.toolsets.len())
148            .field("sub_agents_count", &self.sub_agents.len())
149            .finish()
150    }
151}
152
153/// Builder for RealtimeAgent.
154pub struct RealtimeAgentBuilder {
155    name: String,
156    description: Option<String>,
157    model: Option<BoxedRealtimeModel>,
158    instruction: Option<String>,
159    instruction_provider: Option<Arc<InstructionProvider>>,
160    global_instruction: Option<String>,
161    global_instruction_provider: Option<Arc<GlobalInstructionProvider>>,
162    voice: Option<String>,
163    vad_config: Option<VadConfig>,
164    modalities: Vec<String>,
165    tools: Vec<Arc<dyn Tool>>,
166    toolsets: Vec<Arc<dyn Toolset>>,
167    sub_agents: Vec<Arc<dyn Agent>>,
168    before_callbacks: Vec<BeforeAgentCallback>,
169    after_callbacks: Vec<AfterAgentCallback>,
170    before_tool_callbacks: Vec<BeforeToolCallback>,
171    after_tool_callbacks: Vec<AfterToolCallback>,
172    on_audio: Option<AudioCallback>,
173    on_transcript: Option<TranscriptCallback>,
174    on_speech_started: Option<SpeechCallback>,
175    on_speech_stopped: Option<SpeechCallback>,
176
177    #[cfg(feature = "video-avatar")]
178    avatar_config: Option<crate::avatar::AvatarConfig>,
179
180    #[cfg(feature = "video-avatar")]
181    avatar_provider: Option<std::sync::Arc<dyn crate::avatar::AvatarProvider>>,
182}
183
184impl RealtimeAgentBuilder {
185    /// Create a new builder with the given agent name.
186    pub fn new(name: impl Into<String>) -> Self {
187        Self {
188            name: name.into(),
189            description: None,
190            model: None,
191            instruction: None,
192            instruction_provider: None,
193            global_instruction: None,
194            global_instruction_provider: None,
195            voice: None,
196            vad_config: None,
197            modalities: vec!["text".to_string(), "audio".to_string()],
198            tools: Vec::new(),
199            toolsets: Vec::new(),
200            sub_agents: Vec::new(),
201            before_callbacks: Vec::new(),
202            after_callbacks: Vec::new(),
203            before_tool_callbacks: Vec::new(),
204            after_tool_callbacks: Vec::new(),
205            on_audio: None,
206            on_transcript: None,
207            on_speech_started: None,
208            on_speech_stopped: None,
209            #[cfg(feature = "video-avatar")]
210            avatar_config: None,
211            #[cfg(feature = "video-avatar")]
212            avatar_provider: None,
213        }
214    }
215
216    /// Set the agent description.
217    pub fn description(mut self, desc: impl Into<String>) -> Self {
218        self.description = Some(desc.into());
219        self
220    }
221
222    /// Set the realtime model.
223    pub fn model(mut self, model: BoxedRealtimeModel) -> Self {
224        self.model = Some(model);
225        self
226    }
227
228    /// Set a static instruction.
229    pub fn instruction(mut self, instruction: impl Into<String>) -> Self {
230        self.instruction = Some(instruction.into());
231        self
232    }
233
234    /// Set a dynamic instruction provider.
235    pub fn instruction_provider(mut self, provider: InstructionProvider) -> Self {
236        self.instruction_provider = Some(Arc::new(provider));
237        self
238    }
239
240    /// Set a static global instruction.
241    pub fn global_instruction(mut self, instruction: impl Into<String>) -> Self {
242        self.global_instruction = Some(instruction.into());
243        self
244    }
245
246    /// Set a dynamic global instruction provider.
247    pub fn global_instruction_provider(mut self, provider: GlobalInstructionProvider) -> Self {
248        self.global_instruction_provider = Some(Arc::new(provider));
249        self
250    }
251
252    /// Set the voice for audio output.
253    pub fn voice(mut self, voice: impl Into<String>) -> Self {
254        self.voice = Some(voice.into());
255        self
256    }
257
258    /// Set voice activity detection configuration.
259    pub fn vad(mut self, config: VadConfig) -> Self {
260        self.vad_config = Some(config);
261        self
262    }
263
264    /// Enable server-side VAD with default settings.
265    pub fn server_vad(mut self) -> Self {
266        self.vad_config = Some(VadConfig {
267            mode: VadMode::ServerVad,
268            threshold: Some(0.5),
269            prefix_padding_ms: Some(300),
270            silence_duration_ms: Some(500),
271            interrupt_response: Some(true),
272            eagerness: None,
273        });
274        self
275    }
276
277    /// Set output modalities (e.g., ["text", "audio"]).
278    pub fn modalities(mut self, modalities: Vec<String>) -> Self {
279        self.modalities = modalities;
280        self
281    }
282
283    /// Add a tool.
284    pub fn tool(mut self, tool: Arc<dyn Tool>) -> Self {
285        self.tools.push(tool);
286        self
287    }
288
289    /// Register a dynamic toolset for per-invocation tool resolution.
290    ///
291    /// Toolsets are resolved at the start of each `run()` call using the
292    /// invocation's `ReadonlyContext`. This enables context-dependent tools
293    /// like per-user browser sessions from a pool.
294    pub fn toolset(mut self, toolset: Arc<dyn Toolset>) -> Self {
295        self.toolsets.push(toolset);
296        self
297    }
298
299    /// Add a sub-agent for handoffs.
300    pub fn sub_agent(mut self, agent: Arc<dyn Agent>) -> Self {
301        self.sub_agents.push(agent);
302        self
303    }
304
305    /// Add a before-agent callback.
306    pub fn before_agent_callback(mut self, callback: BeforeAgentCallback) -> Self {
307        self.before_callbacks.push(callback);
308        self
309    }
310
311    /// Add an after-agent callback.
312    pub fn after_agent_callback(mut self, callback: AfterAgentCallback) -> Self {
313        self.after_callbacks.push(callback);
314        self
315    }
316
317    /// Add a before-tool callback.
318    pub fn before_tool_callback(mut self, callback: BeforeToolCallback) -> Self {
319        self.before_tool_callbacks.push(callback);
320        self
321    }
322
323    /// Add an after-tool callback.
324    pub fn after_tool_callback(mut self, callback: AfterToolCallback) -> Self {
325        self.after_tool_callbacks.push(callback);
326        self
327    }
328
329    /// Set callback for audio output events.
330    pub fn on_audio(mut self, callback: AudioCallback) -> Self {
331        self.on_audio = Some(callback);
332        self
333    }
334
335    /// Set callback for transcript events.
336    pub fn on_transcript(mut self, callback: TranscriptCallback) -> Self {
337        self.on_transcript = Some(callback);
338        self
339    }
340
341    /// Set callback for speech started events.
342    pub fn on_speech_started(mut self, callback: SpeechCallback) -> Self {
343        self.on_speech_started = Some(callback);
344        self
345    }
346
347    /// Set callback for speech stopped events.
348    pub fn on_speech_stopped(mut self, callback: SpeechCallback) -> Self {
349        self.on_speech_stopped = Some(callback);
350        self
351    }
352
353    /// Set the video avatar configuration for this agent.
354    ///
355    /// When set, the avatar configuration is included in the session setup
356    /// payload sent to the realtime provider. If the provider does not support
357    /// video avatars, a warning is logged and the session proceeds audio-only.
358    ///
359    /// Requires the `video-avatar` feature flag.
360    #[cfg(feature = "video-avatar")]
361    pub fn avatar(mut self, config: crate::avatar::AvatarConfig) -> Self {
362        self.avatar_config = Some(config);
363        self
364    }
365
366    /// Set the video avatar provider for this agent.
367    ///
368    /// When both an `AvatarConfig` (with a provider kind) and an `AvatarProvider`
369    /// instance are set, the runner routes audio through the avatar provider
370    /// for lip-sync rendering instead of sending raw audio to the client.
371    ///
372    /// Requires the `video-avatar` feature flag.
373    ///
374    /// # Example
375    ///
376    /// ```rust,ignore
377    /// use std::sync::Arc;
378    /// use adk_realtime::avatar::heygen::{HeyGenConfig, HeyGenProvider};
379    ///
380    /// let provider = Arc::new(HeyGenProvider::new(HeyGenConfig::new("key")));
381    /// let agent = RealtimeAgentBuilder::new("assistant")
382    ///     .avatar(avatar_config)
383    ///     .avatar_provider(provider)
384    ///     .build()?;
385    /// ```
386    #[cfg(feature = "video-avatar")]
387    pub fn avatar_provider(
388        mut self,
389        provider: std::sync::Arc<dyn crate::avatar::AvatarProvider>,
390    ) -> Self {
391        self.avatar_provider = Some(provider);
392        self
393    }
394
395    /// Build the RealtimeAgent.
396    pub fn build(self) -> Result<RealtimeAgent> {
397        let model =
398            self.model.ok_or_else(|| AdkError::agent("RealtimeModel is required".to_string()))?;
399
400        Ok(RealtimeAgent {
401            name: self.name,
402            description: self.description.unwrap_or_default(),
403            model,
404            instruction: self.instruction,
405            instruction_provider: self.instruction_provider,
406            global_instruction: self.global_instruction,
407            global_instruction_provider: self.global_instruction_provider,
408            voice: self.voice,
409            vad_config: self.vad_config,
410            modalities: self.modalities,
411            tools: self.tools,
412            toolsets: self.toolsets,
413            sub_agents: self.sub_agents,
414            before_callbacks: Arc::new(self.before_callbacks),
415            after_callbacks: Arc::new(self.after_callbacks),
416            before_tool_callbacks: Arc::new(self.before_tool_callbacks),
417            after_tool_callbacks: Arc::new(self.after_tool_callbacks),
418            on_audio: self.on_audio,
419            on_transcript: self.on_transcript,
420            on_speech_started: self.on_speech_started,
421            on_speech_stopped: self.on_speech_stopped,
422            #[cfg(feature = "video-avatar")]
423            avatar_config: self.avatar_config,
424            #[cfg(feature = "video-avatar")]
425            avatar_provider: self.avatar_provider,
426        })
427    }
428}
429
430impl RealtimeAgent {
431    /// Create a new builder.
432    pub fn builder(name: impl Into<String>) -> RealtimeAgentBuilder {
433        RealtimeAgentBuilder::new(name)
434    }
435
436    /// Get the static instruction, if set.
437    pub fn instruction(&self) -> Option<&String> {
438        self.instruction.as_ref()
439    }
440
441    /// Get the voice setting, if set.
442    pub fn voice(&self) -> Option<&String> {
443        self.voice.as_ref()
444    }
445
446    /// Get the VAD configuration, if set.
447    pub fn vad_config(&self) -> Option<&VadConfig> {
448        self.vad_config.as_ref()
449    }
450
451    /// Get the list of tools.
452    pub fn tools(&self) -> &[Arc<dyn Tool>] {
453        &self.tools
454    }
455
456    /// Get the avatar configuration, if set.
457    ///
458    /// Requires the `video-avatar` feature flag.
459    #[cfg(feature = "video-avatar")]
460    pub fn avatar_config(&self) -> Option<&crate::avatar::AvatarConfig> {
461        self.avatar_config.as_ref()
462    }
463
464    /// Get the avatar provider, if set.
465    ///
466    /// Requires the `video-avatar` feature flag.
467    #[cfg(feature = "video-avatar")]
468    pub fn avatar_provider(&self) -> Option<&std::sync::Arc<dyn crate::avatar::AvatarProvider>> {
469        self.avatar_provider.as_ref()
470    }
471
472    /// Build the realtime configuration from agent settings.
473    async fn build_config(
474        &self,
475        ctx: &Arc<dyn InvocationContext>,
476        resolved_tools: &[Arc<dyn Tool>],
477    ) -> Result<RealtimeConfig> {
478        let mut config = RealtimeConfig::default();
479
480        // Build instruction from providers or static value
481        if let Some(provider) = &self.global_instruction_provider {
482            let global_inst = provider(ctx.clone() as Arc<dyn ReadonlyContext>).await?;
483            if !global_inst.is_empty() {
484                config.instruction = Some(global_inst);
485            }
486        } else if let Some(ref template) = self.global_instruction {
487            let processed = adk_core::inject_session_state(ctx.as_ref(), template).await?;
488            config.instruction = Some(processed);
489        }
490
491        // Add agent-specific instruction
492        if let Some(provider) = &self.instruction_provider {
493            let inst = provider(ctx.clone() as Arc<dyn ReadonlyContext>).await?;
494            if !inst.is_empty() {
495                if let Some(existing) = &mut config.instruction {
496                    existing.push_str("\n\n");
497                    existing.push_str(&inst);
498                } else {
499                    config.instruction = Some(inst);
500                }
501            }
502        } else if let Some(ref template) = self.instruction {
503            let processed = adk_core::inject_session_state(ctx.as_ref(), template).await?;
504            if let Some(existing) = &mut config.instruction {
505                existing.push_str("\n\n");
506                existing.push_str(&processed);
507            } else {
508                config.instruction = Some(processed);
509            }
510        }
511
512        // Voice settings
513        config.voice = self.voice.clone();
514        config.turn_detection = self.vad_config.clone();
515        config.modalities = Some(self.modalities.clone());
516
517        // Convert ADK tools to realtime tool definitions
518        let tool_defs: Vec<ToolDefinition> = resolved_tools
519            .iter()
520            .map(|t| ToolDefinition {
521                name: t.name().to_string(),
522                description: Some(t.enhanced_description().to_string()),
523                parameters: t.parameters_schema(),
524            })
525            .collect();
526
527        if !tool_defs.is_empty() {
528            config.tools = Some(tool_defs);
529        }
530
531        // Add transfer_to_agent tool if sub-agents exist
532        if !self.sub_agents.is_empty() {
533            let mut tools = config.tools.unwrap_or_default();
534            tools.push(ToolDefinition {
535                name: "transfer_to_agent".to_string(),
536                description: Some("Transfer execution to another agent.".to_string()),
537                parameters: Some(serde_json::json!({
538                    "type": "object",
539                    "properties": {
540                        "agent_name": {
541                            "type": "string",
542                            "description": "The name of the agent to transfer to."
543                        }
544                    },
545                    "required": ["agent_name"]
546                })),
547            });
548            config.tools = Some(tools);
549        }
550
551        // Include avatar configuration in session setup if present.
552        // Currently no realtime provider supports video avatars natively,
553        // so we log a warning and proceed audio-only. The config is still
554        // placed in `extra` so future provider implementations can read it.
555        #[cfg(feature = "video-avatar")]
556        if let Some(ref avatar) = self.avatar_config {
557            tracing::warn!(
558                agent = %self.name,
559                source_url = %avatar.source_url,
560                "video avatar configured but the current realtime provider does not support video avatars; proceeding audio-only"
561            );
562            let avatar_json = serde_json::to_value(avatar).unwrap_or_else(|e| {
563                tracing::warn!("failed to serialize avatar config: {e}");
564                serde_json::Value::Null
565            });
566            let extra = config.extra.get_or_insert_with(|| serde_json::json!({}));
567            if let Some(obj) = extra.as_object_mut() {
568                obj.insert("avatarConfig".to_string(), avatar_json);
569            }
570        }
571
572        Ok(config)
573    }
574
575    /// Execute a tool call.
576    #[allow(dead_code)]
577    async fn execute_tool(
578        &self,
579        ctx: &Arc<dyn InvocationContext>,
580        call_id: &str,
581        name: &str,
582        arguments: &str,
583    ) -> (serde_json::Value, EventActions) {
584        // Find the tool
585        let tool = self.tools.iter().find(|t| t.name() == name);
586
587        if let Some(tool) = tool {
588            let args: serde_json::Value =
589                serde_json::from_str(arguments).unwrap_or(serde_json::json!({}));
590
591            // Create tool context
592            let tool_ctx: Arc<dyn ToolContext> =
593                Arc::new(RealtimeToolContext::new(ctx.clone(), call_id.to_string()));
594
595            // Execute before_tool callbacks
596            let tool_cb_ctx =
597                Arc::new(ToolCallbackContext::new(ctx.clone(), name.to_string(), args.clone()));
598            for callback in self.before_tool_callbacks.as_ref() {
599                if let Err(e) = callback(tool_cb_ctx.clone() as Arc<dyn CallbackContext>).await {
600                    return (
601                        serde_json::json!({ "error": e.to_string() }),
602                        EventActions::default(),
603                    );
604                }
605            }
606
607            // Execute the tool
608            let result = match tool.execute(tool_ctx.clone(), args.clone()).await {
609                Ok(result) => result,
610                Err(e) => serde_json::json!({ "error": e.to_string() }),
611            };
612
613            let actions = tool_ctx.actions();
614
615            // Execute after_tool callbacks
616            let tool_cb_ctx =
617                Arc::new(ToolCallbackContext::new(ctx.clone(), name.to_string(), args.clone()));
618            for callback in self.after_tool_callbacks.as_ref() {
619                if let Err(e) = callback(tool_cb_ctx.clone() as Arc<dyn CallbackContext>).await {
620                    return (serde_json::json!({ "error": e.to_string() }), actions);
621                }
622            }
623
624            (result, actions)
625        } else {
626            (
627                serde_json::json!({ "error": format!("Tool {} not found", name) }),
628                EventActions::default(),
629            )
630        }
631    }
632}
633
634#[async_trait]
635impl Agent for RealtimeAgent {
636    fn name(&self) -> &str {
637        &self.name
638    }
639
640    fn description(&self) -> &str {
641        &self.description
642    }
643
644    fn sub_agents(&self) -> &[Arc<dyn Agent>] {
645        &self.sub_agents
646    }
647
648    fn interaction_mode(&self) -> AgentInteractionMode {
649        AgentInteractionMode::Realtime
650    }
651
652    async fn run(&self, ctx: Arc<dyn InvocationContext>) -> Result<EventStream> {
653        let agent_name = self.name.clone();
654        let invocation_id = ctx.invocation_id().to_string();
655        let model = self.model.clone();
656        let _sub_agents = self.sub_agents.clone();
657
658        // Clone callback refs
659        let before_callbacks = self.before_callbacks.clone();
660        let after_callbacks = self.after_callbacks.clone();
661        let before_tool_callbacks = self.before_tool_callbacks.clone();
662        let after_tool_callbacks = self.after_tool_callbacks.clone();
663        let tools = self.tools.clone();
664        let toolsets = self.toolsets.clone();
665
666        // Clone realtime callbacks
667        let on_audio = self.on_audio.clone();
668        let on_transcript = self.on_transcript.clone();
669        let on_speech_started = self.on_speech_started.clone();
670        let on_speech_stopped = self.on_speech_stopped.clone();
671
672        // Clone avatar provider for the stream closure
673        #[cfg(feature = "video-avatar")]
674        let avatar_provider = self.avatar_provider.clone();
675        #[cfg(feature = "video-avatar")]
676        let avatar_config_for_session = self.avatar_config.clone();
677
678        // ===== RESOLVE TOOLSETS =====
679        let mut resolved_tools: Vec<Arc<dyn Tool>> = tools.clone();
680        let static_tool_names: std::collections::HashSet<String> =
681            tools.iter().map(|t| t.name().to_string()).collect();
682        let mut toolset_source: std::collections::HashMap<String, String> =
683            std::collections::HashMap::new();
684
685        for toolset in &toolsets {
686            let toolset_tools = toolset.tools(ctx.clone() as Arc<dyn ReadonlyContext>).await?;
687            for tool in &toolset_tools {
688                let name = tool.name().to_string();
689                if static_tool_names.contains(&name) {
690                    return Err(AdkError::agent(format!(
691                        "Duplicate tool name '{}': conflict between static tool and toolset '{}'",
692                        name,
693                        toolset.name()
694                    )));
695                }
696                if let Some(other_toolset_name) = toolset_source.get(&name) {
697                    return Err(AdkError::agent(format!(
698                        "Duplicate tool name '{}': conflict between toolset '{}' and toolset '{}'",
699                        name,
700                        other_toolset_name,
701                        toolset.name()
702                    )));
703                }
704                toolset_source.insert(name, toolset.name().to_string());
705                resolved_tools.push(tool.clone());
706            }
707        }
708
709        // Build config with resolved tools
710        let config = self.build_config(&ctx, &resolved_tools).await?;
711
712        let s = stream! {
713            // ===== BEFORE AGENT CALLBACKS =====
714            for callback in before_callbacks.as_ref() {
715                match callback(ctx.clone() as Arc<dyn CallbackContext>).await {
716                    Ok(Some(content)) => {
717                        let mut early_event = Event::new(&invocation_id);
718                        early_event.author = agent_name.clone();
719                        early_event.llm_response.content = Some(content);
720                        yield Ok(early_event);
721                        return;
722                    }
723                    Ok(None) => continue,
724                    Err(e) => {
725                        yield Err(e);
726                        return;
727                    }
728                }
729            }
730
731            // ===== CONNECT TO REALTIME SESSION =====
732            let session = match model.connect(config).await {
733                Ok(s) => s,
734                Err(e) => {
735                    yield Err(AdkError::model(format!("Failed to connect: {}", e)));
736                    return;
737                }
738            };
739
740            // Yield session started event
741            let mut start_event = Event::new(&invocation_id);
742            start_event.author = agent_name.clone();
743            start_event.llm_response.content = Some(Content {
744                role: "system".to_string(),
745                parts: vec![Part::Text {
746                    text: format!("Realtime session started: {}", session.session_id()),
747                }],
748            });
749            yield Ok(start_event);
750
751            // ===== START AVATAR SESSION (if configured) =====
752            #[cfg(feature = "video-avatar")]
753            let avatar_session_id: Option<String> = {
754                if let (Some(provider), Some(config)) = (&avatar_provider, &avatar_config_for_session) {
755                    match provider.start_session(config).await {
756                        Ok(session_info) => {
757                            tracing::info!(
758                                provider = %session_info.provider,
759                                session_id = %session_info.session_id,
760                                "avatar session started"
761                            );
762                            // Emit avatar session info as an event for the client
763                            let mut avatar_event = Event::new(&invocation_id);
764                            avatar_event.author = agent_name.clone();
765                            avatar_event.llm_response.content = Some(Content {
766                                role: "system".to_string(),
767                                parts: vec![Part::Text {
768                                    text: serde_json::to_string(&session_info).unwrap_or_default(),
769                                }],
770                            });
771                            yield Ok(avatar_event);
772                            Some(session_info.session_id)
773                        }
774                        Err(e) => {
775                            // Graceful degradation: log warning, continue audio-only
776                            tracing::warn!(
777                                error = %e,
778                                "avatar session creation failed, falling back to audio-only"
779                            );
780                            None
781                        }
782                    }
783                } else {
784                    None
785                }
786            };
787            #[cfg(not(feature = "video-avatar"))]
788            let _avatar_session_id: Option<String> = None;
789
790            // Spawn keep-alive task for avatar session
791            #[cfg(feature = "video-avatar")]
792            let _avatar_keep_alive_handle: Option<tokio::task::JoinHandle<()>> = {
793                if let (Some(provider), Some(sess_id)) = (&avatar_provider, &avatar_session_id) {
794                    Some(crate::avatar::spawn_keep_alive(
795                        provider.clone(),
796                        sess_id.clone(),
797                        std::time::Duration::from_secs(30),
798                    ))
799                } else {
800                    None
801                }
802            };
803
804            // ===== SEND INITIAL USER CONTENT =====
805            // If user provided text input, send it to start the conversation
806            let user_content = ctx.user_content();
807            for part in &user_content.parts {
808                if let Part::Text { text } = part {
809                    if let Err(e) = session.send_text(text).await {
810                        yield Err(AdkError::model(format!("Failed to send text: {}", e)));
811                        return;
812                    }
813                    // Request a response
814                    if let Err(e) = session.create_response().await {
815                        yield Err(AdkError::model(format!("Failed to create response: {}", e)));
816                        return;
817                    }
818                }
819            }
820
821            // ===== PROCESS REALTIME EVENTS =====
822            let mut audio_buffers = std::collections::HashMap::<String, Vec<u8>>::new();
823            let mut oversized_audio = std::collections::HashSet::<String>::new();
824            loop {
825                let event = session.next_event().await;
826
827                match event {
828                    Some(Ok(server_event)) => {
829                        match server_event {
830                            ServerEvent::AudioDelta { delta, item_id, .. } => {
831                                // Route audio through avatar provider if active
832                                #[cfg(feature = "video-avatar")]
833                                if let (Some(provider), Some(sess_id)) = (&avatar_provider, &avatar_session_id) {
834                                    if let Err(e) = provider.send_audio(sess_id, &delta).await {
835                                        tracing::warn!(error = %e, "avatar send_audio failed");
836                                    }
837                                    // Don't yield raw audio to client — avatar provides video+audio
838                                    // Still call the on_audio callback for monitoring
839                                    if let Some(ref cb) = on_audio {
840                                        cb(&delta, &item_id).await;
841                                    }
842                                    continue;
843                                }
844
845                                // No avatar provider — send raw audio to client
846                                if let Some(ref cb) = on_audio {
847                                    cb(&delta, &item_id).await;
848                                }
849
850                                if !oversized_audio.contains(&item_id) {
851                                    let buffer = audio_buffers.entry(item_id.clone()).or_default();
852                                    if buffer.len().saturating_add(delta.len())
853                                        <= MAX_BUFFERED_PLAYBACK_AUDIO_BYTES
854                                    {
855                                        buffer.extend_from_slice(&delta);
856                                    } else {
857                                        audio_buffers.remove(&item_id);
858                                        oversized_audio.insert(item_id.clone());
859                                        tracing::warn!(
860                                            item.id = item_id,
861                                            limit.bytes = MAX_BUFFERED_PLAYBACK_AUDIO_BYTES,
862                                            "realtime playback buffer exceeded its limit; raw audio events continue"
863                                        );
864                                    }
865                                }
866
867                                // Yield audio event (delta is already raw bytes)
868                                let mut audio_event = Event::new(&invocation_id);
869                                audio_event.author = agent_name.clone();
870                                audio_event.provider_metadata.insert(
871                                    "adk.realtime.audio_stream".to_string(),
872                                    "pcm16-24000-mono".to_string(),
873                                );
874                                audio_event.llm_response.content = Some(Content {
875                                    role: "model".to_string(),
876                                    parts: vec![Part::InlineData {
877                                        mime_type: "audio/pcm".to_string(),
878                                        data: delta,
879                                        uri: None,
880                                        annotations: None,
881                                    }],
882                                });
883                                yield Ok(audio_event);
884                            }
885
886                            ServerEvent::AudioDone { item_id, .. } => {
887                                let oversized = oversized_audio.remove(&item_id);
888                                if !oversized
889                                    && let Some(pcm) = audio_buffers.remove(&item_id)
890                                    && !pcm.is_empty()
891                                {
892                                    let mut audio_event = Event::new(&invocation_id);
893                                    audio_event.author = agent_name.clone();
894                                    audio_event.provider_metadata.insert(
895                                        "adk.realtime.audio_playback".to_string(),
896                                        "wav-24000-mono".to_string(),
897                                    );
898                                    audio_event.llm_response.content = Some(Content {
899                                        role: "model".to_string(),
900                                        parts: vec![Part::InlineData {
901                                            mime_type: "audio/wav".to_string(),
902                                            data: pcm16_mono_wav(&pcm, 24_000),
903                                            uri: None,
904                                            annotations: None,
905                                        }],
906                                    });
907                                    yield Ok(audio_event);
908                                }
909                            }
910
911                            ServerEvent::TextDelta { delta, item_id, .. } => {
912                                let mut text_event = Event::with_id(
913                                    format!("{invocation_id}:realtime-text:{item_id}"),
914                                    &invocation_id,
915                                );
916                                text_event.author = agent_name.clone();
917                                text_event.llm_response.partial = true;
918                                text_event.llm_response.content = Some(Content {
919                                    role: "model".to_string(),
920                                    parts: vec![Part::Text { text: delta.clone() }],
921                                });
922                                yield Ok(text_event);
923                            }
924
925                            ServerEvent::TextDone { text, item_id, .. } => {
926                                let mut text_event = Event::with_id(
927                                    format!("{invocation_id}:realtime-text:{item_id}"),
928                                    &invocation_id,
929                                );
930                                text_event.author = agent_name.clone();
931                                text_event.llm_response.content = Some(Content {
932                                    role: "model".to_string(),
933                                    parts: vec![Part::Text { text }],
934                                });
935                                yield Ok(text_event);
936                            }
937
938                            ServerEvent::TranscriptDelta { delta, item_id, .. } => {
939                                if let Some(ref cb) = on_transcript {
940                                    cb(&delta, &item_id).await;
941                                }
942                                let mut transcript_event = Event::with_id(
943                                    format!("{invocation_id}:realtime-transcript:{item_id}"),
944                                    &invocation_id,
945                                );
946                                transcript_event.author = agent_name.clone();
947                                transcript_event.llm_response.partial = true;
948                                transcript_event.provider_metadata.insert(
949                                    "adk.realtime.transcript".to_string(),
950                                    "output".to_string(),
951                                );
952                                transcript_event.llm_response.content = Some(Content {
953                                    role: "model".to_string(),
954                                    parts: vec![Part::Text { text: delta }],
955                                });
956                                yield Ok(transcript_event);
957                            }
958
959                            ServerEvent::TranscriptDone { transcript, item_id, .. } => {
960                                let mut transcript_event = Event::with_id(
961                                    format!("{invocation_id}:realtime-transcript:{item_id}"),
962                                    &invocation_id,
963                                );
964                                transcript_event.author = agent_name.clone();
965                                transcript_event.provider_metadata.insert(
966                                    "adk.realtime.transcript".to_string(),
967                                    "output".to_string(),
968                                );
969                                transcript_event.llm_response.content = Some(Content {
970                                    role: "model".to_string(),
971                                    parts: vec![Part::Text { text: transcript }],
972                                });
973                                yield Ok(transcript_event);
974                            }
975
976                            ServerEvent::SpeechStarted { audio_start_ms, .. } => {
977                                if let Some(ref cb) = on_speech_started {
978                                    cb(audio_start_ms).await;
979                                }
980                            }
981
982                            ServerEvent::SpeechStopped { audio_end_ms, .. } => {
983                                if let Some(ref cb) = on_speech_stopped {
984                                    cb(audio_end_ms).await;
985                                }
986                            }
987
988                            ServerEvent::FunctionCallDone {
989                                call_id,
990                                name,
991                                arguments,
992                                ..
993                            } => {
994                                // Handle transfer_to_agent
995                                if name == "transfer_to_agent" {
996                                    let args: serde_json::Value = serde_json::from_str(&arguments)
997                                        .unwrap_or(serde_json::json!({}));
998                                    let target = args.get("agent_name")
999                                        .and_then(|v| v.as_str())
1000                                        .unwrap_or_default()
1001                                        .to_string();
1002
1003                                    let mut transfer_event = Event::new(&invocation_id);
1004                                    transfer_event.author = agent_name.clone();
1005                                    transfer_event.actions.transfer_to_agent = Some(target);
1006                                    yield Ok(transfer_event);
1007
1008                                    let _ = session.close().await;
1009                                    return;
1010                                }
1011
1012                                // Execute tool
1013                                let tool = resolved_tools.iter().find(|t| t.name() == name);
1014
1015                                let (result, actions) = if let Some(tool) = tool {
1016                                    let args: serde_json::Value = serde_json::from_str(&arguments)
1017                                        .unwrap_or(serde_json::json!({}));
1018
1019                                    let tool_ctx: Arc<dyn ToolContext> = Arc::new(
1020                                        RealtimeToolContext::new(ctx.clone(), call_id.clone())
1021                                    );
1022
1023                                    let cb_ctx: Arc<dyn CallbackContext> =
1024                                        Arc::new(ToolCallbackContext::new(
1025                                            ctx.clone(),
1026                                            name.clone(),
1027                                            args.clone(),
1028                                        ));
1029
1030                                    let result = execute_tool_with_callbacks(
1031                                        tool.as_ref(),
1032                                        tool_ctx.clone(),
1033                                        cb_ctx,
1034                                        args.clone(),
1035                                        before_tool_callbacks.as_ref(),
1036                                        after_tool_callbacks.as_ref(),
1037                                    )
1038                                    .await;
1039
1040                                    (result, tool_ctx.actions())
1041                                } else {
1042                                    (
1043                                        serde_json::json!({ "error": format!("Tool {} not found", name) }),
1044                                        EventActions::default(),
1045                                    )
1046                                };
1047
1048                                // Yield tool event
1049                                let mut tool_event = Event::new(&invocation_id);
1050                                tool_event.author = agent_name.clone();
1051                                tool_event.actions = actions.clone();
1052                                tool_event.llm_response.content = Some(Content {
1053                                    role: "function".to_string(),
1054                                    parts: vec![Part::FunctionResponse {
1055                                        function_response: adk_core::FunctionResponseData::new(name.clone(), result.clone()),
1056                                        id: Some(call_id.clone()),
1057                                        annotations: None,
1058                                    }],
1059                                });
1060                                yield Ok(tool_event);
1061
1062                                // Check for escalation
1063                                if actions.escalate || actions.skip_summarization {
1064                                    let _ = session.close().await;
1065                                    return;
1066                                }
1067
1068                                // Send tool response back to session
1069                                let response = ToolResponse {
1070                                    call_id,
1071                                    output: result,
1072                                };
1073                                if let Err(e) = session.send_tool_response(response).await {
1074                                    yield Err(AdkError::model(format!("Failed to send tool response: {}", e)));
1075                                    let _ = session.close().await;
1076                                    return;
1077                                }
1078                            }
1079
1080                            ServerEvent::ResponseDone { .. } => {
1081                                // Response complete, continue listening
1082                            }
1083
1084                            ServerEvent::Error { error, .. } => {
1085                                yield Err(AdkError::model(format!(
1086                                    "Realtime error: {} - {}",
1087                                    error.code.unwrap_or_default(),
1088                                    error.message
1089                                )));
1090                            }
1091
1092
1093                            _ => {
1094                                // Ignore other events
1095                            }
1096                        }
1097                    }
1098                    Some(Err(e)) => {
1099                        yield Err(AdkError::model(format!("Session error: {}", e)));
1100                        break;
1101                    }
1102                    None => {
1103                        // Session closed
1104                        break;
1105                    }
1106                }
1107            }
1108
1109            // ===== STOP AVATAR SESSION (cleanup) =====
1110            #[cfg(feature = "video-avatar")]
1111            {
1112                // Abort keep-alive task
1113                if let Some(handle) = _avatar_keep_alive_handle {
1114                    handle.abort();
1115                }
1116                // Stop the avatar session
1117                if let (Some(provider), Some(sess_id)) = (&avatar_provider, &avatar_session_id) {
1118                    if let Err(e) = provider.stop_session(sess_id).await {
1119                        tracing::warn!(error = %e, "avatar session cleanup failed");
1120                    }
1121                }
1122            }
1123
1124            // ===== AFTER AGENT CALLBACKS =====
1125            for callback in after_callbacks.as_ref() {
1126                match callback(ctx.clone() as Arc<dyn CallbackContext>).await {
1127                    Ok(Some(content)) => {
1128                        let mut after_event = Event::new(&invocation_id);
1129                        after_event.author = agent_name.clone();
1130                        after_event.llm_response.content = Some(content);
1131                        yield Ok(after_event);
1132                        break;
1133                    }
1134                    Ok(None) => continue,
1135                    Err(e) => {
1136                        yield Err(e);
1137                        return;
1138                    }
1139                }
1140            }
1141        };
1142
1143        Ok(Box::pin(s))
1144    }
1145}
1146
1147fn pcm16_mono_wav(pcm: &[u8], sample_rate: u32) -> Vec<u8> {
1148    let data_len = u32::try_from(pcm.len()).unwrap_or(u32::MAX);
1149    let mut wav = Vec::with_capacity(44 + pcm.len());
1150    wav.extend_from_slice(b"RIFF");
1151    wav.extend_from_slice(&(36_u32.saturating_add(data_len)).to_le_bytes());
1152    wav.extend_from_slice(b"WAVEfmt ");
1153    wav.extend_from_slice(&16_u32.to_le_bytes());
1154    wav.extend_from_slice(&1_u16.to_le_bytes());
1155    wav.extend_from_slice(&1_u16.to_le_bytes());
1156    wav.extend_from_slice(&sample_rate.to_le_bytes());
1157    wav.extend_from_slice(&sample_rate.saturating_mul(2).to_le_bytes());
1158    wav.extend_from_slice(&2_u16.to_le_bytes());
1159    wav.extend_from_slice(&16_u16.to_le_bytes());
1160    wav.extend_from_slice(b"data");
1161    wav.extend_from_slice(&data_len.to_le_bytes());
1162    wav.extend_from_slice(pcm);
1163    wav
1164}
1165
1166/// Tool context for realtime agent tool execution.
1167/// Runs one tool through its before- and after-tool callbacks.
1168///
1169/// The callback contract matches the standard agent loop, which the realtime path did not
1170/// honour: a before-callback returning `Ok(Some(content))` substitutes a result and the tool
1171/// does **not** run, `Ok(None)` allows it, and an error refuses it and skips the after
1172/// callbacks. Previously the loop evaluated `(error_result, EventActions::default())` as a
1173/// discarded expression statement and fell through to `tool.execute`, so a gate could neither
1174/// deny nor substitute — it reported a decision while the tool ran regardless. After-callback
1175/// results were dropped by `let _ =`.
1176async fn execute_tool_with_callbacks(
1177    tool: &dyn Tool,
1178    tool_ctx: Arc<dyn ToolContext>,
1179    cb_ctx: Arc<dyn CallbackContext>,
1180    args: serde_json::Value,
1181    before_tool_callbacks: &[BeforeToolCallback],
1182    after_tool_callbacks: &[AfterToolCallback],
1183) -> serde_json::Value {
1184    let mut short_circuit: Option<serde_json::Value> = None;
1185    let mut run_after_tool_callbacks = true;
1186
1187    for callback in before_tool_callbacks {
1188        match callback(cb_ctx.clone()).await {
1189            Ok(Some(content)) => {
1190                short_circuit = Some(content_to_tool_result(&content));
1191                break;
1192            }
1193            Ok(None) => continue,
1194            Err(e) => {
1195                short_circuit = Some(serde_json::json!({ "error": e.to_string() }));
1196                run_after_tool_callbacks = false;
1197                break;
1198            }
1199        }
1200    }
1201
1202    let mut result = match short_circuit {
1203        Some(result) => result,
1204        None => match tool.execute(tool_ctx, args).await {
1205            Ok(value) => value,
1206            Err(e) => serde_json::json!({ "error": e.to_string() }),
1207        },
1208    };
1209
1210    if run_after_tool_callbacks {
1211        for callback in after_tool_callbacks {
1212            match callback(cb_ctx.clone()).await {
1213                Ok(Some(modified)) => {
1214                    result = content_to_tool_result(&modified);
1215                    break;
1216                }
1217                Ok(None) => continue,
1218                Err(e) => {
1219                    result = serde_json::json!({ "error": e.to_string() });
1220                    break;
1221                }
1222            }
1223        }
1224    }
1225
1226    result
1227}
1228
1229/// Turns a callback's substitute `Content` into the result sent back to the provider.
1230///
1231/// A callback returns `Content` because that is what the standard agent loop puts on the
1232/// event stream. The realtime transport wants a JSON tool result, so a function response is
1233/// unwrapped to its payload and anything else is carried as its text.
1234fn content_to_tool_result(content: &Content) -> serde_json::Value {
1235    for part in &content.parts {
1236        if let Part::FunctionResponse { function_response, .. } = part {
1237            return function_response.response.clone();
1238        }
1239    }
1240
1241    let text: String = content.parts.iter().filter_map(|part| part.text()).collect();
1242    serde_json::json!({ "result": text })
1243}
1244
1245struct RealtimeToolContext {
1246    parent_ctx: Arc<dyn InvocationContext>,
1247    function_call_id: String,
1248    actions: Mutex<EventActions>,
1249}
1250
1251impl RealtimeToolContext {
1252    fn new(parent_ctx: Arc<dyn InvocationContext>, function_call_id: String) -> Self {
1253        Self { parent_ctx, function_call_id, actions: Mutex::new(EventActions::default()) }
1254    }
1255}
1256
1257#[async_trait]
1258impl ReadonlyContext for RealtimeToolContext {
1259    fn invocation_id(&self) -> &str {
1260        self.parent_ctx.invocation_id()
1261    }
1262
1263    fn agent_name(&self) -> &str {
1264        self.parent_ctx.agent_name()
1265    }
1266
1267    fn user_id(&self) -> &str {
1268        self.parent_ctx.user_id()
1269    }
1270
1271    fn app_name(&self) -> &str {
1272        self.parent_ctx.app_name()
1273    }
1274
1275    fn session_id(&self) -> &str {
1276        self.parent_ctx.session_id()
1277    }
1278
1279    fn branch(&self) -> &str {
1280        self.parent_ctx.branch()
1281    }
1282
1283    fn user_content(&self) -> &Content {
1284        self.parent_ctx.user_content()
1285    }
1286}
1287
1288#[async_trait]
1289impl CallbackContext for RealtimeToolContext {
1290    fn artifacts(&self) -> Option<Arc<dyn adk_core::Artifacts>> {
1291        self.parent_ctx.artifacts()
1292    }
1293
1294    /// Shared state from the parent context, so realtime tools coordinate with the rest of
1295    /// the run rather than seeing `None`.
1296    fn shared_state(&self) -> Option<Arc<adk_core::SharedState>> {
1297        self.parent_ctx.shared_state()
1298    }
1299}
1300
1301#[async_trait]
1302impl ToolContext for RealtimeToolContext {
1303    fn function_call_id(&self) -> &str {
1304        &self.function_call_id
1305    }
1306
1307    fn actions(&self) -> EventActions {
1308        self.actions.lock().unwrap().clone()
1309    }
1310
1311    fn set_actions(&self, actions: EventActions) {
1312        *self.actions.lock().unwrap() = actions;
1313    }
1314
1315    async fn search_memory(&self, query: &str) -> Result<Vec<MemoryEntry>> {
1316        if let Some(memory) = self.parent_ctx.memory() {
1317            memory.search(query).await
1318        } else {
1319            Ok(vec![])
1320        }
1321    }
1322
1323    /// The caller's scopes, from the parent context.
1324    ///
1325    /// Without this the trait default returned an empty list, so a scope-checking tool saw an
1326    /// unauthenticated caller in realtime and behaved differently than in the standard loop.
1327    fn user_scopes(&self) -> Vec<String> {
1328        self.parent_ctx.user_scopes()
1329    }
1330
1331    /// Secrets resolved through the parent context.
1332    ///
1333    /// The trait default returns `None`, which a tool cannot distinguish from a secret that is
1334    /// genuinely absent.
1335    async fn get_secret(&self, name: &str) -> Result<Option<String>> {
1336        self.parent_ctx.get_secret(name).await
1337    }
1338}
1339
1340#[cfg(test)]
1341mod tool_safety_tests {
1342    //! A before-tool callback must be able to stop a tool, and a realtime tool must see the
1343    //! same capabilities it sees in the standard loop.
1344    //!
1345    //! The dispatch loop built `(error_result, EventActions::default())` as a discarded
1346    //! expression statement and then fell through to `tool.execute`, so a denying callback
1347    //! reported a decision that had no effect. After-callback results were dropped with
1348    //! `let _ =`. `RealtimeToolContext` implemented only the required methods, inheriting
1349    //! `user_scopes() -> vec![]`, `get_secret() -> None`, and `shared_state() -> None`, so a
1350    //! scope- or secret-checking tool behaved differently in realtime than under a Runner.
1351
1352    use super::*;
1353    use adk_core::{RunConfig, SharedState, State};
1354    use std::collections::HashMap;
1355    use std::sync::atomic::{AtomicUsize, Ordering};
1356
1357    #[test]
1358    fn completed_pcm_audio_is_wrapped_as_a_playable_wav() {
1359        let pcm = [0_u8, 1, 2, 3];
1360        let wav = pcm16_mono_wav(&pcm, 24_000);
1361
1362        assert_eq!(&wav[0..4], b"RIFF");
1363        assert_eq!(&wav[8..12], b"WAVE");
1364        assert_eq!(&wav[12..16], b"fmt ");
1365        assert_eq!(u32::from_le_bytes(wav[24..28].try_into().unwrap()), 24_000);
1366        assert_eq!(&wav[36..40], b"data");
1367        assert_eq!(u32::from_le_bytes(wav[40..44].try_into().unwrap()), pcm.len() as u32);
1368        assert_eq!(&wav[44..], pcm);
1369    }
1370
1371    /// Counts how many times it is executed.
1372    struct CountingTool {
1373        executions: Arc<AtomicUsize>,
1374    }
1375
1376    #[async_trait]
1377    impl Tool for CountingTool {
1378        fn name(&self) -> &str {
1379            "counting"
1380        }
1381        fn description(&self) -> &str {
1382            "counts executions"
1383        }
1384        async fn execute(
1385            &self,
1386            _ctx: Arc<dyn ToolContext>,
1387            _args: serde_json::Value,
1388        ) -> Result<serde_json::Value> {
1389            self.executions.fetch_add(1, Ordering::SeqCst);
1390            Ok(serde_json::json!({ "ran": true }))
1391        }
1392    }
1393
1394    /// The minimum context the callback path needs.
1395    struct TestToolContext {
1396        actions: Mutex<EventActions>,
1397        content: Content,
1398    }
1399
1400    impl TestToolContext {
1401        fn new() -> Self {
1402            Self { actions: Mutex::new(EventActions::default()), content: Content::new("user") }
1403        }
1404    }
1405
1406    #[async_trait]
1407    impl ReadonlyContext for TestToolContext {
1408        fn invocation_id(&self) -> &str {
1409            "inv"
1410        }
1411        fn agent_name(&self) -> &str {
1412            "agent"
1413        }
1414        fn user_id(&self) -> &str {
1415            "user"
1416        }
1417        fn app_name(&self) -> &str {
1418            "app"
1419        }
1420        fn session_id(&self) -> &str {
1421            "session"
1422        }
1423        fn branch(&self) -> &str {
1424            ""
1425        }
1426        fn user_content(&self) -> &Content {
1427            &self.content
1428        }
1429    }
1430
1431    #[async_trait]
1432    impl CallbackContext for TestToolContext {
1433        fn artifacts(&self) -> Option<Arc<dyn adk_core::Artifacts>> {
1434            None
1435        }
1436    }
1437
1438    #[async_trait]
1439    impl ToolContext for TestToolContext {
1440        fn function_call_id(&self) -> &str {
1441            "call-1"
1442        }
1443        fn actions(&self) -> EventActions {
1444            self.actions.lock().unwrap().clone()
1445        }
1446        fn set_actions(&self, actions: EventActions) {
1447            *self.actions.lock().unwrap() = actions;
1448        }
1449        async fn search_memory(&self, _query: &str) -> Result<Vec<MemoryEntry>> {
1450            Ok(vec![])
1451        }
1452    }
1453
1454    /// Runs the tool through the callback gate with the supplied callbacks.
1455    async fn dispatch(
1456        before: Vec<BeforeToolCallback>,
1457        after: Vec<AfterToolCallback>,
1458        executions: Arc<AtomicUsize>,
1459    ) -> serde_json::Value {
1460        let tool = CountingTool { executions };
1461        let ctx = Arc::new(TestToolContext::new());
1462        execute_tool_with_callbacks(
1463            &tool,
1464            ctx.clone() as Arc<dyn ToolContext>,
1465            ctx as Arc<dyn CallbackContext>,
1466            serde_json::json!({}),
1467            &before,
1468            &after,
1469        )
1470        .await
1471    }
1472
1473    #[tokio::test]
1474    async fn a_before_callback_error_prevents_execution() {
1475        let executions = Arc::new(AtomicUsize::new(0));
1476        let before: Vec<BeforeToolCallback> =
1477            vec![Box::new(|_ctx| Box::pin(async { Err(AdkError::tool("denied by policy")) }))];
1478
1479        let result = dispatch(before, vec![], Arc::clone(&executions)).await;
1480
1481        assert_eq!(executions.load(Ordering::SeqCst), 0, "a refused tool must not run: {result}");
1482        assert!(
1483            result["error"].as_str().unwrap_or_default().contains("denied by policy"),
1484            "the refusal reason must reach the provider: {result}"
1485        );
1486    }
1487
1488    #[tokio::test]
1489    async fn a_before_callback_substitution_prevents_execution() {
1490        let executions = Arc::new(AtomicUsize::new(0));
1491        let before: Vec<BeforeToolCallback> = vec![Box::new(|_ctx| {
1492            Box::pin(async {
1493                Ok(Some(Content {
1494                    role: "function".to_string(),
1495                    parts: vec![Part::FunctionResponse {
1496                        function_response: adk_core::FunctionResponseData::new(
1497                            "counting",
1498                            serde_json::json!({ "cached": true }),
1499                        ),
1500                        id: None,
1501                        annotations: None,
1502                    }],
1503                }))
1504            })
1505        })];
1506
1507        let result = dispatch(before, vec![], Arc::clone(&executions)).await;
1508
1509        assert_eq!(
1510            executions.load(Ordering::SeqCst),
1511            0,
1512            "a substituted result must not run the tool"
1513        );
1514        assert_eq!(result, serde_json::json!({ "cached": true }));
1515    }
1516
1517    #[tokio::test]
1518    async fn a_permitting_callback_lets_the_tool_run() {
1519        let executions = Arc::new(AtomicUsize::new(0));
1520        let before: Vec<BeforeToolCallback> = vec![Box::new(|_ctx| Box::pin(async { Ok(None) }))];
1521
1522        let result = dispatch(before, vec![], Arc::clone(&executions)).await;
1523
1524        assert_eq!(executions.load(Ordering::SeqCst), 1);
1525        assert_eq!(result, serde_json::json!({ "ran": true }));
1526    }
1527
1528    #[tokio::test]
1529    async fn an_after_callback_error_becomes_the_result() {
1530        let executions = Arc::new(AtomicUsize::new(0));
1531        let after: Vec<AfterToolCallback> =
1532            vec![Box::new(|_ctx| Box::pin(async { Err(AdkError::tool("post-check failed")) }))];
1533
1534        let result = dispatch(vec![], after, Arc::clone(&executions)).await;
1535
1536        assert_eq!(executions.load(Ordering::SeqCst), 1, "the tool ran, as it should have");
1537        assert!(
1538            result["error"].as_str().unwrap_or_default().contains("post-check failed"),
1539            "an after-callback failure must not be dropped: {result}"
1540        );
1541    }
1542
1543    #[tokio::test]
1544    async fn after_callbacks_are_skipped_when_a_before_callback_refuses() {
1545        let executions = Arc::new(AtomicUsize::new(0));
1546        let after_ran = Arc::new(AtomicUsize::new(0));
1547        let counter = Arc::clone(&after_ran);
1548
1549        let before: Vec<BeforeToolCallback> =
1550            vec![Box::new(|_ctx| Box::pin(async { Err(AdkError::tool("refused")) }))];
1551        let after: Vec<AfterToolCallback> = vec![Box::new(move |_ctx| {
1552            let counter = Arc::clone(&counter);
1553            Box::pin(async move {
1554                counter.fetch_add(1, Ordering::SeqCst);
1555                Ok(None)
1556            })
1557        })];
1558
1559        let result = dispatch(before, after, Arc::clone(&executions)).await;
1560
1561        assert_eq!(executions.load(Ordering::SeqCst), 0);
1562        assert_eq!(after_ran.load(Ordering::SeqCst), 0, "matching the standard loop's ordering");
1563        assert!(result["error"].as_str().unwrap_or_default().contains("refused"), "{result}");
1564    }
1565
1566    // ── Context capabilities ──────────────────────────────────────────
1567
1568    struct TestState;
1569    impl State for TestState {
1570        fn get(&self, _key: &str) -> Option<serde_json::Value> {
1571            None
1572        }
1573        fn set(&mut self, _key: String, _value: serde_json::Value) {}
1574        fn all(&self) -> HashMap<String, serde_json::Value> {
1575            HashMap::new()
1576        }
1577    }
1578
1579    struct TestSession;
1580    impl adk_core::Session for TestSession {
1581        fn id(&self) -> &str {
1582            "session"
1583        }
1584        fn app_name(&self) -> &str {
1585            "app"
1586        }
1587        fn user_id(&self) -> &str {
1588            "user"
1589        }
1590        fn state(&self) -> &dyn State {
1591            &TestState
1592        }
1593        fn conversation_history(&self) -> Vec<Content> {
1594            Vec::new()
1595        }
1596    }
1597
1598    /// A parent context carrying the capabilities a tool should still see in realtime.
1599    struct CapableParent {
1600        content: Content,
1601        config: RunConfig,
1602        session: TestSession,
1603        shared: Arc<SharedState>,
1604    }
1605
1606    #[async_trait]
1607    impl ReadonlyContext for CapableParent {
1608        fn invocation_id(&self) -> &str {
1609            "inv"
1610        }
1611        fn agent_name(&self) -> &str {
1612            "agent"
1613        }
1614        fn user_id(&self) -> &str {
1615            "user"
1616        }
1617        fn app_name(&self) -> &str {
1618            "app"
1619        }
1620        fn session_id(&self) -> &str {
1621            "session"
1622        }
1623        fn branch(&self) -> &str {
1624            ""
1625        }
1626        fn user_content(&self) -> &Content {
1627            &self.content
1628        }
1629    }
1630
1631    #[async_trait]
1632    impl CallbackContext for CapableParent {
1633        fn artifacts(&self) -> Option<Arc<dyn adk_core::Artifacts>> {
1634            None
1635        }
1636
1637        fn shared_state(&self) -> Option<Arc<SharedState>> {
1638            Some(Arc::clone(&self.shared))
1639        }
1640    }
1641
1642    #[async_trait]
1643    impl InvocationContext for CapableParent {
1644        fn agent(&self) -> Arc<dyn Agent> {
1645            unreachable!("not used by these tests")
1646        }
1647        fn memory(&self) -> Option<Arc<dyn adk_core::Memory>> {
1648            None
1649        }
1650        fn session(&self) -> &dyn adk_core::Session {
1651            &self.session
1652        }
1653        fn run_config(&self) -> &RunConfig {
1654            &self.config
1655        }
1656        fn end_invocation(&self) {}
1657        fn ended(&self) -> bool {
1658            false
1659        }
1660        fn user_scopes(&self) -> Vec<String> {
1661            vec!["repo:write".to_string()]
1662        }
1663        async fn get_secret(&self, name: &str) -> Result<Option<String>> {
1664            Ok(Some(format!("secret-for-{name}")))
1665        }
1666    }
1667
1668    #[tokio::test]
1669    async fn the_realtime_tool_context_preserves_parent_capabilities() {
1670        let parent = Arc::new(CapableParent {
1671            content: Content::new("user"),
1672            config: RunConfig::default(),
1673            session: TestSession,
1674            shared: Arc::new(SharedState::new()),
1675        }) as Arc<dyn InvocationContext>;
1676
1677        let ctx = RealtimeToolContext::new(parent, "call-1".to_string());
1678
1679        assert_eq!(
1680            ctx.user_scopes(),
1681            vec!["repo:write".to_string()],
1682            "an empty scope list makes an authenticated caller look anonymous"
1683        );
1684        assert_eq!(ctx.get_secret("api_key").await.unwrap().as_deref(), Some("secret-for-api_key"));
1685        assert!(ctx.shared_state().is_some(), "shared state must reach realtime tools");
1686        assert_eq!(ctx.app_name(), "app", "identity still delegates");
1687    }
1688}