#[cfg(feature = "realtime")]
mod enabled {
use base64::Engine;
use futures_util::{SinkExt, StreamExt};
use reqwest::Method;
use serde::{Deserialize, Serialize};
use serde_json::{json, Map, Value};
use std::sync::{
atomic::{AtomicBool, Ordering},
Arc,
};
use tokio::sync::{mpsc, Mutex, RwLock};
use tokio_tungstenite::{
connect_async,
tungstenite::{client::IntoClientRequest, http::HeaderValue, Message},
};
use crate::{
agent::{
function_map_with_events, graph_with_context, prepare_input, realtime_tool_definitions,
resolve_instructions, resolve_peers, tool_context, validate_graph, validate_output,
Agent, AgentContext, AgentTool, Runner,
},
Error, Input, Result, ToolChoice,
};
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct RealtimeEvent {
#[serde(rename = "type")]
pub event_type: String,
pub delta: Option<String>,
#[serde(default)]
pub output: Value,
#[serde(default)]
pub data: Map<String, Value>,
}
#[derive(Debug, Clone, Deserialize)]
struct Descriptor {
model: String,
endpoint: String,
client_secret: String,
conversation_id: Option<String>,
#[serde(default)]
capabilities: Value,
}
pub struct RealtimeSession {
pub model: String,
pub conversation_id: Option<String>,
pub available_tools: Vec<Value>,
sender: mpsc::Sender<Value>,
receiver: Mutex<mpsc::Receiver<Result<RealtimeEvent>>>,
close: mpsc::Sender<()>,
active_agent: Arc<RwLock<Agent>>,
user_id: Option<String>,
session_id: String,
closed: AtomicBool,
}
impl RealtimeSession {
pub async fn send_message(&self, input: impl Into<Input>) -> Result<()> {
if self.closed.load(Ordering::Acquire) {
return Err(Error::Realtime("session is closed".into()));
}
let agent = self.active_agent.read().await.clone();
let context = context(&agent, self.user_id.as_deref(), &self.session_id);
let input = prepare_input(&agent, input.into(), &context).await?;
if let Some(hooks) = &agent.hooks {
hooks.on_start(&context, &input).await?;
}
let values = match input {
Input::Text(text) => vec![
json!({"type":"message","role":"user","content":[{"type":"input_text","text":text}]}),
],
Input::Message(value) => vec![value],
Input::Messages(values) => values,
};
for mut item in values {
item["type"] = json!("message");
if item.get("role").is_none() {
item["role"] = json!("user");
}
if let Some(text) = item["content"].as_str() {
item["content"] = json!([{"type":"input_text","text":text}]);
}
self.sender
.send(json!({"type":"conversation.item.create","item":item}))
.await
.map_err(|_| Error::Realtime("session is closed".into()))?;
}
self.sender
.send(json!({"type":"response.create"}))
.await
.map_err(|_| Error::Realtime("session is closed".into()))
}
pub async fn send_audio(&self, bytes: &[u8]) -> Result<()> {
if self.closed.load(Ordering::Acquire) {
return Err(Error::Realtime("session is closed".into()));
}
let agent = self.active_agent.read().await;
if !agent.modalities.is_empty() && !agent.modalities.iter().any(|v| v == "audio") {
return Err(Error::Validation("Agent input does not allow audio".into()));
}
self.sender.send(json!({"type":"input_audio_buffer.append","audio":base64::engine::general_purpose::STANDARD.encode(bytes)})).await.map_err(|_| Error::Realtime("session is closed".into()))
}
pub async fn next_event(&self) -> Option<Result<RealtimeEvent>> {
self.receiver.lock().await.recv().await
}
pub async fn close(&self) -> Result<()> {
if self.closed.swap(true, Ordering::AcqRel) {
return Ok(());
}
self.close
.send(())
.await
.map_err(|_| Error::Realtime("session is already closed".into()))
}
}
pub struct RealtimeRunner;
impl RealtimeRunner {
pub async fn connect(
agent: &Agent,
user_id: Option<&str>,
session_id: Option<&str>,
) -> Result<RealtimeSession> {
validate_graph(agent)?;
resolve_peers(agent).await?;
let unsupported = agent
.output_modalities
.iter()
.filter(|value| !matches!(value.as_str(), "text" | "audio"))
.cloned()
.collect::<Vec<_>>();
if !unsupported.is_empty() {
return Err(Error::Validation(format!(
"realtime output does not allow: {}",
unsupported.join(", ")
)));
}
let session_id = session_id.unwrap_or("default");
let conversation_id = format!(
"rust:{}:{}",
url::form_urlencoded::byte_serialize(user_id.unwrap_or("anonymous").as_bytes())
.collect::<String>(),
url::form_urlencoded::byte_serialize(session_id.as_bytes()).collect::<String>()
);
let context = context(agent, user_id, session_id);
let output_modalities = if agent.output_modalities.is_empty() {
vec!["audio".to_owned()]
} else {
agent.output_modalities.clone()
};
let body = json!({
"interface":"api", "user_id":user_id, "conversation_id":conversation_id,
"mode":if agent.modalities.iter().any(|value| matches!(value.as_str(), "image" | "video")){"voice_vision"}else{"voice"},
"model":agent.model, "voice":agent.voice.as_ref().and_then(|value| value.output_voice.as_deref()),
"instructions":resolve_instructions(agent, &context), "output_modalities":output_modalities,
"transport":"websocket", "metadata":{
"voice":agent.voice, "routing":agent.routing, "required_transport":"realtime",
"marona_agent":{"input_modalities":agent.modalities,"output_modalities":agent.output_modalities,"tools_enabled":agent.tool_choice != ToolChoice::None && (agent.hub_tools.is_some() || !agent.tools.is_empty() || !agent.peers.is_empty() || !agent.handoffs.is_empty()),"tool_choice":agent.tool_choice},
"marona_connection_scope":agent.hub_tools.as_ref().map(|value| value.session()),
"marona_multi_agent":graph_with_context(agent, &context),
}
});
let descriptor: Descriptor = agent
.marona
.core
.json(
&agent.marona.core.runtime_url,
Method::POST,
"/runtime/realtime/session",
Some(&body),
)
.await?;
if descriptor.endpoint.is_empty() || descriptor.client_secret.is_empty() {
return Err(Error::Realtime(
"runtime returned incomplete session credentials".into(),
));
}
let mut request = descriptor
.endpoint
.clone()
.into_client_request()
.map_err(|e| Error::Realtime(e.to_string()))?;
request.headers_mut().insert(
"Authorization",
HeaderValue::from_str(&format!("Bearer {}", descriptor.client_secret))
.map_err(|e| Error::Realtime(e.to_string()))?,
);
let (socket, _) = connect_async(request)
.await
.map_err(|e| Error::Realtime(e.to_string()))?;
let (mut sink, mut stream) = socket.split();
let (send_tx, mut send_rx) = mpsc::channel::<Value>(64);
let (event_tx, event_rx) = mpsc::channel::<Result<RealtimeEvent>>(64);
let (close_tx, mut close_rx) = mpsc::channel::<()>(1);
let active_state = Arc::new(RwLock::new(agent.clone()));
let shared_active = active_state.clone();
let root_agent = agent.clone();
let user = user_id.map(str::to_owned);
let session = session_id.to_owned();
let conversation = conversation_id.clone();
let selector_map = descriptor
.capabilities
.get("selector_map")
.and_then(Value::as_object)
.cloned()
.unwrap_or_default();
let initial_tools = descriptor
.capabilities
.get("selector_tools")
.and_then(Value::as_array)
.cloned()
.unwrap_or_default();
let available_tools = initial_tools.clone();
tokio::spawn(async move {
let mut active_agent = root_agent;
let mut initial = realtime_tool_definitions(&active_agent);
if active_agent.tool_choice != ToolChoice::None {
initial.extend(initial_tools.clone());
}
let update = json!({"type":"session.update","session":{"type":"realtime","tools":initial,"tool_choice":active_agent.tool_choice}});
let _ = sink.send(Message::Text(update.to_string().into())).await;
loop {
tokio::select! {
Some(value) = send_rx.recv() => {
if sink.send(Message::Text(value.to_string().into())).await.is_err() { break; }
}
Some(_) = close_rx.recv() => { let _ = sink.close().await; break; }
message = stream.next() => {
let Some(message) = message else { break; };
let value = match message { Ok(Message::Text(text)) => serde_json::from_str::<Value>(&text), Ok(Message::Binary(bytes)) => serde_json::from_slice::<Value>(&bytes), Ok(_) => continue, Err(error) => { let _ = event_tx.send(Err(Error::Realtime(error.to_string()))).await; break; } };
let value = match value { Ok(value) => value, Err(error) => { let _ = event_tx.send(Err(Error::Protocol(error.to_string()))).await; continue; } };
let event_type = value.get("type").and_then(Value::as_str).unwrap_or("");
if event_type == "response.done" {
let calls = realtime_calls(&value);
if !calls.is_empty() {
let mut next_tools: Option<(Vec<Value>, ToolChoice)> = None;
let mut used_hub_tool = false;
for call in calls {
let name = call.get("name").and_then(Value::as_str).unwrap_or("");
let parsed_arguments = call_arguments(&call);
let arguments = parsed_arguments.as_ref().cloned().unwrap_or(Value::Null);
let ctx = self::context(&active_agent,user.as_deref(),&session);
let hook_result = if let Some(hooks) = &active_agent.hooks { hooks.on_tool_call(&ctx,name).await } else { Ok(()) };
let captured = Arc::new(std::sync::Mutex::new(Vec::<Value>::new()));
let capture = captured.clone();
let functions = function_map_with_events(&active_agent, Some(Arc::new(move |event| { capture.lock().unwrap_or_else(|e|e.into_inner()).push(event); })));
let result = if let Err(error) = parsed_arguments {
Err(error)
} else if let Err(error) = hook_result { Err(error)
} else if active_agent.tool_choice == ToolChoice::None {
Err(Error::Tool("Agent tools are disabled".into()))
} else if let Some(selector) = selector_map.get(name) {
match &active_agent.hub_tools {
Some(connection) => {
let selected = selected_capabilities(connection, selector);
let tools = selected.iter().map(realtime_capability).collect::<Vec<_>>();
if !tools.is_empty() {
next_tools = Some((tools, ToolChoice::Required));
}
Ok(json!({
"status":"completed",
"selector_type":selector.get("selector_type"),
"identifier":selector.get("identifier"),
"capabilities":selected,
}))
}
None => Err(Error::Realtime("the Agent has no Marona Hub connection".into())),
}
} else if let Some(AgentTool::Delegation { agent, options, .. }) = active_agent
.tools
.iter()
.find(|value| matches!(value, AgentTool::Delegation { name: value, .. } if value == name))
{
let mut target = (**agent).clone();
target.input_schema = options.input_schema.clone(); target.output_schema = options.output_schema.clone();
target.timeout_ms = options.timeout_ms;
let delegated_input = if target.input_schema.is_some() { Input::Message(arguments.clone()) } else { delegated_input(&arguments) };
let started = RealtimeEvent {
event_type: "agent.delegation.started".into(),
output: Value::Null,
delta: None,
data: Map::from_iter([
("source_agent".into(), json!(active_agent.name)),
("target_agent".into(), json!(target.name)),
]),
};
let _ = event_tx.send(Ok(started)).await;
match Runner::run(&target, delegated_input, user.as_deref(), Some(&session)).await {
Ok(value) => {
let completed = RealtimeEvent {
event_type: "agent.delegation.completed".into(),
output: value.output.clone(),
delta: None,
data: Map::from_iter([
("source_agent".into(), json!(active_agent.name)),
("target_agent".into(), json!(target.name)),
]),
};
let _ = event_tx.send(Ok(completed)).await;
Ok(json!({"status":"completed","agent":target.name,"output":value.output}))
}
Err(error) => Err(error),
}
} else if let Some(transfer) = active_agent.handoffs.iter().find(|value| value.name == name) {
let mut target = (*transfer.agent).clone();
if transfer.voice_transition == "preserve" { target.voice = active_agent.voice.clone(); }
match prepare_handoff(transfer, arguments.clone(), user.as_deref(), &session).await {
Err(error) => Err(error),
Ok(input) => {
let source = active_agent.name.clone();
if let Some(hooks) = &active_agent.hooks { let _ = hooks.on_handoff(&self::context(&active_agent,user.as_deref(),&session), &target.name).await; }
active_agent = target;
*shared_active.write().await = active_agent.clone();
let tools = realtime_tool_definitions(&active_agent);
next_tools = Some((tools, active_agent.tool_choice));
let completed = RealtimeEvent {
event_type: "agent.handoff.completed".into(),
output: Value::Null,
delta: None,
data: Map::from_iter([
("source_agent".into(), json!(source)),
("target_agent".into(), json!(active_agent.name)),
]),
};
let _ = event_tx.send(Ok(completed)).await;
Ok(json!({"status":"completed","active_agent":active_agent.name,"handoff_input":input}))
}
}
} else if let Some((tool, owner)) = functions.get(name) {
tool.invoke(
arguments,
tool_context(owner, user.as_deref(), &session),
)
.await
.map(|value| json!({"status":"completed","tool_name":name,"result":value}))
} else {
used_hub_tool = true;
match &active_agent.hub_tools {
Some(connection) => connection
.call_tool(name, arguments, user.as_deref(), Some(&conversation))
.await
.map(|value| json!({"status":"completed","capability":name,"result":value})),
None => Err(Error::Realtime(format!("tool '{name}' is unavailable"))),
}
};
let output = match result {
Ok(value) => value,
Err(error) => { if let Some(hooks) = &active_agent.hooks { let _ = hooks.on_error(&ctx, &error).await; } json!({"status":"failed","error":error.to_string()}) },
};
let peer_events = std::mem::take(&mut *captured.lock().unwrap_or_else(|e|e.into_inner()));
for value in peer_events {
let event = RealtimeEvent { event_type:value["type"].as_str().unwrap_or("a2a.task.status").into(), delta:None, output:value.get("output").cloned().unwrap_or(Value::Null), data:value.as_object().cloned().unwrap_or_default() };
let _ = event_tx.send(Ok(event)).await;
}
let lifecycle = RealtimeEvent {
event_type: if output.get("status").and_then(Value::as_str)
== Some("failed")
{
"tool.call.failed".into()
} else {
"tool.call.completed".into()
},
output: output.clone(),
delta: None,
data: Map::from_iter([(
"tool_name".into(),
json!(name),
)]),
};
let _ = event_tx.send(Ok(lifecycle)).await;
let event = json!({"type":"conversation.item.create","item":{"type":"function_call_output","call_id":call.get("call_id"),"output":output.to_string()}});
if sink.send(Message::Text(event.to_string().into())).await.is_err() {
break;
}
}
if next_tools.is_none() && used_hub_tool && !initial_tools.is_empty() {
next_tools = Some((initial_tools.clone(), ToolChoice::Auto));
}
if let Some((tools, choice)) = next_tools {
let mut update = json!({"type":"session.update","session":{"type":"realtime","instructions":resolve_instructions(&active_agent, &self::context(&active_agent,user.as_deref(),&session)),"tools":tools,"tool_choice":choice}});
if let Some(voice) = active_agent.voice.as_ref().and_then(|v|v.output_voice.as_ref()) { update["session"]["audio"]["output"]["voice"] = json!(voice); }
let _ = sink.send(Message::Text(update.to_string().into())).await;
}
let _ = sink.send(Message::Text(json!({"type":"response.create","response":{"tool_choice":"auto"}}).to_string().into())).await;
continue;
}
}
if let Some(mut event) = normalize_event(value) {
if event.event_type == "response.completed" {
let ctx = self::context(&active_agent,user.as_deref(),&session);
match validate_output(&active_agent,event.output,&ctx).await {
Ok(output) => event.output = output,
Err(error) => { if let Some(hooks) = &active_agent.hooks { let _ = hooks.on_error(&ctx,&error).await; } let _ = event_tx.send(Err(error)).await; continue; }
}
if let Some(hooks) = &active_agent.hooks {
let result = crate::RunResult { output:event.output.clone(), last_agent:active_agent.name.clone(), active_agent:active_agent.name.clone(), agent_runs:vec![], usage:Map::new(), routing:Map::new(), trace_id:None, a2a_events:vec![], response:crate::RuntimeResponse {status:"completed".into(),output:event.output.clone(),..Default::default()} };
if let Err(error) = hooks.on_complete(&ctx,&result).await { let _ = event_tx.send(Err(error)).await; continue; }
}
}
let _ = event_tx.send(Ok(event)).await;
}
}
}
}
});
Ok(RealtimeSession {
model: descriptor.model,
conversation_id: descriptor.conversation_id,
available_tools,
sender: send_tx,
receiver: Mutex::new(event_rx),
close: close_tx,
active_agent: active_state,
user_id: user_id.map(str::to_owned),
session_id: session_id.into(),
closed: AtomicBool::new(false),
})
}
}
fn context(agent: &Agent, user: Option<&str>, session: &str) -> AgentContext {
AgentContext {
user_id: user.map(str::to_owned),
session_id: session.into(),
agent_name: agent.name.clone(),
metadata: agent.metadata.clone(),
}
}
fn realtime_calls(value: &Value) -> Vec<Value> {
value
.pointer("/response/output")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter(|item| item.get("type").and_then(Value::as_str) == Some("function_call"))
.cloned()
.collect()
}
fn call_arguments(call: &Value) -> Result<Value> {
let value = match call.get("arguments") {
Some(Value::String(value)) => serde_json::from_str(value)
.map_err(|_| Error::Validation("invalid function arguments JSON".into()))?,
Some(value) => value.clone(),
None => json!({}),
};
if !value.is_object() {
return Err(Error::Validation(
"function arguments must be an object".into(),
));
}
Ok(value)
}
fn delegated_input(arguments: &Value) -> Input {
for key in ["input", "request", "task", "message"] {
if let Some(value) = arguments.get(key).and_then(Value::as_str) {
if !value.trim().is_empty() {
return Input::Text(value.trim().to_owned());
}
}
}
Input::Text(arguments.to_string())
}
async fn prepare_handoff(
transfer: &crate::Handoff,
arguments: Value,
user: Option<&str>,
session: &str,
) -> Result<Value> {
if let Some(schema) = &transfer.input_schema {
crate::schema::validate(&arguments, schema, "Handoff input")?;
}
let input = if let Some(filter) = &transfer.input_filter {
filter(arguments)?
} else {
arguments
};
let value = if transfer.agent.input_schema.is_some() {
Input::Message(input.clone())
} else {
delegated_input(&input)
};
prepare_input(
&transfer.agent,
value,
&context(&transfer.agent, user, session),
)
.await?;
Ok(input)
}
fn selected_capabilities(connection: &crate::McpConnection, selector: &Value) -> Vec<Value> {
let selector_type = selector
.get("selector_type")
.and_then(Value::as_str)
.unwrap_or("");
let identifier = selector
.get("identifier")
.and_then(Value::as_str)
.unwrap_or("");
connection
.list_capabilities()
.into_iter()
.filter(|tool| {
let annotations = tool.get("annotations").unwrap_or(&Value::Null);
let capability_type = annotations
.get("type")
.and_then(Value::as_str)
.or_else(|| annotations.get("app_slug").map(|_| "app"))
.or_else(|| annotations.get("skill_name").map(|_| "skill"));
capability_type == Some(selector_type)
&& match selector_type {
"app" => {
annotations.get("app_slug").and_then(Value::as_str) == Some(identifier)
}
"skill" => {
annotations.get("skill_name").and_then(Value::as_str)
== Some(identifier)
}
_ => false,
}
})
.collect()
}
fn realtime_capability(value: &Value) -> Value {
json!({
"type":"function",
"name":value.get("name"),
"description":value.get("description"),
"parameters":value.get("inputSchema").cloned().unwrap_or_else(|| json!({"type":"object","properties":{}})),
})
}
fn normalize_event(value: Value) -> Option<RealtimeEvent> {
let raw = value.get("type").and_then(Value::as_str)?;
let event_type = match raw {
"session.created" => "session.connected",
"session.updated" => "session.updated",
"input_audio_buffer.speech_started" => "input_audio.started",
"input_audio_buffer.speech_stopped" => "input_audio.stopped",
"input_audio_buffer.committed" => "input_audio.committed",
"response.output_audio.delta" => "output_audio.delta",
"response.output_audio_transcript.delta" => "transcript.delta",
"response.output_text.delta" => "output_text.delta",
"response.done" => "response.completed",
"error" => "error",
_ => return None,
}
.to_owned();
let output = if raw == "response.done" {
let parts: Vec<Value> = value
.pointer("/response/output")
.and_then(Value::as_array)
.into_iter()
.flatten()
.flat_map(|item| {
item.get("content")
.and_then(Value::as_array)
.into_iter()
.flatten()
})
.cloned()
.collect();
let texts: Vec<&str> = parts
.iter()
.filter_map(|part| part.get("text").and_then(Value::as_str))
.collect();
if !texts.is_empty() {
json!(texts.join(""))
} else {
Value::Array(parts)
}
} else {
Value::Null
};
Some(RealtimeEvent {
event_type,
delta: value
.get("delta")
.and_then(Value::as_str)
.map(str::to_owned),
output,
data: value.as_object().cloned().unwrap_or_default(),
})
}
}
#[cfg(feature = "realtime")]
pub use enabled::*;
#[cfg(not(feature = "realtime"))]
pub struct RealtimeRunner;