#[cfg(feature = "realtime")]
mod enabled {
use base64::Engine;
use futures_util::{SinkExt, StreamExt};
use reqwest::Method;
use serde::{Deserialize, Serialize};
use serde_json::{json, Map, Value};
use tokio::sync::{mpsc, Mutex};
use tokio_tungstenite::{
connect_async,
tungstenite::{client::IntoClientRequest, http::HeaderValue, Message},
};
use crate::{
agent::{
function_map, graph_value, realtime_tool_definitions, tool_context, Agent, AgentTool,
Runner,
},
Error, Input, Result, ToolChoice,
};
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct RealtimeEvent {
#[serde(rename = "type")]
pub event_type: String,
pub delta: Option<String>,
#[serde(default)]
pub output: Value,
#[serde(default)]
pub data: Map<String, Value>,
}
#[derive(Debug, Clone, Deserialize)]
struct Descriptor {
model: String,
endpoint: String,
client_secret: String,
conversation_id: Option<String>,
#[serde(default)]
capabilities: Value,
}
pub struct RealtimeSession {
pub model: String,
pub conversation_id: Option<String>,
pub available_tools: Vec<Value>,
sender: mpsc::Sender<Value>,
receiver: Mutex<mpsc::Receiver<Result<RealtimeEvent>>>,
close: mpsc::Sender<()>,
}
impl RealtimeSession {
pub async fn send_message(&self, input: impl Into<Input>) -> Result<()> {
let value = match input.into() {
Input::Text(text) => {
json!({"type":"conversation.item.create","item":{"type":"message","role":"user","content":[{"type":"input_text","text":text}]}})
}
Input::Message(value) => json!({"type":"conversation.item.create","item":value}),
Input::Messages(values) => json!({"type":"marona.input.create","items":values}),
};
self.sender
.send(value)
.await
.map_err(|_| Error::Realtime("session is closed".into()))?;
self.sender
.send(json!({"type":"response.create"}))
.await
.map_err(|_| Error::Realtime("session is closed".into()))
}
pub async fn send_audio(&self, bytes: &[u8]) -> Result<()> {
self.sender.send(json!({"type":"input_audio_buffer.append","audio":base64::engine::general_purpose::STANDARD.encode(bytes)})).await.map_err(|_| Error::Realtime("session is closed".into()))
}
pub async fn next_event(&self) -> Option<Result<RealtimeEvent>> {
self.receiver.lock().await.recv().await
}
pub async fn close(&self) -> Result<()> {
self.close
.send(())
.await
.map_err(|_| Error::Realtime("session is already closed".into()))
}
}
pub struct RealtimeRunner;
impl RealtimeRunner {
pub async fn connect(
agent: &Agent,
user_id: Option<&str>,
session_id: Option<&str>,
) -> Result<RealtimeSession> {
let unsupported = agent
.output_modalities
.iter()
.filter(|value| !matches!(value.as_str(), "text" | "audio"))
.cloned()
.collect::<Vec<_>>();
if !unsupported.is_empty() {
return Err(Error::Validation(format!(
"realtime output does not allow: {}",
unsupported.join(", ")
)));
}
let session_id = session_id.unwrap_or("default");
let conversation_id = format!("rust:{}:{}", user_id.unwrap_or("anonymous"), session_id);
let body = json!({
"interface":"api", "user_id":user_id, "conversation_id":conversation_id,
"mode":if agent.modalities.iter().any(|value| matches!(value.as_str(), "image" | "video")){"voice_vision"}else{"voice"},
"model":agent.model, "voice":agent.voice.as_ref().and_then(|value| value.output_voice.as_deref()),
"instructions":agent.instructions, "output_modalities":agent.output_modalities,
"transport":"websocket", "metadata":{
"voice":agent.voice, "routing":agent.routing, "required_transport":"realtime",
"marona_agent":{"input_modalities":agent.modalities,"output_modalities":agent.output_modalities,"tools_enabled":agent.tool_choice != ToolChoice::None && (agent.hub_tools.is_some() || !agent.tools.is_empty() || !agent.peers.is_empty() || !agent.handoffs.is_empty()),"tool_choice":agent.tool_choice},
"marona_connection_scope":agent.hub_tools.as_ref().map(|value| value.session()),
"marona_multi_agent":graph_value(agent),
}
});
let descriptor: Descriptor = agent
.marona
.core
.json(
&agent.marona.core.runtime_url,
Method::POST,
"/runtime/realtime/session",
Some(&body),
)
.await?;
if descriptor.endpoint.is_empty() || descriptor.client_secret.is_empty() {
return Err(Error::Realtime(
"runtime returned incomplete session credentials".into(),
));
}
let mut request = descriptor
.endpoint
.clone()
.into_client_request()
.map_err(|e| Error::Realtime(e.to_string()))?;
request.headers_mut().insert(
"Authorization",
HeaderValue::from_str(&format!("Bearer {}", descriptor.client_secret))
.map_err(|e| Error::Realtime(e.to_string()))?,
);
let (socket, _) = connect_async(request)
.await
.map_err(|e| Error::Realtime(e.to_string()))?;
let (mut sink, mut stream) = socket.split();
let (send_tx, mut send_rx) = mpsc::channel::<Value>(64);
let (event_tx, event_rx) = mpsc::channel::<Result<RealtimeEvent>>(64);
let (close_tx, mut close_rx) = mpsc::channel::<()>(1);
let connection = agent.hub_tools.clone();
let root_agent = agent.clone();
let user = user_id.map(str::to_owned);
let session = session_id.to_owned();
let conversation = conversation_id.clone();
let selector_map = descriptor
.capabilities
.get("selector_map")
.and_then(Value::as_object)
.cloned()
.unwrap_or_default();
let initial_tools = descriptor
.capabilities
.get("selector_tools")
.and_then(Value::as_array)
.cloned()
.unwrap_or_default();
let available_tools = initial_tools.clone();
tokio::spawn(async move {
let mut active_agent = root_agent;
loop {
tokio::select! {
Some(value) = send_rx.recv() => {
if sink.send(Message::Text(value.to_string().into())).await.is_err() { break; }
}
Some(_) = close_rx.recv() => { let _ = sink.close().await; break; }
message = stream.next() => {
let Some(message) = message else { break; };
let value = match message { Ok(Message::Text(text)) => serde_json::from_str::<Value>(&text), Ok(Message::Binary(bytes)) => serde_json::from_slice::<Value>(&bytes), Ok(_) => continue, Err(error) => { let _ = event_tx.send(Err(Error::Realtime(error.to_string()))).await; break; } };
let value = match value { Ok(value) => value, Err(error) => { let _ = event_tx.send(Err(Error::Protocol(error.to_string()))).await; continue; } };
let event_type = value.get("type").and_then(Value::as_str).unwrap_or("");
if event_type == "response.done" {
let calls = realtime_calls(&value);
if !calls.is_empty() {
let mut next_tools: Option<(Vec<Value>, ToolChoice)> = None;
let mut used_hub_tool = false;
for call in calls {
let name = call.get("name").and_then(Value::as_str).unwrap_or("");
let arguments = call_arguments(&call);
let result = if let Some(selector) = selector_map.get(name) {
match &connection {
Some(connection) => {
let selected = selected_capabilities(connection, selector);
let tools = selected.iter().map(realtime_capability).collect::<Vec<_>>();
if !tools.is_empty() {
next_tools = Some((tools, ToolChoice::Required));
}
Ok(json!({
"status":"completed",
"selector_type":selector.get("selector_type"),
"identifier":selector.get("identifier"),
"capabilities":selected,
}))
}
None => Err(Error::Realtime("the Agent has no Marona Hub connection".into())),
}
} else if let Some(AgentTool::Delegation { agent, .. }) = active_agent
.tools
.iter()
.find(|value| matches!(value, AgentTool::Delegation { name: value, .. } if value == name))
{
let target = agent.clone();
let delegated_input = delegated_input(&arguments);
let started = RealtimeEvent {
event_type: "agent.delegation.started".into(),
output: Value::Null,
delta: None,
data: Map::from_iter([
("source_agent".into(), json!(active_agent.name)),
("target_agent".into(), json!(target.name)),
]),
};
let _ = event_tx.send(Ok(started)).await;
match Runner::run(&target, delegated_input, user.as_deref(), Some(&session)).await {
Ok(value) => {
let completed = RealtimeEvent {
event_type: "agent.delegation.completed".into(),
output: value.output.clone(),
delta: None,
data: Map::from_iter([
("source_agent".into(), json!(active_agent.name)),
("target_agent".into(), json!(target.name)),
]),
};
let _ = event_tx.send(Ok(completed)).await;
Ok(json!({"status":"completed","agent":target.name,"output":value.output}))
}
Err(error) => Err(error),
}
} else if let Some(transfer) = active_agent.handoffs.iter().find(|value| value.name == name) {
let source = active_agent.name.clone();
active_agent = (*transfer.agent).clone();
let tools = realtime_tool_definitions(&active_agent);
next_tools = Some((tools, active_agent.tool_choice));
let completed = RealtimeEvent {
event_type: "agent.handoff.completed".into(),
output: Value::Null,
delta: None,
data: Map::from_iter([
("source_agent".into(), json!(source)),
("target_agent".into(), json!(active_agent.name)),
]),
};
let _ = event_tx.send(Ok(completed)).await;
Ok(json!({"status":"completed","active_agent":active_agent.name,"handoff_input":arguments}))
} else if let Some((tool, owner)) = function_map(&active_agent).get(name) {
tool.invoke(
arguments,
tool_context(owner, user.as_deref(), &session),
)
.await
.map(|value| json!({"status":"completed","tool_name":name,"result":value}))
} else {
used_hub_tool = true;
match &connection {
Some(connection) => connection
.call_tool(name, arguments, user.as_deref(), Some(&conversation))
.await
.map(|value| json!({"status":"completed","capability":name,"result":value})),
None => Err(Error::Realtime(format!("tool '{name}' is unavailable"))),
}
};
let output = match result {
Ok(value) => value,
Err(error) => json!({"status":"failed","error":error.to_string()}),
};
let lifecycle = RealtimeEvent {
event_type: if output.get("status").and_then(Value::as_str)
== Some("failed")
{
"tool.call.failed".into()
} else {
"tool.call.completed".into()
},
output: output.clone(),
delta: None,
data: Map::from_iter([(
"tool_name".into(),
json!(name),
)]),
};
let _ = event_tx.send(Ok(lifecycle)).await;
let event = json!({"type":"conversation.item.create","item":{"type":"function_call_output","call_id":call.get("call_id"),"output":output.to_string()}});
if sink.send(Message::Text(event.to_string().into())).await.is_err() {
break;
}
}
if next_tools.is_none() && used_hub_tool && !initial_tools.is_empty() {
next_tools = Some((initial_tools.clone(), ToolChoice::Auto));
}
if let Some((tools, choice)) = next_tools {
let update = json!({"type":"session.update","session":{"type":"realtime","instructions":active_agent.instructions,"tools":tools,"tool_choice":choice}});
let _ = sink.send(Message::Text(update.to_string().into())).await;
}
let _ = sink.send(Message::Text(json!({"type":"response.create","response":{"tool_choice":"auto"}}).to_string().into())).await;
continue;
}
}
if let Some(event) = normalize_event(value) { let _ = event_tx.send(Ok(event)).await; }
}
}
}
});
Ok(RealtimeSession {
model: descriptor.model,
conversation_id: descriptor.conversation_id,
available_tools,
sender: send_tx,
receiver: Mutex::new(event_rx),
close: close_tx,
})
}
}
fn realtime_calls(value: &Value) -> Vec<Value> {
value
.pointer("/response/output")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter(|item| item.get("type").and_then(Value::as_str) == Some("function_call"))
.cloned()
.collect()
}
fn call_arguments(call: &Value) -> Value {
match call.get("arguments") {
Some(Value::String(value)) => serde_json::from_str(value).unwrap_or_else(|_| json!({})),
Some(Value::Object(value)) => Value::Object(value.clone()),
_ => json!({}),
}
}
fn delegated_input(arguments: &Value) -> Input {
for key in ["input", "request", "task", "message"] {
if let Some(value) = arguments.get(key).and_then(Value::as_str) {
if !value.trim().is_empty() {
return Input::Text(value.trim().to_owned());
}
}
}
Input::Text(arguments.to_string())
}
fn selected_capabilities(connection: &crate::McpConnection, selector: &Value) -> Vec<Value> {
let selector_type = selector
.get("selector_type")
.and_then(Value::as_str)
.unwrap_or("");
let identifier = selector
.get("identifier")
.and_then(Value::as_str)
.unwrap_or("");
connection
.list_capabilities()
.into_iter()
.filter(|tool| {
let annotations = tool.get("annotations").unwrap_or(&Value::Null);
let capability_type = annotations
.get("type")
.and_then(Value::as_str)
.or_else(|| annotations.get("app_slug").map(|_| "app"))
.or_else(|| annotations.get("skill_name").map(|_| "skill"));
capability_type == Some(selector_type)
&& match selector_type {
"app" => {
annotations.get("app_slug").and_then(Value::as_str) == Some(identifier)
}
"skill" => {
annotations.get("skill_name").and_then(Value::as_str)
== Some(identifier)
}
_ => false,
}
})
.collect()
}
fn realtime_capability(value: &Value) -> Value {
json!({
"type":"function",
"name":value.get("name"),
"description":value.get("description"),
"parameters":value.get("inputSchema").cloned().unwrap_or_else(|| json!({"type":"object","properties":{}})),
})
}
fn normalize_event(value: Value) -> Option<RealtimeEvent> {
let raw = value.get("type").and_then(Value::as_str)?;
let event_type = match raw {
"session.created" => "session.connected",
"session.updated" => "session.updated",
"input_audio_buffer.speech_started" => "input_audio.started",
"input_audio_buffer.speech_stopped" => "input_audio.stopped",
"input_audio_buffer.committed" => "input_audio.committed",
"response.output_audio.delta" => "output_audio.delta",
"response.output_audio_transcript.delta" => "transcript.delta",
"response.output_text.delta" => "output_text.delta",
"response.done" => "response.completed",
"error" => "error",
_ => return None,
}
.to_owned();
let output = if raw == "response.done" {
value
.pointer("/response/output")
.cloned()
.unwrap_or(Value::Null)
} else {
Value::Null
};
Some(RealtimeEvent {
event_type,
delta: value
.get("delta")
.and_then(Value::as_str)
.map(str::to_owned),
output,
data: value.as_object().cloned().unwrap_or_default(),
})
}
}
#[cfg(feature = "realtime")]
pub use enabled::*;
#[cfg(not(feature = "realtime"))]
pub struct RealtimeRunner;