pub mod actions;
pub mod media;
use crate::server::connection::ConnectionId;
use crate::utils::clock::Instant;
use anyhow::Result;
use std::collections::VecDeque;
use std::net::SocketAddr;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use tokio::net::UdpSocket;
use tokio::sync::mpsc;
use tracing::{error, trace};
use crate::llm::action_helper::call_llm;
use crate::llm::actions::protocol_trait::Protocol;
use crate::llm::event_handler_executor::EventHandlerResult;
use crate::llm::ollama_client::OllamaClient;
use crate::logging::emit::Log;
use crate::protocol::{Event, StartupParams};
use crate::scripting::EventHandlerType;
use crate::server::RtpProtocol;
use crate::state::app_state::AppState;
use actions::{RTCP_RECEIVED_EVENT, RTP_RECEIVED_EVENT};
use media::{AudioCodec, RtpPacketizer};
pub const DEFAULT_LLM_MAX_PER_MINUTE: u32 = 30;
#[derive(Debug)]
pub struct RtpLlmBudget {
max_per_minute: u32,
window: Duration,
hits: VecDeque<Instant>,
suppressed: u64,
}
impl RtpLlmBudget {
pub fn new(max_per_minute: u32) -> Self {
Self {
max_per_minute,
window: Duration::from_secs(60),
hits: VecDeque::new(),
suppressed: 0,
}
}
pub fn max_per_minute(&self) -> u32 {
self.max_per_minute
}
pub fn try_take(&mut self) -> bool {
self.try_take_at(Instant::now())
}
pub fn try_take_at(&mut self, now: Instant) -> bool {
if self.max_per_minute == 0 {
return false;
}
while let Some(front) = self.hits.front() {
if now.duration_since(*front) >= self.window {
self.hits.pop_front();
} else {
break;
}
}
if self.hits.len() as u32 >= self.max_per_minute {
return false;
}
self.hits.push_back(now);
true
}
pub fn note_refusal(&mut self) -> Option<u64> {
self.suppressed += 1;
if self.suppressed == 1 {
Some(0)
} else {
None
}
}
pub fn take_suppressed(&mut self) -> u64 {
std::mem::take(&mut self.suppressed)
}
}
#[derive(Debug, Clone)]
pub struct RtpConfig {
pub llm_max_per_minute: u32,
}
impl Default for RtpConfig {
fn default() -> Self {
Self {
llm_max_per_minute: DEFAULT_LLM_MAX_PER_MINUTE,
}
}
}
impl RtpConfig {
pub fn from_params(params: &Option<StartupParams>) -> Result<Self> {
let mut cfg = Self::default();
let Some(params) = params else {
return Ok(cfg);
};
if let Some(v) = params.get_optional_u64("llm_max_per_minute")? {
if v > u32::MAX as u64 {
anyhow::bail!("llm_max_per_minute {v} is out of range");
}
cfg.llm_max_per_minute = v as u32;
}
Ok(cfg)
}
}
pub struct RtpServer;
impl RtpServer {
pub async fn spawn_with_llm_actions(
listen_addr: SocketAddr,
llm_client: OllamaClient,
app_state: Arc<AppState>,
status_tx: mpsc::UnboundedSender<String>,
server_id: crate::state::ServerId,
cfg: RtpConfig,
) -> Result<SocketAddr> {
let socket = Arc::new(UdpSocket::bind(listen_addr).await?);
let local_addr = socket.local_addr()?;
Log::new(Some(&status_tx)).info(format!(
"RTP server listening on {} (llm_max_per_minute={})",
local_addr, cfg.llm_max_per_minute
));
let protocol = Arc::new(RtpProtocol::new());
let task_registrar = app_state.clone();
let budget = Arc::new(Mutex::new(RtpLlmBudget::new(cfg.llm_max_per_minute)));
let accept_handle = tokio::spawn(async move {
let mut buffer = vec![0u8; 65535];
loop {
match socket.recv_from(&mut buffer).await {
Ok((n, peer_addr)) => {
let data = buffer[..n].to_vec();
let connection_id =
Self::track_peer(&app_state, server_id, peer_addr, local_addr, n).await;
if connection_id.is_none() {
trace!("RTP datagram for a server that is no longer registered");
continue;
}
let connection_id = connection_id.expect("checked is_none above");
let llm_clone = llm_client.clone();
let state_clone = app_state.clone();
let status_clone = status_tx.clone();
let socket_clone = socket.clone();
let protocol_clone = protocol.clone();
let budget_clone = budget.clone();
let task_owner = app_state.clone();
task_owner
.spawn_server_task(server_id, async move {
Self::handle_datagram(
&data,
peer_addr,
local_addr,
connection_id,
server_id,
&llm_clone,
&state_clone,
&status_clone,
&socket_clone,
protocol_clone.as_ref(),
&budget_clone,
)
.await;
})
.await;
}
Err(e) => {
Log::new(Some(&status_tx)).error(format!("RTP recv error: {}", e));
}
}
}
});
task_registrar
.register_server_task(server_id, accept_handle)
.await;
Ok(local_addr)
}
async fn track_peer(
state: &AppState,
server_id: crate::state::ServerId,
peer_addr: SocketAddr,
local_addr: SocketAddr,
bytes: usize,
) -> Option<ConnectionId> {
let existing = state
.with_server_mut(server_id, |s| {
s.connections
.values_mut()
.find(|c| c.remote_addr == peer_addr)
.map(|c| {
c.bytes_received += bytes as u64;
c.packets_received += 1;
c.last_activity = crate::utils::clock::Instant::now();
c.id
})
})
.await;
match existing {
None => None,
Some(Some(id)) => Some(id),
Some(None) => {
use crate::state::server::{
ConnectionState as ServerConnectionState, ConnectionStatus,
ProtocolConnectionInfo,
};
let id = ConnectionId::new(state.get_next_unified_id().await);
let now = crate::utils::clock::Instant::now();
state
.add_connection_to_server(
server_id,
ServerConnectionState {
id,
remote_addr: peer_addr,
local_addr,
bytes_sent: 0,
bytes_received: bytes as u64,
packets_sent: 0,
packets_received: 1,
last_activity: now,
status: ConnectionStatus::Active,
status_changed_at: now,
protocol_info: ProtocolConnectionInfo::empty(),
},
)
.await;
Some(id)
}
}
}
pub async fn configured_handler_kind(
state: &AppState,
server_id: crate::state::ServerId,
event_type_id: &str,
) -> Option<&'static str> {
let config = state.get_event_handler_config(server_id).await?;
match config.find_handler(event_type_id)? {
EventHandlerType::Script { .. } => Some("script"),
EventHandlerType::Static { .. } => Some("static"),
EventHandlerType::Manual { .. } => Some("manual"),
EventHandlerType::Llm { .. } => Some("llm"),
}
}
#[allow(clippy::too_many_arguments)]
async fn handle_datagram(
data: &[u8],
peer_addr: SocketAddr,
local_addr: SocketAddr,
connection_id: ConnectionId,
server_id: crate::state::ServerId,
llm: &OllamaClient,
state: &AppState,
status_tx: &mpsc::UnboundedSender<String>,
socket: &UdpSocket,
protocol: &RtpProtocol,
budget: &Mutex<RtpLlmBudget>,
) {
let is_rtcp = media::is_rtcp(data);
let base = serde_json::json!({
"peer_addr": peer_addr.to_string(),
"local_addr": local_addr.to_string(),
"connection_id": connection_id.to_string(),
});
let event = if is_rtcp {
let mut d = base;
d["packet_type"] = serde_json::json!(data.get(1).copied().unwrap_or(0));
d["length"] = serde_json::json!(data.len());
Event {
event_type: &RTCP_RECEIVED_EVENT,
data: d,
}
} else {
let parsed = match media::parse_rtp(data) {
Some(p) => p,
None => {
Log::new(Some(status_tx)).warn(format!(
"RTP unparseable datagram from {} ({} bytes)",
peer_addr,
data.len()
));
return;
}
};
Log::new(Some(status_tx)).trace(format!(
"RTP in: pt={} seq={} ts={} ssrc={:08x} len={}",
parsed.payload_type,
parsed.sequence,
parsed.timestamp,
parsed.ssrc,
parsed.payload_len
));
let mut d = base;
d["payload_type"] = serde_json::json!(parsed.payload_type);
d["sequence"] = serde_json::json!(parsed.sequence);
d["timestamp"] = serde_json::json!(parsed.timestamp);
d["ssrc"] = serde_json::json!(parsed.ssrc);
d["marker"] = serde_json::json!(parsed.marker);
d["payload_len"] = serde_json::json!(parsed.payload_len);
Event {
event_type: &RTP_RECEIVED_EVENT,
data: d,
}
};
let event_id = event.event_type.id.clone();
let handler_outcome = crate::llm::event_handler_executor::try_execute_event_handler(
state,
server_id,
Some(connection_id),
&event_id,
&event.event_type.description,
Some(event.data.clone()),
Some(protocol),
)
.await;
let (result, answered_by) = match handler_outcome {
Ok(EventHandlerResult::Handled(r)) => {
crate::pipe::dispatch_pipes(state, server_id, &event).await;
state
.record_access_log(
crate::state::AccessLogOwner::Server(server_id.as_u32()),
protocol.protocol_name(),
Some(connection_id.as_u32()),
&event_id,
event.data.clone(),
r.access_log_actions(),
)
.await;
(r, "handler")
}
Err(e) => {
Log::new(Some(status_tx)).error(format!(
"RTP {} from {} decision=fail_closed_handler_error; no media sent: {}",
event_id, peer_addr, e
));
return;
}
Ok(EventHandlerResult::FallbackToLlm { .. }) => {
let refused = {
let mut b = budget.lock().expect("RTP budget mutex poisoned");
if b.try_take() {
None
} else {
let report_now = b.note_refusal().is_some();
Some((b.max_per_minute(), report_now))
}
};
if let Some((max, report_now)) = refused {
let why = match Self::configured_handler_kind(state, server_id, &event_id).await
{
Some(kind @ ("script" | "static" | "manual")) => format!(
" (the {kind} rule for this event did not answer — an unknown \
language, a missing interpreter or a failing script falls \
through to the model, which is charged)"
),
_ => String::new(),
};
let line = format!(
"RTP {} from {} decision=fail_closed_rate_limited; no media sent \
(already used the {} model consultation(s) this minute allows — raise \
llm_max_per_minute, or answer this event with a script/static handler, \
which is never charged){}",
event_id, peer_addr, max, why
);
if report_now {
Log::new(Some(status_tx)).warn(line);
} else {
trace!("{line}");
}
return;
}
let suppressed = {
let mut b = budget.lock().expect("RTP budget mutex poisoned");
b.take_suppressed()
};
if suppressed > 1 {
Log::new(Some(status_tx)).warn(format!(
"RTP dropped {} further datagram(s) over the model budget before this one",
suppressed - 1
));
}
match call_llm(llm, state, server_id, Some(connection_id), &event, protocol).await {
Ok(r) => (r, "model"),
Err(e) => {
let decision = if crate::utils::WireFailure::classify(&e).is_overloaded() {
"fail_closed_backend_overloaded"
} else {
"fail_closed_backend_error"
};
Log::new(Some(status_tx)).error(format!(
"RTP {} from {} decision={}; no media sent: {}",
event_id, peer_addr, decision, e
));
return;
}
}
}
};
if result.raw_actions.is_empty() {
Log::new(Some(status_tx)).info(format!(
"RTP {} from {} decision={}_sent_nothing (no media requested)",
event_id, peer_addr, answered_by
));
}
for action in &result.raw_actions {
Self::execute_send_action(action, peer_addr, socket, status_tx, state, server_id).await;
}
}
async fn execute_send_action(
action: &serde_json::Value,
peer_addr: SocketAddr,
socket: &UdpSocket,
status_tx: &mpsc::UnboundedSender<String>,
state: &AppState,
server_id: crate::state::ServerId,
) {
let action_type = action.get("type").and_then(|v| v.as_str()).unwrap_or("");
match action_type {
"send_rtp_audio" => {
Self::send_rtp_audio(action, peer_addr, socket, status_tx, state, server_id).await;
}
"send_rtcp_sender_report" => {
Self::send_rtcp(action, peer_addr, socket, status_tx).await;
}
"" => {
Log::new(Some(status_tx)).warn(format!(
"RTP action with no \"type\" field, ignored: {}",
action
));
}
other => {
Log::new(Some(status_tx)).debug(format!(
"RTP no wire output for action type \"{}\" (handled by the shared executor \
if it is a common action)",
other
));
}
}
}
async fn send_rtp_audio(
action: &serde_json::Value,
peer_addr: SocketAddr,
socket: &UdpSocket,
status_tx: &mpsc::UnboundedSender<String>,
state: &AppState,
server_id: crate::state::ServerId,
) {
let codec = match action
.get("payload_type")
.and_then(|v| v.as_str())
.map(AudioCodec::parse)
.unwrap_or(Ok(AudioCodec::Pcmu))
{
Ok(c) => c,
Err(e) => {
Log::new(Some(status_tx)).warn(format!("RTP send_rtp_audio: {}", e));
return;
}
};
let content = match media::parse_audio_content(action) {
Ok(c) => c,
Err(e) => {
Log::new(Some(status_tx)).warn(format!("RTP send_rtp_audio content: {}", e));
return;
}
};
let duration_ms = action
.get("duration_ms")
.and_then(|v| v.as_u64())
.unwrap_or(1000);
let payload = match media::synthesize(codec, &content, duration_ms) {
Ok(p) => p,
Err(e) => {
Log::new(Some(status_tx)).warn(format!("RTP synthesis: {}", e));
return;
}
};
let ssrc = action
.get("ssrc")
.and_then(|v| v.as_u64())
.map(|v| v as u32)
.unwrap_or_else(|| rand::random());
let seq = action
.get("start_sequence")
.and_then(|v| v.as_u64())
.map(|v| v as u16);
let ts = action
.get("start_timestamp")
.and_then(|v| v.as_u64())
.map(|v| v as u32);
let mut packetizer = RtpPacketizer::new(ssrc, codec.payload_type(), seq, ts);
let packets = packetizer.packetize(&payload, media::G711_SAMPLES_PER_FRAME);
let mut sent = 0u64;
let mut bytes = 0u64;
for pkt in &packets {
match socket.send_to(pkt, peer_addr).await {
Ok(w) => {
sent += 1;
bytes += w as u64;
}
Err(e) => {
error!("RTP send failed to {}: {}", peer_addr, e);
break;
}
}
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
}
state
.with_server_mut(server_id, |s| {
if let Some(c) = s
.connections
.values_mut()
.find(|c| c.remote_addr == peer_addr)
{
c.bytes_sent += bytes;
c.packets_sent += sent;
}
})
.await;
Log::new(Some(status_tx)).debug(format!(
"RTP sent {} {} packet(s) ({} bytes) to {}",
sent,
codec.rtpmap_name(),
bytes,
peer_addr
));
}
async fn send_rtcp(
action: &serde_json::Value,
peer_addr: SocketAddr,
socket: &UdpSocket,
status_tx: &mpsc::UnboundedSender<String>,
) {
let ssrc = action
.get("ssrc")
.and_then(|v| v.as_u64())
.map(|v| v as u32)
.unwrap_or_else(rand::random);
let rtp_ts = action
.get("rtp_timestamp")
.and_then(|v| v.as_u64())
.unwrap_or(0) as u32;
let packet_count = action
.get("packet_count")
.and_then(|v| v.as_u64())
.unwrap_or(0) as u32;
let octet_count = action
.get("octet_count")
.and_then(|v| v.as_u64())
.unwrap_or(0) as u32;
let sr = media::build_rtcp_sender_report(ssrc, rtp_ts, packet_count, octet_count);
match socket.send_to(&sr, peer_addr).await {
Ok(w) => {
Log::new(Some(status_tx)).debug(format!("RTCP SR to {} ({} bytes)", peer_addr, w));
}
Err(e) => {
Log::new(Some(status_tx)).error(format!("RTCP send failed: {}", e));
}
}
}
}