use std::any::Any;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use serde::{Deserialize, Serialize};
use crate::processor::metrics::MetricsData;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Direction {
Downstream,
Upstream,
}
impl Direction {
pub fn flip(self) -> Direction {
match self {
Direction::Downstream => Direction::Upstream,
Direction::Upstream => Direction::Downstream,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum FrameClass {
System,
Data,
Control,
}
static FRAME_ID: AtomicU64 = AtomicU64::new(1);
pub fn next_frame_id() -> u64 {
FRAME_ID.fetch_add(1, Ordering::Relaxed)
}
#[derive(Debug, Clone)]
pub struct FrameMeta {
pub id: u64,
pub name: &'static str,
pub pts: Option<i64>,
pub broadcast_sibling_id: Option<u64>,
pub extra: Option<Arc<serde_json::Map<String, serde_json::Value>>>,
pub transport_source: Option<Arc<str>>,
pub transport_destination: Option<Arc<str>>,
}
impl FrameMeta {
pub fn new(frame: &Frame) -> Self {
Self {
id: next_frame_id(),
name: frame.name(),
pts: None,
broadcast_sibling_id: None,
extra: None,
transport_source: None,
transport_destination: None,
}
}
}
pub trait CustomFrame: Any + Send + Sync + std::fmt::Debug {
fn frame_class(&self) -> FrameClass;
fn uninterruptible(&self) -> bool {
false
}
fn name(&self) -> &'static str {
"Custom"
}
fn as_any(&self) -> &dyn Any;
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AudioFrame {
pub pcm: Vec<i16>,
pub sample_rate: u32,
pub num_channels: u16,
}
impl AudioFrame {
pub fn mono(pcm: Vec<i16>, sample_rate: u32) -> Self {
Self {
pcm,
sample_rate,
num_channels: 1,
}
}
pub fn len(&self) -> usize {
self.pcm.len()
}
pub fn is_empty(&self) -> bool {
self.pcm.is_empty()
}
}
impl From<crate::types::AudioChunk> for AudioFrame {
fn from(c: crate::types::AudioChunk) -> Self {
AudioFrame::mono(c.pcm, c.sample_rate)
}
}
impl From<&crate::types::AudioChunk> for AudioFrame {
fn from(c: &crate::types::AudioChunk) -> Self {
AudioFrame::mono(c.pcm.clone(), c.sample_rate)
}
}
impl From<&AudioFrame> for crate::types::AudioChunk {
fn from(f: &AudioFrame) -> Self {
crate::types::AudioChunk::new(f.pcm.clone(), f.sample_rate)
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct StartParams {
pub audio_in_sample_rate: u32,
pub audio_out_sample_rate: u32,
pub enable_metrics: bool,
pub enable_usage_metrics: bool,
pub enable_tracing: bool,
pub report_only_initial_ttfb: bool,
}
impl Default for StartParams {
fn default() -> Self {
Self {
audio_in_sample_rate: 16_000,
audio_out_sample_rate: 24_000,
enable_metrics: false,
enable_usage_metrics: false,
enable_tracing: false,
report_only_initial_ttfb: true,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum KeypadEntry {
Zero,
One,
Two,
Three,
Four,
Five,
Six,
Seven,
Eight,
Nine,
Star,
Pound,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct Language(pub String);
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum ServiceKind {
Stt,
Tts,
Llm,
Filter,
Mixer,
All,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct VadParams {
pub confidence: f32,
pub start_secs: f32,
pub stop_secs: f32,
pub min_volume: f32,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct TurnParams {
pub stop_secs: f32,
pub pre_speech_ms: f32,
pub max_duration_secs: Option<f32>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct FunctionCall {
pub function_name: String,
pub tool_call_id: String,
pub arguments: serde_json::Value,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct FunctionCallResult {
pub function_name: String,
pub tool_call_id: String,
pub result: serde_json::Value,
}
#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
pub struct LlmContext {
pub messages: Vec<serde_json::Value>,
pub tools: Vec<serde_json::Value>,
}
#[derive(Debug, Clone)]
pub enum Frame {
Start(StartParams),
Cancel {
reason: Option<String>,
},
Error {
message: String,
fatal: bool,
processor: Option<Arc<str>>,
},
Interruption,
InputAudio(Arc<AudioFrame>),
UserAudio {
audio: Arc<AudioFrame>,
user_id: Arc<str>,
},
InputText(String),
InputDtmf(KeypadEntry),
UserStartedSpeaking,
UserStoppedSpeaking,
UserSpeaking,
BotStartedSpeaking,
BotStoppedSpeaking,
BotSpeaking,
VadUserStartedSpeaking {
start_secs: f32,
},
VadUserStoppedSpeaking {
stop_secs: f32,
},
SttMute(bool),
Metrics(Vec<MetricsData>),
TransportMessage {
payload: serde_json::Value,
urgent: bool,
},
ClientConnected,
BotConnected,
FunctionCallsStarted(Vec<FunctionCall>),
FunctionCallInProgress {
call: FunctionCall,
cancel_on_interruption: bool,
},
FunctionCallCancel {
function_name: String,
tool_call_id: String,
},
OutputAudio(Arc<AudioFrame>),
TtsAudio {
audio: Arc<AudioFrame>,
context_id: Option<Arc<str>>,
},
Text(String),
LlmText(String),
Transcription {
text: String,
user_id: Arc<str>,
language: Option<Language>,
final_: bool,
},
InterimTranscription {
text: String,
user_id: Arc<str>,
language: Option<Language>,
},
TtsSpeak {
text: String,
append_to_context: Option<bool>,
},
TtsText {
text: String,
context_id: Option<Arc<str>>,
},
FunctionCallResult(FunctionCallResult),
LlmRun,
LlmContext(Arc<LlmContext>),
OutputDtmf(Vec<KeypadEntry>),
End {
reason: Option<String>,
},
Stop,
LlmResponseStart,
LlmResponseEnd,
TtsStarted {
context_id: Option<Arc<str>>,
},
TtsStopped {
context_id: Option<Arc<str>>,
},
UpdateSettings {
target: ServiceKind,
settings: serde_json::Value,
},
SpeechControlParams {
vad: Option<VadParams>,
turn: Option<TurnParams>,
},
Heartbeat {
timestamp_ns: i64,
},
OutputTransportReady,
FilterEnable(bool),
MixerEnable(bool),
Custom(Arc<dyn CustomFrame>),
}
impl Frame {
pub fn class(&self) -> FrameClass {
use Frame::*;
match self {
Start(_)
| Cancel { .. }
| Error { .. }
| Interruption
| InputAudio(_)
| UserAudio { .. }
| InputText(_)
| InputDtmf(_)
| UserStartedSpeaking
| UserStoppedSpeaking
| UserSpeaking
| BotStartedSpeaking
| BotStoppedSpeaking
| BotSpeaking
| VadUserStartedSpeaking { .. }
| VadUserStoppedSpeaking { .. }
| SttMute(_)
| Metrics(_)
| TransportMessage { .. }
| ClientConnected
| BotConnected
| FunctionCallsStarted(_)
| FunctionCallInProgress { .. }
| FunctionCallCancel { .. } => FrameClass::System,
End { .. }
| Stop
| LlmResponseStart
| LlmResponseEnd
| TtsStarted { .. }
| TtsStopped { .. }
| UpdateSettings { .. }
| SpeechControlParams { .. }
| Heartbeat { .. }
| OutputTransportReady
| FilterEnable(_)
| MixerEnable(_) => FrameClass::Control,
OutputAudio(_)
| TtsAudio { .. }
| Text(_)
| LlmText(_)
| Transcription { .. }
| InterimTranscription { .. }
| TtsSpeak { .. }
| TtsText { .. }
| FunctionCallResult(_)
| LlmRun
| LlmContext(_)
| OutputDtmf(_) => FrameClass::Data,
Custom(c) => c.frame_class(),
}
}
pub fn uninterruptible(&self) -> bool {
use Frame::*;
match self {
End { .. } | Stop | FunctionCallResult(_) | UpdateSettings { .. } => true,
Custom(c) => c.uninterruptible(),
_ => false,
}
}
pub fn name(&self) -> &'static str {
use Frame::*;
match self {
Start(_) => "Start",
Cancel { .. } => "Cancel",
Error { .. } => "Error",
Interruption => "Interruption",
InputAudio(_) => "InputAudio",
UserAudio { .. } => "UserAudio",
InputText(_) => "InputText",
InputDtmf(_) => "InputDtmf",
UserStartedSpeaking => "UserStartedSpeaking",
UserStoppedSpeaking => "UserStoppedSpeaking",
UserSpeaking => "UserSpeaking",
BotStartedSpeaking => "BotStartedSpeaking",
BotStoppedSpeaking => "BotStoppedSpeaking",
BotSpeaking => "BotSpeaking",
VadUserStartedSpeaking { .. } => "VadUserStartedSpeaking",
VadUserStoppedSpeaking { .. } => "VadUserStoppedSpeaking",
SttMute(_) => "SttMute",
Metrics(_) => "Metrics",
TransportMessage { .. } => "TransportMessage",
ClientConnected => "ClientConnected",
BotConnected => "BotConnected",
FunctionCallsStarted(_) => "FunctionCallsStarted",
FunctionCallInProgress { .. } => "FunctionCallInProgress",
FunctionCallCancel { .. } => "FunctionCallCancel",
OutputAudio(_) => "OutputAudio",
TtsAudio { .. } => "TtsAudio",
Text(_) => "Text",
LlmText(_) => "LlmText",
Transcription { .. } => "Transcription",
InterimTranscription { .. } => "InterimTranscription",
TtsSpeak { .. } => "TtsSpeak",
TtsText { .. } => "TtsText",
FunctionCallResult(_) => "FunctionCallResult",
LlmRun => "LlmRun",
LlmContext(_) => "LlmContext",
OutputDtmf(_) => "OutputDtmf",
End { .. } => "End",
Stop => "Stop",
LlmResponseStart => "LlmResponseStart",
LlmResponseEnd => "LlmResponseEnd",
TtsStarted { .. } => "TtsStarted",
TtsStopped { .. } => "TtsStopped",
UpdateSettings { .. } => "UpdateSettings",
SpeechControlParams { .. } => "SpeechControlParams",
Heartbeat { .. } => "Heartbeat",
OutputTransportReady => "OutputTransportReady",
FilterEnable(_) => "FilterEnable",
MixerEnable(_) => "MixerEnable",
Custom(c) => c.name(),
}
}
pub fn is_terminal(&self) -> bool {
matches!(self, Frame::End { .. } | Frame::Stop | Frame::Cancel { .. })
}
pub fn kind(&self) -> FrameKind {
use Frame::*;
match self {
Start(_) => FrameKind::Start,
Cancel { .. } => FrameKind::Cancel,
Error { .. } => FrameKind::Error,
Interruption => FrameKind::Interruption,
InputAudio(_) => FrameKind::InputAudio,
UserAudio { .. } => FrameKind::UserAudio,
InputText(_) => FrameKind::InputText,
InputDtmf(_) => FrameKind::InputDtmf,
UserStartedSpeaking => FrameKind::UserStartedSpeaking,
UserStoppedSpeaking => FrameKind::UserStoppedSpeaking,
UserSpeaking => FrameKind::UserSpeaking,
BotStartedSpeaking => FrameKind::BotStartedSpeaking,
BotStoppedSpeaking => FrameKind::BotStoppedSpeaking,
BotSpeaking => FrameKind::BotSpeaking,
VadUserStartedSpeaking { .. } => FrameKind::VadUserStartedSpeaking,
VadUserStoppedSpeaking { .. } => FrameKind::VadUserStoppedSpeaking,
SttMute(_) => FrameKind::SttMute,
Metrics(_) => FrameKind::Metrics,
TransportMessage { .. } => FrameKind::TransportMessage,
ClientConnected => FrameKind::ClientConnected,
BotConnected => FrameKind::BotConnected,
FunctionCallsStarted(_) => FrameKind::FunctionCallsStarted,
FunctionCallInProgress { .. } => FrameKind::FunctionCallInProgress,
FunctionCallCancel { .. } => FrameKind::FunctionCallCancel,
OutputAudio(_) => FrameKind::OutputAudio,
TtsAudio { .. } => FrameKind::TtsAudio,
Text(_) => FrameKind::Text,
LlmText(_) => FrameKind::LlmText,
Transcription { .. } => FrameKind::Transcription,
InterimTranscription { .. } => FrameKind::InterimTranscription,
TtsSpeak { .. } => FrameKind::TtsSpeak,
TtsText { .. } => FrameKind::TtsText,
FunctionCallResult(_) => FrameKind::FunctionCallResult,
LlmRun => FrameKind::LlmRun,
LlmContext(_) => FrameKind::LlmContext,
OutputDtmf(_) => FrameKind::OutputDtmf,
End { .. } => FrameKind::End,
Stop => FrameKind::Stop,
LlmResponseStart => FrameKind::LlmResponseStart,
LlmResponseEnd => FrameKind::LlmResponseEnd,
TtsStarted { .. } => FrameKind::TtsStarted,
TtsStopped { .. } => FrameKind::TtsStopped,
UpdateSettings { .. } => FrameKind::UpdateSettings,
SpeechControlParams { .. } => FrameKind::SpeechControlParams,
Heartbeat { .. } => FrameKind::Heartbeat,
OutputTransportReady => FrameKind::OutputTransportReady,
FilterEnable(_) => FrameKind::FilterEnable,
MixerEnable(_) => FrameKind::MixerEnable,
Custom(_) => FrameKind::Custom,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub enum FrameKind {
Start,
Cancel,
Error,
Interruption,
InputAudio,
UserAudio,
InputText,
InputDtmf,
UserStartedSpeaking,
UserStoppedSpeaking,
UserSpeaking,
BotStartedSpeaking,
BotStoppedSpeaking,
BotSpeaking,
VadUserStartedSpeaking,
VadUserStoppedSpeaking,
SttMute,
Metrics,
TransportMessage,
ClientConnected,
BotConnected,
FunctionCallsStarted,
FunctionCallInProgress,
FunctionCallCancel,
OutputAudio,
TtsAudio,
Text,
LlmText,
Transcription,
InterimTranscription,
TtsSpeak,
TtsText,
FunctionCallResult,
LlmRun,
LlmContext,
OutputDtmf,
End,
Stop,
LlmResponseStart,
LlmResponseEnd,
TtsStarted,
TtsStopped,
UpdateSettings,
SpeechControlParams,
Heartbeat,
OutputTransportReady,
FilterEnable,
MixerEnable,
Custom,
}
#[cfg(test)]
mod tests {
use super::*;
#[derive(Debug)]
struct MyCustom {
class: FrameClass,
keep: bool,
payload: u32,
}
impl CustomFrame for MyCustom {
fn frame_class(&self) -> FrameClass {
self.class
}
fn uninterruptible(&self) -> bool {
self.keep
}
fn name(&self) -> &'static str {
"MyCustom"
}
fn as_any(&self) -> &dyn Any {
self
}
}
#[test]
fn class_table_covers_every_variant() {
let a = Arc::new(AudioFrame::mono(vec![0], 16_000));
let uid: Arc<str> = Arc::from("u");
let cases: Vec<(Frame, FrameClass)> = vec![
(Frame::Start(StartParams::default()), FrameClass::System),
(Frame::Cancel { reason: None }, FrameClass::System),
(
Frame::Error {
message: "x".into(),
fatal: false,
processor: None,
},
FrameClass::System,
),
(Frame::Interruption, FrameClass::System),
(Frame::InputAudio(a.clone()), FrameClass::System),
(
Frame::UserAudio {
audio: a.clone(),
user_id: uid.clone(),
},
FrameClass::System,
),
(Frame::InputText("hi".into()), FrameClass::System),
(Frame::InputDtmf(KeypadEntry::One), FrameClass::System),
(Frame::UserStartedSpeaking, FrameClass::System),
(Frame::UserStoppedSpeaking, FrameClass::System),
(Frame::UserSpeaking, FrameClass::System),
(Frame::BotStartedSpeaking, FrameClass::System),
(Frame::BotStoppedSpeaking, FrameClass::System),
(Frame::BotSpeaking, FrameClass::System),
(
Frame::VadUserStartedSpeaking { start_secs: 0.2 },
FrameClass::System,
),
(
Frame::VadUserStoppedSpeaking { stop_secs: 0.5 },
FrameClass::System,
),
(Frame::SttMute(true), FrameClass::System),
(Frame::Metrics(vec![]), FrameClass::System),
(
Frame::TransportMessage {
payload: serde_json::json!({}),
urgent: true,
},
FrameClass::System,
),
(Frame::ClientConnected, FrameClass::System),
(Frame::BotConnected, FrameClass::System),
(Frame::FunctionCallsStarted(vec![]), FrameClass::System),
(
Frame::FunctionCallInProgress {
call: FunctionCall {
function_name: "f".into(),
tool_call_id: "1".into(),
arguments: serde_json::json!({}),
},
cancel_on_interruption: false,
},
FrameClass::System,
),
(
Frame::FunctionCallCancel {
function_name: "f".into(),
tool_call_id: "1".into(),
},
FrameClass::System,
),
(Frame::OutputAudio(a.clone()), FrameClass::Data),
(
Frame::TtsAudio {
audio: a.clone(),
context_id: None,
},
FrameClass::Data,
),
(Frame::Text("t".into()), FrameClass::Data),
(Frame::LlmText("t".into()), FrameClass::Data),
(
Frame::Transcription {
text: "t".into(),
user_id: uid.clone(),
language: None,
final_: true,
},
FrameClass::Data,
),
(
Frame::InterimTranscription {
text: "t".into(),
user_id: uid.clone(),
language: None,
},
FrameClass::Data,
),
(
Frame::TtsSpeak {
text: "t".into(),
append_to_context: None,
},
FrameClass::Data,
),
(
Frame::TtsText {
text: "t".into(),
context_id: None,
},
FrameClass::Data,
),
(
Frame::FunctionCallResult(FunctionCallResult {
function_name: "f".into(),
tool_call_id: "1".into(),
result: serde_json::json!({}),
}),
FrameClass::Data,
),
(Frame::LlmRun, FrameClass::Data),
(
Frame::LlmContext(Arc::new(LlmContext::default())),
FrameClass::Data,
),
(
Frame::OutputDtmf(vec![KeypadEntry::Pound]),
FrameClass::Data,
),
(Frame::End { reason: None }, FrameClass::Control),
(Frame::Stop, FrameClass::Control),
(Frame::LlmResponseStart, FrameClass::Control),
(Frame::LlmResponseEnd, FrameClass::Control),
(Frame::TtsStarted { context_id: None }, FrameClass::Control),
(Frame::TtsStopped { context_id: None }, FrameClass::Control),
(
Frame::UpdateSettings {
target: ServiceKind::Llm,
settings: serde_json::json!({}),
},
FrameClass::Control,
),
(
Frame::SpeechControlParams {
vad: None,
turn: None,
},
FrameClass::Control,
),
(Frame::Heartbeat { timestamp_ns: 0 }, FrameClass::Control),
(Frame::OutputTransportReady, FrameClass::Control),
(Frame::FilterEnable(true), FrameClass::Control),
(Frame::MixerEnable(false), FrameClass::Control),
];
for (frame, expected) in cases {
assert_eq!(
frame.class(),
expected,
"class() mismatch for {}",
frame.name()
);
}
}
#[test]
fn uninterruptible_table() {
assert!(Frame::End { reason: None }.uninterruptible());
assert!(Frame::Stop.uninterruptible());
assert!(Frame::FunctionCallResult(FunctionCallResult {
function_name: "f".into(),
tool_call_id: "1".into(),
result: serde_json::json!({}),
})
.uninterruptible());
assert!(Frame::UpdateSettings {
target: ServiceKind::All,
settings: serde_json::json!({}),
}
.uninterruptible());
assert!(!Frame::Interruption.uninterruptible());
assert!(!Frame::Text("t".into()).uninterruptible());
assert!(!Frame::OutputAudio(Arc::new(AudioFrame::mono(vec![0], 8000))).uninterruptible());
assert!(!Frame::Start(StartParams::default()).uninterruptible());
}
#[test]
fn custom_frame_delegates_class_and_uninterruptible() {
let c = Frame::Custom(Arc::new(MyCustom {
class: FrameClass::Control,
keep: true,
payload: 7,
}));
assert_eq!(c.class(), FrameClass::Control);
assert!(c.uninterruptible());
assert_eq!(c.name(), "MyCustom");
assert_eq!(c.kind(), FrameKind::Custom);
let d = Frame::Custom(Arc::new(MyCustom {
class: FrameClass::Data,
keep: false,
payload: 9,
}));
assert_eq!(d.class(), FrameClass::Data);
assert!(!d.uninterruptible());
if let Frame::Custom(inner) = &d {
let mc = inner.as_any().downcast_ref::<MyCustom>().expect("downcast");
assert_eq!(mc.payload, 9);
} else {
panic!("expected Custom");
}
}
#[test]
fn frame_ids_are_monotonic_and_unique() {
let a = next_frame_id();
let b = next_frame_id();
let c = next_frame_id();
assert!(a < b && b < c);
}
#[test]
fn audio_chunk_roundtrips_to_audio_frame() {
let chunk = crate::types::AudioChunk::new(vec![1, 2, 3], 16_000);
let frame: AudioFrame = (&chunk).into();
assert_eq!(frame.num_channels, 1);
assert_eq!(frame.pcm, vec![1, 2, 3]);
let back: crate::types::AudioChunk = (&frame).into();
assert_eq!(back, chunk);
}
}