use std::sync::{Arc, Mutex};
use async_trait::async_trait;
use serde_json::{json, Value};
use tokio::time::Instant;
use crate::brain::AgentBrain;
use crate::codec::Resampler;
use crate::error::{FlowcatError, Result};
use crate::processor::frame::{Frame, LlmContext, StartParams};
use crate::processor::{Envelope, FrameProcessor, Link, ProcessorSetup};
use crate::service::adapters::{LlmProcessor, SttProcessor, TtsProcessor};
use crate::service::{LlmService, SttService, TtsService};
use crate::session::SessionSource;
use crate::transport::MediaTransport;
use crate::types::AudioChunk;
use super::s2s::{
advance_playout, spawn_transport_pump, BrainProcessor, FinalizeProcessor, LiveState,
ModelToolCall, RecorderProcessor, Reprompt, SessionToolRelay, SharedState, SharedTransport,
ToolRelay, ToolResult, TranscriptProcessor, TransportInput, MAX_PLAYOUT_DRAIN,
};
use super::{Pipeline, PipelineTask, PipelineTaskParams, SourcePump};
#[derive(Default)]
struct RollingContext {
messages: Vec<Value>,
system_prompt: Option<String>,
tools: Vec<Value>,
}
impl RollingContext {
fn new(system_prompt: Option<String>, tools: Vec<Value>) -> Self {
Self {
messages: Vec::new(),
system_prompt,
tools,
}
}
fn push(&mut self, role: &str, content: &str) {
self.messages
.push(json!({ "role": role, "content": content }));
}
fn snapshot(&self) -> LlmContext {
let mut messages = Vec::with_capacity(self.messages.len() + 1);
if let Some(sys) = &self.system_prompt {
messages.push(json!({ "role": "system", "content": sys }));
}
messages.extend(self.messages.iter().cloned());
LlmContext {
messages,
tools: self.tools.clone(),
}
}
fn len(&self) -> usize {
self.messages.len()
}
fn reprompt(&mut self, system_prompt: String, tools: Vec<Value>) {
self.system_prompt = Some(system_prompt);
self.tools = tools;
}
fn push_tool_result(&mut self, tool_call_id: &str, content: &Value) {
self.messages.push(json!({
"role": "tool",
"tool_call_id": tool_call_id,
"content": content,
}));
}
}
type SharedContext = Arc<Mutex<RollingContext>>;
#[derive(Clone)]
struct TurnMute(Arc<TurnMuteInner>);
struct TurnMuteInner {
muted: std::sync::atomic::AtomicBool,
generation: std::sync::atomic::AtomicU64,
bot_until: Mutex<Option<Instant>>,
end_tx: tokio::sync::mpsc::UnboundedSender<Frame>,
no_audio_timeout: std::time::Duration,
}
impl TurnMute {
fn new(
end_tx: tokio::sync::mpsc::UnboundedSender<Frame>,
no_audio_timeout: std::time::Duration,
) -> Self {
TurnMute(Arc::new(TurnMuteInner {
muted: std::sync::atomic::AtomicBool::new(false),
generation: std::sync::atomic::AtomicU64::new(0),
bot_until: Mutex::new(None),
end_tx,
no_audio_timeout,
}))
}
fn begin(&self) {
use std::sync::atomic::Ordering;
if self.0.muted.swap(true, Ordering::SeqCst) {
return; }
tracing::debug!("half-duplex: muting STT (turn started)");
let _ = self.0.end_tx.send(Frame::SttMute(true));
*self.0.bot_until.lock().unwrap() = None;
let my_gen = self.0.generation.fetch_add(1, Ordering::SeqCst) + 1;
let inner = self.0.clone();
tokio::spawn(async move {
let deadline = Instant::now() + inner.no_audio_timeout;
loop {
if inner.generation.load(Ordering::SeqCst) != my_gen {
return; }
let until = *inner.bot_until.lock().unwrap();
let now = Instant::now();
match until {
Some(t) if t > now => tokio::time::sleep(t - now).await, Some(_) => break, None if now >= deadline => break, None => tokio::time::sleep(std::time::Duration::from_millis(200)).await,
}
}
if inner.generation.load(Ordering::SeqCst) == my_gen
&& inner.muted.swap(false, Ordering::SeqCst)
{
tracing::debug!("half-duplex: unmuting STT (turn complete)");
let _ = inner.end_tx.send(Frame::SttMute(false));
}
});
}
fn note_bot_audio(&self, samples: usize, carrier_rate: u32) {
let mut until = self.0.bot_until.lock().unwrap();
*until = Some(advance_playout(
*until,
Instant::now(),
samples,
carrier_rate,
));
}
fn take_bot_until(&self) -> Option<Instant> {
self.0.bot_until.lock().unwrap().take()
}
}
#[async_trait]
pub trait ContextSummarizer: Send + Sync {
async fn summarize(&self, messages: &[Value]) -> Option<String>;
}
#[derive(Debug, Clone)]
pub struct SummarizerConfig {
pub trigger_after_messages: usize,
}
impl Default for SummarizerConfig {
fn default() -> Self {
Self {
trigger_after_messages: 20,
}
}
}
struct NoopSummarizer;
#[async_trait]
impl ContextSummarizer for NoopSummarizer {
async fn summarize(&self, _messages: &[Value]) -> Option<String> {
None
}
}
pub struct UserContextAggregator {
ctx: SharedContext,
summarizer: Arc<dyn ContextSummarizer>,
summarizer_cfg: SummarizerConfig,
summarize_in_flight: Arc<std::sync::atomic::AtomicBool>,
transcript_state: Option<SharedState>,
turn_mute: Option<TurnMute>,
}
impl UserContextAggregator {
fn new(
ctx: SharedContext,
summarizer: Arc<dyn ContextSummarizer>,
summarizer_cfg: SummarizerConfig,
) -> Self {
Self {
ctx,
summarizer,
summarizer_cfg,
summarize_in_flight: Arc::new(std::sync::atomic::AtomicBool::new(false)),
transcript_state: None,
turn_mute: None,
}
}
fn with_transcript_state(mut self, state: SharedState) -> Self {
self.transcript_state = Some(state);
self
}
fn with_turn_mute(mut self, tm: TurnMute) -> Self {
self.turn_mute = Some(tm);
self
}
fn maybe_summarize(&self, len: usize) {
use std::sync::atomic::Ordering;
if len < self.summarizer_cfg.trigger_after_messages {
return;
}
if self
.summarize_in_flight
.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
.is_err()
{
return;
}
let ctx = self.ctx.clone();
let summarizer = self.summarizer.clone();
let in_flight = self.summarize_in_flight.clone();
tokio::spawn(async move {
let history = {
let c = ctx.lock().unwrap();
c.messages.clone()
};
if let Some(summary) = summarizer.summarize(&history).await {
let mut c = ctx.lock().unwrap();
c.messages = vec![
json!({ "role": "system", "content": format!("Summary so far: {summary}") }),
];
}
in_flight.store(false, Ordering::SeqCst);
});
}
}
#[async_trait]
impl FrameProcessor for UserContextAggregator {
fn name(&self) -> &str {
"UserContextAggregator"
}
async fn process_frame(&mut self, env: Envelope, link: &Link) -> Result<()> {
if let Frame::Transcription {
text, final_: true, ..
} = &env.frame
{
if let Some(st) = &self.transcript_state {
st.lock().unwrap().transcript.push_user(text);
}
let (snapshot, len) = {
let mut c = self.ctx.lock().unwrap();
c.push("user", text);
(c.snapshot(), c.len())
};
self.maybe_summarize(len);
if let Some(tm) = &self.turn_mute {
tm.begin();
}
link.push_down(Frame::LlmContext(Arc::new(snapshot))).await;
return Ok(());
}
link.push(env.meta, env.frame, env.direction).await;
Ok(())
}
}
pub struct AssistantContextAggregator {
ctx: SharedContext,
buffer: String,
in_response: bool,
transcript_state: Option<SharedState>,
}
impl AssistantContextAggregator {
fn new(ctx: SharedContext) -> Self {
Self {
ctx,
buffer: String::new(),
in_response: false,
transcript_state: None,
}
}
fn with_transcript_state(mut self, state: SharedState) -> Self {
self.transcript_state = Some(state);
self
}
}
#[async_trait]
impl FrameProcessor for AssistantContextAggregator {
fn name(&self) -> &str {
"AssistantContextAggregator"
}
async fn process_frame(&mut self, env: Envelope, link: &Link) -> Result<()> {
match &env.frame {
Frame::LlmResponseStart => {
self.in_response = true;
self.buffer.clear();
link.push(env.meta, env.frame, env.direction).await;
}
Frame::LlmText(chunk) => {
self.buffer.push_str(chunk);
}
Frame::LlmResponseEnd => {
self.in_response = false;
let reply = std::mem::take(&mut self.buffer);
if !reply.is_empty() {
if let Some(st) = &self.transcript_state {
st.lock().unwrap().transcript.push_bot(&reply);
}
{
let mut c = self.ctx.lock().unwrap();
c.push("assistant", &reply);
}
link.push_down(Frame::TtsSpeak {
text: reply,
append_to_context: Some(false),
})
.await;
}
link.push(env.meta, env.frame, env.direction).await;
}
_ => link.push(env.meta, env.frame, env.direction).await,
}
Ok(())
}
}
pub struct CascadedConfig {
pub system_prompt: Option<String>,
pub tools: Vec<Value>,
pub summarizer: Option<Arc<dyn ContextSummarizer>>,
pub summarizer_cfg: SummarizerConfig,
pub task_params: PipelineTaskParams,
}
impl Default for CascadedConfig {
fn default() -> Self {
Self {
system_prompt: None,
tools: Vec::new(),
summarizer: None,
summarizer_cfg: SummarizerConfig::default(),
task_params: PipelineTaskParams {
idle_timeout: None,
..Default::default()
},
}
}
}
fn assemble<S, L, T>(
stt: S,
llm: L,
tts: T,
config: CascadedConfig,
) -> (Pipeline, PipelineTaskParams)
where
S: SttService + 'static,
L: LlmService + 'static,
T: TtsService + 'static,
{
let ctx: SharedContext = Arc::new(Mutex::new(RollingContext::new(
config.system_prompt,
config.tools,
)));
let summarizer = config
.summarizer
.unwrap_or_else(|| Arc::new(NoopSummarizer) as Arc<dyn ContextSummarizer>);
let processors: Vec<Box<dyn FrameProcessor>> = vec![
Box::new(SttProcessor::new(stt)),
Box::new(UserContextAggregator::new(
ctx.clone(),
summarizer,
config.summarizer_cfg,
)),
Box::new(LlmProcessor::new(llm)),
Box::new(AssistantContextAggregator::new(ctx.clone())),
Box::new(TtsProcessor::new(tts)),
];
(Pipeline::new(processors), config.task_params)
}
pub fn build_cascaded_pipeline<S, L, T>(
stt: S,
llm: L,
tts: T,
config: CascadedConfig,
) -> PipelineTask
where
S: SttService + 'static,
L: LlmService + 'static,
T: TtsService + 'static,
{
build_cascaded_task_with_observers(stt, llm, tts, config, vec![])
}
pub fn build_cascaded_task_with_observers<S, L, T>(
stt: S,
llm: L,
tts: T,
config: CascadedConfig,
observers: Vec<Arc<dyn crate::observer::FrameObserver>>,
) -> PipelineTask
where
S: SttService + 'static,
L: LlmService + 'static,
T: TtsService + 'static,
{
let (pipeline, params) = assemble(stt, llm, tts, config);
PipelineTask::new(pipeline, params, observers)
}
fn is_transition_ack(result: &Value) -> bool {
result
.get("status")
.and_then(Value::as_str)
.is_some_and(|s| matches!(s, "moved" | "stay" | "ended"))
}
struct CascadedToolBridge {
ctx: SharedContext,
}
impl CascadedToolBridge {
fn new(ctx: SharedContext) -> Self {
Self { ctx }
}
}
#[async_trait]
impl FrameProcessor for CascadedToolBridge {
fn name(&self) -> &str {
"CascadedToolBridge"
}
async fn process_frame(&mut self, env: Envelope, link: &Link) -> Result<()> {
match &env.frame {
Frame::FunctionCallsStarted(calls) => {
for c in calls {
link.push_down(Frame::Custom(Arc::new(ModelToolCall {
id: c.tool_call_id.clone(),
name: c.function_name.clone(),
args: c.arguments.clone(),
})))
.await;
}
}
Frame::Custom(c) if c.as_any().is::<Reprompt>() => {
let rp = c.as_any().downcast_ref::<Reprompt>().unwrap();
let tools_json: Vec<Value> = rp
.tools
.iter()
.map(|t| serde_json::to_value(t).unwrap_or(Value::Null))
.collect();
let snapshot = {
let mut ctx = self.ctx.lock().unwrap();
ctx.reprompt(rp.prompt.clone(), tools_json);
ctx.snapshot()
};
tracing::debug!("cascaded transition: reprompt → re-running LLM for new node");
link.push_up(Frame::LlmContext(Arc::new(snapshot))).await;
}
Frame::Custom(c) if c.as_any().is::<ToolResult>() => {
let tr = c.as_any().downcast_ref::<ToolResult>().unwrap();
if !is_transition_ack(&tr.result) {
let snapshot = {
let mut ctx = self.ctx.lock().unwrap();
ctx.push_tool_result(&tr.id, &tr.result);
ctx.snapshot()
};
link.push_up(Frame::LlmContext(Arc::new(snapshot))).await;
}
}
_ => link.push(env.meta, env.frame, env.direction).await,
}
Ok(())
}
}
struct CascadedTransportOutput<T: MediaTransport> {
transport: T,
out_resampler: Option<Resampler>,
tts_rate: u32,
carrier_rate: u32,
state: SharedState,
end_tx: tokio::sync::mpsc::UnboundedSender<Frame>,
turn_mute: TurnMute,
}
impl<T: MediaTransport> CascadedTransportOutput<T> {
fn new(
transport: T,
tts_rate: u32,
carrier_rate: u32,
state: SharedState,
end_tx: tokio::sync::mpsc::UnboundedSender<Frame>,
turn_mute: TurnMute,
) -> Self {
Self {
transport,
out_resampler: None,
tts_rate,
carrier_rate,
state,
end_tx,
turn_mute,
}
}
fn fail_transport(&self, e: FlowcatError) {
let newly_dead = {
let mut st = self.state.lock().unwrap();
let was = st.transport_dead;
st.record_error(e); st.transport_dead && !was
};
if newly_dead {
let _ = self.end_tx.send(Frame::End { reason: None });
}
}
}
#[async_trait]
impl<T: MediaTransport + 'static> FrameProcessor for CascadedTransportOutput<T> {
fn name(&self) -> &str {
"CascadedTransportOutput"
}
async fn start(&mut self, _setup: &ProcessorSetup, _params: &StartParams) -> Result<()> {
self.out_resampler = Some(Resampler::new(self.tts_rate, self.carrier_rate)?);
Ok(())
}
async fn process_frame(&mut self, env: Envelope, link: &Link) -> Result<()> {
match &env.frame {
Frame::OutputAudio(audio) | Frame::TtsAudio { audio, .. } => {
if self.state.lock().unwrap().transport_dead {
return Ok(());
}
let chunk = AudioChunk::from(audio.as_ref());
self.state.lock().unwrap().recorder.push_outbound(&chunk);
let resampler = self
.out_resampler
.as_mut()
.expect("out_resampler set in start()");
match resampler.process(&chunk) {
Ok(down) => {
if !down.is_empty() {
let samples = down.pcm.len();
if let Err(e) = self.transport.send_audio(down).await {
self.fail_transport(e);
} else {
self.turn_mute.note_bot_audio(samples, self.carrier_rate);
}
}
}
Err(e) => self.state.lock().unwrap().record_error(e),
}
}
Frame::Interruption => {
if let Err(e) = self.transport.send_clear().await {
self.fail_transport(e);
}
let _ = self.turn_mute.take_bot_until();
link.push(env.meta, env.frame, env.direction).await;
}
Frame::End { .. } => {
let until = self.turn_mute.take_bot_until();
if let Some(until) = until {
let now = Instant::now();
if until > now {
tokio::time::sleep((until - now).min(MAX_PLAYOUT_DRAIN)).await;
}
}
link.push(env.meta, env.frame, env.direction).await;
}
_ => link.push(env.meta, env.frame, env.direction).await,
}
Ok(())
}
}
pub struct CascadedTask {
pub task: PipelineTask,
pump: SourcePump,
}
impl CascadedTask {
pub async fn run(self) -> Result<()> {
let res = self.task.run().await;
self.pump.abort();
res
}
}
struct CascadedKickoffProcessor {
ctx: SharedContext,
kicked_off: bool,
turn_mute: Option<TurnMute>,
}
impl CascadedKickoffProcessor {
fn new(ctx: SharedContext) -> Self {
Self {
ctx,
kicked_off: false,
turn_mute: None,
}
}
fn with_turn_mute(mut self, tm: TurnMute) -> Self {
self.turn_mute = Some(tm);
self
}
}
#[async_trait]
impl FrameProcessor for CascadedKickoffProcessor {
fn name(&self) -> &str {
"CascadedKickoff"
}
async fn process_frame(&mut self, env: Envelope, link: &Link) -> Result<()> {
if !self.kicked_off && matches!(env.frame, Frame::ClientConnected) {
self.kicked_off = true;
tracing::info!("cascaded kickoff: greeting on ClientConnected");
if let Some(tm) = &self.turn_mute {
tm.begin();
}
let mut snapshot = self.ctx.lock().unwrap().snapshot();
snapshot.tools.clear();
link.push_down(Frame::LlmContext(Arc::new(snapshot))).await;
}
link.push(env.meta, env.frame, env.direction).await;
Ok(())
}
}
#[allow(clippy::too_many_arguments)]
pub async fn build_cascaded_task<Tr, St, L, Ts, B, Se>(
transport: Tr,
stt: St,
llm: L,
tts: Ts,
brain: B,
session: Se,
run_id: i64,
token: String,
config: CascadedConfig,
) -> Result<CascadedTask>
where
Tr: MediaTransport + 'static,
St: SttService + 'static,
L: LlmService + 'static,
Ts: TtsService + 'static,
B: AgentBrain + 'static,
Se: SessionSource + 'static,
{
build_cascaded_call_with_observers(
transport,
stt,
llm,
tts,
brain,
session,
run_id,
token,
config,
vec![],
)
.await
}
#[allow(clippy::too_many_arguments)]
pub async fn build_cascaded_call_with_observers<Tr, St, L, Ts, B, Se>(
transport: Tr,
stt: St,
llm: L,
tts: Ts,
brain: B,
session: Se,
run_id: i64,
token: String,
config: CascadedConfig,
observers: Vec<Arc<dyn crate::observer::FrameObserver>>,
) -> Result<CascadedTask>
where
Tr: MediaTransport + 'static,
St: SttService + 'static,
L: LlmService + 'static,
Ts: TtsService + 'static,
B: AgentBrain + 'static,
Se: SessionSource + 'static,
{
let carrier_rate = transport.carrier_rate();
let tts_rate = tts.sample_rate();
let session = Arc::new(session);
let relay = Arc::new(SessionToolRelay::new(
session.clone(),
run_id,
token.clone(),
));
let node_id = brain.current_node_id();
let mcp = relay.node_tools(&node_id).await;
let mcp_names: std::collections::HashSet<String> = mcp.iter().map(|t| t.name.clone()).collect();
let mut initial_tools = brain.tools();
initial_tools.extend(mcp);
let initial_tools_json: Vec<Value> = initial_tools
.iter()
.map(|t| serde_json::to_value(t).unwrap_or(Value::Null))
.collect();
let ctx: SharedContext = Arc::new(Mutex::new(RollingContext::new(
Some(brain.system_prompt()),
initial_tools_json,
)));
let state: SharedState = Arc::new(Mutex::new(LiveState::new(carrier_rate)));
let shared = SharedTransport::new(transport);
let (end_tx, mut end_rx) = tokio::sync::mpsc::unbounded_channel::<Frame>();
let turn_mute = TurnMute::new(end_tx.clone(), std::time::Duration::from_secs(12));
let summarizer = config
.summarizer
.unwrap_or_else(|| Arc::new(NoopSummarizer) as Arc<dyn ContextSummarizer>);
let processors: Vec<Box<dyn FrameProcessor>> = vec![
Box::new(TransportInput::new()),
Box::new(SttProcessor::new(stt)),
Box::new(
UserContextAggregator::new(ctx.clone(), summarizer, config.summarizer_cfg)
.with_transcript_state(state.clone())
.with_turn_mute(turn_mute.clone()),
),
Box::new(CascadedKickoffProcessor::new(ctx.clone()).with_turn_mute(turn_mute.clone())),
Box::new(LlmProcessor::new(llm)),
Box::new(CascadedToolBridge::new(ctx.clone())),
Box::new(BrainProcessor::new(
brain,
relay,
mcp_names,
state.clone(),
end_tx.clone(),
)),
Box::new(AssistantContextAggregator::new(ctx.clone()).with_transcript_state(state.clone())),
Box::new(TtsProcessor::new(tts)),
Box::new(CascadedTransportOutput::new(
shared.clone(),
tts_rate,
carrier_rate,
state.clone(),
end_tx.clone(),
turn_mute.clone(),
)),
Box::new(RecorderProcessor::new(state.clone())),
Box::new(TranscriptProcessor::new(state.clone())),
Box::new(FinalizeProcessor::new(
session,
run_id,
token,
state.clone(),
)),
];
let pipeline = Pipeline::new(processors);
let params = PipelineTaskParams {
idle_timeout: None,
..config.task_params
};
let task = PipelineTask::new(pipeline, params, observers);
let head = task.queue_sender();
tokio::spawn(async move {
while let Some(f) = end_rx.recv().await {
if head.send(f).is_err() {
break;
}
}
});
let pump = spawn_transport_pump(shared, task.queue_sender());
Ok(CascadedTask { task, pump })
}
#[cfg(test)]
mod tests {
use super::*;
use crate::observer::{FrameEvent, FrameObserver};
use crate::processor::frame::AudioFrame;
use crate::service::{MockLlm, MockStt, MockTts};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::time::Duration;
#[derive(Default)]
struct Tap {
names: Mutex<Vec<&'static str>>,
saw_output_audio: AtomicBool,
tts_speak: Mutex<std::collections::BTreeMap<u64, String>>,
}
impl Tap {
fn tts_speak_texts(&self) -> Vec<String> {
self.tts_speak.lock().unwrap().values().cloned().collect()
}
}
#[async_trait]
impl FrameObserver for Tap {
async fn on_process(&self, e: &FrameEvent<'_>) {
self.names.lock().unwrap().push(e.frame.name());
match &e.frame {
Frame::OutputAudio(_) => self.saw_output_audio.store(true, Ordering::Relaxed),
Frame::TtsSpeak { text, .. } => {
self.tts_speak
.lock()
.unwrap()
.insert(e.meta.id, text.clone());
}
_ => {}
}
}
}
#[tokio::test]
async fn cascaded_pipeline_runs_a_turn_end_to_end() {
let tap = Arc::new(Tap::default());
let task = build_cascaded_task_with_observers(
MockStt::new("book a dentist appointment"),
MockLlm::new("you said: "),
MockTts::new(24_000),
CascadedConfig::default(),
vec![tap.clone() as Arc<dyn FrameObserver>],
);
let audio = Arc::new(AudioFrame::mono(vec![1, 2, 3, 4], 16_000));
task.queue_frame(Frame::InputAudio(audio)).await;
task.stop_when_done().await;
tokio::time::timeout(Duration::from_secs(5), task.run())
.await
.expect("cascaded pipeline timed out")
.expect("run ok");
let names = tap.names.lock().unwrap().clone();
assert!(
tap.saw_output_audio.load(Ordering::Relaxed),
"cascaded pipeline must emit OutputAudio; saw {names:?}"
);
assert!(
names.contains(&"Transcription"),
"STT must emit Transcription"
);
assert!(
names.contains(&"LlmContext"),
"user aggregator must emit LlmContext"
);
assert!(names.contains(&"LlmText"), "LLM must emit LlmText");
assert!(
names.contains(&"TtsSpeak"),
"assistant aggregator must emit TtsSpeak"
);
let speaks = tap.tts_speak_texts();
assert_eq!(
speaks,
vec!["you said: book a dentist appointment".to_string()]
);
}
#[tokio::test]
async fn kickoff_greets_on_connect_once() {
let tap = Arc::new(Tap::default());
let ctx: SharedContext = Arc::new(Mutex::new(RollingContext::new(
Some("Greet the caller.".into()),
vec![],
)));
let pipeline = Pipeline::new(vec![
Box::new(CascadedKickoffProcessor::new(ctx.clone())),
Box::new(LlmProcessor::new(MockLlm::new("reply: "))),
Box::new(AssistantContextAggregator::new(ctx.clone())),
Box::new(TtsProcessor::new(MockTts::new(24_000))),
]);
let task = PipelineTask::new(
pipeline,
PipelineTaskParams {
idle_timeout: None,
..Default::default()
},
vec![tap.clone() as Arc<dyn FrameObserver>],
);
task.queue_frame(Frame::ClientConnected).await;
task.queue_frame(Frame::ClientConnected).await;
task.stop_when_done().await;
tokio::time::timeout(Duration::from_secs(5), task.run())
.await
.expect("kickoff pipeline timed out")
.expect("run ok");
let names = tap.names.lock().unwrap().clone();
assert!(
tap.saw_output_audio.load(Ordering::Relaxed),
"kickoff must produce a greeting OutputAudio; saw {names:?}"
);
assert!(
!names.contains(&"Transcription"),
"the greeting must require no user input; saw {names:?}"
);
let speaks = tap.tts_speak_texts();
assert_eq!(
speaks,
vec!["reply: Greet the caller.".to_string()],
"greet exactly once"
);
}
#[tokio::test]
async fn kickoff_greeting_is_tool_free() {
let tap = Arc::new(CtxTap::default());
let ctx: SharedContext = Arc::new(Mutex::new(RollingContext::new(
Some("Greet the caller.".into()),
vec![
json!({"name": "end_call", "description": "end the call", "params": {"type":"object"}}),
],
)));
let pipeline = Pipeline::new(vec![Box::new(CascadedKickoffProcessor::new(ctx.clone()))]);
let task = PipelineTask::new(
pipeline,
PipelineTaskParams {
idle_timeout: None,
..Default::default()
},
vec![tap.clone() as Arc<dyn FrameObserver>],
);
task.queue_frame(Frame::ClientConnected).await;
task.stop_when_done().await;
tokio::time::timeout(Duration::from_secs(5), task.run())
.await
.expect("timed out")
.expect("run ok");
let ctxs = tap.contexts.lock().unwrap().clone();
assert_eq!(ctxs.len(), 1, "kickoff emits exactly one greeting context");
assert!(
ctxs[0].tools.is_empty(),
"greeting context must carry NO tools"
);
assert_eq!(
ctxs[0].messages[0]["role"], "system",
"system prompt still present so it can greet"
);
}
#[tokio::test]
async fn turn_mute_locks_then_unmutes_after_playout() {
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<Frame>();
let tm = TurnMute::new(tx, Duration::from_secs(5)); tm.begin();
assert!(
matches!(rx.recv().await, Some(Frame::SttMute(true))),
"turn start mutes STT"
);
tm.begin(); tm.note_bot_audio(800, 8000);
let unmute = tokio::time::timeout(Duration::from_secs(2), rx.recv())
.await
.expect("unmute did not arrive");
assert!(
matches!(unmute, Some(Frame::SttMute(false))),
"unmutes after playout drains"
);
}
#[tokio::test]
async fn turn_mute_unmutes_via_safety_timeout_when_no_audio() {
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<Frame>();
let tm = TurnMute::new(tx, Duration::from_millis(200)); tm.begin();
assert!(matches!(rx.recv().await, Some(Frame::SttMute(true))));
let unmute = tokio::time::timeout(Duration::from_secs(2), rx.recv())
.await
.expect("safety-timeout unmute did not arrive");
assert!(matches!(unmute, Some(Frame::SttMute(false))));
}
#[derive(Default)]
struct CtxTap {
contexts: Mutex<Vec<LlmContext>>,
}
#[async_trait]
impl FrameObserver for CtxTap {
async fn on_process(&self, e: &FrameEvent<'_>) {
if let Frame::LlmContext(c) = &e.frame {
self.contexts.lock().unwrap().push((**c).clone());
}
}
}
#[tokio::test]
async fn user_aggregator_accrues_transcription_into_context() {
let tap = Arc::new(CtxTap::default());
let ctx: SharedContext = Arc::new(Mutex::new(RollingContext::new(
Some("be helpful".into()),
vec![],
)));
let pipeline = Pipeline::new(vec![Box::new(UserContextAggregator::new(
ctx.clone(),
Arc::new(NoopSummarizer),
SummarizerConfig::default(),
))]);
let task = PipelineTask::new(
pipeline,
PipelineTaskParams {
idle_timeout: None,
..Default::default()
},
vec![tap.clone() as Arc<dyn FrameObserver>],
);
let uid: Arc<str> = Arc::from("user");
task.queue_frame(Frame::Transcription {
text: "hello".into(),
user_id: uid.clone(),
language: None,
final_: true,
})
.await;
task.queue_frame(Frame::InterimTranscription {
text: "wor".into(),
user_id: uid.clone(),
language: None,
})
.await;
task.queue_frame(Frame::Transcription {
text: "world".into(),
user_id: uid.clone(),
language: None,
final_: true,
})
.await;
task.stop_when_done().await;
tokio::time::timeout(Duration::from_secs(5), task.run())
.await
.expect("timed out")
.expect("run ok");
let contexts = tap.contexts.lock().unwrap().clone();
assert_eq!(contexts.len(), 2, "one LlmContext per final transcription");
assert_eq!(contexts[0].messages.len(), 2);
assert_eq!(contexts[0].messages[0]["role"], "system");
assert_eq!(contexts[0].messages[1]["content"], "hello");
assert_eq!(contexts[1].messages.len(), 3);
assert_eq!(contexts[1].messages[2]["content"], "world");
}
#[derive(Default)]
struct SpeakTap {
texts: Mutex<Vec<String>>,
}
#[async_trait]
impl FrameObserver for SpeakTap {
async fn on_process(&self, e: &FrameEvent<'_>) {
if let Frame::TtsSpeak { text, .. } = &e.frame {
self.texts.lock().unwrap().push(text.clone());
}
}
}
#[tokio::test]
async fn assistant_aggregator_accrues_response_into_one_utterance() {
let tap = Arc::new(SpeakTap::default());
let ctx: SharedContext = Arc::new(Mutex::new(RollingContext::default()));
let pipeline = Pipeline::new(vec![Box::new(AssistantContextAggregator::new(ctx.clone()))]);
let task = PipelineTask::new(
pipeline,
PipelineTaskParams {
idle_timeout: None,
..Default::default()
},
vec![tap.clone() as Arc<dyn FrameObserver>],
);
task.queue_frame(Frame::LlmResponseStart).await;
task.queue_frame(Frame::LlmText("Hello ".into())).await;
task.queue_frame(Frame::LlmText("there, ".into())).await;
task.queue_frame(Frame::LlmText("friend".into())).await;
task.queue_frame(Frame::LlmResponseEnd).await;
task.stop_when_done().await;
tokio::time::timeout(Duration::from_secs(5), task.run())
.await
.expect("timed out")
.expect("run ok");
let texts = tap.texts.lock().unwrap().clone();
assert_eq!(texts, vec!["Hello there, friend".to_string()]);
let c = ctx.lock().unwrap();
assert_eq!(c.messages.len(), 1);
assert_eq!(c.messages[0]["role"], "assistant");
assert_eq!(c.messages[0]["content"], "Hello there, friend");
}
struct CountingSummarizer {
fired: Arc<AtomicUsize>,
}
#[async_trait]
impl ContextSummarizer for CountingSummarizer {
async fn summarize(&self, _messages: &[Value]) -> Option<String> {
self.fired.fetch_add(1, Ordering::SeqCst);
Some("compressed".into())
}
}
#[tokio::test]
async fn summarizer_fires_once_past_threshold_and_compresses() {
let fired = Arc::new(AtomicUsize::new(0));
let ctx: SharedContext = Arc::new(Mutex::new(RollingContext::default()));
let pipeline = Pipeline::new(vec![Box::new(UserContextAggregator::new(
ctx.clone(),
Arc::new(CountingSummarizer {
fired: fired.clone(),
}),
SummarizerConfig {
trigger_after_messages: 3,
},
))]);
let task = PipelineTask::new(
pipeline,
PipelineTaskParams {
idle_timeout: None,
..Default::default()
},
vec![],
);
let uid: Arc<str> = Arc::from("user");
for i in 0..4 {
task.queue_frame(Frame::Transcription {
text: format!("msg {i}"),
user_id: uid.clone(),
language: None,
final_: true,
})
.await;
}
task.stop_when_done().await;
tokio::time::timeout(Duration::from_secs(5), task.run())
.await
.expect("timed out")
.expect("run ok");
tokio::time::sleep(Duration::from_millis(50)).await;
assert!(
fired.load(Ordering::SeqCst) >= 1,
"summarizer must fire once past the threshold"
);
let c = ctx.lock().unwrap();
assert_eq!(c.messages.len(), 1, "history collapsed to the summary");
assert!(c.messages[0]["content"]
.as_str()
.unwrap()
.contains("compressed"));
}
use crate::pipeline::s2s_test_mocks::{Captured, MockSession, MockSocket, CARRIER_RATE};
use crate::processor::frame::FunctionCall;
use crate::serializer::PlivoSerializer;
use crate::transport::WsCarrierTransport;
use crate::types::{BrainAction, ToolDecl, WsOut};
use serde_json::json;
const TRANSITION_TOOL: &str = "go_to_booking";
const END_TOOL: &str = "end_call";
struct TransitioningMockBrain {
node_id: Arc<Mutex<String>>,
seen_tools: Arc<Mutex<Vec<String>>>,
actions: Arc<Mutex<Vec<&'static str>>>,
}
impl AgentBrain for TransitioningMockBrain {
fn system_prompt(&self) -> String {
"You are a test agent (start node).".into()
}
fn tools(&self) -> Vec<ToolDecl> {
vec![ToolDecl {
name: TRANSITION_TOOL.into(),
description: "Move to the booking node.".into(),
params: json!({ "type": "object", "properties": {} }),
}]
}
fn current_node_id(&self) -> String {
self.node_id.lock().unwrap().clone()
}
fn on_tool_call(&mut self, name: &str, _args: &serde_json::Value) -> BrainAction {
self.seen_tools.lock().unwrap().push(name.to_string());
if name == END_TOOL {
self.actions.lock().unwrap().push("end");
BrainAction::End {
disposition: Some("booked".into()),
}
} else if name == TRANSITION_TOOL && *self.node_id.lock().unwrap() == "start" {
*self.node_id.lock().unwrap() = "booking".into();
self.actions.lock().unwrap().push("transition");
BrainAction::Transition {
system_prompt: "You are at the booking node.".into(),
tools: vec![ToolDecl {
name: END_TOOL.into(),
description: "End the call.".into(),
params: json!({ "type": "object", "properties": {} }),
}],
say: Some("Sure, let's get you booked.".into()),
}
} else {
self.actions.lock().unwrap().push("stay");
BrainAction::Stay
}
}
fn is_finished(&self) -> bool {
false
}
fn collected_vars(&self) -> serde_json::Value {
json!({ "name": "Ada", "intent": "booking" })
}
}
struct ScriptedCascadedLlm {
node_id: Arc<Mutex<String>>,
runs: Arc<AtomicUsize>,
booking_reply: String,
}
#[async_trait]
impl LlmService for ScriptedCascadedLlm {
fn name(&self) -> &str {
"ScriptedCascadedLlm"
}
async fn start(&mut self, _params: &StartParams) -> Result<()> {
Ok(())
}
async fn run_llm<'a>(
&'a mut self,
_ctx: &'a LlmContext,
) -> Result<futures::stream::BoxStream<'a, Frame>> {
use futures::StreamExt;
self.runs.fetch_add(1, Ordering::SeqCst);
let at_start = *self.node_id.lock().unwrap() == "start";
let frames: Vec<Frame> = if at_start {
vec![
Frame::LlmResponseStart,
Frame::FunctionCallsStarted(vec![FunctionCall {
function_name: TRANSITION_TOOL.into(),
tool_call_id: "fc-tr-1".into(),
arguments: json!({}),
}]),
Frame::LlmResponseEnd,
]
} else {
vec![
Frame::LlmResponseStart,
Frame::LlmText(self.booking_reply.clone()),
Frame::LlmResponseEnd,
Frame::FunctionCallsStarted(vec![FunctionCall {
function_name: END_TOOL.into(),
tool_call_id: "fc-end-1".into(),
arguments: json!({}),
}]),
]
};
Ok(futures::stream::iter(frames).boxed())
}
fn set_tools(&mut self, _tools: Vec<crate::service::Tool>) {}
}
fn play_audio_frames(sent: &[WsOut]) -> Vec<String> {
sent.iter()
.filter_map(|o| match o {
WsOut::Text(t) if t.contains("playAudio") => Some(t.clone()),
_ => None,
})
.collect()
}
#[tokio::test]
async fn cascaded_task_runs_a_turn_with_a_transition_and_finalizes() {
let sent = Arc::new(Mutex::new(Vec::<WsOut>::new()));
let captured = Arc::new(Mutex::new(Captured::default()));
let seen_tools = Arc::new(Mutex::new(Vec::<String>::new()));
let actions = Arc::new(Mutex::new(Vec::<&'static str>::new()));
let runs = Arc::new(AtomicUsize::new(0));
let transport = WsCarrierTransport::new(
MockSocket::new(sent.clone()),
PlivoSerializer::new(CARRIER_RATE),
);
let node_id = Arc::new(Mutex::new(String::from("start")));
let brain = TransitioningMockBrain {
node_id: node_id.clone(),
seen_tools: seen_tools.clone(),
actions: actions.clone(),
};
let task = build_cascaded_task(
transport,
MockStt::new("book a dentist appointment"),
ScriptedCascadedLlm {
node_id: node_id.clone(),
runs: runs.clone(),
booking_reply: "Sure, let's get you booked in. ".repeat(12),
},
MockTts::new(24_000),
brain,
MockSession::new(captured.clone()),
7777,
"tok-cascaded".into(),
CascadedConfig::default(),
)
.await
.expect("build_cascaded_task");
tokio::time::timeout(Duration::from_secs(5), task.run())
.await
.expect("cascaded task timed out")
.expect("cascaded task errored");
let play = play_audio_frames(&sent.lock().unwrap());
assert!(
!play.is_empty(),
"expected ≥1 playAudio frame to the carrier (STT→LLM→brain→TTS→output)"
);
let seen = seen_tools.lock().unwrap().clone();
assert!(
seen.contains(&TRANSITION_TOOL.to_string()),
"the transition tool-call must reach the brain; saw {seen:?}"
);
let acts = actions.lock().unwrap().clone();
assert!(
acts.contains(&"transition"),
"a tool-call must drive a TRANSITION, not just a stay; actions={acts:?}"
);
assert!(
acts.contains(&"end"),
"the end tool-call must end the call; actions={acts:?}"
);
assert_eq!(
acts.iter().filter(|a| **a == "transition").count(),
1,
"exactly one graph transition; actions={acts:?}"
);
assert!(
runs.load(Ordering::SeqCst) >= 2,
"the transition must re-run the LLM (≥2 runs); got {}",
runs.load(Ordering::SeqCst)
);
let c = captured.lock().unwrap();
assert_eq!(
c.complete_calls, 1,
"session.complete must fire exactly once"
);
let fin = c.finalize.as_ref().expect("finalize payload present");
assert_eq!(fin.recording_url.as_deref(), Some("runs/4242/recording"));
assert_eq!(fin.transcript_url.as_deref(), Some("runs/4242/transcript"));
assert_eq!(fin.collected_vars["name"], "Ada");
assert_eq!(fin.collected_vars["intent"], "booking");
assert_eq!(
fin.collected_vars["call_disposition"], "booked",
"the End disposition must fold into collected_vars"
);
assert!(fin.usage.is_object(), "usage must be a JSON object");
assert!(
fin.usage["duration_seconds"].is_number(),
"finalize usage must carry duration_seconds"
);
}
#[test]
fn is_transition_ack_matches_only_brain_vocabulary() {
assert!(is_transition_ack(&json!({ "status": "moved" })));
assert!(is_transition_ack(&json!({ "status": "stay" })));
assert!(is_transition_ack(&json!({ "status": "ended" })));
assert!(!is_transition_ack(&json!({ "status": "ok" })));
assert!(!is_transition_ack(
&json!({ "status": "confirmed", "booking_id": "B1" })
));
assert!(!is_transition_ack(&json!({ "booking_id": "B1" })));
assert!(!is_transition_ack(&json!({ "status": 200 })));
assert!(!is_transition_ack(&json!("moved")));
}
}