flowcat_core/pipeline/cascaded.rs
1// SPDX-License-Identifier: Apache-2.0
2//
3//! Cascaded STT→LLM→TTS topology builder + context aggregators.
4//!
5//! Builds the cascaded [`Pipeline`](crate::pipeline::Pipeline) from concrete
6//! service processors (the [`adapters`](crate::service::adapters) wrapping the
7//! frozen `SttService`/`LlmService`/`TtsService` traits), wiring in the
8//! user/assistant **context aggregators** and an optional summarizer hook. This is
9//! the cascaded counterpart to the live realtime/S2S path ([`crate::pipeline::s2s`]);
10//! the service traits it builds against are frozen in [`crate::service`].
11//!
12//! ## Topology (pipecat `pipeline_builder.build_pipeline` shape)
13//!
14//! ```text
15//! transport.input → STT → UserContextAggregator → LLM → AssistantContextAggregator → TTS → transport.output
16//! ```
17//!
18//! - **STT** ([`SttProcessor`]) turns `InputAudio` → `Transcription`.
19//! - The **user aggregator** ([`UserContextAggregator`]) accrues each final
20//! `Transcription` into the shared rolling [`LlmContext`] as a `user` message and,
21//! on turn completion, emits a [`Frame::LlmContext`] that runs the LLM (mirrors
22//! pipecat `LLMUserContextAggregator`).
23//! - **LLM** ([`LlmProcessor`]) streams `LlmResponseStart` / `LlmText`* /
24//! `LlmResponseEnd`.
25//! - The **assistant aggregator** ([`AssistantContextAggregator`]) accrues the
26//! streamed `LlmText` between the start/end framing, appends the completed reply to
27//! the shared context as an `assistant` message, and emits a [`Frame::TtsSpeak`]
28//! to synthesize it (mirrors pipecat `LLMAssistantContextAggregator`).
29//! - **TTS** ([`TtsProcessor`]) turns `TtsSpeak` → `OutputAudio` (mapped from
30//! `TtsAudio`) for the transport-out sink.
31//!
32//! A `transport.input` source and `transport.output` sink are **not** owned by this
33//! builder (the live media path / transport layer provides them, fed via
34//! [`SourcePump`](crate::pipeline::SourcePump)); the builder produces the inner
35//! STT→…→TTS chain as a [`PipelineTask`], and the caller queues `InputAudio` at the
36//! head and consumes `OutputAudio` at the tail. The mock fixture below drives it
37//! exactly that way.
38//!
39//! ## The summarizer hook (fire-and-forget, pipecat parity)
40//!
41//! Per pipecat's `LLMContextSummarizer`, when the rolling context grows past a
42//! threshold (or a turn boundary fires) the aggregator emits an
43//! **on-transition summarize** signal: a fire-and-forget call into an injected
44//! [`ContextSummarizer`] that compresses old history off the hot path. The builder
45//! wires an optional summarizer; if none is given, the hook is a no-op (the mock
46//! path). This is the cascaded analogue of the realtime path's transition re-prompt.
47
48use std::sync::{Arc, Mutex};
49
50use async_trait::async_trait;
51use serde_json::{json, Value};
52use tokio::time::Instant;
53
54use crate::brain::AgentBrain;
55use crate::codec::Resampler;
56use crate::error::{FlowcatError, Result};
57use crate::processor::frame::{Frame, LlmContext, StartParams};
58use crate::processor::{Envelope, FrameProcessor, Link, ProcessorSetup};
59use crate::service::adapters::{LlmProcessor, SttProcessor, TtsProcessor};
60use crate::service::{LlmService, SttService, TtsService};
61use crate::session::SessionSource;
62use crate::transport::MediaTransport;
63use crate::types::AudioChunk;
64
65// Reuse the **realtime** path's outer processors verbatim (no parallel impl): the
66// brain/transition wiring, the recorder/transcript/finalize taps, the shared
67// live-call state, the transport head pump, and the tool-relay seam. These are the
68// `pub(crate)` items in [`super::s2s`] — the cascaded task differs only in its
69// *inner* chain (STT→agg→LLM→agg→TTS) and the LLM→brain tool-call bridge.
70use super::s2s::{
71 advance_playout, spawn_transport_pump, BrainProcessor, FinalizeProcessor, LiveState,
72 ModelToolCall, RecorderProcessor, Reprompt, SessionToolRelay, SharedState, SharedTransport,
73 ToolRelay, ToolResult, TranscriptProcessor, TransportInput, MAX_PLAYOUT_DRAIN,
74};
75use super::{Pipeline, PipelineTask, PipelineTaskParams, SourcePump};
76
77// ===========================================================================
78// Shared rolling LLM context — the cascaded analogue of S2S's LiveState.
79// ===========================================================================
80
81/// The rolling conversation context shared by the user + assistant aggregators.
82/// Holds the message history (the same `messages` shape [`LlmContext`] carries) and
83/// the current tool set. Guarded by a **`std::sync::Mutex`** that is only ever held
84/// for the brief synchronous mutate-and-clone — **never across an `.await`** (every
85/// aggregator snapshots under the lock, drops the guard, then awaits the push).
86#[derive(Default)]
87struct RollingContext {
88 /// Message history (role/content JSON objects), oldest-first.
89 messages: Vec<Value>,
90 /// Optional system prompt prepended when building the run context.
91 system_prompt: Option<String>,
92 /// Tools advertised to the LLM (carried into each `LlmContext`).
93 tools: Vec<Value>,
94}
95
96impl RollingContext {
97 fn new(system_prompt: Option<String>, tools: Vec<Value>) -> Self {
98 Self {
99 messages: Vec::new(),
100 system_prompt,
101 tools,
102 }
103 }
104
105 /// Append a `{role, content}` message.
106 fn push(&mut self, role: &str, content: &str) {
107 self.messages
108 .push(json!({ "role": role, "content": content }));
109 }
110
111 /// Build an immutable [`LlmContext`] snapshot to run the LLM over (system
112 /// prompt first if set, then the rolling history).
113 fn snapshot(&self) -> LlmContext {
114 let mut messages = Vec::with_capacity(self.messages.len() + 1);
115 if let Some(sys) = &self.system_prompt {
116 messages.push(json!({ "role": "system", "content": sys }));
117 }
118 messages.extend(self.messages.iter().cloned());
119 LlmContext {
120 messages,
121 tools: self.tools.clone(),
122 }
123 }
124
125 /// Number of non-system messages in the rolling history.
126 fn len(&self) -> usize {
127 self.messages.len()
128 }
129
130 /// Swap the active system prompt + tool set on a graph **transition** (the
131 /// cascaded analogue of the realtime path's `update_system`). Subsequent
132 /// [`snapshot`](RollingContext::snapshot)s carry the new prompt + tools, so the
133 /// re-run LLM speaks for the destination node. `tools` arrive as the brain's
134 /// [`ToolDecl`] shape and are stored as the opaque JSON the [`LlmContext`]
135 /// carries.
136 fn reprompt(&mut self, system_prompt: String, tools: Vec<Value>) {
137 self.system_prompt = Some(system_prompt);
138 self.tools = tools;
139 }
140
141 /// Append a workflow (MCP/HTTP) **tool result** to the rolling history as a
142 /// `tool` message so the re-run LLM sees the result and continues the turn
143 /// (the cascaded analogue of the realtime `send_tool_result`). The content is
144 /// stored verbatim (a bare string, matching the brain's relay contract).
145 fn push_tool_result(&mut self, tool_call_id: &str, content: &Value) {
146 self.messages.push(json!({
147 "role": "tool",
148 "tool_call_id": tool_call_id,
149 "content": content,
150 }));
151 }
152}
153
154type SharedContext = Arc<Mutex<RollingContext>>;
155
156// ===========================================================================
157// TurnMute — half-duplex turn lock (serializes turns, no self-echo).
158// ===========================================================================
159
160/// Half-duplex turn lock: STT stays muted from turn start (a final `Transcription`
161/// → `LlmContext`, or the kickoff greeting) until the bot's reply has played out, so
162/// a second question can't queue a concurrent LLM run and the bot can't transcribe
163/// its own echo. Shared (Clone/`Arc`) by the user + kickoff processors (which
164/// `begin` a turn) and the sink (which extends the playout estimate). No barge-in.
165#[derive(Clone)]
166struct TurnMute(Arc<TurnMuteInner>);
167
168struct TurnMuteInner {
169 muted: std::sync::atomic::AtomicBool,
170 /// Bumped each `begin()` so a stale unmute watchdog supersedes itself.
171 generation: std::sync::atomic::AtomicU64,
172 /// When the bot's audio sent so far finishes playing at the carrier.
173 bot_until: Mutex<Option<Instant>>,
174 /// Injects `SttMute` at the pipeline head (→ down to the `SttProcessor`).
175 end_tx: tokio::sync::mpsc::UnboundedSender<Frame>,
176 /// Unmute after this long if a turn produces NO bot audio (a tool-only / no-reply
177 /// turn), so the lock can never deadlock.
178 no_audio_timeout: std::time::Duration,
179}
180
181impl TurnMute {
182 fn new(
183 end_tx: tokio::sync::mpsc::UnboundedSender<Frame>,
184 no_audio_timeout: std::time::Duration,
185 ) -> Self {
186 TurnMute(Arc::new(TurnMuteInner {
187 muted: std::sync::atomic::AtomicBool::new(false),
188 generation: std::sync::atomic::AtomicU64::new(0),
189 bot_until: Mutex::new(None),
190 end_tx,
191 no_audio_timeout,
192 }))
193 }
194
195 /// Mute the STT (once) and spawn the unmute watchdog. No-op if a turn is already
196 /// in flight — the lock holds the whole turn (covering the LLM-thinking gap too).
197 fn begin(&self) {
198 use std::sync::atomic::Ordering;
199 if self.0.muted.swap(true, Ordering::SeqCst) {
200 return; // already muted — a turn is in flight
201 }
202 tracing::debug!("half-duplex: muting STT (turn started)");
203 let _ = self.0.end_tx.send(Frame::SttMute(true));
204 *self.0.bot_until.lock().unwrap() = None;
205 let my_gen = self.0.generation.fetch_add(1, Ordering::SeqCst) + 1;
206 let inner = self.0.clone();
207 tokio::spawn(async move {
208 let deadline = Instant::now() + inner.no_audio_timeout;
209 loop {
210 if inner.generation.load(Ordering::SeqCst) != my_gen {
211 return; // superseded by a newer turn
212 }
213 let until = *inner.bot_until.lock().unwrap();
214 let now = Instant::now();
215 match until {
216 Some(t) if t > now => tokio::time::sleep(t - now).await, // bot still playing
217 Some(_) => break, // reply done → unmute
218 None if now >= deadline => break, // no audio (tool-only turn) → safety unmute
219 None => tokio::time::sleep(std::time::Duration::from_millis(200)).await,
220 }
221 }
222 if inner.generation.load(Ordering::SeqCst) == my_gen
223 && inner.muted.swap(false, Ordering::SeqCst)
224 {
225 tracing::debug!("half-duplex: unmuting STT (turn complete)");
226 let _ = inner.end_tx.send(Frame::SttMute(false));
227 }
228 });
229 }
230
231 /// Extend the playout estimate as the reply plays (the watchdog waits on it).
232 fn note_bot_audio(&self, samples: usize, carrier_rate: u32) {
233 let mut until = self.0.bot_until.lock().unwrap();
234 *until = Some(advance_playout(
235 *until,
236 Instant::now(),
237 samples,
238 carrier_rate,
239 ));
240 }
241
242 /// Take (and clear) the current playout estimate — for the sink's End-of-call drain.
243 fn take_bot_until(&self) -> Option<Instant> {
244 self.0.bot_until.lock().unwrap().take()
245 }
246}
247
248// ===========================================================================
249// Context summarizer hook (pipecat `LLMContextSummarizer`).
250// ===========================================================================
251
252/// A fire-and-forget context summarizer (pipecat `LLMContextSummarizer`). When the
253/// rolling context crosses [`SummarizerConfig::trigger_after_messages`], the user
254/// aggregator spawns [`summarize`](ContextSummarizer::summarize) off the hot path so
255/// it never delays the turn. The default impl is a no-op (the mock/no-summarizer
256/// path); a real impl compresses old history into a summary message.
257#[async_trait]
258pub trait ContextSummarizer: Send + Sync {
259 /// Summarize `messages` (the rolling history at the transition) into a single
260 /// replacement summary string, or `None` to leave the context untouched. Called
261 /// fire-and-forget — its result is applied opportunistically and never blocks
262 /// the next turn.
263 async fn summarize(&self, messages: &[Value]) -> Option<String>;
264}
265
266/// Tuning for the summarizer hook (pipecat `LLMAutoContextSummarizationConfig`).
267#[derive(Debug, Clone)]
268pub struct SummarizerConfig {
269 /// Fire the summarize hook once the rolling history reaches this many messages.
270 pub trigger_after_messages: usize,
271}
272
273impl Default for SummarizerConfig {
274 fn default() -> Self {
275 // Mirrors pipecat's default of summarizing on a long-ish context; small
276 // enough that a multi-turn call exercises it, large enough not to thrash.
277 Self {
278 trigger_after_messages: 20,
279 }
280 }
281}
282
283/// A no-op summarizer — the default when no summarizer is wired (the mock path).
284struct NoopSummarizer;
285
286#[async_trait]
287impl ContextSummarizer for NoopSummarizer {
288 async fn summarize(&self, _messages: &[Value]) -> Option<String> {
289 None
290 }
291}
292
293// ===========================================================================
294// UserContextAggregator (pipecat LLMUserContextAggregator).
295// ===========================================================================
296
297/// Accrues final user `Transcription` frames into the shared rolling context as
298/// `user` messages and, on each completed user turn, emits a [`Frame::LlmContext`]
299/// snapshot downstream to run the LLM. Also fires the fire-and-forget summarizer
300/// hook when the context grows past the configured threshold (pipecat
301/// `LLMUserContextAggregator` + `LLMContextSummarizer` trigger).
302///
303/// The final transcription is **consumed** (transformed into the `LlmContext`, not
304/// forwarded) so the downstream [`LlmProcessor`] is triggered exactly once per turn
305/// — by the `LlmContext`, never also by a raw `Transcription` (which would
306/// double-run the LLM). Observers still see the transcription via `on_process` at
307/// the STT processor and here. Only a `final_` transcription closes the user turn
308/// (the cascaded turn boundary); interim transcriptions and everything else pass
309/// through unchanged.
310pub struct UserContextAggregator {
311 ctx: SharedContext,
312 summarizer: Arc<dyn ContextSummarizer>,
313 summarizer_cfg: SummarizerConfig,
314 /// Guards against re-firing the summarizer for the same threshold crossing.
315 summarize_in_flight: Arc<std::sync::atomic::AtomicBool>,
316 /// Optional shared call state: when set, the final user transcription is tapped
317 /// into the transcript here (it is *consumed* into the LlmContext below, so it
318 /// never reaches the end-of-chain `TranscriptProcessor`). `None` in the bare
319 /// inner pipeline / unit tests.
320 transcript_state: Option<SharedState>,
321 /// Optional half-duplex turn lock: `begin()` is called as each user turn starts
322 /// (when the `LlmContext` is emitted), so a second question can't queue a
323 /// concurrent LLM run. `None` in the bare inner pipeline / unit tests.
324 turn_mute: Option<TurnMute>,
325}
326
327impl UserContextAggregator {
328 fn new(
329 ctx: SharedContext,
330 summarizer: Arc<dyn ContextSummarizer>,
331 summarizer_cfg: SummarizerConfig,
332 ) -> Self {
333 Self {
334 ctx,
335 summarizer,
336 summarizer_cfg,
337 summarize_in_flight: Arc::new(std::sync::atomic::AtomicBool::new(false)),
338 transcript_state: None,
339 turn_mute: None,
340 }
341 }
342
343 /// Tap the final user transcription into this shared call state's transcript
344 /// (the cascaded path's user side; the bot side rides `LlmText` to the
345 /// `TranscriptProcessor`). Builder-style so unit tests need no change.
346 fn with_transcript_state(mut self, state: SharedState) -> Self {
347 self.transcript_state = Some(state);
348 self
349 }
350
351 /// Wire the turn lock (live builder sets it; unit tests omit it).
352 fn with_turn_mute(mut self, tm: TurnMute) -> Self {
353 self.turn_mute = Some(tm);
354 self
355 }
356
357 /// Fire the summarizer off the hot path if the context grew past the threshold
358 /// (pipecat: summarize on transition, fire-and-forget). Applies the returned
359 /// summary by collapsing the history into a single summary message.
360 fn maybe_summarize(&self, len: usize) {
361 use std::sync::atomic::Ordering;
362 if len < self.summarizer_cfg.trigger_after_messages {
363 return;
364 }
365 // One in-flight summarize at a time (the pipecat `_summarization_in_progress`
366 // guard) — a compare-exchange so we never double-spawn.
367 if self
368 .summarize_in_flight
369 .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst)
370 .is_err()
371 {
372 return;
373 }
374 let ctx = self.ctx.clone();
375 let summarizer = self.summarizer.clone();
376 let in_flight = self.summarize_in_flight.clone();
377 tokio::spawn(async move {
378 // Snapshot the history under the std lock, drop the guard before await.
379 let history = {
380 let c = ctx.lock().unwrap();
381 c.messages.clone()
382 };
383 if let Some(summary) = summarizer.summarize(&history).await {
384 // Apply: replace the history with a single summary message (the
385 // pipecat "compress old history into a summary" result).
386 let mut c = ctx.lock().unwrap();
387 c.messages = vec![
388 json!({ "role": "system", "content": format!("Summary so far: {summary}") }),
389 ];
390 }
391 in_flight.store(false, Ordering::SeqCst);
392 });
393 }
394}
395
396#[async_trait]
397impl FrameProcessor for UserContextAggregator {
398 fn name(&self) -> &str {
399 "UserContextAggregator"
400 }
401
402 async fn process_frame(&mut self, env: Envelope, link: &Link) -> Result<()> {
403 if let Frame::Transcription {
404 text, final_: true, ..
405 } = &env.frame
406 {
407 // Tap the user line into the transcript (it's consumed below, so the
408 // end-of-chain TranscriptProcessor never sees it).
409 if let Some(st) = &self.transcript_state {
410 st.lock().unwrap().transcript.push_user(text);
411 }
412 // Accrue the user message + snapshot the context under the lock; drop the
413 // guard before awaiting the push (no std Mutex held across await).
414 let (snapshot, len) = {
415 let mut c = self.ctx.lock().unwrap();
416 c.push("user", text);
417 (c.snapshot(), c.len())
418 };
419 // Fire the summarizer hook off the hot path (pipecat transition summarize).
420 self.maybe_summarize(len);
421 // Lock the turn (mute STT until the reply plays out) so a second question
422 // can't queue a concurrent LLM run.
423 if let Some(tm) = &self.turn_mute {
424 tm.begin();
425 }
426 // Run the LLM over the aggregated context (pipecat: aggregator → LLMRun).
427 // The transcription is CONSUMED here (transformed into the LlmContext, not
428 // forwarded) so the LLM fires exactly once per turn — never also on a raw
429 // Transcription. Observers already saw it via on_process at STT + here.
430 link.push_down(Frame::LlmContext(Arc::new(snapshot))).await;
431 return Ok(());
432 }
433 link.push(env.meta, env.frame, env.direction).await;
434 Ok(())
435 }
436}
437
438// ===========================================================================
439// AssistantContextAggregator (pipecat LLMAssistantContextAggregator).
440// ===========================================================================
441
442/// Accrues the streamed LLM response (`LlmText` between `LlmResponseStart` and
443/// `LlmResponseEnd`) into a single assistant turn, appends the completed reply to
444/// the shared rolling context as an `assistant` message, and emits a
445/// [`Frame::TtsSpeak`] to synthesize it (pipecat `LLMAssistantContextAggregator`).
446///
447/// `LlmText` chunks are buffered (not forwarded individually); the assembled reply
448/// is emitted once as a `TtsSpeak` on `LlmResponseEnd`, so the TTS service speaks
449/// whole utterances. The framing + text frames are still forwarded so downstream
450/// transcript taps observe the response. Everything else passes through unchanged.
451pub struct AssistantContextAggregator {
452 ctx: SharedContext,
453 /// The in-progress assistant turn (accrued `LlmText`).
454 buffer: String,
455 /// Whether we are inside a `LlmResponseStart`/`End` span.
456 in_response: bool,
457 /// Optional shared call state: when set, the completed bot reply is tapped into
458 /// the transcript here (the cascaded bot side; the user side is tapped in
459 /// `UserContextAggregator`). `None` in the bare inner pipeline / unit tests.
460 transcript_state: Option<SharedState>,
461}
462
463impl AssistantContextAggregator {
464 fn new(ctx: SharedContext) -> Self {
465 Self {
466 ctx,
467 buffer: String::new(),
468 in_response: false,
469 transcript_state: None,
470 }
471 }
472
473 /// Tap the completed bot reply into this shared call state's transcript.
474 /// Builder-style so unit tests need no change.
475 fn with_transcript_state(mut self, state: SharedState) -> Self {
476 self.transcript_state = Some(state);
477 self
478 }
479}
480
481#[async_trait]
482impl FrameProcessor for AssistantContextAggregator {
483 fn name(&self) -> &str {
484 "AssistantContextAggregator"
485 }
486
487 async fn process_frame(&mut self, env: Envelope, link: &Link) -> Result<()> {
488 match &env.frame {
489 Frame::LlmResponseStart => {
490 self.in_response = true;
491 self.buffer.clear();
492 link.push(env.meta, env.frame, env.direction).await;
493 }
494 Frame::LlmText(chunk) => {
495 // Accrue the streamed token and CONSUME it (do NOT forward). The
496 // assembled reply is emitted once as a `TtsSpeak` on `LlmResponseEnd`,
497 // so the TTS service speaks the whole utterance exactly once.
498 // Forwarding each token would make `TtsProcessor` synthesize every
499 // word individually (it also speaks raw `LlmText` — its no-aggregator
500 // fallback), producing the choppy word-by-word "hi. how. can. i…"
501 // double-speak. The bot transcript is tapped directly below
502 // (`push_bot`), so nothing downstream needs the raw tokens. Symmetric
503 // with `UserContextAggregator`, which consumes the `Transcription`.
504 self.buffer.push_str(chunk);
505 }
506 Frame::LlmResponseEnd => {
507 self.in_response = false;
508 let reply = std::mem::take(&mut self.buffer);
509 if !reply.is_empty() {
510 // Tap the bot reply into the transcript (the cascaded bot side).
511 if let Some(st) = &self.transcript_state {
512 st.lock().unwrap().transcript.push_bot(&reply);
513 }
514 // Append the completed reply to the rolling context (under the
515 // lock, then drop before awaiting).
516 {
517 let mut c = self.ctx.lock().unwrap();
518 c.push("assistant", &reply);
519 }
520 // Drive TTS with the whole assembled utterance.
521 link.push_down(Frame::TtsSpeak {
522 text: reply,
523 append_to_context: Some(false),
524 })
525 .await;
526 }
527 link.push(env.meta, env.frame, env.direction).await;
528 }
529 _ => link.push(env.meta, env.frame, env.direction).await,
530 }
531 Ok(())
532 }
533}
534
535// ===========================================================================
536// The builder.
537// ===========================================================================
538
539/// Optional knobs for [`build_cascaded_pipeline`]. Defaults give the mock-friendly
540/// shape (no system prompt, no tools, no summarizer, default task params).
541pub struct CascadedConfig {
542 /// Initial system prompt prepended to every LLM run context.
543 pub system_prompt: Option<String>,
544 /// Initial tool set advertised to the LLM (carried into each `LlmContext`).
545 pub tools: Vec<Value>,
546 /// Optional context summarizer (fire-and-forget on transition). `None` ⇒ no-op.
547 pub summarizer: Option<Arc<dyn ContextSummarizer>>,
548 /// Summarizer trigger tuning.
549 pub summarizer_cfg: SummarizerConfig,
550 /// Pipeline task params (audio rates, idle timeout, …).
551 pub task_params: PipelineTaskParams,
552}
553
554impl Default for CascadedConfig {
555 fn default() -> Self {
556 Self {
557 system_prompt: None,
558 tools: Vec::new(),
559 summarizer: None,
560 summarizer_cfg: SummarizerConfig::default(),
561 // The cascaded path has its own turn boundaries; keep the idle gate off
562 // by default so a fixture run never trips the 300 s timer.
563 task_params: PipelineTaskParams {
564 idle_timeout: None,
565 ..Default::default()
566 },
567 }
568 }
569}
570
571/// Assemble the cascaded STT→user-agg→LLM→assistant-agg→TTS [`Pipeline`] +
572/// `PipelineTaskParams` from the service impls + config (the shared inner builder
573/// behind [`build_cascaded_pipeline`] / [`build_cascaded_task_with_observers`]).
574fn assemble<S, L, T>(
575 stt: S,
576 llm: L,
577 tts: T,
578 config: CascadedConfig,
579) -> (Pipeline, PipelineTaskParams)
580where
581 S: SttService + 'static,
582 L: LlmService + 'static,
583 T: TtsService + 'static,
584{
585 let ctx: SharedContext = Arc::new(Mutex::new(RollingContext::new(
586 config.system_prompt,
587 config.tools,
588 )));
589 let summarizer = config
590 .summarizer
591 .unwrap_or_else(|| Arc::new(NoopSummarizer) as Arc<dyn ContextSummarizer>);
592
593 let processors: Vec<Box<dyn FrameProcessor>> = vec![
594 Box::new(SttProcessor::new(stt)),
595 Box::new(UserContextAggregator::new(
596 ctx.clone(),
597 summarizer,
598 config.summarizer_cfg,
599 )),
600 Box::new(LlmProcessor::new(llm)),
601 Box::new(AssistantContextAggregator::new(ctx.clone())),
602 Box::new(TtsProcessor::new(tts)),
603 ];
604 (Pipeline::new(processors), config.task_params)
605}
606
607/// Build the cascaded STT→LLM→TTS [`PipelineTask`]
608/// (`transport.input → STT → user-agg → LLM → assistant-agg → TTS → transport.output`).
609///
610/// `stt`/`llm`/`tts` are any impls of the frozen
611/// [`SttService`]/[`LlmService`]/[`TtsService`] traits — the mock impls for tests,
612/// real providers (Deepgram/OpenAI/Cartesia in `flowcat-services`) in production.
613/// The returned task is driven exactly like the S2S task: queue `InputAudio` at the
614/// head (or feed a transport via [`SourcePump`](crate::pipeline::SourcePump)) and
615/// consume `OutputAudio` at the tail.
616///
617/// This is generic over the three service traits and pulls **no network dependency**
618/// — flowcat-core stays dep-light; the providers live in `flowcat-services`.
619pub fn build_cascaded_pipeline<S, L, T>(
620 stt: S,
621 llm: L,
622 tts: T,
623 config: CascadedConfig,
624) -> PipelineTask
625where
626 S: SttService + 'static,
627 L: LlmService + 'static,
628 T: TtsService + 'static,
629{
630 build_cascaded_task_with_observers(stt, llm, tts, config, vec![])
631}
632
633/// As [`build_cascaded_pipeline`], but attaches `observers` to the task (the
634/// fixture path provider tests use to assert frame flow, and the live path uses to wire
635/// metrics/RTVI/transcript observers).
636pub fn build_cascaded_task_with_observers<S, L, T>(
637 stt: S,
638 llm: L,
639 tts: T,
640 config: CascadedConfig,
641 observers: Vec<Arc<dyn crate::observer::FrameObserver>>,
642) -> PipelineTask
643where
644 S: SttService + 'static,
645 L: LlmService + 'static,
646 T: TtsService + 'static,
647{
648 let (pipeline, params) = assemble(stt, llm, tts, config);
649 PipelineTask::new(pipeline, params, observers)
650}
651
652// ===========================================================================
653// `build_cascaded_task`: the cascaded analogue of `build_s2s_task`.
654//
655// The realtime `build_s2s_task` composes the realtime model with the brain /
656// recorder / transcript / finalize outer processors. The cascaded path needs the
657// SAME outer processors around the cascaded INNER chain (STT→user-agg→LLM→
658// assistant-agg→TTS). The one new piece is the LLM→brain **tool-call bridge**:
659//
660// ```text
661// [pump]→TransportInput→STT→UserAgg→LLM→CascadedToolBridge→Brain→AssistantAgg→TTS→CascadedOutput→Recorder→Transcript→Finalize
662// │ (FunctionCallsStarted → ModelToolCall, downstream) ▲
663// ▲ (Reprompt/ToolResult upstream → context update + LLM re-run)│
664// ```
665//
666// In realtime mode the model's tool-calls drive both workflow tools (via
667// `ToolRelay`) and graph transitions (via the `AgentBrain` seam on a tool call); in
668// cascaded mode the **cascaded LLM's** function-calls drive the exact same brain —
669// the only difference is the *source* of the tool-call (the LLM, not the realtime
670// model) and how the brain's result is consumed (fed back into the rolling LLM
671// context + a fresh LLM run, rather than `realtime.send_tool_result`).
672//
673// Generic over the four seams + the three services, so it works with both the mocks
674// and the real flowcat-services providers, and flowcat-core stays embedder-agnostic.
675// ===========================================================================
676
677// ---------------------------------------------------------------------------
678// CascadedToolBridge — the LLM↔brain glue.
679// ---------------------------------------------------------------------------
680
681/// The bridge that lets the **reused realtime [`BrainProcessor`]** drive the
682/// cascaded LLM. It sits between the [`LlmProcessor`] (upstream) and the
683/// [`BrainProcessor`] (downstream) and translates in both directions:
684///
685/// - **Downstream (LLM → brain):** the cascaded LLM emits
686/// [`Frame::FunctionCallsStarted`] when it calls a function. The bridge converts
687/// each [`FunctionCall`](crate::processor::frame::FunctionCall) into a
688/// [`ModelToolCall`] custom frame — the **exact**
689/// frame the realtime `RealtimeServiceProcessor` emits — so the downstream
690/// `BrainProcessor` treats a cascaded LLM tool-call identically to a realtime
691/// model tool-call (workflow-relay vs. transition vs. end). The
692/// `FunctionCallsStarted` frame itself is **consumed** (not forwarded) so the
693/// assistant aggregator / TTS never speak a tool-call as if it were a reply.
694///
695/// - **Upstream (brain → LLM):** the `BrainProcessor` pushes its decision **upstream**
696/// as [`Reprompt`] / [`ToolResult`] custom frames (toward where the realtime
697/// service would consume them). The bridge intercepts them and applies the
698/// cascaded analogue:
699/// * [`Reprompt`] (a graph **transition**) → swap the rolling context's system
700/// prompt + tools, then re-run the LLM (emit a fresh [`Frame::LlmContext`]
701/// upstream to the [`LlmProcessor`]) so the bot speaks for the new node.
702/// * [`ToolResult`] whose payload is a bare value (a **workflow/MCP** result,
703/// per the brain's relay contract — `json!(content)`, not a `{status}`
704/// envelope) → append it to the rolling context as a `tool` message and re-run
705/// the LLM so the model continues the turn with the result in hand.
706/// * [`ToolResult`] carrying a `{status: moved|stay|ended}` ack → a no-op (the
707/// transition is already driven by the paired `Reprompt`; `ended` is driven by
708/// the brain's head-`End`; `stay` needs no re-run).
709///
710/// This is the cascaded counterpart of the realtime `RealtimeServiceProcessor`'s
711/// upstream-frame consumer (`send_tool_result`/`update_system`) — same brain, same
712/// custom frames, a different back-end.
713///
714/// True iff a [`ToolResult`] payload is a graph transition/stay/end **ACK** (the
715/// brain's `{status: moved|stay|ended}` envelope), as opposed to a workflow/MCP tool
716/// result that must be fed back to the LLM. We match the brain's exact ACK vocabulary
717/// — NOT merely the presence of a `status` key — because MCP tool results are JSON
718/// objects (see [`s2s::mcp_result_struct`]) and many carry their own
719/// `{"status": "ok"/"confirmed"}`; those must reach the LLM, not be swallowed as acks.
720fn is_transition_ack(result: &Value) -> bool {
721 result
722 .get("status")
723 .and_then(Value::as_str)
724 .is_some_and(|s| matches!(s, "moved" | "stay" | "ended"))
725}
726
727struct CascadedToolBridge {
728 /// The shared rolling LLM context (also held by the user/assistant aggregators),
729 /// updated on transitions + tool results.
730 ctx: SharedContext,
731}
732
733impl CascadedToolBridge {
734 fn new(ctx: SharedContext) -> Self {
735 Self { ctx }
736 }
737}
738
739#[async_trait]
740impl FrameProcessor for CascadedToolBridge {
741 fn name(&self) -> &str {
742 "CascadedToolBridge"
743 }
744
745 async fn process_frame(&mut self, env: Envelope, link: &Link) -> Result<()> {
746 match &env.frame {
747 // ---- Downstream: LLM tool-calls → the brain (as ModelToolCall) ----
748 // Convert each FunctionCall to the realtime path's ModelToolCall frame so
749 // the reused BrainProcessor consumes it unchanged. Consume the original
750 // (don't forward it past the brain into the assistant aggregator / TTS).
751 Frame::FunctionCallsStarted(calls) => {
752 for c in calls {
753 link.push_down(Frame::Custom(Arc::new(ModelToolCall {
754 id: c.tool_call_id.clone(),
755 name: c.function_name.clone(),
756 args: c.arguments.clone(),
757 })))
758 .await;
759 }
760 }
761
762 // ---- Upstream: a transition re-prompt from the brain ----
763 // Swap the rolling context's prompt + tools, then re-run the LLM so the
764 // bot speaks for the destination node (cascaded analogue of update_system
765 // + the model continuing). Tools carry as the opaque JSON the LlmContext
766 // holds.
767 Frame::Custom(c) if c.as_any().is::<Reprompt>() => {
768 let rp = c.as_any().downcast_ref::<Reprompt>().unwrap();
769 let tools_json: Vec<Value> = rp
770 .tools
771 .iter()
772 .map(|t| serde_json::to_value(t).unwrap_or(Value::Null))
773 .collect();
774 let snapshot = {
775 let mut ctx = self.ctx.lock().unwrap();
776 ctx.reprompt(rp.prompt.clone(), tools_json);
777 ctx.snapshot()
778 };
779 // Re-run the LLM (upstream) for the new node. A transition → a second
780 // utterance; this log helps spot an auto-transition right after the greeting.
781 tracing::debug!("cascaded transition: reprompt → re-running LLM for new node");
782 link.push_up(Frame::LlmContext(Arc::new(snapshot))).await;
783 }
784
785 // ---- Upstream: a tool result from the brain ----
786 Frame::Custom(c) if c.as_any().is::<ToolResult>() => {
787 let tr = c.as_any().downcast_ref::<ToolResult>().unwrap();
788 // A `{status: moved|stay|ended}` envelope is a transition/stay/end
789 // ACK (no re-run here — the transition re-runs via its paired
790 // Reprompt; `ended` via the brain's head-End). Any other payload is a
791 // workflow/MCP result → feed it back + re-run.
792 if !is_transition_ack(&tr.result) {
793 let snapshot = {
794 let mut ctx = self.ctx.lock().unwrap();
795 ctx.push_tool_result(&tr.id, &tr.result);
796 ctx.snapshot()
797 };
798 link.push_up(Frame::LlmContext(Arc::new(snapshot))).await;
799 }
800 }
801
802 // Everything else (LlmResponseStart/Text/End, audio, lifecycle) flows on
803 // to the brain → assistant aggregator → TTS unchanged.
804 _ => link.push(env.meta, env.frame, env.direction).await,
805 }
806 Ok(())
807 }
808}
809
810// ---------------------------------------------------------------------------
811// CascadedTransportOutput — the sink, parameterized by the TTS source rate.
812// ---------------------------------------------------------------------------
813
814/// The cascaded sink-side stage. Like the realtime [`super::s2s::TransportOutput`]
815/// but the source rate is the **TTS service's** sample rate (not Gemini's fixed
816/// 24 kHz): it records the bot leg at the TTS rate and resamples TTS-rate→carrier
817/// before `send_audio`. Consumes `Interruption` → `send_clear`. Records into the
818/// **same** shared [`LiveState`] the reused [`RecorderProcessor`]/[`FinalizeProcessor`]
819/// read.
820struct CascadedTransportOutput<T: MediaTransport> {
821 transport: T,
822 out_resampler: Option<Resampler>,
823 tts_rate: u32,
824 carrier_rate: u32,
825 state: SharedState,
826 /// Sends `Frame::End` once on transport death.
827 end_tx: tokio::sync::mpsc::UnboundedSender<Frame>,
828 /// Turn lock: extend its playout estimate per chunk (drives unmute + the End-drain).
829 turn_mute: TurnMute,
830}
831
832impl<T: MediaTransport> CascadedTransportOutput<T> {
833 fn new(
834 transport: T,
835 tts_rate: u32,
836 carrier_rate: u32,
837 state: SharedState,
838 end_tx: tokio::sync::mpsc::UnboundedSender<Frame>,
839 turn_mute: TurnMute,
840 ) -> Self {
841 Self {
842 transport,
843 out_resampler: None,
844 tts_rate,
845 carrier_rate,
846 state,
847 end_tx,
848 turn_mute,
849 }
850 }
851
852 /// Terminal transport error (peer gone): record it (de-duped to one log) and end
853 /// the call **once**. The `OutputAudio` guard then drops remaining buffered frames
854 /// instead of re-failing on the dead transport.
855 fn fail_transport(&self, e: FlowcatError) {
856 let newly_dead = {
857 let mut st = self.state.lock().unwrap();
858 let was = st.transport_dead;
859 st.record_error(e); // logs once + sets transport_dead for a transport error
860 st.transport_dead && !was
861 };
862 if newly_dead {
863 let _ = self.end_tx.send(Frame::End { reason: None });
864 }
865 }
866}
867
868#[async_trait]
869impl<T: MediaTransport + 'static> FrameProcessor for CascadedTransportOutput<T> {
870 fn name(&self) -> &str {
871 "CascadedTransportOutput"
872 }
873
874 async fn start(&mut self, _setup: &ProcessorSetup, _params: &StartParams) -> Result<()> {
875 self.out_resampler = Some(Resampler::new(self.tts_rate, self.carrier_rate)?);
876 Ok(())
877 }
878
879 async fn process_frame(&mut self, env: Envelope, link: &Link) -> Result<()> {
880 match &env.frame {
881 // Bot audio out (the TtsProcessor maps TtsAudio → OutputAudio): record at
882 // the TTS rate, resample TTS-rate→carrier, send to the carrier.
883 Frame::OutputAudio(audio) | Frame::TtsAudio { audio, .. } => {
884 // Transport already dead → drop buffered bot audio (no re-send, no
885 // repeat WARN); the call is tearing down.
886 if self.state.lock().unwrap().transport_dead {
887 return Ok(());
888 }
889 let chunk = AudioChunk::from(audio.as_ref());
890 self.state.lock().unwrap().recorder.push_outbound(&chunk);
891 let resampler = self
892 .out_resampler
893 .as_mut()
894 .expect("out_resampler set in start()");
895 match resampler.process(&chunk) {
896 Ok(down) => {
897 if !down.is_empty() {
898 let samples = down.pcm.len();
899 if let Err(e) = self.transport.send_audio(down).await {
900 self.fail_transport(e);
901 } else {
902 // Keep STT muted until this reply finishes playing.
903 self.turn_mute.note_bot_audio(samples, self.carrier_rate);
904 }
905 }
906 }
907 Err(e) => self.state.lock().unwrap().record_error(e),
908 }
909 }
910 // Barge-in: flush the carrier playback.
911 Frame::Interruption => {
912 if let Err(e) = self.transport.send_clear().await {
913 self.fail_transport(e);
914 }
915 let _ = self.turn_mute.take_bot_until();
916 link.push(env.meta, env.frame, env.direction).await;
917 }
918 // End-of-call: wait out the final bot utterance still playing at the
919 // carrier before teardown (mirrors the realtime sink — see s2s.rs).
920 Frame::End { .. } => {
921 let until = self.turn_mute.take_bot_until();
922 if let Some(until) = until {
923 let now = Instant::now();
924 if until > now {
925 tokio::time::sleep((until - now).min(MAX_PLAYOUT_DRAIN)).await;
926 }
927 }
928 link.push(env.meta, env.frame, env.direction).await;
929 }
930 _ => link.push(env.meta, env.frame, env.direction).await,
931 }
932 Ok(())
933 }
934}
935
936// ---------------------------------------------------------------------------
937// The assembler.
938// ---------------------------------------------------------------------------
939
940/// A built cascaded task plus its transport pump handle — the cascaded analogue of
941/// [`super::s2s::S2sTask`].
942///
943/// Drive it with `task.run().await`; the pump feeds the transport's `recv()` into
944/// the pipeline head and aborts when `run()` returns.
945pub struct CascadedTask {
946 /// The wired pipeline task — run it to completion.
947 pub task: PipelineTask,
948 /// The transport-pump source reader (aborted on drop / after `run`).
949 pump: SourcePump,
950}
951
952impl CascadedTask {
953 /// Run the cascaded pipeline to completion, then stop the transport pump.
954 pub async fn run(self) -> Result<()> {
955 let res = self.task.run().await;
956 self.pump.abort();
957 res
958 }
959}
960
961// ===========================================================================
962// CascadedKickoffProcessor — the cascaded analogue of the realtime `kickoff`.
963// ===========================================================================
964
965/// Makes the bot **greet on connect** (pipecat parity). The transport pump emits
966/// [`Frame::ClientConnected`] when the media stream starts; the realtime path
967/// consumes it and calls `realtime.kickoff()` so the model speaks first
968/// ([`super::s2s`] `ClientConnected → kickoff`). The cascaded chain had no such
969/// consumer, so the bot stayed silent until the user spoke. This processor closes
970/// that gap: on the **first** `ClientConnected` it snapshots the already-seeded
971/// rolling context (system prompt + start-node tools, empty history) and emits a
972/// [`Frame::LlmContext`] — the exact frame the user aggregator emits per turn — so
973/// the downstream `LlmProcessor → BrainProcessor → AssistantContextAggregator →
974/// TtsProcessor` path generates and speaks the greeting from the node prompt. The
975/// assistant aggregator appends the greeting to the rolling context, so the first
976/// real user turn carries correct history. Idempotent (greets once per call).
977struct CascadedKickoffProcessor {
978 ctx: SharedContext,
979 kicked_off: bool,
980 turn_mute: Option<TurnMute>,
981}
982
983impl CascadedKickoffProcessor {
984 fn new(ctx: SharedContext) -> Self {
985 Self {
986 ctx,
987 kicked_off: false,
988 turn_mute: None,
989 }
990 }
991
992 /// Wire the turn lock (live builder sets it; unit tests omit it).
993 fn with_turn_mute(mut self, tm: TurnMute) -> Self {
994 self.turn_mute = Some(tm);
995 self
996 }
997}
998
999#[async_trait]
1000impl FrameProcessor for CascadedKickoffProcessor {
1001 fn name(&self) -> &str {
1002 "CascadedKickoff"
1003 }
1004
1005 async fn process_frame(&mut self, env: Envelope, link: &Link) -> Result<()> {
1006 if !self.kicked_off && matches!(env.frame, Frame::ClientConnected) {
1007 self.kicked_off = true;
1008 tracing::info!("cascaded kickoff: greeting on ClientConnected");
1009 // The greeting is a bot turn → lock it.
1010 if let Some(tm) = &self.turn_mute {
1011 tm.begin();
1012 }
1013 // Run the LLM once over the seeded context (system prompt only). Lock is
1014 // dropped before the await (no std Mutex held across `.await`).
1015 let mut snapshot = self.ctx.lock().unwrap().snapshot();
1016 // Text-only greeting: clear tools so the opening turn can't `endCall`/
1017 // transition and hang up before the bot speaks. Tools return on user turns.
1018 snapshot.tools.clear();
1019 link.push_down(Frame::LlmContext(Arc::new(snapshot))).await;
1020 }
1021 // Always forward the original frame so `ClientConnected` keeps propagating.
1022 link.push(env.meta, env.frame, env.direction).await;
1023 Ok(())
1024 }
1025}
1026
1027/// Assemble the cascaded processor task beside [`build_s2s_task`](super::s2s::build_s2s_task):
1028/// the cascaded **inner** chain (STT → user-agg → LLM → assistant-agg → TTS, the
1029/// [`build_cascaded_pipeline`] topology) wrapped with the **same outer processors**
1030/// `build_s2s_task` uses — `TransportInput` (fed by the transport [`SourcePump`]),
1031/// the brain/transition wiring ([`BrainProcessor`] + [`SessionToolRelay`]), the
1032/// recorder / transcript / finalize taps, and a TTS-rate-aware transport sink. The
1033/// LLM↔brain [`CascadedToolBridge`] integrates the cascaded LLM's function-calls into
1034/// the brain exactly as the realtime model's tool-calls feed it in `build_s2s_task`.
1035///
1036/// Mirrors `build_s2s_task`'s structure: resolve the start node's tools, build the
1037/// shared state, share the transport between the pump and the sink, route an inner
1038/// `End` request through the head, and spawn the pump. Generic over the four
1039/// flowcat-core seams (`MediaTransport`/`AgentBrain`/`SessionSource` + the three
1040/// services), so it drives both mocks and real providers; flowcat-core stays
1041/// embedder-agnostic.
1042///
1043/// Delegates to [`build_cascaded_task_with_observers`] with no observers
1044/// (historical signature — unchanged for every existing caller + test).
1045#[allow(clippy::too_many_arguments)]
1046pub async fn build_cascaded_task<Tr, St, L, Ts, B, Se>(
1047 transport: Tr,
1048 stt: St,
1049 llm: L,
1050 tts: Ts,
1051 brain: B,
1052 session: Se,
1053 run_id: i64,
1054 token: String,
1055 config: CascadedConfig,
1056) -> Result<CascadedTask>
1057where
1058 Tr: MediaTransport + 'static,
1059 St: SttService + 'static,
1060 L: LlmService + 'static,
1061 Ts: TtsService + 'static,
1062 B: AgentBrain + 'static,
1063 Se: SessionSource + 'static,
1064{
1065 build_cascaded_call_with_observers(
1066 transport,
1067 stt,
1068 llm,
1069 tts,
1070 brain,
1071 session,
1072 run_id,
1073 token,
1074 config,
1075 vec![],
1076 )
1077 .await
1078}
1079
1080/// `build_cascaded_task` with external pipeline `observers` (e.g. an `RtviObserver`
1081/// streaming live transcript/RTF events). The full-call (outer) counterpart of the
1082/// inner [`build_cascaded_task_with_observers`]; this is the one the media host's
1083/// `run_call` uses.
1084#[allow(clippy::too_many_arguments)]
1085pub async fn build_cascaded_call_with_observers<Tr, St, L, Ts, B, Se>(
1086 transport: Tr,
1087 stt: St,
1088 llm: L,
1089 tts: Ts,
1090 brain: B,
1091 session: Se,
1092 run_id: i64,
1093 token: String,
1094 config: CascadedConfig,
1095 observers: Vec<Arc<dyn crate::observer::FrameObserver>>,
1096) -> Result<CascadedTask>
1097where
1098 Tr: MediaTransport + 'static,
1099 St: SttService + 'static,
1100 L: LlmService + 'static,
1101 Ts: TtsService + 'static,
1102 B: AgentBrain + 'static,
1103 Se: SessionSource + 'static,
1104{
1105 let carrier_rate = transport.carrier_rate();
1106 let tts_rate = tts.sample_rate();
1107 let session = Arc::new(session);
1108 let relay = Arc::new(SessionToolRelay::new(
1109 session.clone(),
1110 run_id,
1111 token.clone(),
1112 ));
1113
1114 // 1. Resolve the start node's tool set (transitions + node MCP tools) and the
1115 // MCP-name branch set — verbatim build_s2s_task's resolve ordering.
1116 let node_id = brain.current_node_id();
1117 let mcp = relay.node_tools(&node_id).await;
1118 let mcp_names: std::collections::HashSet<String> = mcp.iter().map(|t| t.name.clone()).collect();
1119 let mut initial_tools = brain.tools();
1120 initial_tools.extend(mcp);
1121 let initial_tools_json: Vec<Value> = initial_tools
1122 .iter()
1123 .map(|t| serde_json::to_value(t).unwrap_or(Value::Null))
1124 .collect();
1125
1126 // 2. The shared rolling LLM context (the cascaded analogue of S2S's LiveState
1127 // re-prompt): seeded with the brain's opening system prompt + the resolved
1128 // start-node tools, shared by the user/assistant aggregators + the bridge.
1129 let ctx: SharedContext = Arc::new(Mutex::new(RollingContext::new(
1130 Some(brain.system_prompt()),
1131 initial_tools_json,
1132 )));
1133
1134 // 3. The shared live-call state (reused from the realtime path).
1135 let state: SharedState = Arc::new(Mutex::new(LiveState::new(carrier_rate)));
1136
1137 // 4. Share the one transport between the pump (recv) and the sink (send).
1138 let shared = SharedTransport::new(transport);
1139
1140 // 5. The end-request channel (reused pattern): an inner processor (brain End)
1141 // requests a clean drain through the pipeline head.
1142 let (end_tx, mut end_rx) = tokio::sync::mpsc::unbounded_channel::<Frame>();
1143
1144 // 5b. The half-duplex turn lock (12 s safety timeout for a no-reply turn).
1145 let turn_mute = TurnMute::new(end_tx.clone(), std::time::Duration::from_secs(12));
1146
1147 // 6. The summarizer (reuse the cascaded builder's no-op default unless wired).
1148 let summarizer = config
1149 .summarizer
1150 .unwrap_or_else(|| Arc::new(NoopSummarizer) as Arc<dyn ContextSummarizer>);
1151
1152 // 7. Build the processor chain: the cascaded inner chain wrapped with the
1153 // realtime path's outer processors + the LLM↔brain bridge.
1154 let processors: Vec<Box<dyn FrameProcessor>> = vec![
1155 Box::new(TransportInput::new()),
1156 Box::new(SttProcessor::new(stt)),
1157 Box::new(
1158 UserContextAggregator::new(ctx.clone(), summarizer, config.summarizer_cfg)
1159 .with_transcript_state(state.clone())
1160 .with_turn_mute(turn_mute.clone()),
1161 ),
1162 // Greet on connect (pipecat parity): on ClientConnected, run the LLM once
1163 // over the seeded system prompt so the bot speaks first. Placed right before
1164 // the LLM so its emitted LlmContext reaches LlmProcessor directly.
1165 Box::new(CascadedKickoffProcessor::new(ctx.clone()).with_turn_mute(turn_mute.clone())),
1166 Box::new(LlmProcessor::new(llm)),
1167 Box::new(CascadedToolBridge::new(ctx.clone())),
1168 Box::new(BrainProcessor::new(
1169 brain,
1170 relay,
1171 mcp_names,
1172 state.clone(),
1173 end_tx.clone(),
1174 )),
1175 Box::new(AssistantContextAggregator::new(ctx.clone()).with_transcript_state(state.clone())),
1176 Box::new(TtsProcessor::new(tts)),
1177 Box::new(CascadedTransportOutput::new(
1178 shared.clone(),
1179 tts_rate,
1180 carrier_rate,
1181 state.clone(),
1182 end_tx.clone(),
1183 turn_mute.clone(),
1184 )),
1185 Box::new(RecorderProcessor::new(state.clone())),
1186 Box::new(TranscriptProcessor::new(state.clone())),
1187 Box::new(FinalizeProcessor::new(
1188 session,
1189 run_id,
1190 token,
1191 state.clone(),
1192 )),
1193 ];
1194 let pipeline = Pipeline::new(processors);
1195
1196 // 8. Build the task. The cascaded path has its own turn boundaries; keep the
1197 // idle gate off (the call ends on brain End / transport Stop), mirroring
1198 // build_s2s_task. Honor the caller's task_params but force the idle gate off.
1199 let params = PipelineTaskParams {
1200 idle_timeout: None,
1201 ..config.task_params
1202 };
1203 let task = PipelineTask::new(pipeline, params, observers);
1204
1205 // 9. Forward end-requests into the task head (Source breaks → clean drain).
1206 let head = task.queue_sender();
1207 tokio::spawn(async move {
1208 while let Some(f) = end_rx.recv().await {
1209 if head.send(f).is_err() {
1210 break;
1211 }
1212 }
1213 });
1214
1215 // 10. Spawn the transport pump feeding the task head (reused from the realtime
1216 // path — emits ClientConnected / InputAudio / End at the head).
1217 let pump = spawn_transport_pump(shared, task.queue_sender());
1218
1219 Ok(CascadedTask { task, pump })
1220}
1221
1222// ===========================================================================
1223// Tests — the cascaded pipeline integration tests.
1224// ===========================================================================
1225#[cfg(test)]
1226mod tests {
1227 use super::*;
1228 use crate::observer::{FrameEvent, FrameObserver};
1229 use crate::processor::frame::AudioFrame;
1230 use crate::service::{MockLlm, MockStt, MockTts};
1231 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
1232 use std::time::Duration;
1233
1234 /// An observer that records every processed frame name + flags output audio.
1235 /// `on_process` fires once **per processor** that receives a frame, so a
1236 /// `TtsSpeak` is observed at both the TTS processor and the internal Sink; the
1237 /// tap dedupes the captured TtsSpeak texts by the frame's monotonic id so it
1238 /// records one entry per *distinct emitted* utterance.
1239 #[derive(Default)]
1240 struct Tap {
1241 names: Mutex<Vec<&'static str>>,
1242 saw_output_audio: AtomicBool,
1243 tts_speak: Mutex<std::collections::BTreeMap<u64, String>>,
1244 }
1245 impl Tap {
1246 /// Distinct emitted TtsSpeak texts, in id order.
1247 fn tts_speak_texts(&self) -> Vec<String> {
1248 self.tts_speak.lock().unwrap().values().cloned().collect()
1249 }
1250 }
1251 #[async_trait]
1252 impl FrameObserver for Tap {
1253 async fn on_process(&self, e: &FrameEvent<'_>) {
1254 self.names.lock().unwrap().push(e.frame.name());
1255 match &e.frame {
1256 Frame::OutputAudio(_) => self.saw_output_audio.store(true, Ordering::Relaxed),
1257 Frame::TtsSpeak { text, .. } => {
1258 self.tts_speak
1259 .lock()
1260 .unwrap()
1261 .insert(e.meta.id, text.clone());
1262 }
1263 _ => {}
1264 }
1265 }
1266 }
1267
1268 /// The §9 step-10 gate, cascaded form: mock-STT → mock-LLM → mock-TTS runs a
1269 /// full turn end-to-end through a **real** [`PipelineTask`] built by
1270 /// [`build_cascaded_pipeline`] — caller audio → transcription → LLM text → TTS
1271 /// audio → output. Provider tests reuse this fixture against real providers.
1272 #[tokio::test]
1273 async fn cascaded_pipeline_runs_a_turn_end_to_end() {
1274 let tap = Arc::new(Tap::default());
1275 let task = build_cascaded_task_with_observers(
1276 MockStt::new("book a dentist appointment"),
1277 MockLlm::new("you said: "),
1278 MockTts::new(24_000),
1279 CascadedConfig::default(),
1280 vec![tap.clone() as Arc<dyn FrameObserver>],
1281 );
1282
1283 // Drive a turn: feed one InputAudio chunk, then end.
1284 let audio = Arc::new(AudioFrame::mono(vec![1, 2, 3, 4], 16_000));
1285 task.queue_frame(Frame::InputAudio(audio)).await;
1286 task.stop_when_done().await;
1287
1288 tokio::time::timeout(Duration::from_secs(5), task.run())
1289 .await
1290 .expect("cascaded pipeline timed out")
1291 .expect("run ok");
1292
1293 // The turn produced output audio (STT→user-agg→LLM→assistant-agg→TTS fired).
1294 let names = tap.names.lock().unwrap().clone();
1295 assert!(
1296 tap.saw_output_audio.load(Ordering::Relaxed),
1297 "cascaded pipeline must emit OutputAudio; saw {names:?}"
1298 );
1299 assert!(
1300 names.contains(&"Transcription"),
1301 "STT must emit Transcription"
1302 );
1303 assert!(
1304 names.contains(&"LlmContext"),
1305 "user aggregator must emit LlmContext"
1306 );
1307 assert!(names.contains(&"LlmText"), "LLM must emit LlmText");
1308 assert!(
1309 names.contains(&"TtsSpeak"),
1310 "assistant aggregator must emit TtsSpeak"
1311 );
1312
1313 // The assistant aggregator assembled the whole reply into one TtsSpeak.
1314 let speaks = tap.tts_speak_texts();
1315 assert_eq!(
1316 speaks,
1317 vec!["you said: book a dentist appointment".to_string()]
1318 );
1319 }
1320
1321 /// The kickoff makes the bot **greet on connect**: a `ClientConnected` with NO
1322 /// user audio still runs the LLM once over the seeded system prompt and produces
1323 /// a spoken greeting (TtsSpeak → OutputAudio). Idempotent — a second
1324 /// `ClientConnected` does not greet again. (Closes the cascaded "silent until the
1325 /// user speaks" gap; the realtime path's `kickoff` analogue.)
1326 #[tokio::test]
1327 async fn kickoff_greets_on_connect_once() {
1328 let tap = Arc::new(Tap::default());
1329 // The context is seeded exactly as build_cascaded_task seeds it: the brain's
1330 // opening system prompt + start-node tools, with empty history.
1331 let ctx: SharedContext = Arc::new(Mutex::new(RollingContext::new(
1332 Some("Greet the caller.".into()),
1333 vec![],
1334 )));
1335 let pipeline = Pipeline::new(vec![
1336 Box::new(CascadedKickoffProcessor::new(ctx.clone())),
1337 Box::new(LlmProcessor::new(MockLlm::new("reply: "))),
1338 Box::new(AssistantContextAggregator::new(ctx.clone())),
1339 Box::new(TtsProcessor::new(MockTts::new(24_000))),
1340 ]);
1341 let task = PipelineTask::new(
1342 pipeline,
1343 PipelineTaskParams {
1344 idle_timeout: None,
1345 ..Default::default()
1346 },
1347 vec![tap.clone() as Arc<dyn FrameObserver>],
1348 );
1349 // Two ClientConnected, no user input at all.
1350 task.queue_frame(Frame::ClientConnected).await;
1351 task.queue_frame(Frame::ClientConnected).await;
1352 task.stop_when_done().await;
1353 tokio::time::timeout(Duration::from_secs(5), task.run())
1354 .await
1355 .expect("kickoff pipeline timed out")
1356 .expect("run ok");
1357
1358 let names = tap.names.lock().unwrap().clone();
1359 // Greeted with NO user turn: output audio produced, but no Transcription.
1360 assert!(
1361 tap.saw_output_audio.load(Ordering::Relaxed),
1362 "kickoff must produce a greeting OutputAudio; saw {names:?}"
1363 );
1364 assert!(
1365 !names.contains(&"Transcription"),
1366 "the greeting must require no user input; saw {names:?}"
1367 );
1368 // Idempotent: exactly one greeting utterance (the MockLlm echoes the system
1369 // prompt as the last message's content).
1370 let speaks = tap.tts_speak_texts();
1371 assert_eq!(
1372 speaks,
1373 vec!["reply: Greet the caller.".to_string()],
1374 "greet exactly once"
1375 );
1376 }
1377
1378 /// The kickoff greeting must be **tool-free**: even though the context is seeded
1379 /// with tools (transitions/endCall), the emitted greeting `LlmContext` carries NO
1380 /// tools — so the opening turn can't call `endCall`/a transition and hang up before
1381 /// the bot speaks (the live "AI can't speak back" regression).
1382 #[tokio::test]
1383 async fn kickoff_greeting_is_tool_free() {
1384 let tap = Arc::new(CtxTap::default());
1385 let ctx: SharedContext = Arc::new(Mutex::new(RollingContext::new(
1386 Some("Greet the caller.".into()),
1387 vec![
1388 json!({"name": "end_call", "description": "end the call", "params": {"type":"object"}}),
1389 ],
1390 )));
1391 let pipeline = Pipeline::new(vec![Box::new(CascadedKickoffProcessor::new(ctx.clone()))]);
1392 let task = PipelineTask::new(
1393 pipeline,
1394 PipelineTaskParams {
1395 idle_timeout: None,
1396 ..Default::default()
1397 },
1398 vec![tap.clone() as Arc<dyn FrameObserver>],
1399 );
1400 task.queue_frame(Frame::ClientConnected).await;
1401 task.stop_when_done().await;
1402 tokio::time::timeout(Duration::from_secs(5), task.run())
1403 .await
1404 .expect("timed out")
1405 .expect("run ok");
1406 let ctxs = tap.contexts.lock().unwrap().clone();
1407 assert_eq!(ctxs.len(), 1, "kickoff emits exactly one greeting context");
1408 assert!(
1409 ctxs[0].tools.is_empty(),
1410 "greeting context must carry NO tools"
1411 );
1412 assert_eq!(
1413 ctxs[0].messages[0]["role"], "system",
1414 "system prompt still present so it can greet"
1415 );
1416 }
1417
1418 /// TurnMute locks at turn start (SttMute(true)) and unmutes once the bot reply has
1419 /// played out. A second `begin()` while muted is a no-op (the lock holds the whole
1420 /// turn — covering the LLM-thinking gap, not just while the bot speaks).
1421 #[tokio::test]
1422 async fn turn_mute_locks_then_unmutes_after_playout() {
1423 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<Frame>();
1424 let tm = TurnMute::new(tx, Duration::from_secs(5)); // long safety timeout (not used here)
1425 tm.begin();
1426 assert!(
1427 matches!(rx.recv().await, Some(Frame::SttMute(true))),
1428 "turn start mutes STT"
1429 );
1430 tm.begin(); // already muted → no-op (no second SttMute)
1431 // A short bot reply: 100 ms of carrier audio (800 samples @ 8 kHz).
1432 tm.note_bot_audio(800, 8000);
1433 let unmute = tokio::time::timeout(Duration::from_secs(2), rx.recv())
1434 .await
1435 .expect("unmute did not arrive");
1436 assert!(
1437 matches!(unmute, Some(Frame::SttMute(false))),
1438 "unmutes after playout drains"
1439 );
1440 }
1441
1442 /// A turn that produces NO bot audio (tool-only / no reply) still unmutes via the
1443 /// safety timeout — the lock never deadlocks.
1444 #[tokio::test]
1445 async fn turn_mute_unmutes_via_safety_timeout_when_no_audio() {
1446 let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<Frame>();
1447 let tm = TurnMute::new(tx, Duration::from_millis(200)); // short safety timeout
1448 tm.begin();
1449 assert!(matches!(rx.recv().await, Some(Frame::SttMute(true))));
1450 // No note_bot_audio at all → unmute fires via the timeout.
1451 let unmute = tokio::time::timeout(Duration::from_secs(2), rx.recv())
1452 .await
1453 .expect("safety-timeout unmute did not arrive");
1454 assert!(matches!(unmute, Some(Frame::SttMute(false))));
1455 }
1456
1457 // ---- Context aggregator unit tests --------------------------------------
1458
1459 /// A tap that captures the LlmContext snapshots the user aggregator emits.
1460 #[derive(Default)]
1461 struct CtxTap {
1462 contexts: Mutex<Vec<LlmContext>>,
1463 }
1464 #[async_trait]
1465 impl FrameObserver for CtxTap {
1466 async fn on_process(&self, e: &FrameEvent<'_>) {
1467 if let Frame::LlmContext(c) = &e.frame {
1468 self.contexts.lock().unwrap().push((**c).clone());
1469 }
1470 }
1471 }
1472
1473 /// The user aggregator accrues each final transcription into the rolling context
1474 /// and emits a growing `LlmContext` per turn (transcription accrual).
1475 #[tokio::test]
1476 async fn user_aggregator_accrues_transcription_into_context() {
1477 let tap = Arc::new(CtxTap::default());
1478 let ctx: SharedContext = Arc::new(Mutex::new(RollingContext::new(
1479 Some("be helpful".into()),
1480 vec![],
1481 )));
1482 let pipeline = Pipeline::new(vec![Box::new(UserContextAggregator::new(
1483 ctx.clone(),
1484 Arc::new(NoopSummarizer),
1485 SummarizerConfig::default(),
1486 ))]);
1487 let task = PipelineTask::new(
1488 pipeline,
1489 PipelineTaskParams {
1490 idle_timeout: None,
1491 ..Default::default()
1492 },
1493 vec![tap.clone() as Arc<dyn FrameObserver>],
1494 );
1495 let uid: Arc<str> = Arc::from("user");
1496 task.queue_frame(Frame::Transcription {
1497 text: "hello".into(),
1498 user_id: uid.clone(),
1499 language: None,
1500 final_: true,
1501 })
1502 .await;
1503 // An interim transcription must NOT trigger a run.
1504 task.queue_frame(Frame::InterimTranscription {
1505 text: "wor".into(),
1506 user_id: uid.clone(),
1507 language: None,
1508 })
1509 .await;
1510 task.queue_frame(Frame::Transcription {
1511 text: "world".into(),
1512 user_id: uid.clone(),
1513 language: None,
1514 final_: true,
1515 })
1516 .await;
1517 task.stop_when_done().await;
1518 tokio::time::timeout(Duration::from_secs(5), task.run())
1519 .await
1520 .expect("timed out")
1521 .expect("run ok");
1522
1523 let contexts = tap.contexts.lock().unwrap().clone();
1524 assert_eq!(contexts.len(), 2, "one LlmContext per final transcription");
1525 // First run: system + "hello".
1526 assert_eq!(contexts[0].messages.len(), 2);
1527 assert_eq!(contexts[0].messages[0]["role"], "system");
1528 assert_eq!(contexts[0].messages[1]["content"], "hello");
1529 // Second run accrues: system + "hello" + "world".
1530 assert_eq!(contexts[1].messages.len(), 3);
1531 assert_eq!(contexts[1].messages[2]["content"], "world");
1532 }
1533
1534 /// A tap that captures the TtsSpeak texts the assistant aggregator emits.
1535 #[derive(Default)]
1536 struct SpeakTap {
1537 texts: Mutex<Vec<String>>,
1538 }
1539 #[async_trait]
1540 impl FrameObserver for SpeakTap {
1541 async fn on_process(&self, e: &FrameEvent<'_>) {
1542 if let Frame::TtsSpeak { text, .. } = &e.frame {
1543 self.texts.lock().unwrap().push(text.clone());
1544 }
1545 }
1546 }
1547
1548 /// The assistant aggregator accrues streamed `LlmText` between the start/end
1549 /// framing into one `assistant` message + a single `TtsSpeak` (response accrual).
1550 #[tokio::test]
1551 async fn assistant_aggregator_accrues_response_into_one_utterance() {
1552 let tap = Arc::new(SpeakTap::default());
1553 let ctx: SharedContext = Arc::new(Mutex::new(RollingContext::default()));
1554 let pipeline = Pipeline::new(vec![Box::new(AssistantContextAggregator::new(ctx.clone()))]);
1555 let task = PipelineTask::new(
1556 pipeline,
1557 PipelineTaskParams {
1558 idle_timeout: None,
1559 ..Default::default()
1560 },
1561 vec![tap.clone() as Arc<dyn FrameObserver>],
1562 );
1563 // Stream a response in three tokens.
1564 task.queue_frame(Frame::LlmResponseStart).await;
1565 task.queue_frame(Frame::LlmText("Hello ".into())).await;
1566 task.queue_frame(Frame::LlmText("there, ".into())).await;
1567 task.queue_frame(Frame::LlmText("friend".into())).await;
1568 task.queue_frame(Frame::LlmResponseEnd).await;
1569 task.stop_when_done().await;
1570 tokio::time::timeout(Duration::from_secs(5), task.run())
1571 .await
1572 .expect("timed out")
1573 .expect("run ok");
1574
1575 // One assembled utterance.
1576 let texts = tap.texts.lock().unwrap().clone();
1577 assert_eq!(texts, vec!["Hello there, friend".to_string()]);
1578 // And it was appended to the rolling context as an assistant message.
1579 let c = ctx.lock().unwrap();
1580 assert_eq!(c.messages.len(), 1);
1581 assert_eq!(c.messages[0]["role"], "assistant");
1582 assert_eq!(c.messages[0]["content"], "Hello there, friend");
1583 }
1584
1585 /// A summarizer that records it fired and returns a fixed summary.
1586 struct CountingSummarizer {
1587 fired: Arc<AtomicUsize>,
1588 }
1589 #[async_trait]
1590 impl ContextSummarizer for CountingSummarizer {
1591 async fn summarize(&self, _messages: &[Value]) -> Option<String> {
1592 self.fired.fetch_add(1, Ordering::SeqCst);
1593 Some("compressed".into())
1594 }
1595 }
1596
1597 /// The summarizer hook fires (fire-and-forget) once the rolling context crosses
1598 /// the threshold, and its result collapses the history (summarizer fire).
1599 #[tokio::test]
1600 async fn summarizer_fires_once_past_threshold_and_compresses() {
1601 let fired = Arc::new(AtomicUsize::new(0));
1602 let ctx: SharedContext = Arc::new(Mutex::new(RollingContext::default()));
1603 let pipeline = Pipeline::new(vec![Box::new(UserContextAggregator::new(
1604 ctx.clone(),
1605 Arc::new(CountingSummarizer {
1606 fired: fired.clone(),
1607 }),
1608 // Fire after just 3 messages so the test is quick.
1609 SummarizerConfig {
1610 trigger_after_messages: 3,
1611 },
1612 ))]);
1613 let task = PipelineTask::new(
1614 pipeline,
1615 PipelineTaskParams {
1616 idle_timeout: None,
1617 ..Default::default()
1618 },
1619 vec![],
1620 );
1621 let uid: Arc<str> = Arc::from("user");
1622 for i in 0..4 {
1623 task.queue_frame(Frame::Transcription {
1624 text: format!("msg {i}"),
1625 user_id: uid.clone(),
1626 language: None,
1627 final_: true,
1628 })
1629 .await;
1630 }
1631 task.stop_when_done().await;
1632 tokio::time::timeout(Duration::from_secs(5), task.run())
1633 .await
1634 .expect("timed out")
1635 .expect("run ok");
1636
1637 // Give the fire-and-forget summarize task a beat to apply.
1638 tokio::time::sleep(Duration::from_millis(50)).await;
1639 assert!(
1640 fired.load(Ordering::SeqCst) >= 1,
1641 "summarizer must fire once past the threshold"
1642 );
1643 // The history was compressed to the single summary message.
1644 let c = ctx.lock().unwrap();
1645 assert_eq!(c.messages.len(), 1, "history collapsed to the summary");
1646 assert!(c.messages[0]["content"]
1647 .as_str()
1648 .unwrap()
1649 .contains("compressed"));
1650 }
1651
1652 // =======================================================================
1653 // `build_cascaded_task` turn fixture.
1654 //
1655 // Drives `build_cascaded_task` with a MOCK transport + mock STT/LLM/TTS +
1656 // mock brain/session through a full turn and asserts the turn flows end to
1657 // end, a cascaded-LLM tool-call drives a brain **transition** (not a stay),
1658 // and `session.complete` fires once with the expected finalize payload. No
1659 // live key needed.
1660 // =======================================================================
1661
1662 use crate::pipeline::s2s_test_mocks::{Captured, MockSession, MockSocket, CARRIER_RATE};
1663 use crate::processor::frame::FunctionCall;
1664 use crate::serializer::PlivoSerializer;
1665 use crate::transport::WsCarrierTransport;
1666 use crate::types::{BrainAction, ToolDecl, WsOut};
1667 use serde_json::json;
1668
1669 const TRANSITION_TOOL: &str = "go_to_booking";
1670 const END_TOOL: &str = "end_call";
1671
1672 /// A brain that **transitions** on `TRANSITION_TOOL` (the cascaded gate's key
1673 /// assertion: a tool-call moves the graph, not just a `Stay`) and **ends** on
1674 /// `END_TOOL`. Records every tool-call name + the actions it returned.
1675 struct TransitioningMockBrain {
1676 node_id: Arc<Mutex<String>>,
1677 seen_tools: Arc<Mutex<Vec<String>>>,
1678 actions: Arc<Mutex<Vec<&'static str>>>,
1679 }
1680 impl AgentBrain for TransitioningMockBrain {
1681 fn system_prompt(&self) -> String {
1682 "You are a test agent (start node).".into()
1683 }
1684 fn tools(&self) -> Vec<ToolDecl> {
1685 vec![ToolDecl {
1686 name: TRANSITION_TOOL.into(),
1687 description: "Move to the booking node.".into(),
1688 params: json!({ "type": "object", "properties": {} }),
1689 }]
1690 }
1691 fn current_node_id(&self) -> String {
1692 self.node_id.lock().unwrap().clone()
1693 }
1694 fn on_tool_call(&mut self, name: &str, _args: &serde_json::Value) -> BrainAction {
1695 self.seen_tools.lock().unwrap().push(name.to_string());
1696 if name == END_TOOL {
1697 self.actions.lock().unwrap().push("end");
1698 BrainAction::End {
1699 disposition: Some("booked".into()),
1700 }
1701 } else if name == TRANSITION_TOOL && *self.node_id.lock().unwrap() == "start" {
1702 // Move the active node so the post-transition tool set + node_id
1703 // reflect the booking node (the relay scopes by node_id). Idempotent:
1704 // a repeat transition-call once already at booking is a no-op `Stay`.
1705 *self.node_id.lock().unwrap() = "booking".into();
1706 self.actions.lock().unwrap().push("transition");
1707 BrainAction::Transition {
1708 system_prompt: "You are at the booking node.".into(),
1709 tools: vec![ToolDecl {
1710 name: END_TOOL.into(),
1711 description: "End the call.".into(),
1712 params: json!({ "type": "object", "properties": {} }),
1713 }],
1714 say: Some("Sure, let's get you booked.".into()),
1715 }
1716 } else {
1717 self.actions.lock().unwrap().push("stay");
1718 BrainAction::Stay
1719 }
1720 }
1721 fn is_finished(&self) -> bool {
1722 false
1723 }
1724 fn collected_vars(&self) -> serde_json::Value {
1725 json!({ "name": "Ada", "intent": "booking" })
1726 }
1727 }
1728
1729 /// A scripted cascaded LLM driven by the brain's **current node** (a shared
1730 /// `node_id`, so the script is deterministic no matter how many user turns the
1731 /// mock socket produces):
1732 /// - while at the **start** node: emit *only* a
1733 /// `FunctionCallsStarted([TRANSITION_TOOL])` (no spoken text) — a tool-call the
1734 /// bridge feeds to the brain to drive a **transition** to the booking node.
1735 /// - while at the **booking** node (after the transition re-prompt re-runs the
1736 /// LLM): speak the booking reply, then **after `LlmResponseEnd`** emit a
1737 /// `FunctionCallsStarted([END_TOOL])` to end the call. Emitting the end-call
1738 /// *after* the response framing means the assistant aggregator has already
1739 /// produced the `TtsSpeak` (→TTS→`OutputAudio`, downstream toward the carrier)
1740 /// before the brain's terminal `End` is injected at the head — so the reply is
1741 /// spoken before teardown (the cascaded analogue of the s2s script emitting
1742 /// `AudioOut` before the end tool-call).
1743 struct ScriptedCascadedLlm {
1744 node_id: Arc<Mutex<String>>,
1745 runs: Arc<AtomicUsize>,
1746 booking_reply: String,
1747 }
1748 #[async_trait]
1749 impl LlmService for ScriptedCascadedLlm {
1750 fn name(&self) -> &str {
1751 "ScriptedCascadedLlm"
1752 }
1753 async fn start(&mut self, _params: &StartParams) -> Result<()> {
1754 Ok(())
1755 }
1756 async fn run_llm<'a>(
1757 &'a mut self,
1758 _ctx: &'a LlmContext,
1759 ) -> Result<futures::stream::BoxStream<'a, Frame>> {
1760 use futures::StreamExt;
1761 self.runs.fetch_add(1, Ordering::SeqCst);
1762 let at_start = *self.node_id.lock().unwrap() == "start";
1763 let frames: Vec<Frame> = if at_start {
1764 // Start node: a transition tool-call, no spoken text.
1765 vec![
1766 Frame::LlmResponseStart,
1767 Frame::FunctionCallsStarted(vec![FunctionCall {
1768 function_name: TRANSITION_TOOL.into(),
1769 tool_call_id: "fc-tr-1".into(),
1770 arguments: json!({}),
1771 }]),
1772 Frame::LlmResponseEnd,
1773 ]
1774 } else {
1775 // Booking node: speak the reply, THEN (after the response framing
1776 // flushed the TtsSpeak downstream) emit the end tool-call.
1777 vec![
1778 Frame::LlmResponseStart,
1779 Frame::LlmText(self.booking_reply.clone()),
1780 Frame::LlmResponseEnd,
1781 Frame::FunctionCallsStarted(vec![FunctionCall {
1782 function_name: END_TOOL.into(),
1783 tool_call_id: "fc-end-1".into(),
1784 arguments: json!({}),
1785 }]),
1786 ]
1787 };
1788 Ok(futures::stream::iter(frames).boxed())
1789 }
1790 fn set_tools(&mut self, _tools: Vec<crate::service::Tool>) {}
1791 }
1792
1793 /// Extract the `playAudio` carrier frames (the bot audio that reached the
1794 /// carrier) from the captured outbound socket frames.
1795 fn play_audio_frames(sent: &[WsOut]) -> Vec<String> {
1796 sent.iter()
1797 .filter_map(|o| match o {
1798 WsOut::Text(t) if t.contains("playAudio") => Some(t.clone()),
1799 _ => None,
1800 })
1801 .collect()
1802 }
1803
1804 #[tokio::test]
1805 async fn cascaded_task_runs_a_turn_with_a_transition_and_finalizes() {
1806 let sent = Arc::new(Mutex::new(Vec::<WsOut>::new()));
1807 let captured = Arc::new(Mutex::new(Captured::default()));
1808 let seen_tools = Arc::new(Mutex::new(Vec::<String>::new()));
1809 let actions = Arc::new(Mutex::new(Vec::<&'static str>::new()));
1810 let runs = Arc::new(AtomicUsize::new(0));
1811
1812 let transport = WsCarrierTransport::new(
1813 MockSocket::new(sent.clone()),
1814 PlivoSerializer::new(CARRIER_RATE),
1815 );
1816 // The brain + the scripted LLM share the active node id — the LLM follows
1817 // the (re-prompted) node exactly as a real cascaded LLM follows its prompt.
1818 let node_id = Arc::new(Mutex::new(String::from("start")));
1819 let brain = TransitioningMockBrain {
1820 node_id: node_id.clone(),
1821 seen_tools: seen_tools.clone(),
1822 actions: actions.clone(),
1823 };
1824 let task = build_cascaded_task(
1825 transport,
1826 MockStt::new("book a dentist appointment"),
1827 ScriptedCascadedLlm {
1828 node_id: node_id.clone(),
1829 runs: runs.clone(),
1830 // Long enough that MockTts (one PCM sample per char) yields ≥ the
1831 // resampler's 256-sample block, so the 24k→8k carrier audio is
1832 // non-empty and actually reaches the socket as a `playAudio` frame.
1833 booking_reply: "Sure, let's get you booked in. ".repeat(12),
1834 },
1835 MockTts::new(24_000),
1836 brain,
1837 MockSession::new(captured.clone()),
1838 7777,
1839 "tok-cascaded".into(),
1840 CascadedConfig::default(),
1841 )
1842 .await
1843 .expect("build_cascaded_task");
1844
1845 tokio::time::timeout(Duration::from_secs(5), task.run())
1846 .await
1847 .expect("cascaded task timed out")
1848 .expect("cascaded task errored");
1849
1850 // --- (1) The turn flowed end-to-end: bot audio reached the carrier. ------
1851 let play = play_audio_frames(&sent.lock().unwrap());
1852 assert!(
1853 !play.is_empty(),
1854 "expected ≥1 playAudio frame to the carrier (STT→LLM→brain→TTS→output)"
1855 );
1856
1857 // --- (2) A cascaded-LLM tool-call drove a brain TRANSITION (not a stay). --
1858 let seen = seen_tools.lock().unwrap().clone();
1859 assert!(
1860 seen.contains(&TRANSITION_TOOL.to_string()),
1861 "the transition tool-call must reach the brain; saw {seen:?}"
1862 );
1863 let acts = actions.lock().unwrap().clone();
1864 assert!(
1865 acts.contains(&"transition"),
1866 "a tool-call must drive a TRANSITION, not just a stay; actions={acts:?}"
1867 );
1868 assert!(
1869 acts.contains(&"end"),
1870 "the end tool-call must end the call; actions={acts:?}"
1871 );
1872 // Exactly one real transition (the brain is idempotent once at booking).
1873 assert_eq!(
1874 acts.iter().filter(|a| **a == "transition").count(),
1875 1,
1876 "exactly one graph transition; actions={acts:?}"
1877 );
1878 // ≥2 LLM runs fired: the start-node turn (transition) + the post-transition
1879 // re-prompt re-run that produces the spoken booking reply.
1880 assert!(
1881 runs.load(Ordering::SeqCst) >= 2,
1882 "the transition must re-run the LLM (≥2 runs); got {}",
1883 runs.load(Ordering::SeqCst)
1884 );
1885
1886 // --- (3) session.complete fired ONCE with the expected finalize payload. -
1887 let c = captured.lock().unwrap();
1888 assert_eq!(
1889 c.complete_calls, 1,
1890 "session.complete must fire exactly once"
1891 );
1892 let fin = c.finalize.as_ref().expect("finalize payload present");
1893 // Recording + transcript stored keys (the run-scoped keys, never the
1894 // secret-bearing presigned URL).
1895 assert_eq!(fin.recording_url.as_deref(), Some("runs/4242/recording"));
1896 assert_eq!(fin.transcript_url.as_deref(), Some("runs/4242/transcript"));
1897 // collected_vars from the brain + the folded disposition from the End.
1898 assert_eq!(fin.collected_vars["name"], "Ada");
1899 assert_eq!(fin.collected_vars["intent"], "booking");
1900 assert_eq!(
1901 fin.collected_vars["call_disposition"], "booked",
1902 "the End disposition must fold into collected_vars"
1903 );
1904 // Usage is present in the finalize payload (default totals — no provider
1905 // usage wired into the mock LLM, but the field is always serialized).
1906 assert!(fin.usage.is_object(), "usage must be a JSON object");
1907 // Wall-clock duration rides in usage_metrics — the control plane reads
1908 // `usage_metrics.duration_seconds` for the Interactions duration/cost.
1909 assert!(
1910 fin.usage["duration_seconds"].is_number(),
1911 "finalize usage must carry duration_seconds"
1912 );
1913 }
1914
1915 // ---- is_transition_ack: don't swallow MCP results that carry a `status` -----
1916 // Regression guard for the cascaded counterpart of `s2s::mcp_result_struct`:
1917 // MCP tool results are now JSON objects, and an object with `{"status": "ok"}`
1918 // must still be fed back to the LLM, not misread as a `{status: moved}` ack.
1919
1920 #[test]
1921 fn is_transition_ack_matches_only_brain_vocabulary() {
1922 // The brain's three real ACK envelopes:
1923 assert!(is_transition_ack(&json!({ "status": "moved" })));
1924 assert!(is_transition_ack(&json!({ "status": "stay" })));
1925 assert!(is_transition_ack(&json!({ "status": "ended" })));
1926
1927 // MCP/workflow results that happen to carry a `status` are NOT acks:
1928 assert!(!is_transition_ack(&json!({ "status": "ok" })));
1929 assert!(!is_transition_ack(
1930 &json!({ "status": "confirmed", "booking_id": "B1" })
1931 ));
1932 // …nor a tool result with no status, nor a non-object/non-string status:
1933 assert!(!is_transition_ack(&json!({ "booking_id": "B1" })));
1934 assert!(!is_transition_ack(&json!({ "status": 200 })));
1935 assert!(!is_transition_ack(&json!("moved")));
1936 }
1937}