use std::fmt;
use std::path::Path;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use std::time::{Duration, Instant};
use futures_util::StreamExt;
use ringbuf::traits::{Consumer, Observer};
use ringbuf::HeapCons;
use tokio::sync::mpsc;
use tokio_util::sync::CancellationToken;
use tracing::{debug, info, info_span, warn, Instrument};
use crate::audio::{
resample_offline, AudioInputConfig, AudioOutputConfig, MicCapture, Playback, PlaybackHandle,
CAPTURE_RATE,
};
use crate::config::Config;
use crate::error::{LlmError, Result, SkadooshError, SttError};
use crate::llm::client::{ensure_success, SseLineBuffer, CLAUSE_MAX_LEN, CLAUSE_MIN_LEN};
use crate::llm::{parse_sse_line, ClauseSplitter, LlmClient};
use crate::stt::{SttConfig, WhisperStt};
use crate::tts::{build_engine, TtsClip, TtsEngine, TTS_SAMPLE_RATE};
use crate::vad::{SileroVad, VadEvent, VadSegmenter, FRAME_LEN};
const VAD_EVENTS_CAP: usize = 8;
const SEGMENT_CAP: usize = 4;
const TEXT_CAP: usize = 8;
const TURN_CAP: usize = 8;
const CLAUSE_CAP: usize = 16;
const TURN_DONE_CAP: usize = 8;
const FATAL_CAP: usize = 8;
const AUDIBLE_POLL: Duration = Duration::from_millis(2);
const AUDIBLE_TIMEOUT: Duration = Duration::from_secs(3);
#[derive(Debug, Clone)]
pub struct SelftestReport {
pub segment_ms: u64,
pub stt_ms: u64,
pub llm_ttft_ms: u64,
pub first_clause_ms: u64,
pub tts_ms: u64,
pub total_ms: u64,
pub transcript: String,
}
impl fmt::Display for SelftestReport {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
writeln!(f, "skadoosh selftest — latency report")?;
writeln!(f, " {:<30} {:>8} ms", "vad segmentation", self.segment_ms)?;
writeln!(f, " {:<30} {:>8} ms", "stt (whisper)", self.stt_ms)?;
writeln!(
f,
" {:<30} {:>8} ms",
"llm time-to-first-token", self.llm_ttft_ms
)?;
writeln!(
f,
" {:<30} {:>8} ms",
"llm first clause", self.first_clause_ms
)?;
writeln!(f, " {:<30} {:>8} ms", "tts first clip", self.tts_ms)?;
writeln!(f, " {:<30} {:>8} ms", "total", self.total_ms)?;
write!(f, " transcript: {:?}", self.transcript)
}
}
pub struct Pipeline {
config: Config,
shutdown: CancellationToken,
}
impl Pipeline {
pub fn new(config: Config) -> Result<Self> {
Ok(Self {
config,
shutdown: CancellationToken::new(),
})
}
pub fn shutdown_token(&self) -> CancellationToken {
self.shutdown.clone()
}
pub fn run(self) -> Result<()> {
let Self { config, shutdown } = self;
let (capture, cons) = MicCapture::start(&AudioInputConfig {
device_name: config.input_device.clone(),
})?;
let (playback, handle) = Playback::start(&AudioOutputConfig {
device_name: config.output_device.clone(),
})?;
let vad = SileroVad::new(&config.vad_model)?;
let segmenter = VadSegmenter::new(config.vad_threshold, config.silence_ms);
let stt = WhisperStt::start(&config.whisper_model, &SttConfig::default())?;
let tts_engine = build_engine(&config)?;
let llm = LlmClient::new(
&config.llm_url,
&config.llm_model,
&config.system_prompt,
config.max_history_turns,
);
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.map_err(|err| anyhow::anyhow!("failed to start tokio runtime: {err}"))?;
let result = runtime.block_on(async move {
let (vad_tx, vad_rx) = mpsc::channel(VAD_EVENTS_CAP);
let (fatal_tx, fatal_rx) = mpsc::channel(FATAL_CAP);
let vad_join = tokio::spawn(
vad_task(VadParts {
capture,
cons,
vad,
segmenter,
threshold: config.vad_threshold,
sink: handle.clone(),
events_tx: vad_tx,
fatal_tx: fatal_tx.clone(),
shutdown: shutdown.clone(),
})
.instrument(info_span!("vad")),
);
let result = run_orchestrator(Topology {
vad_events: vad_rx,
fatal_tx,
fatal_rx,
stt,
llm,
tts_engine,
sink: handle.clone(),
shutdown,
})
.await;
match vad_join.await {
Ok(()) => {}
Err(join_err) => {
warn!(error = %join_err, "VAD task panicked");
if result.is_ok() {
return Err(anyhow::anyhow!("VAD task panicked: {join_err}").into());
}
}
}
result
});
playback.stop();
result
}
pub fn run_selftest(self, wav: &Path, out_wav: &Path) -> Result<SelftestReport> {
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.map_err(|err| anyhow::anyhow!("failed to start tokio runtime: {err}"))?;
runtime.block_on(self.selftest_async(wav, out_wav))
}
}
#[doc(hidden)]
#[derive(Debug)]
pub enum VadEventMsg {
SpeechStart,
Segment {
samples: Vec<f32>,
t_speech_end: Instant,
},
}
#[doc(hidden)]
pub trait SpeechToText: Send + 'static {
fn transcribe(
&self,
samples: Vec<f32>,
) -> impl std::future::Future<Output = Result<String>> + Send;
fn dropped_jobs(&self) -> u64 {
0
}
fn stop(self)
where
Self: Sized,
{
}
}
impl SpeechToText for WhisperStt {
fn transcribe(
&self,
samples: Vec<f32>,
) -> impl std::future::Future<Output = Result<String>> + Send {
let reply = WhisperStt::transcribe(self, samples);
async move {
match reply.await {
Ok(result) => result,
Err(_) => Err(SttError::WorkerGone.into()),
}
}
}
fn dropped_jobs(&self) -> u64 {
WhisperStt::dropped_jobs(self)
}
fn stop(self) {
WhisperStt::stop(self);
}
}
#[doc(hidden)]
pub trait ClipSink: Clone + Send + 'static {
fn queue_clip(&self, clip: TtsClip) -> impl std::future::Future<Output = Result<()>> + Send;
fn flush(&self);
fn is_playing(&self) -> bool;
}
impl ClipSink for PlaybackHandle {
fn queue_clip(&self, clip: TtsClip) -> impl std::future::Future<Output = Result<()>> + Send {
PlaybackHandle::queue_clip(self, clip)
}
fn flush(&self) {
PlaybackHandle::flush(self);
}
fn is_playing(&self) -> bool {
PlaybackHandle::is_playing(self)
}
}
#[doc(hidden)]
pub struct Topology<S: SpeechToText, C: ClipSink> {
pub vad_events: mpsc::Receiver<VadEventMsg>,
pub fatal_tx: mpsc::Sender<SkadooshError>,
pub fatal_rx: mpsc::Receiver<SkadooshError>,
pub stt: S,
pub llm: LlmClient,
pub tts_engine: Box<dyn TtsEngine>,
pub sink: C,
pub shutdown: CancellationToken,
}
struct SegmentMsg {
turn_id: u64,
token: CancellationToken,
samples: Vec<f32>,
t_speech_end: Instant,
}
struct TextMsg {
turn_id: u64,
token: CancellationToken,
text: String,
t_speech_end: Instant,
t_text: Instant,
}
struct TurnMsg {
turn_id: u64,
token: CancellationToken,
clauses: mpsc::Receiver<String>,
t_speech_end: Instant,
t_text: Instant,
}
#[derive(Debug, Clone)]
struct TurnTiming {
t_speech_end: Instant,
t_text: Instant,
t_first_clause: Option<Instant>,
t_first_clip: Option<Instant>,
}
#[derive(Debug, Default)]
struct OnsetGate {
pending: bool,
}
impl OnsetGate {
fn filter(&mut self, is_start: bool, is_speech: bool, playing: bool) -> bool {
if self.pending {
self.pending = false;
return is_speech;
}
if is_start && playing {
self.pending = true;
return false;
}
is_start
}
}
struct VadParts<C: ClipSink> {
capture: MicCapture,
cons: HeapCons<f32>,
vad: SileroVad,
segmenter: VadSegmenter,
threshold: f32,
sink: C,
events_tx: mpsc::Sender<VadEventMsg>,
fatal_tx: mpsc::Sender<SkadooshError>,
shutdown: CancellationToken,
}
async fn vad_task<C: ClipSink>(parts: VadParts<C>) {
let VadParts {
capture,
mut cons,
mut vad,
mut segmenter,
threshold,
sink,
events_tx,
fatal_tx,
shutdown,
} = parts;
let _capture = capture;
let mut gate = OnsetGate::default();
loop {
if shutdown.is_cancelled() {
break;
}
let occupied = cons.occupied_len();
if occupied < FRAME_LEN {
let deficit = (FRAME_LEN - occupied) as u64;
let wait = Duration::from_millis((deficit / 16).clamp(1, 10));
tokio::select! {
biased;
_ = shutdown.cancelled() => break,
_ = tokio::time::sleep(wait) => {}
}
continue;
}
let mut frame = [0.0f32; FRAME_LEN];
let popped = cons.pop_slice(&mut frame);
debug_assert_eq!(popped, FRAME_LEN);
let prob = match vad.process(&frame) {
Ok(prob) => prob,
Err(err) => {
let _ = fatal_tx.send(err).await;
break;
}
};
let is_speech = prob >= threshold;
let event = segmenter.push(&frame, prob);
let is_start = matches!(event, Some(VadEvent::SpeechStart));
let forward_start = gate.filter(is_start, is_speech, sink.is_playing());
let msg = match event {
Some(VadEvent::Segment(samples)) => {
vad.reset_state();
Some(VadEventMsg::Segment {
samples,
t_speech_end: Instant::now(),
})
}
_ if forward_start => Some(VadEventMsg::SpeechStart),
_ => None,
};
if let Some(msg) = msg {
let sent = tokio::select! {
biased;
_ = shutdown.cancelled() => break,
sent = events_tx.send(msg) => sent,
};
if sent.is_err() {
debug!("orchestrator gone; VAD task exiting");
break;
}
}
}
}
async fn stt_bridge<S: SpeechToText>(
mut segment_rx: mpsc::Receiver<SegmentMsg>,
text_tx: mpsc::Sender<TextMsg>,
stt: S,
current_turn: Arc<AtomicU64>,
shutdown: CancellationToken,
fatal_tx: mpsc::Sender<SkadooshError>,
) {
loop {
let msg = tokio::select! {
biased;
_ = shutdown.cancelled() => break,
msg = segment_rx.recv() => match msg {
Some(msg) => msg,
None => break, },
};
let SegmentMsg {
turn_id,
token,
samples,
t_speech_end,
} = msg;
if turn_id != current_turn.load(Ordering::SeqCst) {
debug!(turn_id, "dropping stale segment before transcription");
continue;
}
let dropped_before = stt.dropped_jobs();
let result = tokio::select! {
biased;
_ = shutdown.cancelled() => break,
result = stt.transcribe(samples) => result,
};
let text = match result {
Ok(text) => text,
Err(err) => {
let evicted = stt.dropped_jobs() > dropped_before;
if shutdown.is_cancelled() || evicted {
debug!(
turn_id,
evicted, "STT job dropped during drain/eviction (benign)"
);
continue;
}
warn!(turn_id, error = %err, "fatal STT error");
let _ = fatal_tx.send(err).await;
break;
}
};
if text.trim().is_empty() {
debug!(turn_id, "empty transcript; skipping turn");
continue;
}
if token.is_cancelled() || turn_id != current_turn.load(Ordering::SeqCst) {
debug!(turn_id, "dropping stale transcript");
continue;
}
let msg = TextMsg {
turn_id,
token,
text,
t_speech_end,
t_text: Instant::now(),
};
let sent = tokio::select! {
biased;
_ = shutdown.cancelled() => break,
sent = text_tx.send(msg) => sent,
};
if sent.is_err() {
if shutdown.is_cancelled() {
break;
}
let _ = fatal_tx
.send(anyhow::anyhow!("LLM task channel closed unexpectedly").into())
.await;
break;
}
}
let stopped = tokio::task::spawn_blocking(move || stt.stop()).await;
if let Err(join_err) = stopped {
warn!(error = %join_err, "STT stop panicked");
}
}
async fn llm_task(
mut text_rx: mpsc::Receiver<TextMsg>,
turn_tx: mpsc::Sender<TurnMsg>,
turn_done_tx: mpsc::Sender<u64>,
mut client: LlmClient,
current_turn: Arc<AtomicU64>,
shutdown: CancellationToken,
fatal_tx: mpsc::Sender<SkadooshError>,
) {
loop {
let msg = tokio::select! {
biased;
_ = shutdown.cancelled() => break,
msg = text_rx.recv() => match msg {
Some(msg) => msg,
None => break,
},
};
let TextMsg {
turn_id,
token,
text,
t_speech_end,
t_text,
} = msg;
if token.is_cancelled() || turn_id != current_turn.load(Ordering::SeqCst) {
debug!(turn_id, "dropping stale transcript before LLM request");
continue;
}
let (clause_tx, clause_rx) = mpsc::channel(CLAUSE_CAP);
let turn = TurnMsg {
turn_id,
token: token.clone(),
clauses: clause_rx,
t_speech_end,
t_text,
};
let sent = tokio::select! {
biased;
_ = shutdown.cancelled() => break,
sent = turn_tx.send(turn) => sent,
};
if sent.is_err() {
if shutdown.is_cancelled() {
break;
}
let _ = fatal_tx
.send(anyhow::anyhow!("TTS task channel closed unexpectedly").into())
.await;
break;
}
info!(turn_id, %text, "LLM turn started");
let result = client.stream_reply(&text, clause_tx, token).await;
let _ = turn_done_tx.try_send(turn_id);
match result {
Ok(()) => {}
Err(SkadooshError::Llm(LlmError::Cancelled)) => {
debug!(turn_id, "LLM stream cancelled (barge-in or shutdown)");
}
Err(err) if shutdown.is_cancelled() => {
debug!(turn_id, error = %err, "LLM stream error during shutdown (benign)");
}
Err(err) => {
warn!(turn_id, error = %err, "fatal LLM error");
let _ = fatal_tx.send(err).await;
break;
}
}
}
}
async fn tts_task<C: ClipSink>(
mut turn_rx: mpsc::Receiver<TurnMsg>,
mut engine: Box<dyn TtsEngine>,
sink: C,
current_turn: Arc<AtomicU64>,
shutdown: CancellationToken,
fatal_tx: mpsc::Sender<SkadooshError>,
) {
'outer: loop {
let turn = tokio::select! {
biased;
_ = shutdown.cancelled() => break,
turn = turn_rx.recv() => match turn {
Some(turn) => turn,
None => break,
},
};
let TurnMsg {
turn_id,
token,
mut clauses,
t_speech_end,
t_text,
} = turn;
let mut timing = TurnTiming {
t_speech_end,
t_text,
t_first_clause: None,
t_first_clip: None,
};
'turn: loop {
let clause = tokio::select! {
biased;
_ = shutdown.cancelled() => break 'outer,
_ = token.cancelled() => break 'turn,
clause = clauses.recv() => match clause {
Some(clause) => clause,
None => break 'turn, },
};
let t_clause = Instant::now(); if token.is_cancelled() || turn_id != current_turn.load(Ordering::SeqCst) {
debug!(turn_id, "dropping stale clause");
continue;
}
let (returned_engine, result) = match synthesize_clause(engine, clause).await {
Ok(pair) => pair,
Err(join_err) => {
if !shutdown.is_cancelled() {
warn!(turn_id, error = %join_err, "TTS synthesis panicked");
let _ = fatal_tx
.send(anyhow::anyhow!("TTS synthesis panicked: {join_err}").into())
.await;
}
break 'outer;
}
};
engine = returned_engine;
let clip = match result {
Ok(clip) => clip,
Err(err) => {
if shutdown.is_cancelled() || token.is_cancelled() {
break 'turn; }
warn!(turn_id, error = %err, "fatal TTS error");
let _ = fatal_tx.send(err).await;
break 'outer;
}
};
if token.is_cancelled() || turn_id != current_turn.load(Ordering::SeqCst) {
debug!(turn_id, "discarding clip synthesized after cancel");
continue;
}
timing.t_first_clause.get_or_insert(t_clause);
let queued = tokio::select! {
biased;
_ = shutdown.cancelled() => break 'outer,
_ = token.cancelled() => break 'turn,
queued = sink.queue_clip(clip) => queued,
};
if let Err(err) = queued {
if shutdown.is_cancelled() || token.is_cancelled() {
break 'turn;
}
warn!(turn_id, error = %err, "fatal playback error");
let _ = fatal_tx.send(err).await;
break 'outer;
}
if timing.t_first_clip.is_none() {
timing.t_first_clip = Some(Instant::now());
tokio::spawn(
audible_watcher(sink.clone(), token.clone(), turn_id, timing.clone())
.instrument(info_span!("playback")),
);
}
}
}
}
async fn synthesize_clause(
engine: Box<dyn TtsEngine>,
clause: String,
) -> std::result::Result<(Box<dyn TtsEngine>, Result<TtsClip>), tokio::task::JoinError> {
tokio::task::spawn_blocking(move || {
let mut engine = engine;
let result = engine.synthesize(&clause);
(engine, result)
})
.await
}
async fn audible_watcher<C: ClipSink>(
sink: C,
token: CancellationToken,
turn_id: u64,
timing: TurnTiming,
) {
let started = Instant::now();
loop {
tokio::select! {
biased;
_ = token.cancelled() => {
debug!(turn_id, "turn cancelled before first audible sample");
return;
}
_ = tokio::time::sleep(AUDIBLE_POLL) => {
if sink.is_playing() {
let t_audible = Instant::now();
let (Some(t_clause), Some(t_clip)) =
(timing.t_first_clause, timing.t_first_clip)
else {
return; };
let stt_ms = millis(timing.t_text - timing.t_speech_end);
let llm_ms = millis(t_clause - timing.t_text);
let tts_ms = millis(t_clip - t_clause);
let playback_ms = millis(t_audible - t_clip);
let total_ms = millis(t_audible - timing.t_speech_end);
info!(
turn_id,
stt_ms,
llm_ms,
tts_ms,
playback_ms,
total_ms,
"turn latency: speech-end → first audible sample"
);
return;
}
if started.elapsed() > AUDIBLE_TIMEOUT {
debug!(turn_id, "first-audible wait timed out (stalled device?)");
return;
}
}
}
}
}
fn millis(d: Duration) -> u64 {
d.as_millis() as u64
}
struct ActiveTurn {
turn_id: u64,
token: CancellationToken,
llm_done: bool,
}
#[doc(hidden)]
pub async fn run_orchestrator<S: SpeechToText, C: ClipSink>(
topology: Topology<S, C>,
) -> Result<()> {
let Topology {
vad_events: mut vad_rx,
fatal_tx,
mut fatal_rx,
stt,
llm,
tts_engine,
sink,
shutdown,
} = topology;
let current_turn = Arc::new(AtomicU64::new(0));
let (segment_tx, segment_rx) = mpsc::channel(SEGMENT_CAP);
let (text_tx, text_rx) = mpsc::channel(TEXT_CAP);
let (turn_tx, turn_rx) = mpsc::channel(TURN_CAP);
let (turn_done_tx, mut turn_done_rx) = mpsc::channel(TURN_DONE_CAP);
let mut tasks = tokio::task::JoinSet::new();
tasks.spawn(
stt_bridge(
segment_rx,
text_tx,
stt,
Arc::clone(¤t_turn),
shutdown.clone(),
fatal_tx.clone(),
)
.instrument(info_span!("stt")),
);
tasks.spawn(
llm_task(
text_rx,
turn_tx,
turn_done_tx,
llm,
Arc::clone(¤t_turn),
shutdown.clone(),
fatal_tx.clone(),
)
.instrument(info_span!("llm")),
);
tasks.spawn(
tts_task(
turn_rx,
tts_engine,
sink.clone(),
Arc::clone(¤t_turn),
shutdown.clone(),
fatal_tx.clone(),
)
.instrument(info_span!("tts")),
);
let mut active_turn: Option<ActiveTurn> = None;
let mut fatal: Option<SkadooshError> = None;
loop {
tokio::select! {
biased;
_ = shutdown.cancelled() => {
debug!("shutdown requested");
break;
}
err = fatal_rx.recv() => {
match err {
Some(err) => {
warn!(error = %err, "fatal error; shutting down");
fatal = Some(err);
}
None => {
debug!("fatal channel closed; tasks exited");
}
}
break;
}
done = turn_done_rx.recv() => {
if let Some(done_id) = done {
if let Some(turn) = &mut active_turn {
if turn.turn_id == done_id {
turn.llm_done = true;
}
}
}
}
event = vad_rx.recv() => {
let Some(event) = event else {
if shutdown.is_cancelled() {
break;
}
fatal = Some(
anyhow::anyhow!("VAD event stream closed unexpectedly").into(),
);
break;
};
match event {
VadEventMsg::SpeechStart => {
if sink.is_playing() {
sink.flush();
if let Some(turn) = active_turn.take() {
info!(
turn_id = turn.turn_id,
llm_done = turn.llm_done,
"barge-in: cancelled turn, flushed playback"
);
turn.token.cancel();
} else {
debug!("barge-in flush with no active LLM turn");
}
}
}
VadEventMsg::Segment { samples, t_speech_end } => {
if let Some(turn) = active_turn.take() {
debug!(
turn_id = turn.turn_id,
llm_done = turn.llm_done,
"superseding in-flight turn"
);
turn.token.cancel();
}
let turn_id = current_turn.fetch_add(1, Ordering::SeqCst) + 1;
let token = shutdown.child_token();
active_turn = Some(ActiveTurn {
turn_id,
token: token.clone(),
llm_done: false,
});
info!(
turn_id,
audio_ms = samples.len() as u64 * 1000 / u64::from(CAPTURE_RATE),
"speech segment captured"
);
let msg = SegmentMsg {
turn_id,
token,
samples,
t_speech_end,
};
let sent = tokio::select! {
biased;
_ = shutdown.cancelled() => break,
sent = segment_tx.send(msg) => sent,
};
if sent.is_err() {
if shutdown.is_cancelled() {
break;
}
fatal = Some(
anyhow::anyhow!("STT bridge channel closed unexpectedly").into(),
);
break;
}
}
}
}
}
}
shutdown.cancel();
if let Some(turn) = active_turn.take() {
turn.token.cancel();
}
drop(segment_tx);
drop(fatal_tx);
while let Some(joined) = tasks.join_next().await {
if let Err(join_err) = joined {
warn!(error = %join_err, "pipeline task panicked");
if fatal.is_none() {
fatal = Some(anyhow::anyhow!("pipeline task panicked: {join_err}").into());
}
}
}
match fatal {
Some(err) => Err(err),
None => Ok(()),
}
}
impl Pipeline {
async fn selftest_async(self, wav: &Path, out_wav: &Path) -> Result<SelftestReport> {
let t_start = Instant::now();
let (mono, src_rate) = read_wav(wav)?;
let samples = resample_offline(&mono, src_rate, CAPTURE_RATE);
let t_loaded = Instant::now();
let mut vad = SileroVad::new(&self.config.vad_model)?;
let mut segmenter = VadSegmenter::new(self.config.vad_threshold, self.config.silence_ms);
let silence_frames = (self.config.silence_ms / 32 + 2) as usize;
let mut segment = None;
let mut feed = samples;
feed.extend(std::iter::repeat_n(0.0, silence_frames * FRAME_LEN));
for chunk in feed.chunks_exact(FRAME_LEN) {
let frame: &[f32; FRAME_LEN] = chunk.try_into().expect("chunks_exact(FRAME_LEN)");
let prob = vad.process(frame)?;
if let Some(VadEvent::Segment(audio)) = segmenter.push(frame, prob) {
segment = Some(audio);
break;
}
}
let segment = segment.ok_or_else(|| {
anyhow::anyhow!(
"no speech segment detected in {} (needs audible speech followed by \
> {} ms of silence)",
wav.display(),
self.config.silence_ms
)
})?;
let t_segment = Instant::now();
let stt = WhisperStt::start(&self.config.whisper_model, &SttConfig::default())?;
let transcript = stt
.transcribe(segment)
.await
.map_err(|_| SttError::WorkerGone)??;
stt.stop();
let t_text = Instant::now();
if transcript.trim().is_empty() {
return Err(anyhow::anyhow!("STT produced an empty transcript").into());
}
let http = reqwest::Client::new();
let url = format!(
"{}/chat/completions",
self.config.llm_url.trim_end_matches('/')
);
let body = serde_json::json!({
"model": self.config.llm_model,
"messages": [
{"role": "system", "content": self.config.system_prompt},
{"role": "user", "content": transcript},
],
"stream": true,
});
let t_llm = Instant::now();
let resp = http
.post(&url)
.json(&body)
.send()
.await
.map_err(LlmError::Http)?;
let resp = ensure_success(resp).await?;
let mut engine = build_engine(&self.config)?;
let mut splitter = ClauseSplitter::new(CLAUSE_MIN_LEN, CLAUSE_MAX_LEN);
let mut clips: Vec<TtsClip> = Vec::new();
let mut clause_texts: Vec<String> = Vec::new();
let mut t_first_token: Option<Instant> = None;
let mut t_first_clause: Option<Instant> = None;
let mut t_first_clip: Option<Instant> = None;
let mut stream = resp.bytes_stream();
let mut lines = SseLineBuffer::default();
let mut done = false;
let mut eof = false;
while !done && !eof {
match stream.next().await {
Some(Ok(bytes)) => lines.feed(&bytes),
Some(Err(err)) => return Err(LlmError::Http(err).into()),
None => {
lines.close();
eof = true;
}
}
while let Some(line) = lines.next_line() {
match parse_sse_line(&line) {
None => {}
Some(Ok(None)) => {
done = true;
break;
}
Some(Ok(Some(token))) => {
t_first_token.get_or_insert_with(Instant::now);
for clause in splitter.push(&token) {
t_first_clause.get_or_insert_with(Instant::now);
let (e, result) = synthesize_clause(engine, clause.clone())
.await
.map_err(|err| anyhow::anyhow!("TTS synthesis panicked: {err}"))?;
engine = e;
let clip = result?;
t_first_clip.get_or_insert_with(Instant::now);
clause_texts.push(clause);
clips.push(clip);
}
}
Some(Err(err)) => {
warn!(error = %err, "skipping malformed SSE data line");
}
}
}
}
if let Some(rest) = splitter.flush() {
t_first_clause.get_or_insert_with(Instant::now);
let (_engine, result) = synthesize_clause(engine, rest.clone())
.await
.map_err(|err| anyhow::anyhow!("TTS synthesis panicked: {err}"))?;
let clip = result?;
t_first_clip.get_or_insert_with(Instant::now);
clause_texts.push(rest);
clips.push(clip);
}
if clips.is_empty() {
return Err(anyhow::anyhow!("LLM reply produced no clauses").into());
}
let t_llm_done = Instant::now();
let total_samples: usize = clips.iter().map(|c| c.samples.len()).sum();
let mut pcm = Vec::with_capacity(total_samples);
for clip in &clips {
pcm.extend_from_slice(&clip.samples);
}
write_wav16(out_wav, &pcm, TTS_SAMPLE_RATE)?;
let total_ms = millis(t_start.elapsed());
info!(
clauses = clause_texts.len(),
audio_ms = pcm.len() as u64 * 1000 / u64::from(TTS_SAMPLE_RATE),
out_wav = %out_wav.display(),
"selftest complete"
);
Ok(SelftestReport {
segment_ms: millis(t_segment - t_loaded),
stt_ms: millis(t_text - t_segment),
llm_ttft_ms: millis(t_first_token.unwrap_or(t_llm_done) - t_llm),
first_clause_ms: millis(t_first_clause.unwrap_or(t_llm_done) - t_llm),
tts_ms: millis(
t_first_clip.unwrap_or(t_llm_done) - t_first_clause.unwrap_or(t_llm_done),
),
total_ms,
transcript,
})
}
}
fn read_wav(path: &Path) -> Result<(Vec<f32>, u32)> {
let bytes = std::fs::read(path)
.map_err(|err| anyhow::anyhow!("failed to read {}: {err}", path.display()))?;
if bytes.len() < 12 || &bytes[0..4] != b"RIFF" || &bytes[8..12] != b"WAVE" {
return Err(anyhow::anyhow!("{} is not a RIFF/WAVE file", path.display()).into());
}
let mut fmt: Option<(u16, u16, u32, u16)> = None; let mut data: Option<&[u8]> = None;
let mut pos = 12usize;
while pos + 8 <= bytes.len() {
let id = &bytes[pos..pos + 4];
let size =
u32::from_le_bytes(bytes[pos + 4..pos + 8].try_into().expect("4 bytes")) as usize;
let body_start = pos + 8;
let body_end = body_start.saturating_add(size).min(bytes.len());
match id {
b"fmt " if body_end - body_start >= 16 => {
let b = &bytes[body_start..body_end];
let format = u16::from_le_bytes(b[0..2].try_into().expect("2 bytes"));
let channels = u16::from_le_bytes(b[2..4].try_into().expect("2 bytes"));
let rate = u32::from_le_bytes(b[4..8].try_into().expect("4 bytes"));
let bits = u16::from_le_bytes(b[14..16].try_into().expect("2 bytes"));
fmt = Some((format, channels, rate, bits));
}
b"data" => data = Some(&bytes[body_start..body_end]),
_ => {}
}
pos = body_start + size + (size & 1);
}
let (format, channels, rate, bits) =
fmt.ok_or_else(|| anyhow::anyhow!("{}: missing fmt chunk", path.display()))?;
let data = data.ok_or_else(|| anyhow::anyhow!("{}: missing data chunk", path.display()))?;
let channels = usize::from(channels);
if channels == 0 || rate == 0 {
return Err(anyhow::anyhow!("{}: bad fmt chunk", path.display()).into());
}
let per_sample = |b: &[u8]| -> Result<f32> {
match (format, bits) {
(1, 8) => Ok((f32::from(b[0]) - 128.0) / 128.0),
(1, 16) => {
Ok(f32::from(i16::from_le_bytes(b[0..2].try_into().expect("2 bytes"))) / 32768.0)
}
(1, 24) => {
let v =
i32::from_le_bytes([b[0], b[1], b[2], if b[2] & 0x80 != 0 { 0xFF } else { 0 }]);
Ok(v as f32 / 8_388_608.0)
}
(1, 32) => Ok(
i32::from_le_bytes(b[0..4].try_into().expect("4 bytes")) as f32 / 2_147_483_648.0,
),
(3, 32) => Ok(f32::from_le_bytes(b[0..4].try_into().expect("4 bytes"))),
(format, bits) => Err(anyhow::anyhow!(
"{}: unsupported wav format (format {format}, {bits} bits); \
supported: PCM 8/16/24/32-bit and 32-bit float",
path.display()
)
.into()),
}
};
let sample_bytes = usize::from(bits / 8);
let frame_bytes = sample_bytes * channels;
if frame_bytes == 0 {
return Err(anyhow::anyhow!("{}: bad fmt chunk", path.display()).into());
}
let frames = data.len() / frame_bytes;
let mut mono = Vec::with_capacity(frames);
for frame in 0..frames {
let base = frame * frame_bytes;
let mut acc = 0.0f32;
for ch in 0..channels {
acc += per_sample(&data[base + ch * sample_bytes..])?;
}
mono.push(acc / channels as f32);
}
Ok((mono, rate))
}
fn write_wav16(path: &Path, samples: &[f32], rate: u32) -> Result<()> {
let data_len = (samples.len() * 2) as u32;
let mut out = Vec::with_capacity(44 + data_len as usize);
out.extend_from_slice(b"RIFF");
out.extend_from_slice(&(36 + data_len).to_le_bytes());
out.extend_from_slice(b"WAVE");
out.extend_from_slice(b"fmt ");
out.extend_from_slice(&16u32.to_le_bytes()); out.extend_from_slice(&1u16.to_le_bytes()); out.extend_from_slice(&1u16.to_le_bytes()); out.extend_from_slice(&rate.to_le_bytes());
out.extend_from_slice(&(rate * 2).to_le_bytes()); out.extend_from_slice(&2u16.to_le_bytes()); out.extend_from_slice(&16u16.to_le_bytes()); out.extend_from_slice(b"data");
out.extend_from_slice(&data_len.to_le_bytes());
for &s in samples {
let v = (s.clamp(-1.0, 1.0) * 32767.0) as i16;
out.extend_from_slice(&v.to_le_bytes());
}
std::fs::write(path, &out)
.map_err(|err| anyhow::anyhow!("failed to write {}: {err}", path.display()).into())
}