Skip to main content

rust_voice_assistant/stt/
whisper.rs

1//! Whisper speech recognition via whisper.cpp (whisper-rs), running ggml
2//! models on CPU.
3
4use super::traits::{SpeechToText, TranscriptionResult};
5use crate::audio::resample_linear;
6use crate::error::{Error, Result};
7use async_trait::async_trait;
8use std::path::PathBuf;
9use std::sync::{Arc, Mutex};
10use whisper_rs::{FullParams, SamplingStrategy, WhisperContext, WhisperContextParameters};
11
12const WHISPER_RATE: u32 = 16000;
13
14pub struct WhisperEngine {
15    pub model_name: String,
16    pub language: String,
17    pub model_path: Option<PathBuf>,
18    ctx: Mutex<Option<Arc<WhisperContext>>>,
19}
20
21impl WhisperEngine {
22    /// Engine with no model configured — `transcribe_pcm` reports an honest
23    /// error until a model path is provided.
24    pub fn new(model_name: impl Into<String>, language: impl Into<String>) -> Self {
25        Self {
26            model_name: model_name.into(),
27            language: language.into(),
28            model_path: None,
29            ctx: Mutex::new(None),
30        }
31    }
32
33    /// Engine backed by a ggml whisper model file. The file must exist;
34    /// inference context is loaded lazily on first use.
35    pub fn new_with_path(
36        model_name: impl Into<String>,
37        language: impl Into<String>,
38        model_path: PathBuf,
39    ) -> Result<Self> {
40        if !model_path.is_file() {
41            return Err(Error::ModelNotFound(model_path));
42        }
43        Ok(Self {
44            model_name: model_name.into(),
45            language: language.into(),
46            model_path: Some(model_path),
47            ctx: Mutex::new(None),
48        })
49    }
50
51    fn context(&self) -> Result<Arc<WhisperContext>> {
52        let mut guard = self.ctx.lock().unwrap();
53        if let Some(ctx) = guard.as_ref() {
54            return Ok(ctx.clone());
55        }
56        let path = self.model_path.clone().ok_or_else(|| {
57            Error::ModelMissing(format!("{} model path not set", self.model_name))
58        })?;
59        if !path.is_file() {
60            return Err(Error::ModelNotFound(path));
61        }
62        let ctx = WhisperContext::new_with_params(&path, WhisperContextParameters::default())
63            .map_err(|e| Error::Transcription(format!("load {}: {}", path.display(), e)))?;
64        let ctx = Arc::new(ctx);
65        *guard = Some(ctx.clone());
66        Ok(ctx)
67    }
68}
69
70#[async_trait]
71impl SpeechToText for WhisperEngine {
72    async fn transcribe_pcm(
73        &self,
74        samples: &[f32],
75        sample_rate: u32,
76    ) -> Result<TranscriptionResult> {
77        if samples.is_empty() {
78            return Ok(TranscriptionResult {
79                text: String::new(),
80                is_final: true,
81                confidence: 0,
82            });
83        }
84
85        let ctx = self.context()?;
86        let language = self.language.clone();
87        let pcm = resample_linear(samples, sample_rate, WHISPER_RATE);
88
89        tokio::task::spawn_blocking(move || {
90            let mut state = ctx
91                .create_state()
92                .map_err(|e| Error::Transcription(format!("whisper state: {}", e)))?;
93            let mut params = FullParams::new(SamplingStrategy::Greedy { best_of: 1 });
94            params.set_language(Some(language.as_str()));
95            params.set_no_timestamps(true);
96            params.set_print_special(false);
97            params.set_print_progress(false);
98            params.set_print_realtime(false);
99            params.set_print_timestamps(false);
100
101            state
102                .full(params, &pcm)
103                .map_err(|e| Error::Transcription(format!("whisper inference: {}", e)))?;
104
105            let n = state.full_n_segments();
106            let mut text = String::new();
107            let mut prob_sum = 0.0f32;
108            let mut prob_count = 0u32;
109            let mut prev_seg = String::new();
110            for i in 0..n {
111                if let Some(seg) = state.get_segment(i) {
112                    let seg_text = seg.to_str_lossy()?;
113                    let trimmed = seg_text.trim();
114                    // Whisper captions non-speech audio as "(wind blowing)",
115                    // "[BLANK_AUDIO]", "*music*" — drop it rather than feeding
116                    // noise hallucinations to the router.
117                    if is_non_speech_segment(trimmed) {
118                        continue;
119                    }
120                    // Consecutive identical segments are the classic noise
121                    // hallucination loop — collapse to one occurrence.
122                    if trimmed.eq_ignore_ascii_case(&prev_seg) {
123                        continue;
124                    }
125                    prev_seg = trimmed.to_string();
126                    text.push_str(&seg_text);
127                    for t in 0..seg.n_tokens() {
128                        if let Some(tok) = seg.get_token(t) {
129                            prob_sum += tok.token_probability();
130                            prob_count += 1;
131                        }
132                    }
133                }
134            }
135            let confidence = if prob_count > 0 {
136                ((prob_sum / prob_count as f32) * 100.0).clamp(0.0, 100.0) as u8
137            } else {
138                0
139            };
140            Ok(TranscriptionResult {
141                text: text.trim().to_string(),
142                is_final: true,
143                confidence,
144            })
145        })
146        .await
147        .map_err(|e| Error::Transcription(format!("whisper task: {}", e)))?
148    }
149}
150
151/// True when a segment is entirely non-speech captions: one or more
152/// `()`, `[]`, or `*…*` groups — e.g. `(wind blowing)`, `[BLANK_AUDIO]`,
153/// or a hallucination loop like `(mumbling) (mumbling) (mumbling)`.
154fn is_non_speech_segment(s: &str) -> bool {
155    let mut rest = s;
156    let mut stripped_any = false;
157    loop {
158        let t = rest.trim_start();
159        let close = match t.as_bytes().first() {
160            Some(b'(') => b')',
161            Some(b'[') => b']',
162            Some(b'*') => b'*',
163            _ => break,
164        };
165        let Some(rel) = t[1..].find(close as char) else {
166            break;
167        };
168        let end = 1 + rel;
169        if t[1..end].trim().is_empty() {
170            break;
171        }
172        stripped_any = true;
173        rest = &t[end + 1..];
174    }
175    stripped_any && rest.trim().is_empty()
176}
177
178impl From<whisper_rs::WhisperError> for Error {
179    fn from(e: whisper_rs::WhisperError) -> Self {
180        Error::Transcription(e.to_string())
181    }
182}
183
184#[cfg(test)]
185mod tests {
186    use super::is_non_speech_segment;
187
188    #[test]
189    fn single_caption_is_non_speech() {
190        for s in ["(wind blowing)", "[BLANK_AUDIO]", "*music*", "(silence)"] {
191            assert!(is_non_speech_segment(s), "{s}");
192        }
193    }
194
195    #[test]
196    fn repeated_caption_loop_is_non_speech() {
197        assert!(is_non_speech_segment(
198            "(mumbling) (mumbling) (mumbling) (mumbling) (mumbling) (mumbling)"
199        ));
200        assert!(is_non_speech_segment("[noise][noise]"));
201        assert!(is_non_speech_segment("*music* (applause)"));
202    }
203
204    #[test]
205    fn real_speech_is_not_non_speech() {
206        for s in [
207            "switch to workspace 2",
208            "hello (world)",
209            "(aside) but real words follow",
210            "what's the weather",
211            "x",
212            "()",
213            "(unclosed",
214        ] {
215            assert!(!is_non_speech_segment(s), "{s}");
216        }
217    }
218}