rust_voice_assistant/stt/
whisper.rs1use super::traits::{SpeechToText, TranscriptionResult};
5use crate::audio::resample_linear;
6use crate::error::{Error, Result};
7use async_trait::async_trait;
8use std::path::PathBuf;
9use std::sync::{Arc, Mutex};
10use whisper_rs::{FullParams, SamplingStrategy, WhisperContext, WhisperContextParameters};
11
12const WHISPER_RATE: u32 = 16000;
13
14pub struct WhisperEngine {
15 pub model_name: String,
16 pub language: String,
17 pub model_path: Option<PathBuf>,
18 ctx: Mutex<Option<Arc<WhisperContext>>>,
19}
20
21impl WhisperEngine {
22 pub fn new(model_name: impl Into<String>, language: impl Into<String>) -> Self {
25 Self {
26 model_name: model_name.into(),
27 language: language.into(),
28 model_path: None,
29 ctx: Mutex::new(None),
30 }
31 }
32
33 pub fn new_with_path(
36 model_name: impl Into<String>,
37 language: impl Into<String>,
38 model_path: PathBuf,
39 ) -> Result<Self> {
40 if !model_path.is_file() {
41 return Err(Error::ModelNotFound(model_path));
42 }
43 Ok(Self {
44 model_name: model_name.into(),
45 language: language.into(),
46 model_path: Some(model_path),
47 ctx: Mutex::new(None),
48 })
49 }
50
51 fn context(&self) -> Result<Arc<WhisperContext>> {
52 let mut guard = self.ctx.lock().unwrap();
53 if let Some(ctx) = guard.as_ref() {
54 return Ok(ctx.clone());
55 }
56 let path = self.model_path.clone().ok_or_else(|| {
57 Error::ModelMissing(format!("{} model path not set", self.model_name))
58 })?;
59 if !path.is_file() {
60 return Err(Error::ModelNotFound(path));
61 }
62 let ctx = WhisperContext::new_with_params(&path, WhisperContextParameters::default())
63 .map_err(|e| Error::Transcription(format!("load {}: {}", path.display(), e)))?;
64 let ctx = Arc::new(ctx);
65 *guard = Some(ctx.clone());
66 Ok(ctx)
67 }
68}
69
70#[async_trait]
71impl SpeechToText for WhisperEngine {
72 async fn transcribe_pcm(
73 &self,
74 samples: &[f32],
75 sample_rate: u32,
76 ) -> Result<TranscriptionResult> {
77 if samples.is_empty() {
78 return Ok(TranscriptionResult {
79 text: String::new(),
80 is_final: true,
81 confidence: 0,
82 });
83 }
84
85 let ctx = self.context()?;
86 let language = self.language.clone();
87 let pcm = resample_linear(samples, sample_rate, WHISPER_RATE);
88
89 tokio::task::spawn_blocking(move || {
90 let mut state = ctx
91 .create_state()
92 .map_err(|e| Error::Transcription(format!("whisper state: {}", e)))?;
93 let mut params = FullParams::new(SamplingStrategy::Greedy { best_of: 1 });
94 params.set_language(Some(language.as_str()));
95 params.set_no_timestamps(true);
96 params.set_print_special(false);
97 params.set_print_progress(false);
98 params.set_print_realtime(false);
99 params.set_print_timestamps(false);
100
101 state
102 .full(params, &pcm)
103 .map_err(|e| Error::Transcription(format!("whisper inference: {}", e)))?;
104
105 let n = state.full_n_segments();
106 let mut text = String::new();
107 let mut prob_sum = 0.0f32;
108 let mut prob_count = 0u32;
109 let mut prev_seg = String::new();
110 for i in 0..n {
111 if let Some(seg) = state.get_segment(i) {
112 let seg_text = seg.to_str_lossy()?;
113 let trimmed = seg_text.trim();
114 if is_non_speech_segment(trimmed) {
118 continue;
119 }
120 if trimmed.eq_ignore_ascii_case(&prev_seg) {
123 continue;
124 }
125 prev_seg = trimmed.to_string();
126 text.push_str(&seg_text);
127 for t in 0..seg.n_tokens() {
128 if let Some(tok) = seg.get_token(t) {
129 prob_sum += tok.token_probability();
130 prob_count += 1;
131 }
132 }
133 }
134 }
135 let confidence = if prob_count > 0 {
136 ((prob_sum / prob_count as f32) * 100.0).clamp(0.0, 100.0) as u8
137 } else {
138 0
139 };
140 Ok(TranscriptionResult {
141 text: text.trim().to_string(),
142 is_final: true,
143 confidence,
144 })
145 })
146 .await
147 .map_err(|e| Error::Transcription(format!("whisper task: {}", e)))?
148 }
149}
150
151fn is_non_speech_segment(s: &str) -> bool {
155 let mut rest = s;
156 let mut stripped_any = false;
157 loop {
158 let t = rest.trim_start();
159 let close = match t.as_bytes().first() {
160 Some(b'(') => b')',
161 Some(b'[') => b']',
162 Some(b'*') => b'*',
163 _ => break,
164 };
165 let Some(rel) = t[1..].find(close as char) else {
166 break;
167 };
168 let end = 1 + rel;
169 if t[1..end].trim().is_empty() {
170 break;
171 }
172 stripped_any = true;
173 rest = &t[end + 1..];
174 }
175 stripped_any && rest.trim().is_empty()
176}
177
178impl From<whisper_rs::WhisperError> for Error {
179 fn from(e: whisper_rs::WhisperError) -> Self {
180 Error::Transcription(e.to_string())
181 }
182}
183
184#[cfg(test)]
185mod tests {
186 use super::is_non_speech_segment;
187
188 #[test]
189 fn single_caption_is_non_speech() {
190 for s in ["(wind blowing)", "[BLANK_AUDIO]", "*music*", "(silence)"] {
191 assert!(is_non_speech_segment(s), "{s}");
192 }
193 }
194
195 #[test]
196 fn repeated_caption_loop_is_non_speech() {
197 assert!(is_non_speech_segment(
198 "(mumbling) (mumbling) (mumbling) (mumbling) (mumbling) (mumbling)"
199 ));
200 assert!(is_non_speech_segment("[noise][noise]"));
201 assert!(is_non_speech_segment("*music* (applause)"));
202 }
203
204 #[test]
205 fn real_speech_is_not_non_speech() {
206 for s in [
207 "switch to workspace 2",
208 "hello (world)",
209 "(aside) but real words follow",
210 "what's the weather",
211 "x",
212 "()",
213 "(unclosed",
214 ] {
215 assert!(!is_non_speech_segment(s), "{s}");
216 }
217 }
218}