Skip to main content

loqui_whisper/
lib.rs

1//! Whisper speech-to-text on whisper.cpp.
2//!
3//! Decoding follows what open-speech asks of faster-whisper: beam search of
4//! width 5, temperature 0, an optional initial prompt, and a language that
5//! is detected unless given. Input is 16 kHz mono `f32`
6//! (`loqui_audio::decode_mono_16k` produces it from any upload).
7//!
8//! Weights are GGML files such as `ggml-large-v3-turbo.bin` from
9//! `ggerganov/whisper.cpp`; they are not bundled.
10
11use std::num::NonZeroUsize;
12use std::path::Path;
13use std::sync::Mutex;
14use std::thread::available_parallelism;
15
16use whisper_rs::{FullParams, SamplingStrategy, WhisperContext, WhisperContextParameters, WhisperState};
17
18/// whisper.cpp timestamps are in hundredths of a second.
19const TIMESTAMP_UNIT_SECS: f64 = 0.01;
20const SAMPLE_RATE: f64 = 16_000.0;
21/// whisper.cpp's own default thread count, which it caps at the machine's
22/// CPUs rather than the ones this process may use.
23const DEFAULT_THREADS: usize = 4;
24
25#[derive(Debug, thiserror::Error)]
26pub enum Error {
27    #[error("loading Whisper model: {0}")]
28    Load(String),
29    #[error("transcription failed: {0}")]
30    Inference(String),
31    #[error("language {0:?} is not supported by this English-only model")]
32    Language(String),
33}
34
35/// Where inference runs.
36#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
37pub enum Device {
38    #[default]
39    Cpu,
40    /// A GPU by ordinal. Needs the `cuda` or `vulkan` feature; without one,
41    /// loading fails rather than silently running on the CPU.
42    Gpu(i32),
43}
44
45#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
46pub enum Task {
47    #[default]
48    Transcribe,
49    /// Transcribe into English, whatever the spoken language.
50    Translate,
51}
52
53#[derive(Debug, Clone, Default)]
54pub struct Options {
55    /// An ISO 639-1 code such as `"en"`; `None` or `"auto"` detects it.
56    pub language: Option<String>,
57    /// Text that primes vocabulary and style (Whisper's initial prompt).
58    pub prompt: Option<String>,
59    pub task: Task,
60    /// 0 decodes deterministically, which is open-speech's default.
61    pub temperature: f32,
62    /// CPU threads; `None` (or 0) means whisper.cpp's four, or fewer when
63    /// this process may use fewer CPUs (its affinity mask, cpuset or CPU
64    /// quota).
65    pub threads: Option<u16>,
66}
67
68#[derive(Debug, Clone, PartialEq)]
69pub struct Segment {
70    /// Seconds from the start of the audio.
71    pub start: f64,
72    pub end: f64,
73    pub text: String,
74}
75
76#[derive(Debug, Clone, PartialEq)]
77pub struct Transcription {
78    /// Segment texts joined, as faster-whisper joins them.
79    pub text: String,
80    /// The language spoken (requested or detected), when known.
81    pub language: Option<String>,
82    /// Length of the audio in seconds.
83    pub duration: f64,
84    pub segments: Vec<Segment>,
85}
86
87pub struct Whisper {
88    // The context owns the weights; the state is the scratch space for one
89    // inference at a time. Field order drops the state first.
90    state: Mutex<WhisperState>,
91    _context: WhisperContext,
92    multilingual: bool,
93}
94
95impl std::fmt::Debug for Whisper {
96    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
97        f.debug_struct("Whisper").field("multilingual", &self.multilingual).finish_non_exhaustive()
98    }
99}
100
101impl Whisper {
102    /// Loads a GGML model file.
103    pub fn load(path: &Path, device: Device) -> Result<Self, Error> {
104        whisper_rs::install_logging_hooks();
105        let mut params = WhisperContextParameters::default();
106        match device {
107            Device::Cpu => {
108                params.use_gpu(false);
109            }
110            Device::Gpu(ordinal) => {
111                if !cfg!(any(feature = "cuda", feature = "vulkan")) {
112                    return Err(Error::Load(
113                        "a GPU was requested but loqui-whisper was built without the `cuda` or `vulkan` feature".into(),
114                    ));
115                }
116                params.use_gpu(true).gpu_device(ordinal);
117            }
118        }
119        let path_str = path.to_str().ok_or_else(|| Error::Load(format!("{} is not UTF-8", path.display())))?;
120        let context = WhisperContext::new_with_params(path_str, params).map_err(|e| Error::Load(format!("{}: {e}", path.display())))?;
121        let state = context.create_state().map_err(|e| Error::Load(e.to_string()))?;
122        let multilingual = context.is_multilingual();
123        Ok(Self { state: Mutex::new(state), _context: context, multilingual })
124    }
125
126    pub fn is_multilingual(&self) -> bool {
127        self.multilingual
128    }
129
130    /// Transcribes 16 kHz mono audio. One call at a time per model; others
131    /// wait.
132    pub fn transcribe(&self, samples: &[f32], options: &Options) -> Result<Transcription, Error> {
133        let language = options.language.as_deref().filter(|l| !l.eq_ignore_ascii_case("auto"));
134        if let Some(lang) = language
135            && !self.multilingual
136            && !lang.eq_ignore_ascii_case("en")
137        {
138            return Err(Error::Language(lang.to_owned()));
139        }
140
141        let mut params = FullParams::new(SamplingStrategy::BeamSearch { beam_size: 5, patience: -1.0 });
142        params.set_language(language);
143        params.set_translate(options.task == Task::Translate);
144        params.set_temperature(options.temperature);
145        if let Some(prompt) = options.prompt.as_deref().filter(|p| !p.is_empty()) {
146            params.set_initial_prompt(prompt);
147        }
148        params.set_n_threads(n_threads(options.threads, available_parallelism().map_or(1, NonZeroUsize::get)));
149        params.set_print_special(false);
150        params.set_print_progress(false);
151        params.set_print_realtime(false);
152        params.set_print_timestamps(false);
153
154        let mut state = self.state.lock().map_err(|_| Error::Inference("model lock poisoned".into()))?;
155        state.full(params, samples).map_err(|e| Error::Inference(e.to_string()))?;
156
157        let mut segments = Vec::new();
158        let mut text = String::new();
159        for segment in state.as_iter() {
160            let piece = segment.to_str_lossy().map_err(|e| Error::Inference(e.to_string()))?;
161            text.push_str(&piece);
162            let start = segment.start_timestamp() as f64 * TIMESTAMP_UNIT_SECS;
163            let end = (segment.end_timestamp() as f64 * TIMESTAMP_UNIT_SECS).max(start);
164            if !piece.trim().is_empty() {
165                segments.push(Segment { start, end, text: piece.into_owned() });
166            }
167        }
168        let detected = whisper_rs::get_lang_str(state.full_lang_id_from_state()).map(str::to_owned);
169        let language = match language {
170            Some(lang) => Some(lang.to_owned()),
171            None if !self.multilingual => Some("en".to_owned()),
172            None => detected,
173        };
174        Ok(Transcription { text: text.trim().to_owned(), language, duration: samples.len() as f64 / SAMPLE_RATE, segments })
175    }
176}
177
178/// The threads for one transcription: as many as asked for, else
179/// whisper.cpp's default counted over the `cpus` this process may use.
180fn n_threads(requested: Option<u16>, cpus: usize) -> i32 {
181    match requested.filter(|&t| t > 0) {
182        Some(t) => i32::from(t),
183        None => DEFAULT_THREADS.min(cpus) as i32,
184    }
185}
186
187#[cfg(test)]
188mod tests {
189    use super::*;
190
191    #[test]
192    fn threads_default_to_whisper_cpps_four_within_the_cpus_allowed() {
193        assert_eq!(n_threads(None, 16), 4);
194        assert_eq!(n_threads(None, 2), 2, "two CPUs, as under a cpuset or --cpus=2");
195        assert_eq!(n_threads(None, 1), 1);
196        assert_eq!(n_threads(Some(0), 2), 2, "0 is the default, as before");
197        assert_eq!(n_threads(Some(8), 2), 8, "an explicit count is the caller's to choose");
198    }
199}