joshua 0.1.0

An mmap-based LLM inference engine for the Rust ecosystem — a Rust clone of Cactus
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
//! Speech-to-text with Whisper — pure Rust.
//!
//! Wraps candle's Whisper implementation (encoder, decoder, and the mel
//! spectrogram code are all Rust) behind a small engine mirroring
//! [`crate::engine::Engine`]'s layout conventions:
//!
//! ```text
//! whisper-tiny/
//! ├── model.safetensors    ← weights (e.g. from openai/whisper-tiny)
//! ├── config.json          ← model dimensions
//! └── tokenizer.json       ← HuggingFace tokenizer
//! ```
//!
//! The mel filterbank is generated at load time (librosa-compatible Slaney
//! filters — the same ones Whisper trains with), so no auxiliary data files
//! are needed.  Audio arrives either as raw 16 kHz mono PCM or as a WAV file
//! (any sample rate/channel count; mixed down and linearly resampled).
//!
//! Decoding is greedy per 30-second segment, matching candle's reference
//! decode loop: the full token sequence is re-fed each step (Whisper's KV
//! cache only covers cross-attention) and generation stops at
//! `<|endoftext|>` or the model's target-position cap.

use std::io::Cursor;
use std::path::Path;
use std::sync::Mutex;

use candle_core::{Device, IndexOp, Tensor};
use candle_nn::VarBuilder;
use candle_transformers::models::whisper::{self as wm, audio, model::Whisper, Config};
use tokenizers::Tokenizer;

use crate::error::{JoshuaError, Result};

/// Lowest WAV sample rate accepted (Hz).  A crafted low rate would otherwise
/// amplify a small upload into a huge one during upsampling to 16 kHz; this
/// floor caps that factor to 4× (and 0 would divide by zero).
const MIN_SAMPLE_RATE: u32 = 4_000;

/// Highest WAV sample rate accepted (Hz).  Above pro-audio rates; rejects
/// clearly bogus headers.
const MAX_SAMPLE_RATE: u32 = 384_000;

/// Maximum number of 16 kHz samples processed per request — ~30 minutes of
/// audio.  Caps both the resampler output and the mel-spectrogram tensor so
/// a long or amplified upload cannot exhaust memory.
const MAX_OUTPUT_SAMPLES: usize = wm::SAMPLE_RATE * 60 * 30;

/// A transcription result.
#[derive(Debug, Clone)]
pub struct Transcription {
    /// The transcribed (or translated) text.
    pub text: String,
    /// Audio duration in seconds.
    pub duration: f64,
    /// Language token used (e.g. `"en"`), when the model is multilingual.
    pub language: Option<String>,
}

/// Whisper speech-to-text engine.
///
/// `Send + Sync`; the model (which owns KV caches) is guarded by a mutex, so
/// transcriptions serialise — matching the one-model-instance memory profile
/// expected of a sidecar STT model.
pub struct WhisperEngine {
    model: Mutex<Whisper>,
    tokenizer: Tokenizer,
    config: Config,
    mel_filters: Vec<f32>,
    device: Device,
    model_name: String,
    sot_token: u32,
    eot_token: u32,
    transcribe_token: Option<u32>,
    translate_token: Option<u32>,
    no_timestamps_token: Option<u32>,
}

impl WhisperEngine {
    /// Load a Whisper model directory (`model.safetensors` + `config.json`
    /// + `tokenizer.json`).
    pub fn new(dir: impl AsRef<Path>) -> Result<Self> {
        let dir = dir.as_ref();
        let config: Config = serde_json::from_str(
            &std::fs::read_to_string(dir.join("config.json"))
                .map_err(|e| JoshuaError::ModelLoad(format!("whisper config.json: {e}")))?,
        )
        .map_err(|e| JoshuaError::ModelLoad(format!("whisper config.json parse: {e}")))?;

        let tokenizer = Tokenizer::from_file(dir.join("tokenizer.json"))
            .map_err(|e| JoshuaError::ModelLoad(format!("whisper tokenizer load: {e}")))?;

        let weights = find_safetensors(dir)?;
        // The weights are mapped, so the same compression traps apply as for a
        // GGUF (see `crate::compression`).  There is no explicit mmap request
        // to honour on this path, so an unmappable file is only ever a warning
        // — candle's loader reports the resulting failure itself.
        if let Ok(file) = std::fs::File::open(&weights) {
            if let Some(found) = crate::compression::detect(&file, None) {
                tracing::warn!("{weights:?} cannot be memory-mapped usefully: {found}.");
            }
        }
        let device = Device::Cpu;
        // SAFETY: same contract as the GGUF mmap — weight files are treated
        // as immutable once downloaded.
        let vb = unsafe { VarBuilder::from_mmaped_safetensors(&[&weights], wm::DTYPE, &device) }
            .map_err(|e| JoshuaError::ModelLoad(format!("whisper weights: {e}")))?;
        let model = Whisper::load(&vb, config.clone())
            .map_err(|e| JoshuaError::ModelLoad(format!("whisper model init: {e}")))?;

        let mel_filters = mel_filterbank(config.num_mel_bins, wm::N_FFT, wm::SAMPLE_RATE);

        let token = |s: &str| tokenizer.token_to_id(s);
        let sot_token = token(wm::SOT_TOKEN).ok_or_else(|| {
            JoshuaError::ModelLoad(format!("whisper tokenizer lacks {}", wm::SOT_TOKEN))
        })?;
        let eot_token = token(wm::EOT_TOKEN).ok_or_else(|| {
            JoshuaError::ModelLoad(format!("whisper tokenizer lacks {}", wm::EOT_TOKEN))
        })?;

        let model_name = dir
            .file_name()
            .and_then(|s| s.to_str())
            .unwrap_or("whisper")
            .to_string();

        tracing::info!(
            "Whisper model '{}' ready (mels={}, vocab={})",
            model_name,
            config.num_mel_bins,
            config.vocab_size
        );

        Ok(Self {
            model: Mutex::new(model),
            transcribe_token: token(wm::TRANSCRIBE_TOKEN),
            translate_token: token(wm::TRANSLATE_TOKEN),
            no_timestamps_token: token(wm::NO_TIMESTAMPS_TOKEN),
            tokenizer,
            config,
            mel_filters,
            device,
            model_name,
            sot_token,
            eot_token,
        })
    }

    /// The model directory name, used as the API model identifier.
    pub fn model_name(&self) -> &str {
        &self.model_name
    }

    /// Transcribe a WAV file (any sample rate / channel count).
    pub fn transcribe_wav(
        &self,
        wav_bytes: &[u8],
        language: Option<&str>,
        translate: bool,
    ) -> Result<Transcription> {
        let samples = wav_to_pcm16k(wav_bytes)?;
        self.transcribe(&samples, language, translate)
    }

    /// Transcribe raw 16 kHz mono PCM samples in `[-1, 1]`.
    pub fn transcribe(
        &self,
        pcm: &[f32],
        language: Option<&str>,
        translate: bool,
    ) -> Result<Transcription> {
        if pcm.is_empty() {
            return Err(JoshuaError::InvalidRequest("empty audio".to_string()));
        }
        if pcm.len() > MAX_OUTPUT_SAMPLES {
            return Err(JoshuaError::InvalidRequest(format!(
                "audio is too long ({} samples > {MAX_OUTPUT_SAMPLES} at 16 kHz)",
                pcm.len()
            )));
        }
        let duration = pcm.len() as f64 / wm::SAMPLE_RATE as f64;

        let mel = audio::pcm_to_mel(&self.config, pcm, &self.mel_filters);
        let n_frames = mel.len() / self.config.num_mel_bins;
        let mel = Tensor::from_vec(mel, (1, self.config.num_mel_bins, n_frames), &self.device)
            .map_err(|e| JoshuaError::Inference(e.to_string()))?;

        // Resolve the language token for multilingual checkpoints.
        let language = language.map(str::to_lowercase);
        let language_token = match &language {
            Some(lang) => Some(self.language_token(lang)?),
            // English-only checkpoints have no language tokens at all.
            None => self.tokenizer.token_to_id("<|en|>"),
        };

        // Recover from poisoning rather than failing permanently: the KV
        // cache is flushed per segment below, so a prior panic leaves no
        // state that would corrupt this transcription.
        let mut model = self
            .model
            .lock()
            .unwrap_or_else(|poisoned| poisoned.into_inner());

        let mut text = String::new();
        let mut seek = 0;
        while seek < n_frames {
            let segment_len = usize::min(n_frames - seek, wm::N_FRAMES);
            let segment = mel
                .narrow(2, seek, segment_len)
                .map_err(|e| JoshuaError::Inference(e.to_string()))?;
            let piece = self.decode_segment(&mut model, &segment, language_token, translate)?;
            text.push_str(&piece);
            seek += segment_len;
        }

        Ok(Transcription {
            text: text.trim().to_string(),
            duration,
            language: language.or_else(|| language_token.map(|_| "en".to_string())),
        })
    }

    /// Greedy-decode one ≤30 s mel segment.
    fn decode_segment(
        &self,
        model: &mut Whisper,
        mel: &Tensor,
        language_token: Option<u32>,
        translate: bool,
    ) -> Result<String> {
        let features = model
            .encoder
            .forward(mel, true)
            .map_err(|e| JoshuaError::Inference(e.to_string()))?;

        let mut tokens: Vec<u32> = vec![self.sot_token];
        if let Some(lang) = language_token {
            tokens.push(lang);
            let task = if translate {
                self.translate_token
            } else {
                self.transcribe_token
            };
            if let Some(task) = task {
                tokens.push(task);
            }
        }
        if let Some(no_ts) = self.no_timestamps_token {
            tokens.push(no_ts);
        }
        let n_prompt = tokens.len();

        let sample_len = self.config.max_target_positions / 2;
        for i in 0..sample_len {
            let input = Tensor::new(tokens.as_slice(), &self.device)
                .and_then(|t| t.unsqueeze(0))
                .map_err(|e| JoshuaError::Inference(e.to_string()))?;
            // The KV cache only covers cross-attention, so the full sequence
            // is re-fed each step, flushing on the first.
            let ys = model
                .decoder
                .forward(&input, &features, i == 0)
                .map_err(|e| JoshuaError::Inference(e.to_string()))?;
            let logits = model
                .decoder
                .final_linear(
                    &ys.i((..1, tokens.len() - 1..))
                        .map_err(|e| JoshuaError::Inference(e.to_string()))?,
                )
                .and_then(|l| l.i(0)?.i(0))
                .and_then(|l| l.to_vec1::<f32>())
                .map_err(|e| JoshuaError::Inference(e.to_string()))?;

            let next = self.argmax_suppressed(&logits);
            if next == self.eot_token || tokens.len() >= self.config.max_target_positions {
                break;
            }
            tokens.push(next);
        }

        // Skip the prompt tokens and strip any specials from the output.
        self.tokenizer
            .decode(&tokens[n_prompt..], true)
            .map_err(|e| JoshuaError::Inference(e.to_string()))
    }

    /// Greedy argmax honouring the model's suppress list and never emitting
    /// the start-of-transcript token.
    fn argmax_suppressed(&self, logits: &[f32]) -> u32 {
        let mut best = self.eot_token;
        let mut best_val = f32::NEG_INFINITY;
        for (i, &v) in logits.iter().enumerate() {
            let id = i as u32;
            if id == self.sot_token || self.config.suppress_tokens.contains(&id) {
                continue;
            }
            if v > best_val {
                best_val = v;
                best = id;
            }
        }
        best
    }

    /// Resolve `<|xx|>` for a two-letter language code.
    fn language_token(&self, lang: &str) -> Result<u32> {
        self.tokenizer
            .token_to_id(&format!("<|{lang}|>"))
            .ok_or_else(|| {
                JoshuaError::InvalidRequest(format!(
                    "language '{lang}' is not supported by this whisper model"
                ))
            })
    }
}

// ─── Audio ingest ───────────────────────────────────────────────────────────

/// Decode a WAV file to 16 kHz mono f32 PCM.
///
/// Channels are averaged; other sample rates are linearly resampled — fine
/// for speech (Whisper's mel front-end low-passes well below Nyquist).
pub fn wav_to_pcm16k(bytes: &[u8]) -> Result<Vec<f32>> {
    let mut reader = hound::WavReader::new(Cursor::new(bytes))
        .map_err(|e| JoshuaError::InvalidRequest(format!("invalid WAV: {e}")))?;
    let spec = reader.spec();
    let channels = spec.channels.max(1) as usize;

    // Reject header values that would drive pathological allocation or
    // arithmetic on attacker-controlled uploads.  The floor also bounds the
    // upsampling amplification (16 kHz / rate) to a small factor, on top of
    // the absolute `MAX_OUTPUT_SAMPLES` cap applied below.
    if spec.sample_rate < MIN_SAMPLE_RATE || spec.sample_rate > MAX_SAMPLE_RATE {
        return Err(JoshuaError::InvalidRequest(format!(
            "unsupported WAV sample rate {} (must be {MIN_SAMPLE_RATE}..={MAX_SAMPLE_RATE})",
            spec.sample_rate
        )));
    }
    if !matches!(spec.bits_per_sample, 8 | 16 | 24 | 32) {
        return Err(JoshuaError::InvalidRequest(format!(
            "unsupported WAV bit depth {} (must be 8, 16, 24, or 32)",
            spec.bits_per_sample
        )));
    }

    let interleaved: Vec<f32> = match spec.sample_format {
        hound::SampleFormat::Float => reader
            .samples::<f32>()
            .collect::<std::result::Result<_, _>>()
            .map_err(|e| JoshuaError::InvalidRequest(format!("WAV read: {e}")))?,
        hound::SampleFormat::Int => {
            // Safe now that bits_per_sample is validated to 8/16/24/32.
            let max = (1i64 << (spec.bits_per_sample - 1)) as f32;
            reader
                .samples::<i32>()
                .map(|s| s.map(|v| v as f32 / max))
                .collect::<std::result::Result<_, _>>()
                .map_err(|e| JoshuaError::InvalidRequest(format!("WAV read: {e}")))?
        }
    };

    // Mix down to mono.
    let mono: Vec<f32> = interleaved
        .chunks(channels)
        .map(|frame| frame.iter().sum::<f32>() / channels as f32)
        .collect();

    // Resample to 16 kHz.
    let src_rate = spec.sample_rate as usize;
    if src_rate == wm::SAMPLE_RATE {
        return Ok(mono);
    }
    let ratio = src_rate as f64 / wm::SAMPLE_RATE as f64;
    // Cap the resampled length: a low declared sample rate would otherwise
    // amplify a small upload into a huge allocation (e.g. sample_rate=1 →
    // 16000× blow-up).  MAX_SAMPLE_RATE bounds ratio away from zero, and this
    // caps the absolute output so the audio-duration limit is respected.
    let out_len = ((mono.len() as f64 / ratio) as usize).min(MAX_OUTPUT_SAMPLES);
    if mono.is_empty() {
        return Ok(Vec::new());
    }
    let resampled = (0..out_len)
        .map(|i| {
            let pos = i as f64 * ratio;
            let idx = pos as usize;
            let frac = (pos - idx as f64) as f32;
            let a = mono[idx.min(mono.len() - 1)];
            let b = mono[(idx + 1).min(mono.len() - 1)];
            a * (1.0 - frac) + b * frac
        })
        .collect();
    Ok(resampled)
}

// ─── Mel filterbank ─────────────────────────────────────────────────────────

/// Librosa-compatible Slaney mel filterbank (`htk=False, norm="slaney"`) —
/// the filters Whisper was trained with.  Row-major `n_mels × (n_fft/2+1)`.
fn mel_filterbank(n_mels: usize, n_fft: usize, sample_rate: usize) -> Vec<f32> {
    let n_freqs = n_fft / 2 + 1;
    let f_max = sample_rate as f64 / 2.0;

    // Slaney mel scale: linear below 1 kHz, logarithmic above.
    let hz_to_mel = |hz: f64| {
        if hz < 1000.0 {
            hz * 3.0 / 200.0
        } else {
            15.0 + (hz / 1000.0).ln() * 27.0 / 6.4f64.ln()
        }
    };
    let mel_to_hz = |mel: f64| {
        if mel < 15.0 {
            mel * 200.0 / 3.0
        } else {
            1000.0 * ((mel - 15.0) * 6.4f64.ln() / 27.0).exp()
        }
    };

    let mel_max = hz_to_mel(f_max);
    let band_hz: Vec<f64> = (0..n_mels + 2)
        .map(|i| mel_to_hz(mel_max * i as f64 / (n_mels + 1) as f64))
        .collect();
    let fft_hz: Vec<f64> = (0..n_freqs)
        .map(|k| k as f64 * sample_rate as f64 / n_fft as f64)
        .collect();

    let mut filters = vec![0f32; n_mels * n_freqs];
    for m in 0..n_mels {
        let (lower, center, upper) = (band_hz[m], band_hz[m + 1], band_hz[m + 2]);
        // Slaney area normalisation.
        let enorm = 2.0 / (upper - lower);
        for (k, &hz) in fft_hz.iter().enumerate() {
            let rising = (hz - lower) / (center - lower);
            let falling = (upper - hz) / (upper - center);
            let weight = rising.min(falling).max(0.0);
            filters[m * n_freqs + k] = (weight * enorm) as f32;
        }
    }
    filters
}

// ─── Helpers ────────────────────────────────────────────────────────────────

/// Find the first `.safetensors` file in `dir`.
fn find_safetensors(dir: &Path) -> Result<std::path::PathBuf> {
    let preferred = dir.join("model.safetensors");
    if preferred.exists() {
        return Ok(preferred);
    }
    for entry in std::fs::read_dir(dir)? {
        let path = entry?.path();
        if path.extension().and_then(|e| e.to_str()) == Some("safetensors") {
            return Ok(path);
        }
    }
    Err(JoshuaError::ModelLoad(format!(
        "no .safetensors weights found in {dir:?}"
    )))
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn mel_filterbank_shape_and_energy() {
        let filters = mel_filterbank(80, 400, 16000);
        assert_eq!(filters.len(), 80 * 201);
        // Every band must have some energy, and no negative weights.
        for m in 0..80 {
            let row = &filters[m * 201..(m + 1) * 201];
            assert!(row.iter().all(|&w| w >= 0.0));
            assert!(row.iter().sum::<f32>() > 0.0, "band {m} is empty");
        }
    }

    #[test]
    fn wav_roundtrip_and_resample() {
        // 1 kHz sine at 8 kHz stereo — should mix down and upsample to 16 kHz.
        let spec = hound::WavSpec {
            channels: 2,
            sample_rate: 8000,
            bits_per_sample: 16,
            sample_format: hound::SampleFormat::Int,
        };
        let mut buf = Cursor::new(Vec::new());
        {
            let mut writer = hound::WavWriter::new(&mut buf, spec).unwrap();
            for i in 0..8000 {
                let v = (i as f32 / 8000.0 * 1000.0 * std::f32::consts::TAU).sin();
                let s = (v * i16::MAX as f32 * 0.5) as i16;
                writer.write_sample(s).unwrap();
                writer.write_sample(s).unwrap();
            }
            writer.finalize().unwrap();
        }
        let pcm = wav_to_pcm16k(&buf.into_inner()).unwrap();
        assert_eq!(pcm.len(), 16000, "1 s of audio at 16 kHz");
        assert!(pcm.iter().any(|&v| v.abs() > 0.1), "signal survived");
    }
}