whisper-apr 0.3.0

WASM-first automatic speech recognition engine implementing OpenAI Whisper
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
//! Whisper TUI Application State
//!
//! Manages the application state for the pipeline visualization dashboard.
//! Follows the state machine defined in WAPR-TUI-001.

// MelFilterbank is available via crate::audio but not needed for TUI state

/// Active panel in the whisper dashboard
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum WhisperPanel {
    /// Raw audio waveform visualization
    #[default]
    Waveform,
    /// Mel spectrogram heatmap
    Mel,
    /// Encoder layer activations
    Encoder,
    /// Decoder token generation
    Decoder,
    /// Cross-attention weights
    Attention,
    /// Final transcription output
    Transcription,
    /// Performance metrics
    Metrics,
    /// Help and keyboard bindings
    Help,
}

impl WhisperPanel {
    /// Get panel titles for tab bar
    pub fn titles() -> Vec<&'static str> {
        vec![
            "Waveform [1]",
            "Mel [2]",
            "Encoder [3]",
            "Decoder [4]",
            "Attention [5]",
            "Transcription [6]",
            "Metrics [7]",
            "Help [?]",
        ]
    }

    /// Get panel index
    pub fn index(self) -> usize {
        match self {
            Self::Waveform => 0,
            Self::Mel => 1,
            Self::Encoder => 2,
            Self::Decoder => 3,
            Self::Attention => 4,
            Self::Transcription => 5,
            Self::Metrics => 6,
            Self::Help => 7,
        }
    }

    /// Create from index
    pub fn from_index(index: usize) -> Self {
        match index {
            0 => Self::Waveform,
            1 => Self::Mel,
            2 => Self::Encoder,
            3 => Self::Decoder,
            4 => Self::Attention,
            5 => Self::Transcription,
            6 => Self::Metrics,
            _ => Self::Help,
        }
    }
}

/// Pipeline state machine
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum WhisperState {
    /// No audio loaded
    #[default]
    Idle,
    /// Audio loaded, waveform ready
    WaveformReady,
    /// Mel spectrogram computed
    MelReady,
    /// Encoder processing
    Encoding,
    /// Decoder generating tokens
    Decoding,
    /// Transcription complete
    Complete,
    /// Error state
    Error,
}

/// Encoder layer metrics for visualization
#[derive(Debug, Clone, Default)]
pub struct EncoderLayerMetrics {
    /// Layer index
    pub layer: usize,
    /// Mean activation magnitude
    pub mean_activation: f32,
    /// Max activation magnitude
    pub max_activation: f32,
    /// Self-attention entropy
    pub attention_entropy: f32,
}

/// Decoder token for visualization
#[derive(Debug, Clone)]
pub struct DecoderToken {
    /// Token ID
    pub id: u32,
    /// Token text
    pub text: String,
    /// Log probability
    pub log_prob: f32,
    /// Cross-attention weights (to audio frames)
    pub attention_weights: Vec<f32>,
}

/// Performance metrics
#[derive(Debug, Clone, Default)]
pub struct PipelineMetrics {
    /// Audio duration in seconds
    pub audio_duration_secs: f32,
    /// Mel computation time in ms
    pub mel_time_ms: f32,
    /// Encoder time in ms
    pub encoder_time_ms: f32,
    /// Decoder time in ms
    pub decoder_time_ms: f32,
    /// Total processing time in ms
    pub total_time_ms: f32,
    /// Real-time factor
    pub rtf: f32,
    /// Tokens generated
    pub tokens_generated: usize,
    /// Memory used in bytes
    pub memory_bytes: usize,
}

impl PipelineMetrics {
    /// Compute RTF from audio duration and processing time
    pub fn compute_rtf(&mut self) {
        if self.audio_duration_secs > 0.0 {
            self.rtf = (self.total_time_ms / 1000.0) / self.audio_duration_secs;
        }
    }
}

/// Jidoka alerts for Poka-Yoke visual feedback (WAPR-CLI-001 Section 3.3)
///
/// Implements Toyota Way principle of highlighting abnormalities immediately:
/// - High Drift: YELLOW if processing lag > 500ms
/// - Low Confidence: RED if log-prob < -1.0
/// - Audio Clipping: RED if amplitude > 1.0
#[derive(Debug, Clone, Default)]
pub struct JidokaAlerts {
    /// Processing drift in milliseconds (processing time - audio duration)
    pub drift_ms: f32,
    /// Minimum log probability from decoder tokens
    pub min_log_prob: f32,
    /// Maximum audio amplitude (clipping if > 1.0)
    pub max_amplitude: f32,
}

impl JidokaAlerts {
    /// Check if processing drift is too high (> 500ms)
    pub fn is_high_drift(&self) -> bool {
        self.drift_ms > 500.0
    }

    /// Check if decoder confidence is too low (log_prob < -1.0)
    pub fn is_low_confidence(&self) -> bool {
        self.min_log_prob < -1.0
    }

    /// Check if audio is clipping (amplitude > 1.0)
    pub fn is_clipping(&self) -> bool {
        self.max_amplitude > 1.0
    }

    /// Check if any alert is active
    pub fn has_alerts(&self) -> bool {
        self.is_high_drift() || self.is_low_confidence() || self.is_clipping()
    }
}

/// Whisper dashboard application state
#[derive(Debug, Clone)]
#[allow(clippy::struct_excessive_bools)] // TUI state naturally uses boolean flags
pub struct WhisperApp {
    /// Current active panel
    pub current_panel: WhisperPanel,
    /// Pipeline state
    pub state: WhisperState,
    /// Should quit flag
    pub should_quit: bool,
    /// Paused flag
    pub paused: bool,
    /// Raw audio data
    pub audio_data: Vec<f32>,
    /// Sample rate
    pub sample_rate: u32,
    /// Mel spectrogram data (80 bins x N frames)
    pub mel_data: Vec<f32>,
    /// Number of mel frames
    pub mel_frames: usize,
    /// Encoder layer metrics
    pub encoder_metrics: Vec<EncoderLayerMetrics>,
    /// Decoder tokens
    pub decoder_tokens: Vec<DecoderToken>,
    /// Cross-attention weights (tokens x frames)
    pub attention_weights: Vec<Vec<f32>>,
    /// Final transcription
    pub transcription: String,
    /// Performance metrics
    pub metrics: PipelineMetrics,
    /// Jidoka alerts (Poka-Yoke visual feedback)
    pub alerts: JidokaAlerts,
    /// Error message (if in error state)
    pub error_message: Option<String>,
    /// Status message
    pub status_message: Option<String>,
    /// Horizontal scroll position for panels
    pub scroll_x: usize,
    /// Vertical scroll position for panels
    pub scroll_y: usize,
    /// Selected layer for encoder view
    pub selected_layer: usize,
    /// Show trace overlay (t key)
    pub show_trace_overlay: bool,
    /// Show VAD overlay (v key)
    pub show_vad_overlay: bool,
}

impl Default for WhisperApp {
    fn default() -> Self {
        Self::new()
    }
}

impl WhisperApp {
    /// Create new application state
    pub fn new() -> Self {
        Self {
            current_panel: WhisperPanel::Waveform,
            state: WhisperState::Idle,
            should_quit: false,
            paused: false,
            audio_data: Vec::new(),
            sample_rate: 16000,
            mel_data: Vec::new(),
            mel_frames: 0,
            encoder_metrics: Vec::new(),
            decoder_tokens: Vec::new(),
            attention_weights: Vec::new(),
            transcription: String::new(),
            metrics: PipelineMetrics::default(),
            alerts: JidokaAlerts::default(),
            error_message: None,
            status_message: None,
            scroll_x: 0,
            scroll_y: 0,
            selected_layer: 0,
            show_trace_overlay: false,
            show_vad_overlay: false,
        }
    }

    /// Set current panel
    pub fn set_panel(&mut self, panel: WhisperPanel) {
        self.current_panel = panel;
    }

    /// Transition to a new pipeline state with a status message.
    fn transition(&mut self, state: WhisperState, message: String) {
        self.state = state;
        self.status_message = Some(message);
    }

    /// Record elapsed time in milliseconds from an `Instant`.
    fn elapsed_ms(start: std::time::Instant) -> f32 {
        start.elapsed().as_secs_f32() * 1000.0
    }

    /// Handle keyboard input
    pub fn handle_key(&mut self, key: char) {
        match key {
            '1' => self.current_panel = WhisperPanel::Waveform,
            '2' => self.current_panel = WhisperPanel::Mel,
            '3' => self.current_panel = WhisperPanel::Encoder,
            '4' => self.current_panel = WhisperPanel::Decoder,
            '5' => self.current_panel = WhisperPanel::Attention,
            '6' => self.current_panel = WhisperPanel::Transcription,
            '7' => self.current_panel = WhisperPanel::Metrics,
            '?' => self.current_panel = WhisperPanel::Help,
            ' ' => self.paused = !self.paused,
            't' => self.show_trace_overlay = !self.show_trace_overlay,
            'v' => self.show_vad_overlay = !self.show_vad_overlay,
            'r' => self.reset(),
            'q' => self.should_quit = true,
            _ => {}
        }
    }

    /// Reset to initial state
    #[allow(clippy::no_effect_underscore_binding)]
    pub fn reset(&mut self) {
        let _span = crate::trace_enter!("tui.reset");
        self.audio_data.clear();
        self.mel_data.clear();
        self.mel_frames = 0;
        self.encoder_metrics.clear();
        self.decoder_tokens.clear();
        self.attention_weights.clear();
        self.transcription.clear();
        self.metrics = PipelineMetrics::default();
        self.alerts = JidokaAlerts::default();
        self.error_message = None;
        self.scroll_x = 0;
        self.scroll_y = 0;
        self.selected_layer = 0;
        self.show_trace_overlay = false;
        self.show_vad_overlay = false;
        self.transition(WhisperState::Idle, "Reset to idle".to_string());
    }

    /// Load audio data
    #[allow(clippy::no_effect_underscore_binding)]
    pub fn load_audio(&mut self, audio: &[f32]) {
        let _span = crate::trace_enter!("tui.load_audio");
        self.audio_data = audio.to_vec();
        self.metrics.audio_duration_secs = audio.len() as f32 / self.sample_rate as f32;
        // Compute max amplitude for clipping detection (Jidoka)
        self.alerts.max_amplitude = audio.iter().map(|s| s.abs()).fold(0.0_f32, f32::max);
        self.transition(
            WhisperState::WaveformReady,
            format!(
                "Loaded {} samples ({:.2}s)",
                audio.len(),
                self.metrics.audio_duration_secs
            ),
        );
    }

    /// Compute mel spectrogram (mock for TUI testing)
    #[allow(clippy::no_effect_underscore_binding)]
    pub fn compute_mel(&mut self) {
        const N_MELS: usize = 80;
        const HOP_LENGTH: usize = 160; // 10ms at 16kHz

        let _span = crate::trace_enter!("tui.compute_mel");
        if self.audio_data.is_empty() {
            self.error_message = Some("No audio loaded".to_string());
            self.transition(WhisperState::Error, "No audio loaded".to_string());
            return;
        }

        let start = std::time::Instant::now();

        // Mock mel computation - in real impl would use MelFilterBank
        let n_frames = (self.audio_data.len() / HOP_LENGTH).max(1);

        // Generate mock mel data
        self.mel_data = vec![0.0; N_MELS * n_frames];
        for frame in 0..n_frames {
            for mel_bin in 0..N_MELS {
                // Create a pattern that looks like real mel spectrogram
                let frame_energy = self
                    .audio_data
                    .get(frame * HOP_LENGTH..(frame + 1) * HOP_LENGTH)
                    .map_or(0.0, |s| s.iter().map(|x| x.powi(2)).sum::<f32>());
                let log_energy = (frame_energy + 1e-10).ln();
                // Lower frequencies have more energy
                let freq_weight = 1.0 - (mel_bin as f32 / N_MELS as f32);
                self.mel_data[frame * N_MELS + mel_bin] = log_energy * freq_weight;
            }
        }
        self.mel_frames = n_frames;

        self.metrics.mel_time_ms = Self::elapsed_ms(start);
        self.transition(
            WhisperState::MelReady,
            format!(
                "Computed {n_frames} mel frames in {:.2}ms",
                self.metrics.mel_time_ms
            ),
        );
    }

    /// Start encoding (mock for TUI)
    #[allow(clippy::no_effect_underscore_binding)]
    pub fn start_encoding(&mut self) {
        let _span = crate::trace_enter!("tui.start_encoding");
        if self.state != WhisperState::MelReady {
            return;
        }

        let start = std::time::Instant::now();

        // Mock encoder layers (tiny model has 4 layers)
        self.encoder_metrics = (0..4)
            .map(|layer| EncoderLayerMetrics {
                layer,
                mean_activation: 0.5 + (layer as f32 * 0.1),
                max_activation: 2.0 + (layer as f32 * 0.2),
                attention_entropy: 3.5 - (layer as f32 * 0.3),
            })
            .collect();

        self.metrics.encoder_time_ms = Self::elapsed_ms(start);
        self.transition(
            WhisperState::Encoding,
            format!(
                "Encoded through {} layers in {:.2}ms",
                self.encoder_metrics.len(),
                self.metrics.encoder_time_ms,
            ),
        );
    }

    /// Start decoding (mock for TUI)
    #[allow(clippy::no_effect_underscore_binding)]
    pub fn start_decoding(&mut self) {
        let _span = crate::trace_enter!("tui.start_decoding");
        if self.state != WhisperState::Encoding {
            return;
        }

        let start = std::time::Instant::now();

        // Mock decoder tokens
        let sample_tokens = [
            "<|startoftranscript|>",
            "<|en|>",
            "Hello",
            ",",
            " world",
            ".",
            "<|endoftext|>",
        ];

        self.decoder_tokens = sample_tokens
            .iter()
            .enumerate()
            .map(|(i, text)| {
                let attention = (0..self.mel_frames.max(10))
                    .map(|f| {
                        // Peak attention around expected position
                        let peak = (i as f32 / sample_tokens.len() as f32) * self.mel_frames as f32;
                        let dist = (f as f32 - peak).abs();
                        (-dist / 10.0).exp()
                    })
                    .collect();

                DecoderToken {
                    id: i as u32 + 50000,
                    text: text.to_string(),
                    log_prob: -0.1 - (i as f32 * 0.05),
                    attention_weights: attention,
                }
            })
            .collect();

        // Build attention weights matrix
        self.attention_weights = self
            .decoder_tokens
            .iter()
            .map(|t| t.attention_weights.clone())
            .collect();

        self.metrics.decoder_time_ms = Self::elapsed_ms(start);
        self.metrics.tokens_generated = self.decoder_tokens.len();
        self.transition(
            WhisperState::Decoding,
            format!(
                "Generated {} tokens in {:.2}ms",
                self.decoder_tokens.len(),
                self.metrics.decoder_time_ms,
            ),
        );
    }

    /// Complete transcription
    #[allow(clippy::no_effect_underscore_binding)]
    pub fn complete(&mut self) {
        let _span = crate::trace_enter!("tui.complete");
        if self.state != WhisperState::Decoding {
            return;
        }

        // Build transcription from tokens
        self.transcription = self
            .decoder_tokens
            .iter()
            .filter(|t| !t.text.starts_with("<|"))
            .map(|t| t.text.as_str())
            .collect::<Vec<_>>()
            .join("");

        // Compute total time and RTF
        self.metrics.total_time_ms =
            self.metrics.mel_time_ms + self.metrics.encoder_time_ms + self.metrics.decoder_time_ms;
        self.metrics.compute_rtf();

        self.transition(
            WhisperState::Complete,
            format!(
                "Complete: '{}' (RTF: {:.2}x)",
                self.transcription.trim(),
                self.metrics.rtf,
            ),
        );
    }

    /// Get state description
    pub fn state_description(&self) -> &'static str {
        match self.state {
            WhisperState::Idle => "Idle - Load audio to begin",
            WhisperState::WaveformReady => "Waveform ready - Compute mel spectrogram",
            WhisperState::MelReady => "Mel ready - Start encoding",
            WhisperState::Encoding => "Encoding audio features",
            WhisperState::Decoding => "Decoding to text",
            WhisperState::Complete => "Transcription complete",
            WhisperState::Error => "Error occurred",
        }
    }
}