1mod agreement;
4pub mod layer;
5mod native;
6mod session;
7
8pub use layer::{Layer, Layered, RemoveHesitations, Vocabulary};
9
10use std::path::PathBuf;
11
12#[cfg(not(target_family = "wasm"))]
13use crate::audio::RecordedAudio;
14use crate::audio::{CANONICAL_SAMPLE_RATE, CaptureError, RecordingReader};
15use crate::backend::{BackendError, ModelError};
16use crate::cancel::CancelToken;
17pub use agreement::Cadence;
18pub use session::Session;
19
20pub mod prelude {
25 #[cfg(all(feature = "silero", not(target_family = "wasm")))]
26 pub use super::session::HeardSession;
27 pub use super::{
28 Language, Layer, RemoveHesitations, Session, Streaming, Transcript, TranscriptSegment,
29 Transcription, TranscriptionError, Vocabulary, Whole,
30 };
31 pub use crate::speaker::{Diarizer, Embedder};
32 #[cfg(all(feature = "silero", not(target_family = "wasm")))]
33 pub use crate::speaker::{Heard, SegmentLabeler, SegmentReader, Verdict};
34 #[cfg(all(feature = "silero", not(target_family = "wasm")))]
35 pub use crate::voice_activity::speech_gate;
36}
37
38const MAX_LANGUAGE_CODE_BYTES: usize = 64;
39
40pub trait Transcriber: Send + Sync + 'static {
54 fn validate_language(
55 &self,
56 language: &Language,
57 cancel: &CancelToken,
58 ) -> Result<(), TranscriptionError>;
59
60 fn transcribe_with(
63 &self,
64 audio: TranscriptionAudio<'_>,
65 language: &Language,
66 cancel: &CancelToken,
67 ) -> Result<Hypothesis, TranscriptionError>;
68
69 fn open_stream(
75 &self,
76 language: &Language,
77 cancel: &CancelToken,
78 ) -> Result<Option<Box<dyn ChunkStream>>, TranscriptionError> {
79 let _ = (language, cancel);
80 Ok(None)
81 }
82}
83
84pub trait ChunkStream: Send {
87 fn chunk_samples(&self) -> usize;
89
90 fn feed(&mut self, chunk: &[f32]) -> Result<Emitted, TranscriptionError>;
93}
94
95pub struct Emitted {
100 pub words: Hypothesis,
101 pub joins_previous: bool,
102}
103
104#[derive(Clone, Copy)]
107pub enum Stream {
108 Agreement(Cadence),
109 Native,
110}
111
112pub trait Engine: Transcriber + Sized {
114 const NAME: &'static str;
115
116 const INPUT: AudioInputKind;
119
120 type Config;
121
122 fn check_from(config: &Self::Config) -> Result<PathBuf, TranscriptionError>;
125
126 fn load_from(config: Self::Config, cancel: &CancelToken) -> Result<Self, TranscriptionError>;
127}
128
129pub trait StreamingEngine: Engine {
131 const STREAM: Stream;
132}
133
134impl<T: Transcriber + ?Sized> Transcriber for std::sync::Arc<T> {
135 fn validate_language(
136 &self,
137 language: &Language,
138 cancel: &CancelToken,
139 ) -> Result<(), TranscriptionError> {
140 (**self).validate_language(language, cancel)
141 }
142
143 fn transcribe_with(
144 &self,
145 audio: TranscriptionAudio<'_>,
146 language: &Language,
147 cancel: &CancelToken,
148 ) -> Result<Hypothesis, TranscriptionError> {
149 (**self).transcribe_with(audio, language, cancel)
150 }
151
152 fn open_stream(
153 &self,
154 language: &Language,
155 cancel: &CancelToken,
156 ) -> Result<Option<Box<dyn ChunkStream>>, TranscriptionError> {
157 (**self).open_stream(language, cancel)
158 }
159}
160
161impl<T: Transcriber + ?Sized> Transcriber for Box<T> {
162 fn validate_language(
163 &self,
164 language: &Language,
165 cancel: &CancelToken,
166 ) -> Result<(), TranscriptionError> {
167 (**self).validate_language(language, cancel)
168 }
169
170 fn transcribe_with(
171 &self,
172 audio: TranscriptionAudio<'_>,
173 language: &Language,
174 cancel: &CancelToken,
175 ) -> Result<Hypothesis, TranscriptionError> {
176 (**self).transcribe_with(audio, language, cancel)
177 }
178
179 fn open_stream(
180 &self,
181 language: &Language,
182 cancel: &CancelToken,
183 ) -> Result<Option<Box<dyn ChunkStream>>, TranscriptionError> {
184 (**self).open_stream(language, cancel)
185 }
186}
187
188#[derive(Debug, Clone, Copy, PartialEq, Eq)]
189pub enum AudioInputKind {
190 Samples,
191 Wav,
192}
193
194impl std::fmt::Display for AudioInputKind {
195 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
196 match self {
197 Self::Samples => formatter.write_str("canonical samples"),
198 Self::Wav => formatter.write_str("a completed WAV"),
199 }
200 }
201}
202
203pub enum TranscriptionAudio<'a> {
205 Samples(&'a [f32]),
206 #[cfg(not(target_family = "wasm"))]
207 Wav(&'a RecordedAudio),
208}
209
210impl<'a> TranscriptionAudio<'a> {
211 pub fn kind(&self) -> AudioInputKind {
212 match self {
213 Self::Samples(_) => AudioInputKind::Samples,
214 #[cfg(not(target_family = "wasm"))]
215 Self::Wav(_) => AudioInputKind::Wav,
216 }
217 }
218
219 pub fn require_samples(self, backend: &'static str) -> Result<&'a [f32], TranscriptionError> {
220 #[cfg(target_family = "wasm")]
221 let _ = backend;
222 match self {
223 Self::Samples(samples) => Ok(samples),
224 #[cfg(not(target_family = "wasm"))]
225 Self::Wav(_) => Err(TranscriptionError::UnsupportedAudioInput {
226 backend,
227 received: AudioInputKind::Wav,
228 required: AudioInputKind::Samples,
229 }),
230 }
231 }
232
233 #[cfg(not(target_family = "wasm"))]
234 pub fn require_wav(
235 self,
236 backend: &'static str,
237 ) -> Result<&'a RecordedAudio, TranscriptionError> {
238 match self {
239 Self::Wav(audio) => Ok(audio),
240 Self::Samples(_) => Err(TranscriptionError::UnsupportedAudioInput {
241 backend,
242 received: AudioInputKind::Samples,
243 required: AudioInputKind::Wav,
244 }),
245 }
246 }
247}
248
249impl std::fmt::Debug for TranscriptionAudio<'_> {
250 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
251 formatter
252 .debug_struct("TranscriptionAudio")
253 .field("kind", &self.kind())
254 .finish_non_exhaustive()
255 }
256}
257
258#[derive(Debug, Clone, Copy, PartialEq)]
260pub struct TimedWord {
261 pub start: f32,
262 pub end: f32,
263}
264
265impl TimedWord {
266 pub fn span_from(&self, origin: u64) -> std::ops::Range<u64> {
268 let position = |seconds: f32| {
269 let samples = (f64::from(seconds) * f64::from(CANONICAL_SAMPLE_RATE)).round() as u64;
270 origin.saturating_add(samples)
271 };
272 let start = position(self.start);
273 start..position(self.end).max(start)
274 }
275}
276
277pub struct Hypothesis {
279 text: String,
280 words: Vec<TimedWord>,
281}
282
283impl Hypothesis {
284 pub fn new(text: impl Into<String>) -> Self {
285 Self {
286 text: text.into().trim().to_owned(),
287 words: Vec::new(),
288 }
289 }
290
291 pub fn with_words(text: impl Into<String>, words: Vec<TimedWord>) -> Self {
294 let mut hypothesis = Self::new(text);
295 if hypothesis.text.split_whitespace().count() == words.len() {
296 hypothesis.words = words;
297 }
298 hypothesis
299 }
300
301 pub fn as_str(&self) -> &str {
302 &self.text
303 }
304
305 pub fn words(&self) -> &[TimedWord] {
307 &self.words
308 }
309
310 pub fn into_text(self) -> String {
311 self.text
312 }
313}
314
315pub fn word_timings<'a>(tokens: impl IntoIterator<Item = (&'a str, TimedWord)>) -> Vec<TimedWord> {
322 let mut words: Vec<TimedWord> = Vec::new();
323 let mut open = false;
324 for (text, timing) in tokens {
325 for character in text.chars() {
326 if character.is_whitespace() || character == '\u{2581}' {
327 open = false;
328 continue;
329 }
330 match words.last_mut() {
331 Some(word) if open => word.end = word.end.max(timing.end),
332 _ => {
333 words.push(timing);
334 open = true;
335 }
336 }
337 }
338 }
339 words
340}
341
342impl std::fmt::Debug for Hypothesis {
343 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
344 formatter
345 .debug_struct("Hypothesis")
346 .field("bytes", &self.text.len())
347 .field("timed_words", &self.words.len())
348 .finish()
349 }
350}
351
352pub struct Transcription<T, M: Mode, L = ()> {
353 config: M::Config,
354 transcriber: T,
355 layers: L,
356}
357
358pub struct Operation<'a, T, L = ()> {
359 transcriber: &'a T,
360 language: &'a Language,
361 cancel: CancelToken,
362 strategy: Strategy,
363 transcript: Option<Transcript>,
364 layers: &'a L,
365 output: Option<layer::Output>,
366}
367
368use sealed::{Strategy, WholeRecordingInput};
369
370struct ChunkStep {
371 update: Option<TranscriptUpdate>,
372 transcript: Option<Transcript>,
373}
374
375mod sealed {
377 use super::{CancelToken, Language, Transcriber, TranscriptionError, agreement, native};
378
379 pub enum Strategy {
380 WholeRecording(WholeRecordingInput),
381 Agreement(Box<agreement::AgreementState>),
382 Native(Box<native::NativeState>),
383 }
384
385 #[derive(Clone, Copy)]
386 pub enum WholeRecordingInput {
387 Samples,
388 #[cfg(not(target_family = "wasm"))]
389 Wav,
390 }
391
392 pub trait Sealed {
394 type Config: Copy + Send + Sync + 'static;
395
396 fn strategy<T: Transcriber>(
399 config: Self::Config,
400 transcriber: &T,
401 language: &Language,
402 cancel: &CancelToken,
403 ) -> Result<Strategy, TranscriptionError>;
404 }
405}
406
407#[doc(hidden)]
410pub trait Mode: sealed::Sealed + 'static {
411 const INCREMENTAL: bool;
412}
413
414#[derive(Clone, Copy)]
416pub struct Whole;
417
418impl Mode for Whole {
419 const INCREMENTAL: bool = false;
420}
421
422impl sealed::Sealed for Whole {
423 type Config = WholeRecordingInput;
424
425 fn strategy<T: Transcriber>(
426 config: Self::Config,
427 _transcriber: &T,
428 _language: &Language,
429 _cancel: &CancelToken,
430 ) -> Result<Strategy, TranscriptionError> {
431 Ok(Strategy::WholeRecording(config))
432 }
433}
434
435#[derive(Clone, Copy)]
438pub struct Streaming;
439
440impl Mode for Streaming {
441 const INCREMENTAL: bool = true;
442}
443
444impl sealed::Sealed for Streaming {
445 type Config = Stream;
446
447 fn strategy<T: Transcriber>(
448 config: Self::Config,
449 transcriber: &T,
450 language: &Language,
451 cancel: &CancelToken,
452 ) -> Result<Strategy, TranscriptionError> {
453 match config {
454 Stream::Agreement(cadence) => Ok(Strategy::Agreement(Box::new(
455 agreement::AgreementState::new(cadence),
456 ))),
457 Stream::Native => match transcriber.open_stream(language, cancel)? {
458 Some(stream) => Ok(Strategy::Native(Box::new(native::NativeState::new(stream)))),
459 None => Err(TranscriptionError::NoNativeStream),
460 },
461 }
462 }
463}
464
465impl<T> Transcription<T, Whole> {
466 pub fn complete_samples(transcriber: T) -> Self {
467 Self {
468 config: WholeRecordingInput::Samples,
469 transcriber,
470 layers: (),
471 }
472 }
473
474 #[cfg(not(target_family = "wasm"))]
475 pub fn complete_wav(transcriber: T) -> Self {
476 Self {
477 config: WholeRecordingInput::Wav,
478 transcriber,
479 layers: (),
480 }
481 }
482
483 #[cfg(not(target_family = "wasm"))]
486 pub fn complete(input: AudioInputKind, transcriber: T) -> Self {
487 match input {
488 AudioInputKind::Samples => Self::complete_samples(transcriber),
489 AudioInputKind::Wav => Self::complete_wav(transcriber),
490 }
491 }
492}
493
494impl<T> Transcription<T, Streaming> {
495 pub fn agreement(cadence: Cadence, transcriber: T) -> Self {
499 Self::stream(Stream::Agreement(cadence), transcriber)
500 }
501
502 pub fn stream(stream: Stream, transcriber: T) -> Self {
505 Self {
506 config: stream,
507 transcriber,
508 layers: (),
509 }
510 }
511}
512
513impl<T: StreamingEngine> Transcription<T, Streaming> {
514 pub fn streaming(engine: T) -> Self {
515 Self::stream(T::STREAM, engine)
516 }
517}
518
519impl<T, M: Mode, L: Layer> Transcription<T, M, L> {
520 pub fn with<N: Layer>(self, layer: N) -> Transcription<T, M, Layered<N, L>> {
521 Transcription {
522 config: self.config,
523 transcriber: self.transcriber,
524 layers: Layered::new(self.layers, layer),
525 }
526 }
527}
528
529impl<T: Transcriber, M: Mode, L: Layer> Transcription<T, M, L> {
530 pub fn start<'a>(
532 &'a self,
533 language: &'a Language,
534 ) -> Result<Operation<'a, T, L>, TranscriptionError> {
535 self.start_with(language, &CancelToken::never())
536 }
537
538 pub fn start_with<'a>(
539 &'a self,
540 language: &'a Language,
541 cancel: &CancelToken,
542 ) -> Result<Operation<'a, T, L>, TranscriptionError> {
543 check_cancelled(cancel)?;
544 self.transcriber.validate_language(language, cancel)?;
545 check_cancelled(cancel)?;
546 let strategy = M::strategy(self.config, &self.transcriber, language, cancel)?;
547 Ok(Operation {
548 transcriber: &self.transcriber,
549 language,
550 cancel: cancel.clone(),
551 strategy,
552 transcript: None,
553 layers: &self.layers,
554 output: self.layers.enabled().then(layer::Output::default),
555 })
556 }
557
558 pub fn transcribe_with(
559 &self,
560 recording: &mut RecordingReader,
561 language: &Language,
562 cancel: &CancelToken,
563 ) -> Result<Transcript, TranscriptionError> {
564 self.start_with(language, cancel)?.finish(recording)
565 }
566
567 pub fn transcribe(
568 &self,
569 recording: &mut RecordingReader,
570 language: &Language,
571 ) -> Result<Transcript, TranscriptionError> {
572 self.transcribe_with(recording, language, &CancelToken::never())
573 }
574
575 #[cfg(not(target_family = "wasm"))]
576 pub fn transcribe_file(
577 &self,
578 path: impl AsRef<std::path::Path>,
579 ) -> Result<Transcript, TranscriptionError> {
580 let mut recording =
581 RecordingReader::from_wav_file(path).map_err(TranscriptionError::Capture)?;
582 self.transcribe(&mut recording, &Language::Automatic)
583 }
584}
585
586impl<T: Transcriber, L: Layer> Operation<'_, T, L> {
587 pub fn next_finalized(
588 &mut self,
589 recording: &mut RecordingReader,
590 ) -> Result<Option<TranscriptSegment>, TranscriptionError> {
591 loop {
592 match self.next(recording, false)? {
593 Some(TranscriptUpdate::Finalized(segment)) => return Ok(Some(segment)),
594 Some(TranscriptUpdate::Provisional(_)) => {}
595 None => return Ok(None),
596 }
597 }
598 }
599
600 pub fn completed(&self) -> Option<&Transcript> {
605 self.transcript.as_ref()
606 }
607
608 pub fn retained_from(&self) -> u64 {
614 let strategy = self.strategy_retained_from();
615 self.output
616 .as_ref()
617 .and_then(|output| output.retained_from)
618 .map_or(strategy, |retained| retained.min(strategy))
619 }
620
621 fn strategy_retained_from(&self) -> u64 {
622 match &self.strategy {
623 Strategy::WholeRecording(_) => 0,
624 Strategy::Agreement(state) => state.retained_from(),
625 Strategy::Native(state) => state.retained_from(),
626 }
627 }
628
629 pub fn next_update(
630 &mut self,
631 recording: &mut RecordingReader,
632 ) -> Result<Option<TranscriptUpdate>, TranscriptionError> {
633 self.next(recording, true)
634 }
635
636 pub fn next(
639 &mut self,
640 recording: &mut RecordingReader,
641 previews: bool,
642 ) -> Result<Option<TranscriptUpdate>, TranscriptionError> {
643 let Some(mut output) = self.output.take() else {
644 return self.next_raw_step(recording, previews);
645 };
646 let result = check_cancelled(&self.cancel)
647 .and_then(|()| self.next_processed(recording, previews, &mut output));
648 if result.is_err() {
649 output.finished = true;
650 output.pending.clear();
651 output.text.clear();
652 self.transcript = None;
653 }
654 self.output = Some(output);
655 result
656 }
657
658 fn next_processed(
659 &mut self,
660 recording: &mut RecordingReader,
661 previews: bool,
662 output: &mut layer::Output,
663 ) -> Result<Option<TranscriptUpdate>, TranscriptionError> {
664 loop {
665 if let Some(update) = output.pending.pop_front() {
666 return Ok(Some(update));
667 }
668 if output.finished {
669 return Ok(None);
670 }
671 let update = self.next_raw_step(recording, previews)?;
672 let finished = update.is_none() || self.transcript.is_some();
673 if let Some(update) = update {
674 output.push(update, self.layers, &self.cancel)?;
675 }
676 if finished {
677 output.retained_from = Some(self.strategy_retained_from());
679 while let Some(update) = self.next_raw_step(recording, previews)? {
680 output.push(update, self.layers, &self.cancel)?;
681 }
682 self.transcript = if output.text.is_empty() {
683 None
684 } else {
685 Some(Transcript(std::mem::take(&mut output.text)))
686 };
687 output.finished = true;
688 }
689 }
690 }
691
692 fn next_raw_step(
693 &mut self,
694 recording: &mut RecordingReader,
695 previews: bool,
696 ) -> Result<Option<TranscriptUpdate>, TranscriptionError> {
697 let step = match &mut self.strategy {
698 Strategy::WholeRecording(input) => {
699 if self.transcript.is_some() {
700 return Ok(None);
701 }
702 let completed = transcribe_complete(
703 *input,
704 recording,
705 self.transcriber,
706 self.language,
707 &self.cancel,
708 )?;
709 let segment = TranscriptSegment {
710 text: completed.as_str().to_owned(),
711 span: None,
712 };
713 self.transcript = Some(completed);
714 ChunkStep {
715 update: Some(TranscriptUpdate::Finalized(segment)),
716 transcript: None,
717 }
718 }
719 Strategy::Agreement(state) => {
720 let cancel = self.cancel.clone();
721 match state.next_with(recording, &cancel, previews, |samples| {
722 self.transcriber.transcribe_with(
723 TranscriptionAudio::Samples(samples),
724 self.language,
725 &cancel,
726 )
727 })? {
728 Some(step) => step,
729 None => return Ok(None),
730 }
731 }
732 Strategy::Native(state) => match state.next_finalized_with(recording, &self.cancel)? {
733 Some(step) => step,
734 None => return Ok(None),
735 },
736 };
737 if let Some(transcript) = step.transcript {
738 self.transcript = Some(transcript);
739 }
740 Ok(step.update)
741 }
742
743 pub fn finish(
744 &mut self,
745 recording: &mut RecordingReader,
746 ) -> Result<Transcript, TranscriptionError> {
747 while self.next_finalized(recording)?.is_some() {}
748 self.transcript.take().ok_or(TranscriptionError::NoSpeech)
749 }
750}
751
752fn transcribe_complete<T: Transcriber>(
753 input: WholeRecordingInput,
754 recording: &mut RecordingReader,
755 transcriber: &T,
756 language: &Language,
757 cancel: &CancelToken,
758) -> Result<Transcript, TranscriptionError> {
759 check_cancelled(cancel)?;
760 let hypothesis = match input {
761 WholeRecordingInput::Samples => {
762 recording.fill().map_err(TranscriptionError::Capture)?;
763 check_cancelled(cancel)?;
764 transcriber.transcribe_with(
765 TranscriptionAudio::Samples(recording.samples()),
766 language,
767 cancel,
768 )?
769 }
770 #[cfg(not(target_family = "wasm"))]
771 WholeRecordingInput::Wav => {
772 let audio = recording
773 .finished_audio()
774 .map_err(TranscriptionError::Capture)?;
775 check_cancelled(cancel)?;
776 transcriber.transcribe_with(TranscriptionAudio::Wav(audio), language, cancel)?
777 }
778 };
779 check_cancelled(cancel)?;
780 Transcript::new(hypothesis.into_text())
781}
782
783pub fn check_cancelled(cancel: &CancelToken) -> Result<(), TranscriptionError> {
785 if cancel.is_cancelled() {
786 return Err(TranscriptionError::Cancelled);
787 }
788 Ok(())
789}
790
791pub enum TranscriptUpdate {
792 Finalized(TranscriptSegment),
793 Provisional(String),
794}
795
796#[derive(Clone)]
797pub struct TranscriptSegment {
798 text: String,
799 span: Option<std::ops::Range<u64>>,
800}
801
802impl TranscriptSegment {
803 fn spanned(words: &[String], span: Option<std::ops::Range<u64>>) -> Option<Self> {
804 let text = words.join(" ");
805 (!text.is_empty()).then_some(Self { text, span })
806 }
807
808 pub fn as_str(&self) -> &str {
809 &self.text
810 }
811
812 pub fn span(&self) -> Option<std::ops::Range<u64>> {
815 self.span.clone()
816 }
817
818 pub fn into_text(self) -> String {
819 self.text
820 }
821}
822
823impl std::fmt::Debug for TranscriptSegment {
824 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
825 formatter
826 .debug_struct("TranscriptSegment")
827 .field("bytes", &self.text.len())
828 .field("span", &self.span)
829 .finish()
830 }
831}
832
833#[derive(Debug, Clone, PartialEq, Eq)]
834pub enum Language {
835 Automatic,
836 Specific(LanguageCode),
837}
838
839impl Language {
840 const AUTOMATIC: &str = "auto";
841
842 pub fn as_str(&self) -> &str {
843 match self {
844 Self::Automatic => Self::AUTOMATIC,
845 Self::Specific(code) => code.as_str(),
846 }
847 }
848}
849
850impl std::str::FromStr for Language {
852 type Err = LanguageCodeError;
853
854 fn from_str(value: &str) -> Result<Self, Self::Err> {
855 if value.trim().eq_ignore_ascii_case(Self::AUTOMATIC) {
856 return Ok(Self::Automatic);
857 }
858 LanguageCode::new(value).map(Self::Specific)
859 }
860}
861
862#[derive(Debug, Clone, PartialEq, Eq)]
863pub struct LanguageCode(String);
864
865impl LanguageCode {
866 pub fn english() -> Self {
867 Self("en".to_owned())
868 }
869
870 pub fn new(value: impl Into<String>) -> Result<Self, LanguageCodeError> {
871 let value = value.into();
872 let value = value.trim();
873 if value.is_empty() {
874 return Err(LanguageCodeError::Empty);
875 }
876 if value.len() > MAX_LANGUAGE_CODE_BYTES {
877 return Err(LanguageCodeError::TooLong {
878 actual_bytes: value.len(),
879 maximum_bytes: MAX_LANGUAGE_CODE_BYTES,
880 });
881 }
882 let mut segments = value.split('-');
883 let valid_primary = segments.next().is_some_and(|segment| {
884 !segment.is_empty() && segment.bytes().all(|byte| byte.is_ascii_alphabetic())
885 });
886 let valid_subtags = segments.all(|segment| {
887 !segment.is_empty() && segment.bytes().all(|byte| byte.is_ascii_alphanumeric())
888 });
889 if !valid_primary || !valid_subtags {
890 return Err(LanguageCodeError::Malformed(value.to_owned()));
891 }
892 Ok(Self(value.to_ascii_lowercase()))
893 }
894
895 pub fn as_str(&self) -> &str {
896 &self.0
897 }
898}
899
900#[derive(Debug, thiserror::Error)]
901pub enum LanguageCodeError {
902 #[error("recognition language code must not be empty")]
903 Empty,
904
905 #[error(
906 "recognition language code contains {actual_bytes} bytes; maximum is {maximum_bytes} bytes"
907 )]
908 TooLong {
909 actual_bytes: usize,
910 maximum_bytes: usize,
911 },
912
913 #[error("recognition language code is malformed: {0:?}")]
914 Malformed(String),
915}
916
917#[derive(Clone, PartialEq, Eq)]
918pub struct Transcript(String);
919
920impl Transcript {
921 pub fn new(text: impl Into<String>) -> Result<Self, TranscriptionError> {
922 let text = text.into();
923 let text = text.trim();
924 if text.is_empty() {
925 return Err(TranscriptionError::NoSpeech);
926 }
927 Ok(Self(text.to_owned()))
928 }
929
930 pub fn as_str(&self) -> &str {
931 &self.0
932 }
933
934 pub fn into_text(self) -> String {
935 self.0
936 }
937}
938
939impl std::fmt::Debug for Transcript {
940 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
941 formatter
942 .debug_struct("Transcript")
943 .field("bytes", &self.0.len())
944 .finish()
945 }
946}
947
948#[derive(Debug, thiserror::Error)]
949pub enum TranscriptionError {
950 #[error("the {backend} model cannot be used")]
951 Model {
952 backend: &'static str,
953 #[source]
954 source: ModelError,
955 },
956
957 #[error("failed to load the {backend} model {model}")]
958 Load {
959 backend: &'static str,
960 model: String,
961 #[source]
962 source: BackendError,
963 },
964
965 #[error("the recording being transcribed failed")]
966 Capture(#[source] CaptureError),
967
968 #[error("{backend} inference failed")]
969 Inference {
970 backend: &'static str,
971 #[source]
972 source: BackendError,
973 },
974
975 #[error("{backend} detects the spoken language itself and cannot honour {language:?}")]
976 UnsupportedLanguage {
977 backend: &'static str,
978 language: String,
979 },
980
981 #[error("{backend} needs a UTF-8 recording path; got {}", path.display())]
982 UnsupportedAudioPath {
983 backend: &'static str,
984 path: PathBuf,
985 },
986
987 #[error("{backend} requires {required}, but received {received}")]
988 UnsupportedAudioInput {
989 backend: &'static str,
990 received: AudioInputKind,
991 required: AudioInputKind,
992 },
993
994 #[error("the {backend} backend is not available on {}", std::env::consts::OS)]
995 UnsupportedBackend { backend: &'static str },
996
997 #[error("the {backend} backend has no gpu support on {}", std::env::consts::OS)]
998 UnsupportedDevice { backend: &'static str },
999
1000 #[error("the engine was asked to stream natively but offers no stream")]
1001 NoNativeStream,
1002
1003 #[error("the model loader stopped unexpectedly")]
1004 LoadPanicked,
1005
1006 #[error("transcription was cancelled")]
1007 Cancelled,
1008
1009 #[error("no speech was transcribed")]
1010 NoSpeech,
1011
1012 #[error("transcript processing failed")]
1013 Processing(#[source] layer::Error),
1014}
1015
1016#[cfg(not(target_family = "wasm"))]
1018pub fn validate_model_file(
1019 path: PathBuf,
1020 backend: &'static str,
1021) -> Result<PathBuf, TranscriptionError> {
1022 crate::backend::validate_model_file(path)
1023 .map_err(|source| TranscriptionError::Model { backend, source })
1024}
1025
1026#[cfg(not(target_family = "wasm"))]
1027pub fn validate_model_directory(
1028 path: PathBuf,
1029 backend: &'static str,
1030 required: &[&'static str],
1031) -> Result<PathBuf, TranscriptionError> {
1032 crate::backend::validate_model_directory(path, required)
1033 .map_err(|source| TranscriptionError::Model { backend, source })
1034}
1035
1036pub fn english_only(backend: &'static str, language: &Language) -> Result<(), TranscriptionError> {
1039 let Language::Specific(code) = language else {
1040 return Ok(());
1041 };
1042 if code.as_str().split('-').next() == Some(LanguageCode::english().as_str()) {
1043 return Ok(());
1044 }
1045 Err(TranscriptionError::UnsupportedLanguage {
1046 backend,
1047 language: code.as_str().to_owned(),
1048 })
1049}
1050
1051#[cfg(test)]
1052mod word_timing_tests {
1053 use super::*;
1054
1055 fn token(text: &str, start: f32, end: f32) -> (&str, TimedWord) {
1056 (text, TimedWord { start, end })
1057 }
1058
1059 #[test]
1060 fn tokens_group_into_words_at_the_sentencepiece_marker() {
1061 let tokens = [
1062 token("\u{2581}hel", 0.0, 0.2),
1063 token("lo", 0.2, 0.4),
1064 token(",", 0.4, 0.5),
1065 token("\u{2581}world", 0.6, 0.9),
1066 ];
1067 let words = word_timings(tokens);
1068 assert_eq!(words.len(), 2);
1069 assert_eq!((words[0].start, words[0].end), (0.0, 0.5));
1070 assert_eq!((words[1].start, words[1].end), (0.6, 0.9));
1071 }
1072
1073 #[test]
1074 fn tokens_group_into_words_at_a_leading_space() {
1075 let tokens = [
1076 token(" hel", 0.0, 0.2),
1077 token("lo", 0.2, 0.4),
1078 token(",", 0.4, 0.5),
1079 token(" world", 0.6, 0.9),
1080 ];
1081 let words = word_timings(tokens);
1082 assert_eq!(words.len(), 2);
1083 assert_eq!((words[0].start, words[0].end), (0.0, 0.5));
1084 assert_eq!((words[1].start, words[1].end), (0.6, 0.9));
1085 }
1086
1087 #[test]
1088 fn word_timings_count_what_the_text_splits_into() {
1089 let tokens = [
1091 token("\u{2581}", 0.0, 0.1),
1092 token("\u{2581}1", 0.1, 0.2),
1093 token("8", 0.2, 0.3),
1094 token("9", 0.3, 0.4),
1095 token("\u{2581}years old", 0.4, 0.8),
1096 token(".", 0.8, 0.9),
1097 ];
1098 let text: String = tokens
1099 .iter()
1100 .map(|(text, _)| text.replace('\u{2581}', " "))
1101 .collect();
1102 let words = word_timings(tokens);
1103 assert_eq!(words.len(), text.split_whitespace().count());
1104 assert_eq!((words[0].start, words[0].end), (0.1, 0.4));
1105 assert_eq!((words[2].start, words[2].end), (0.4, 0.9));
1106 }
1107
1108 #[test]
1109 fn a_word_timing_rounds_to_the_nearest_sample() {
1110 let word = TimedWord {
1112 start: 0.100_03,
1113 end: 0.200_04,
1114 };
1115
1116 assert_eq!(word.span_from(1_000), 2_600..4_201);
1117 }
1118}
1119
1120#[cfg(test)]
1121mod language_tests {
1122 use super::*;
1123
1124 const BACKEND: &str = "test-backend";
1125
1126 #[test]
1127 fn detection_and_english_are_accepted() {
1128 assert!(english_only(BACKEND, &Language::Automatic).is_ok());
1129 assert!(english_only(BACKEND, &Language::Specific(LanguageCode::english())).is_ok());
1130
1131 let regional = LanguageCode::new("en-GB").expect("en-GB is a valid code");
1132 assert!(english_only(BACKEND, &Language::Specific(regional)).is_ok());
1133 }
1134
1135 #[test]
1136 fn language_parses_what_as_str_writes() {
1137 for language in [
1138 Language::Automatic,
1139 Language::Specific(LanguageCode::new("en-GB").expect("en-GB is a valid code")),
1140 ] {
1141 assert_eq!(language.as_str().parse::<Language>().ok(), Some(language));
1142 }
1143 assert_eq!(" AUTO".parse::<Language>().ok(), Some(Language::Automatic));
1144 assert!(matches!(
1145 "en_GB".parse::<Language>(),
1146 Err(LanguageCodeError::Malformed(_))
1147 ));
1148 }
1149
1150 #[test]
1151 fn another_language_is_refused_by_name() {
1152 let code = LanguageCode::new("de").expect("de is a valid code");
1153
1154 let error = english_only(BACKEND, &Language::Specific(code))
1155 .expect_err("another language must be refused");
1156
1157 assert!(matches!(
1158 error,
1159 TranscriptionError::UnsupportedLanguage { language, .. } if language == "de"
1160 ));
1161 }
1162}
1163
1164#[cfg(test)]
1165mod coordinator_tests {
1166 use std::sync::{Arc, Mutex};
1167
1168 use crate::audio::testing::recorded;
1169
1170 use super::*;
1171
1172 struct FakeTranscriber {
1173 response: &'static str,
1174 reject_language: bool,
1175 inputs: Mutex<Vec<AudioInputKind>>,
1176 }
1177
1178 impl FakeTranscriber {
1179 fn answering(response: &'static str) -> Self {
1180 Self {
1181 response,
1182 reject_language: false,
1183 inputs: Mutex::new(Vec::new()),
1184 }
1185 }
1186
1187 fn inputs(&self) -> Vec<AudioInputKind> {
1188 self.inputs
1189 .lock()
1190 .unwrap_or_else(|poisoned| poisoned.into_inner())
1191 .clone()
1192 }
1193 }
1194
1195 impl Transcriber for FakeTranscriber {
1196 fn validate_language(
1197 &self,
1198 language: &Language,
1199 _cancel: &CancelToken,
1200 ) -> Result<(), TranscriptionError> {
1201 if self.reject_language {
1202 return Err(TranscriptionError::UnsupportedLanguage {
1203 backend: "fake",
1204 language: language.as_str().to_owned(),
1205 });
1206 }
1207 Ok(())
1208 }
1209
1210 fn transcribe_with(
1211 &self,
1212 audio: TranscriptionAudio<'_>,
1213 _language: &Language,
1214 _cancel: &CancelToken,
1215 ) -> Result<Hypothesis, TranscriptionError> {
1216 self.inputs
1217 .lock()
1218 .unwrap_or_else(|poisoned| poisoned.into_inner())
1219 .push(audio.kind());
1220 Ok(Hypothesis::new(self.response))
1221 }
1222 }
1223
1224 struct Chunked;
1226
1227 struct HelloStream;
1228
1229 impl ChunkStream for HelloStream {
1230 fn chunk_samples(&self) -> usize {
1231 2
1232 }
1233
1234 fn feed(&mut self, chunk: &[f32]) -> Result<Emitted, TranscriptionError> {
1235 Ok(Emitted {
1236 words: Hypothesis::new(match chunk.iter().any(|sample| *sample != 0.0) {
1237 true => "hello",
1238 false => "",
1239 }),
1240 joins_previous: false,
1241 })
1242 }
1243 }
1244
1245 impl Transcriber for Chunked {
1246 fn validate_language(
1247 &self,
1248 _language: &Language,
1249 _cancel: &CancelToken,
1250 ) -> Result<(), TranscriptionError> {
1251 Ok(())
1252 }
1253
1254 fn transcribe_with(
1255 &self,
1256 _audio: TranscriptionAudio<'_>,
1257 _language: &Language,
1258 _cancel: &CancelToken,
1259 ) -> Result<Hypothesis, TranscriptionError> {
1260 unreachable!("a native stream never decodes a bounded input")
1261 }
1262
1263 fn open_stream(
1264 &self,
1265 _language: &Language,
1266 _cancel: &CancelToken,
1267 ) -> Result<Option<Box<dyn ChunkStream>>, TranscriptionError> {
1268 Ok(Some(Box::new(HelloStream)))
1269 }
1270 }
1271
1272 #[test]
1273 fn a_boxed_or_shared_transcriber_still_opens_its_stream() {
1274 let (_directory, mut boxed_recording) = recorded(&[1, 2, 3, 4]);
1275 let (_directory, mut shared_recording) = recorded(&[1, 2, 3, 4]);
1276 let boxed: Box<dyn Transcriber> = Box::new(Chunked);
1277 let shared = Arc::new(Chunked);
1278
1279 let from_boxed = Transcription::stream(Stream::Native, boxed)
1280 .transcribe(&mut boxed_recording, &Language::Automatic)
1281 .expect("the boxed stream opens");
1282 let from_shared = Transcription::stream(Stream::Native, shared)
1283 .transcribe(&mut shared_recording, &Language::Automatic)
1284 .expect("the shared stream opens");
1285
1286 assert_eq!(from_boxed.as_str(), "hello hello");
1287 assert_eq!(from_shared.as_str(), "hello hello");
1288 }
1289
1290 #[test]
1291 fn completion_is_available_before_pending_final_segments_and_preview_clear() {
1292 struct Ending(std::sync::atomic::AtomicBool);
1293 impl Transcriber for Ending {
1294 fn validate_language(
1295 &self,
1296 _: &Language,
1297 _: &CancelToken,
1298 ) -> Result<(), TranscriptionError> {
1299 Ok(())
1300 }
1301 fn transcribe_with(
1302 &self,
1303 _: TranscriptionAudio<'_>,
1304 _: &Language,
1305 _: &CancelToken,
1306 ) -> Result<Hypothesis, TranscriptionError> {
1307 Ok(Hypothesis::new(
1308 if self.0.swap(true, std::sync::atomic::Ordering::Relaxed) {
1309 "one two three"
1310 } else {
1311 "one two"
1312 },
1313 ))
1314 }
1315 }
1316 let (mut recording, mut capture) = crate::audio::Recording::open();
1317 let transcription = Transcription::agreement(
1318 Cadence::from_millis(100, 20_000),
1319 Ending(std::sync::atomic::AtomicBool::new(false)),
1320 );
1321 let mut operation = transcription.start(&Language::Automatic).expect("starts");
1322 capture.push(&[0.5; 1600]);
1323 assert!(
1324 matches!(operation.next_update(&mut recording).expect("preview"), Some(TranscriptUpdate::Provisional(text)) if text == "one two")
1325 );
1326 capture.push(&[0.5; 1600]);
1327 capture.finish();
1328 recording.fill().expect("capture ended");
1329 assert!(
1330 matches!(operation.next_update(&mut recording).expect("final"), Some(TranscriptUpdate::Finalized(segment)) if segment.as_str() == "one two")
1331 );
1332 assert_eq!(
1333 operation.completed().expect("already complete").as_str(),
1334 "one two three"
1335 );
1336 assert!(
1337 matches!(operation.next_update(&mut recording).expect("tail"), Some(TranscriptUpdate::Finalized(segment)) if segment.as_str() == "three")
1338 );
1339 assert!(
1340 matches!(operation.next_update(&mut recording).expect("clear"), Some(TranscriptUpdate::Provisional(text)) if text.is_empty())
1341 );
1342 assert!(
1343 operation
1344 .next_update(&mut recording)
1345 .expect("ended")
1346 .is_none()
1347 );
1348 }
1349
1350 #[test]
1351 fn native_and_whole_updates_are_finalized_only() {
1352 fn check<T: Transcriber, M: Mode>(transcription: Transcription<T, M>, expected: &str) {
1353 let (_directory, mut recording) = recorded(&[1, 2, 3, 4]);
1354 let mut operation = transcription.start(&Language::Automatic).expect("starts");
1355 let mut segments = Vec::new();
1356 while let Some(update) = operation.next_update(&mut recording).expect("reads") {
1357 let TranscriptUpdate::Finalized(segment) = update else {
1358 panic!("only agreement exposes provisional words");
1359 };
1360 segments.push(segment.into_text());
1361 }
1362 assert_eq!(segments.join(" "), expected);
1363 assert_eq!(operation.completed().expect("completed").as_str(), expected);
1364 assert_eq!(
1365 operation.finish(&mut recording).expect("finished").as_str(),
1366 expected
1367 );
1368 }
1369 check(
1370 Transcription::stream(Stream::Native, Chunked),
1371 "hello hello",
1372 );
1373 check(
1374 Transcription::complete_samples(FakeTranscriber::answering("hello there")),
1375 "hello there",
1376 );
1377 }
1378
1379 #[test]
1380 fn streaming_natively_without_a_stream_is_an_error() {
1381 let transcription =
1382 Transcription::stream(Stream::Native, FakeTranscriber::answering("hello"));
1383
1384 let outcome = transcription.start(&Language::Automatic).map(|_| ());
1385
1386 assert!(matches!(outcome, Err(TranscriptionError::NoNativeStream)));
1387 }
1388
1389 #[test]
1390 fn complete_samples_invokes_the_engine_once() {
1391 let (_directory, mut recording) = recorded(&[1, 2, 3]);
1392 let transcriber = Arc::new(FakeTranscriber::answering("hello there"));
1393 let transcription = Transcription::complete_samples(Arc::clone(&transcriber));
1394
1395 let transcript = transcription
1396 .transcribe(&mut recording, &Language::Automatic)
1397 .expect("the fake returns speech");
1398
1399 assert_eq!(transcript.as_str(), "hello there");
1400 assert_eq!(transcriber.inputs(), [AudioInputKind::Samples]);
1401 }
1402
1403 #[cfg(not(target_family = "wasm"))]
1404 #[test]
1405 fn complete_wav_preserves_the_file_input_boundary() {
1406 let (_directory, mut recording) = recorded(&[1, 2, 3]);
1407 let transcriber = Arc::new(FakeTranscriber::answering("hello there"));
1408 let transcription = Transcription::complete_wav(Arc::clone(&transcriber));
1409
1410 transcription
1411 .transcribe(&mut recording, &Language::Automatic)
1412 .expect("the fake returns speech");
1413
1414 assert_eq!(transcriber.inputs(), [AudioInputKind::Wav]);
1415 assert!(recording.samples().is_empty());
1416 }
1417
1418 #[test]
1419 fn an_empty_hypothesis_becomes_terminal_no_speech() {
1420 let (_directory, mut recording) = recorded(&[1, 2, 3]);
1421 let transcription = Transcription::complete_samples(FakeTranscriber::answering(" "));
1422
1423 let outcome = transcription.transcribe(&mut recording, &Language::Automatic);
1424
1425 assert!(matches!(outcome, Err(TranscriptionError::NoSpeech)));
1426 }
1427
1428 #[test]
1429 fn cancellation_prevents_complete_inference() {
1430 let (_directory, mut recording) = recorded(&[1, 2, 3]);
1431 let transcriber = Arc::new(FakeTranscriber::answering("unreachable"));
1432 let transcription = Transcription::complete_samples(Arc::clone(&transcriber));
1433 let cancel = CancelToken::new();
1434 cancel.cancel();
1435
1436 let outcome = transcription.transcribe_with(&mut recording, &Language::Automatic, &cancel);
1437
1438 assert!(matches!(outcome, Err(TranscriptionError::Cancelled)));
1439 assert!(transcriber.inputs().is_empty());
1440 }
1441
1442 #[test]
1443 fn language_rejection_prevents_complete_inference() {
1444 let (_directory, mut recording) = recorded(&[1, 2, 3]);
1445 let mut fake = FakeTranscriber::answering("unreachable");
1446 fake.reject_language = true;
1447 let transcriber = Arc::new(fake);
1448 let transcription = Transcription::complete_samples(Arc::clone(&transcriber));
1449
1450 let outcome = transcription.transcribe(&mut recording, &Language::Automatic);
1451
1452 assert!(matches!(
1453 outcome,
1454 Err(TranscriptionError::UnsupportedLanguage { .. })
1455 ));
1456 assert!(transcriber.inputs().is_empty());
1457 }
1458
1459 #[test]
1460 fn sensitive_transcription_types_redact_their_text() {
1461 let secret = "words that must not reach diagnostics";
1462 let hypothesis = Hypothesis::new(secret);
1463 let transcript = Transcript::new(secret).expect("the text is not empty");
1464 let segment =
1465 TranscriptSegment::spanned(&[secret.to_owned()], None).expect("the text is not empty");
1466
1467 assert!(!format!("{hypothesis:?}").contains(secret));
1468 assert!(!format!("{transcript:?}").contains(secret));
1469 assert!(!format!("{segment:?}").contains(secret));
1470 }
1471}