1mod agreement;
4pub mod layer;
5mod native;
6mod session;
7mod spacing;
8
9pub use layer::{Layer, Layered, RemoveHesitations, Vocabulary};
10pub use spacing::Caret;
11
12use std::path::PathBuf;
13
14#[cfg(not(target_family = "wasm"))]
15use crate::audio::RecordedAudio;
16use crate::audio::{CANONICAL_SAMPLE_RATE, CaptureError, RecordingReader};
17use crate::backend::{BackendError, ModelError};
18use crate::cancel::CancelToken;
19pub use agreement::Cadence;
20pub use session::Session;
21
22pub mod prelude {
27 #[cfg(all(feature = "silero", not(target_family = "wasm")))]
28 pub use super::session::HeardSession;
29 pub use super::{
30 Caret, Language, Layer, RemoveHesitations, Session, Streaming, Transcript,
31 TranscriptSegment, Transcription, TranscriptionError, Vocabulary, Whole,
32 };
33 pub use crate::audio::RecordingReader;
34 #[cfg(all(feature = "microphone", not(target_family = "wasm")))]
35 pub use crate::audio::{Microphone, RecordingMode};
36 pub use crate::cancel::CancelToken;
37 pub use crate::speaker::{Diarizer, Embedder};
38 #[cfg(all(feature = "silero", not(target_family = "wasm")))]
39 pub use crate::speaker::{Heard, SegmentLabeler, SegmentReader, Verdict};
40}
41
42const MAX_LANGUAGE_CODE_BYTES: usize = 64;
43
44pub trait Transcriber: Send + Sync + 'static {
58 fn validate_language(
59 &self,
60 language: &Language,
61 cancel: &CancelToken,
62 ) -> Result<(), TranscriptionError>;
63
64 fn transcribe_with(
67 &self,
68 audio: TranscriptionAudio<'_>,
69 language: &Language,
70 cancel: &CancelToken,
71 ) -> Result<Hypothesis, TranscriptionError>;
72
73 fn open_stream(
79 &self,
80 language: &Language,
81 cancel: &CancelToken,
82 ) -> Result<Option<Box<dyn ChunkStream>>, TranscriptionError> {
83 let _ = (language, cancel);
84 Ok(None)
85 }
86}
87
88pub trait ChunkStream: Send {
91 fn chunk_samples(&self) -> usize;
93
94 fn feed(&mut self, chunk: &[f32]) -> Result<Emitted, TranscriptionError>;
97}
98
99pub struct Emitted {
104 pub words: Hypothesis,
105 pub joins_previous: bool,
106}
107
108#[derive(Clone, Copy)]
111pub enum Stream {
112 Agreement(Cadence),
113 Native,
114}
115
116pub trait Engine: Transcriber + Sized {
118 const NAME: &'static str;
119
120 const INPUT: AudioInputKind;
123
124 type Config;
125
126 fn check_from(config: &Self::Config) -> Result<PathBuf, TranscriptionError>;
129
130 fn load_from(config: Self::Config, cancel: &CancelToken) -> Result<Self, TranscriptionError>;
131}
132
133pub trait StreamingEngine: Engine {
135 const STREAM: Stream;
136}
137
138impl<T: Transcriber + ?Sized> Transcriber for std::sync::Arc<T> {
139 fn validate_language(
140 &self,
141 language: &Language,
142 cancel: &CancelToken,
143 ) -> Result<(), TranscriptionError> {
144 (**self).validate_language(language, cancel)
145 }
146
147 fn transcribe_with(
148 &self,
149 audio: TranscriptionAudio<'_>,
150 language: &Language,
151 cancel: &CancelToken,
152 ) -> Result<Hypothesis, TranscriptionError> {
153 (**self).transcribe_with(audio, language, cancel)
154 }
155
156 fn open_stream(
157 &self,
158 language: &Language,
159 cancel: &CancelToken,
160 ) -> Result<Option<Box<dyn ChunkStream>>, TranscriptionError> {
161 (**self).open_stream(language, cancel)
162 }
163}
164
165impl<T: Transcriber + ?Sized> Transcriber for Box<T> {
166 fn validate_language(
167 &self,
168 language: &Language,
169 cancel: &CancelToken,
170 ) -> Result<(), TranscriptionError> {
171 (**self).validate_language(language, cancel)
172 }
173
174 fn transcribe_with(
175 &self,
176 audio: TranscriptionAudio<'_>,
177 language: &Language,
178 cancel: &CancelToken,
179 ) -> Result<Hypothesis, TranscriptionError> {
180 (**self).transcribe_with(audio, language, cancel)
181 }
182
183 fn open_stream(
184 &self,
185 language: &Language,
186 cancel: &CancelToken,
187 ) -> Result<Option<Box<dyn ChunkStream>>, TranscriptionError> {
188 (**self).open_stream(language, cancel)
189 }
190}
191
192#[derive(Debug, Clone, Copy, PartialEq, Eq)]
193pub enum AudioInputKind {
194 Samples,
195 Wav,
196}
197
198impl std::fmt::Display for AudioInputKind {
199 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
200 match self {
201 Self::Samples => formatter.write_str("canonical samples"),
202 Self::Wav => formatter.write_str("a completed WAV"),
203 }
204 }
205}
206
207pub enum TranscriptionAudio<'a> {
209 Samples(&'a [f32]),
210 #[cfg(not(target_family = "wasm"))]
211 Wav(&'a RecordedAudio),
212}
213
214impl<'a> TranscriptionAudio<'a> {
215 pub fn kind(&self) -> AudioInputKind {
216 match self {
217 Self::Samples(_) => AudioInputKind::Samples,
218 #[cfg(not(target_family = "wasm"))]
219 Self::Wav(_) => AudioInputKind::Wav,
220 }
221 }
222
223 pub fn require_samples(self, backend: &'static str) -> Result<&'a [f32], TranscriptionError> {
224 #[cfg(target_family = "wasm")]
225 let _ = backend;
226 match self {
227 Self::Samples(samples) => Ok(samples),
228 #[cfg(not(target_family = "wasm"))]
229 Self::Wav(_) => Err(TranscriptionError::UnsupportedAudioInput {
230 backend,
231 received: AudioInputKind::Wav,
232 required: AudioInputKind::Samples,
233 }),
234 }
235 }
236
237 #[cfg(not(target_family = "wasm"))]
238 pub fn require_wav(
239 self,
240 backend: &'static str,
241 ) -> Result<&'a RecordedAudio, TranscriptionError> {
242 match self {
243 Self::Wav(audio) => Ok(audio),
244 Self::Samples(_) => Err(TranscriptionError::UnsupportedAudioInput {
245 backend,
246 received: AudioInputKind::Samples,
247 required: AudioInputKind::Wav,
248 }),
249 }
250 }
251}
252
253impl std::fmt::Debug for TranscriptionAudio<'_> {
254 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
255 formatter
256 .debug_struct("TranscriptionAudio")
257 .field("kind", &self.kind())
258 .finish_non_exhaustive()
259 }
260}
261
262#[derive(Debug, Clone, Copy, PartialEq)]
264pub struct TimedWord {
265 pub start: f32,
266 pub end: f32,
267}
268
269impl TimedWord {
270 pub fn span_from(&self, origin: u64) -> std::ops::Range<u64> {
272 let position = |seconds: f32| {
273 let samples = (f64::from(seconds) * f64::from(CANONICAL_SAMPLE_RATE)).round() as u64;
274 origin.saturating_add(samples)
275 };
276 let start = position(self.start);
277 start..position(self.end).max(start)
278 }
279}
280
281pub struct Hypothesis {
283 text: String,
284 words: Vec<TimedWord>,
285}
286
287impl Hypothesis {
288 pub fn new(text: impl Into<String>) -> Self {
289 Self {
290 text: text.into().trim().to_owned(),
291 words: Vec::new(),
292 }
293 }
294
295 pub fn with_words(text: impl Into<String>, words: Vec<TimedWord>) -> Self {
298 let mut hypothesis = Self::new(text);
299 if hypothesis.text.split_whitespace().count() == words.len() {
300 hypothesis.words = words;
301 }
302 hypothesis
303 }
304
305 pub fn as_str(&self) -> &str {
306 &self.text
307 }
308
309 pub fn words(&self) -> &[TimedWord] {
311 &self.words
312 }
313
314 pub fn into_text(self) -> String {
315 self.text
316 }
317}
318
319pub fn word_timings<'a>(tokens: impl IntoIterator<Item = (&'a str, TimedWord)>) -> Vec<TimedWord> {
326 let mut words: Vec<TimedWord> = Vec::new();
327 let mut open = false;
328 for (text, timing) in tokens {
329 for character in text.chars() {
330 if character.is_whitespace() || character == '\u{2581}' {
331 open = false;
332 continue;
333 }
334 match words.last_mut() {
335 Some(word) if open => word.end = word.end.max(timing.end),
336 _ => {
337 words.push(timing);
338 open = true;
339 }
340 }
341 }
342 }
343 words
344}
345
346impl std::fmt::Debug for Hypothesis {
347 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
348 formatter
349 .debug_struct("Hypothesis")
350 .field("bytes", &self.text.len())
351 .field("timed_words", &self.words.len())
352 .finish()
353 }
354}
355
356pub struct Transcription<T, M: Mode, L = ()> {
357 config: M::Config,
358 transcriber: T,
359 layers: L,
360}
361
362pub struct Operation<'a, T, L = ()> {
363 transcriber: &'a T,
364 language: &'a Language,
365 cancel: CancelToken,
366 strategy: Strategy,
367 transcript: Option<Transcript>,
368 layers: &'a L,
369 output: Option<layer::Output>,
370}
371
372use sealed::{Strategy, WholeRecordingInput};
373
374struct ChunkStep {
375 update: Option<TranscriptUpdate>,
376 transcript: Option<Transcript>,
377}
378
379mod sealed {
381 use super::{CancelToken, Language, Transcriber, TranscriptionError, agreement, native};
382
383 pub enum Strategy {
384 WholeRecording(WholeRecordingInput),
385 Agreement(Box<agreement::AgreementState>),
386 Native(Box<native::NativeState>),
387 }
388
389 #[derive(Clone, Copy)]
390 pub enum WholeRecordingInput {
391 Samples,
392 #[cfg(not(target_family = "wasm"))]
393 Wav,
394 }
395
396 pub trait Sealed {
398 type Config: Copy + Send + Sync + 'static;
399
400 fn strategy<T: Transcriber>(
403 config: Self::Config,
404 transcriber: &T,
405 language: &Language,
406 cancel: &CancelToken,
407 ) -> Result<Strategy, TranscriptionError>;
408 }
409}
410
411#[doc(hidden)]
414pub trait Mode: sealed::Sealed + 'static {
415 const INCREMENTAL: bool;
416}
417
418#[derive(Clone, Copy)]
420pub struct Whole;
421
422impl Mode for Whole {
423 const INCREMENTAL: bool = false;
424}
425
426impl sealed::Sealed for Whole {
427 type Config = WholeRecordingInput;
428
429 fn strategy<T: Transcriber>(
430 config: Self::Config,
431 _transcriber: &T,
432 _language: &Language,
433 _cancel: &CancelToken,
434 ) -> Result<Strategy, TranscriptionError> {
435 Ok(Strategy::WholeRecording(config))
436 }
437}
438
439#[derive(Clone, Copy)]
442pub struct Streaming;
443
444impl Mode for Streaming {
445 const INCREMENTAL: bool = true;
446}
447
448impl sealed::Sealed for Streaming {
449 type Config = Stream;
450
451 fn strategy<T: Transcriber>(
452 config: Self::Config,
453 transcriber: &T,
454 language: &Language,
455 cancel: &CancelToken,
456 ) -> Result<Strategy, TranscriptionError> {
457 match config {
458 Stream::Agreement(cadence) => Ok(Strategy::Agreement(Box::new(
459 agreement::AgreementState::new(cadence),
460 ))),
461 Stream::Native => match transcriber.open_stream(language, cancel)? {
462 Some(stream) => Ok(Strategy::Native(Box::new(native::NativeState::new(stream)))),
463 None => Err(TranscriptionError::NoNativeStream),
464 },
465 }
466 }
467}
468
469impl<T> Transcription<T, Whole> {
470 pub fn complete_samples(transcriber: T) -> Self {
471 Self {
472 config: WholeRecordingInput::Samples,
473 transcriber,
474 layers: (),
475 }
476 }
477
478 #[cfg(not(target_family = "wasm"))]
479 pub fn complete_wav(transcriber: T) -> Self {
480 Self {
481 config: WholeRecordingInput::Wav,
482 transcriber,
483 layers: (),
484 }
485 }
486
487 #[cfg(not(target_family = "wasm"))]
490 pub fn complete(input: AudioInputKind, transcriber: T) -> Self {
491 match input {
492 AudioInputKind::Samples => Self::complete_samples(transcriber),
493 AudioInputKind::Wav => Self::complete_wav(transcriber),
494 }
495 }
496}
497
498impl<T> Transcription<T, Streaming> {
499 pub fn agreement(cadence: Cadence, transcriber: T) -> Self {
503 Self::stream(Stream::Agreement(cadence), transcriber)
504 }
505
506 pub fn stream(stream: Stream, transcriber: T) -> Self {
509 Self {
510 config: stream,
511 transcriber,
512 layers: (),
513 }
514 }
515}
516
517impl<T: StreamingEngine> Transcription<T, Streaming> {
518 pub fn streaming(engine: T) -> Self {
519 Self::stream(T::STREAM, engine)
520 }
521}
522
523impl<T, M: Mode, L: Layer> Transcription<T, M, L> {
524 pub fn with<N: Layer>(self, layer: N) -> Transcription<T, M, Layered<N, L>> {
525 Transcription {
526 config: self.config,
527 transcriber: self.transcriber,
528 layers: Layered::new(self.layers, layer),
529 }
530 }
531}
532
533impl<T: Transcriber, M: Mode, L: Layer> Transcription<T, M, L> {
534 pub fn start<'a>(
536 &'a self,
537 language: &'a Language,
538 ) -> Result<Operation<'a, T, L>, TranscriptionError> {
539 self.start_with(language, &CancelToken::never())
540 }
541
542 pub fn start_with<'a>(
543 &'a self,
544 language: &'a Language,
545 cancel: &CancelToken,
546 ) -> Result<Operation<'a, T, L>, TranscriptionError> {
547 check_cancelled(cancel)?;
548 self.transcriber.validate_language(language, cancel)?;
549 check_cancelled(cancel)?;
550 let strategy = M::strategy(self.config, &self.transcriber, language, cancel)?;
551 Ok(Operation {
552 transcriber: &self.transcriber,
553 language,
554 cancel: cancel.clone(),
555 strategy,
556 transcript: None,
557 layers: &self.layers,
558 output: self.layers.enabled().then(layer::Output::default),
559 })
560 }
561
562 pub fn transcribe_with(
563 &self,
564 recording: &mut RecordingReader,
565 language: &Language,
566 cancel: &CancelToken,
567 ) -> Result<Transcript, TranscriptionError> {
568 self.start_with(language, cancel)?.finish(recording)
569 }
570
571 pub fn transcribe(
572 &self,
573 recording: &mut RecordingReader,
574 language: &Language,
575 ) -> Result<Transcript, TranscriptionError> {
576 self.transcribe_with(recording, language, &CancelToken::never())
577 }
578
579 #[cfg(not(target_family = "wasm"))]
580 pub fn transcribe_file(
581 &self,
582 path: impl AsRef<std::path::Path>,
583 ) -> Result<Transcript, TranscriptionError> {
584 let mut recording =
585 RecordingReader::from_wav_file(path).map_err(TranscriptionError::Capture)?;
586 self.transcribe(&mut recording, &Language::Automatic)
587 }
588}
589
590impl<T: Transcriber, L: Layer> Operation<'_, T, L> {
591 pub fn next_finalized(
592 &mut self,
593 recording: &mut RecordingReader,
594 ) -> Result<Option<TranscriptSegment>, TranscriptionError> {
595 loop {
596 match self.next(recording, false)? {
597 Some(TranscriptUpdate::Finalized(segment)) => return Ok(Some(segment)),
598 Some(TranscriptUpdate::Provisional(_)) => {}
599 None => return Ok(None),
600 }
601 }
602 }
603
604 pub fn completed(&self) -> Option<&Transcript> {
609 self.transcript.as_ref()
610 }
611
612 pub fn retained_from(&self) -> u64 {
618 let strategy = self.strategy_retained_from();
619 self.output
620 .as_ref()
621 .and_then(|output| output.retained_from)
622 .map_or(strategy, |retained| retained.min(strategy))
623 }
624
625 fn strategy_retained_from(&self) -> u64 {
626 match &self.strategy {
627 Strategy::WholeRecording(_) => 0,
628 Strategy::Agreement(state) => state.retained_from(),
629 Strategy::Native(state) => state.retained_from(),
630 }
631 }
632
633 pub fn next_update(
634 &mut self,
635 recording: &mut RecordingReader,
636 ) -> Result<Option<TranscriptUpdate>, TranscriptionError> {
637 self.next(recording, true)
638 }
639
640 pub fn next(
643 &mut self,
644 recording: &mut RecordingReader,
645 previews: bool,
646 ) -> Result<Option<TranscriptUpdate>, TranscriptionError> {
647 let Some(mut output) = self.output.take() else {
648 return self.next_raw_step(recording, previews);
649 };
650 let result = check_cancelled(&self.cancel)
651 .and_then(|()| self.next_processed(recording, previews, &mut output));
652 if result.is_err() {
653 output.finished = true;
654 output.pending.clear();
655 output.text.clear();
656 self.transcript = None;
657 }
658 self.output = Some(output);
659 result
660 }
661
662 fn next_processed(
663 &mut self,
664 recording: &mut RecordingReader,
665 previews: bool,
666 output: &mut layer::Output,
667 ) -> Result<Option<TranscriptUpdate>, TranscriptionError> {
668 loop {
669 if let Some(update) = output.pending.pop_front() {
670 return Ok(Some(update));
671 }
672 if output.finished {
673 return Ok(None);
674 }
675 let update = self.next_raw_step(recording, previews)?;
676 let finished = update.is_none() || self.transcript.is_some();
677 if let Some(update) = update {
678 output.push(update, self.layers, &self.cancel)?;
679 }
680 if finished {
681 output.retained_from = Some(self.strategy_retained_from());
683 while let Some(update) = self.next_raw_step(recording, previews)? {
684 output.push(update, self.layers, &self.cancel)?;
685 }
686 self.transcript = if output.text.is_empty() {
687 None
688 } else {
689 Some(Transcript(std::mem::take(&mut output.text)))
690 };
691 output.finished = true;
692 }
693 }
694 }
695
696 fn next_raw_step(
697 &mut self,
698 recording: &mut RecordingReader,
699 previews: bool,
700 ) -> Result<Option<TranscriptUpdate>, TranscriptionError> {
701 let step = match &mut self.strategy {
702 Strategy::WholeRecording(input) => {
703 if self.transcript.is_some() {
704 return Ok(None);
705 }
706 let completed = transcribe_complete(
707 *input,
708 recording,
709 self.transcriber,
710 self.language,
711 &self.cancel,
712 )?;
713 let segment = TranscriptSegment {
714 text: completed.as_str().to_owned(),
715 span: None,
716 };
717 self.transcript = Some(completed);
718 ChunkStep {
719 update: Some(TranscriptUpdate::Finalized(segment)),
720 transcript: None,
721 }
722 }
723 Strategy::Agreement(state) => {
724 let cancel = self.cancel.clone();
725 match state.next_with(recording, &cancel, previews, |samples| {
726 self.transcriber.transcribe_with(
727 TranscriptionAudio::Samples(samples),
728 self.language,
729 &cancel,
730 )
731 })? {
732 Some(step) => step,
733 None => return Ok(None),
734 }
735 }
736 Strategy::Native(state) => match state.next_finalized_with(recording, &self.cancel)? {
737 Some(step) => step,
738 None => return Ok(None),
739 },
740 };
741 if let Some(transcript) = step.transcript {
742 self.transcript = Some(transcript);
743 }
744 Ok(step.update)
745 }
746
747 pub fn finish(
748 &mut self,
749 recording: &mut RecordingReader,
750 ) -> Result<Transcript, TranscriptionError> {
751 while self.next_finalized(recording)?.is_some() {}
752 self.transcript.take().ok_or(TranscriptionError::NoSpeech)
753 }
754}
755
756fn transcribe_complete<T: Transcriber>(
757 input: WholeRecordingInput,
758 recording: &mut RecordingReader,
759 transcriber: &T,
760 language: &Language,
761 cancel: &CancelToken,
762) -> Result<Transcript, TranscriptionError> {
763 check_cancelled(cancel)?;
764 let hypothesis = match input {
765 WholeRecordingInput::Samples => {
766 recording.fill().map_err(TranscriptionError::Capture)?;
767 check_cancelled(cancel)?;
768 transcriber.transcribe_with(
769 TranscriptionAudio::Samples(recording.samples()),
770 language,
771 cancel,
772 )?
773 }
774 #[cfg(not(target_family = "wasm"))]
775 WholeRecordingInput::Wav => {
776 let audio = recording
777 .finished_audio()
778 .map_err(TranscriptionError::Capture)?;
779 check_cancelled(cancel)?;
780 transcriber.transcribe_with(TranscriptionAudio::Wav(audio), language, cancel)?
781 }
782 };
783 check_cancelled(cancel)?;
784 Transcript::new(hypothesis.into_text())
785}
786
787pub fn check_cancelled(cancel: &CancelToken) -> Result<(), TranscriptionError> {
789 if cancel.is_cancelled() {
790 return Err(TranscriptionError::Cancelled);
791 }
792 Ok(())
793}
794
795pub enum TranscriptUpdate {
796 Finalized(TranscriptSegment),
797 Provisional(String),
798}
799
800#[derive(Clone)]
801pub struct TranscriptSegment {
802 text: String,
803 span: Option<std::ops::Range<u64>>,
804}
805
806impl TranscriptSegment {
807 fn spanned(words: &[String], span: Option<std::ops::Range<u64>>) -> Option<Self> {
808 let text = words.join(" ");
809 (!text.is_empty()).then_some(Self { text, span })
810 }
811
812 pub fn as_str(&self) -> &str {
813 &self.text
814 }
815
816 pub fn span(&self) -> Option<std::ops::Range<u64>> {
819 self.span.clone()
820 }
821
822 pub fn into_text(self) -> String {
823 self.text
824 }
825}
826
827impl std::fmt::Debug for TranscriptSegment {
828 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
829 formatter
830 .debug_struct("TranscriptSegment")
831 .field("bytes", &self.text.len())
832 .field("span", &self.span)
833 .finish()
834 }
835}
836
837#[derive(Debug, Clone, PartialEq, Eq)]
838pub enum Language {
839 Automatic,
840 Specific(LanguageCode),
841}
842
843impl Language {
844 const AUTOMATIC: &str = "auto";
845
846 pub fn as_str(&self) -> &str {
847 match self {
848 Self::Automatic => Self::AUTOMATIC,
849 Self::Specific(code) => code.as_str(),
850 }
851 }
852}
853
854impl std::str::FromStr for Language {
856 type Err = LanguageCodeError;
857
858 fn from_str(value: &str) -> Result<Self, Self::Err> {
859 if value.trim().eq_ignore_ascii_case(Self::AUTOMATIC) {
860 return Ok(Self::Automatic);
861 }
862 LanguageCode::new(value).map(Self::Specific)
863 }
864}
865
866#[derive(Debug, Clone, PartialEq, Eq)]
867pub struct LanguageCode(String);
868
869impl LanguageCode {
870 pub fn english() -> Self {
871 Self("en".to_owned())
872 }
873
874 pub fn new(value: impl Into<String>) -> Result<Self, LanguageCodeError> {
875 let value = value.into();
876 let value = value.trim();
877 if value.is_empty() {
878 return Err(LanguageCodeError::Empty);
879 }
880 if value.len() > MAX_LANGUAGE_CODE_BYTES {
881 return Err(LanguageCodeError::TooLong {
882 actual_bytes: value.len(),
883 maximum_bytes: MAX_LANGUAGE_CODE_BYTES,
884 });
885 }
886 let mut segments = value.split('-');
887 let valid_primary = segments.next().is_some_and(|segment| {
888 !segment.is_empty() && segment.bytes().all(|byte| byte.is_ascii_alphabetic())
889 });
890 let valid_subtags = segments.all(|segment| {
891 !segment.is_empty() && segment.bytes().all(|byte| byte.is_ascii_alphanumeric())
892 });
893 if !valid_primary || !valid_subtags {
894 return Err(LanguageCodeError::Malformed(value.to_owned()));
895 }
896 Ok(Self(value.to_ascii_lowercase()))
897 }
898
899 pub fn as_str(&self) -> &str {
900 &self.0
901 }
902}
903
904#[derive(Debug, thiserror::Error)]
905pub enum LanguageCodeError {
906 #[error("recognition language code must not be empty")]
907 Empty,
908
909 #[error(
910 "recognition language code contains {actual_bytes} bytes; maximum is {maximum_bytes} bytes"
911 )]
912 TooLong {
913 actual_bytes: usize,
914 maximum_bytes: usize,
915 },
916
917 #[error("recognition language code is malformed: {0:?}")]
918 Malformed(String),
919}
920
921#[derive(Clone, PartialEq, Eq)]
922pub struct Transcript(String);
923
924impl Transcript {
925 pub fn new(text: impl Into<String>) -> Result<Self, TranscriptionError> {
926 let text = text.into();
927 let text = text.trim();
928 if text.is_empty() {
929 return Err(TranscriptionError::NoSpeech);
930 }
931 Ok(Self(text.to_owned()))
932 }
933
934 pub fn as_str(&self) -> &str {
935 &self.0
936 }
937
938 pub fn into_text(self) -> String {
939 self.0
940 }
941}
942
943impl std::fmt::Debug for Transcript {
944 fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
945 formatter
946 .debug_struct("Transcript")
947 .field("bytes", &self.0.len())
948 .finish()
949 }
950}
951
952#[derive(Debug, thiserror::Error)]
953pub enum TranscriptionError {
954 #[error("the {backend} model cannot be used")]
955 Model {
956 backend: &'static str,
957 #[source]
958 source: ModelError,
959 },
960
961 #[error("failed to load the {backend} model {model}")]
962 Load {
963 backend: &'static str,
964 model: String,
965 #[source]
966 source: BackendError,
967 },
968
969 #[error("the recording being transcribed failed")]
970 Capture(#[source] CaptureError),
971
972 #[error("{backend} inference failed")]
973 Inference {
974 backend: &'static str,
975 #[source]
976 source: BackendError,
977 },
978
979 #[error("{backend} cannot transcribe {language:?}")]
980 UnsupportedLanguage {
981 backend: &'static str,
982 language: String,
983 },
984
985 #[error("{backend} needs a UTF-8 recording path; got {}", path.display())]
986 UnsupportedAudioPath {
987 backend: &'static str,
988 path: PathBuf,
989 },
990
991 #[error("{backend} requires {required}, but received {received}")]
992 UnsupportedAudioInput {
993 backend: &'static str,
994 received: AudioInputKind,
995 required: AudioInputKind,
996 },
997
998 #[error("the {backend} backend is not available on {}", std::env::consts::OS)]
999 UnsupportedBackend { backend: &'static str },
1000
1001 #[error("the {backend} backend has no gpu support on {}", std::env::consts::OS)]
1002 UnsupportedDevice { backend: &'static str },
1003
1004 #[error("the engine was asked to stream natively but offers no stream")]
1005 NoNativeStream,
1006
1007 #[error("the model loader stopped unexpectedly")]
1008 LoadPanicked,
1009
1010 #[error("transcription was cancelled")]
1011 Cancelled,
1012
1013 #[error("no speech was transcribed")]
1014 NoSpeech,
1015
1016 #[error("transcript processing failed")]
1017 Processing(#[source] layer::Error),
1018}
1019
1020#[cfg(not(target_family = "wasm"))]
1022pub fn validate_model_file(
1023 path: PathBuf,
1024 backend: &'static str,
1025) -> Result<PathBuf, TranscriptionError> {
1026 crate::backend::validate_model_file(path)
1027 .map_err(|source| TranscriptionError::Model { backend, source })
1028}
1029
1030#[cfg(not(target_family = "wasm"))]
1031pub fn validate_model_directory(
1032 path: PathBuf,
1033 backend: &'static str,
1034 required: &[&'static str],
1035) -> Result<PathBuf, TranscriptionError> {
1036 crate::backend::validate_model_directory(path, required)
1037 .map_err(|source| TranscriptionError::Model { backend, source })
1038}
1039
1040pub fn english_only(backend: &'static str, language: &Language) -> Result<(), TranscriptionError> {
1043 let Language::Specific(code) = language else {
1044 return Ok(());
1045 };
1046 if code.as_str().split('-').next() == Some(LanguageCode::english().as_str()) {
1047 return Ok(());
1048 }
1049 Err(TranscriptionError::UnsupportedLanguage {
1050 backend,
1051 language: code.as_str().to_owned(),
1052 })
1053}
1054
1055#[cfg(test)]
1056mod word_timing_tests {
1057 use super::*;
1058
1059 fn token(text: &str, start: f32, end: f32) -> (&str, TimedWord) {
1060 (text, TimedWord { start, end })
1061 }
1062
1063 #[test]
1064 fn tokens_group_into_words_at_the_sentencepiece_marker() {
1065 let tokens = [
1066 token("\u{2581}hel", 0.0, 0.2),
1067 token("lo", 0.2, 0.4),
1068 token(",", 0.4, 0.5),
1069 token("\u{2581}world", 0.6, 0.9),
1070 ];
1071 let words = word_timings(tokens);
1072 assert_eq!(words.len(), 2);
1073 assert_eq!((words[0].start, words[0].end), (0.0, 0.5));
1074 assert_eq!((words[1].start, words[1].end), (0.6, 0.9));
1075 }
1076
1077 #[test]
1078 fn tokens_group_into_words_at_a_leading_space() {
1079 let tokens = [
1080 token(" hel", 0.0, 0.2),
1081 token("lo", 0.2, 0.4),
1082 token(",", 0.4, 0.5),
1083 token(" world", 0.6, 0.9),
1084 ];
1085 let words = word_timings(tokens);
1086 assert_eq!(words.len(), 2);
1087 assert_eq!((words[0].start, words[0].end), (0.0, 0.5));
1088 assert_eq!((words[1].start, words[1].end), (0.6, 0.9));
1089 }
1090
1091 #[test]
1092 fn word_timings_count_what_the_text_splits_into() {
1093 let tokens = [
1095 token("\u{2581}", 0.0, 0.1),
1096 token("\u{2581}1", 0.1, 0.2),
1097 token("8", 0.2, 0.3),
1098 token("9", 0.3, 0.4),
1099 token("\u{2581}years old", 0.4, 0.8),
1100 token(".", 0.8, 0.9),
1101 ];
1102 let text: String = tokens
1103 .iter()
1104 .map(|(text, _)| text.replace('\u{2581}', " "))
1105 .collect();
1106 let words = word_timings(tokens);
1107 assert_eq!(words.len(), text.split_whitespace().count());
1108 assert_eq!((words[0].start, words[0].end), (0.1, 0.4));
1109 assert_eq!((words[2].start, words[2].end), (0.4, 0.9));
1110 }
1111
1112 #[test]
1113 fn a_word_timing_rounds_to_the_nearest_sample() {
1114 let word = TimedWord {
1116 start: 0.100_03,
1117 end: 0.200_04,
1118 };
1119
1120 assert_eq!(word.span_from(1_000), 2_600..4_201);
1121 }
1122}
1123
1124#[cfg(test)]
1125mod language_tests {
1126 use super::*;
1127
1128 const BACKEND: &str = "test-backend";
1129
1130 #[test]
1131 fn detection_and_english_are_accepted() {
1132 assert!(english_only(BACKEND, &Language::Automatic).is_ok());
1133 assert!(english_only(BACKEND, &Language::Specific(LanguageCode::english())).is_ok());
1134
1135 let regional = LanguageCode::new("en-GB").expect("en-GB is a valid code");
1136 assert!(english_only(BACKEND, &Language::Specific(regional)).is_ok());
1137 }
1138
1139 #[test]
1140 fn language_parses_what_as_str_writes() {
1141 for language in [
1142 Language::Automatic,
1143 Language::Specific(LanguageCode::new("en-GB").expect("en-GB is a valid code")),
1144 ] {
1145 assert_eq!(language.as_str().parse::<Language>().ok(), Some(language));
1146 }
1147 assert_eq!(" AUTO".parse::<Language>().ok(), Some(Language::Automatic));
1148 assert!(matches!(
1149 "en_GB".parse::<Language>(),
1150 Err(LanguageCodeError::Malformed(_))
1151 ));
1152 }
1153
1154 #[test]
1155 fn another_language_is_refused_by_name() {
1156 let code = LanguageCode::new("de").expect("de is a valid code");
1157
1158 let error = english_only(BACKEND, &Language::Specific(code))
1159 .expect_err("another language must be refused");
1160
1161 assert!(matches!(
1162 error,
1163 TranscriptionError::UnsupportedLanguage { language, .. } if language == "de"
1164 ));
1165 }
1166}
1167
1168#[cfg(test)]
1169mod coordinator_tests {
1170 use std::sync::{Arc, Mutex};
1171
1172 use crate::audio::testing::recorded;
1173
1174 use super::*;
1175
1176 struct FakeTranscriber {
1177 response: &'static str,
1178 reject_language: bool,
1179 inputs: Mutex<Vec<AudioInputKind>>,
1180 }
1181
1182 impl FakeTranscriber {
1183 fn answering(response: &'static str) -> Self {
1184 Self {
1185 response,
1186 reject_language: false,
1187 inputs: Mutex::new(Vec::new()),
1188 }
1189 }
1190
1191 fn inputs(&self) -> Vec<AudioInputKind> {
1192 self.inputs
1193 .lock()
1194 .unwrap_or_else(|poisoned| poisoned.into_inner())
1195 .clone()
1196 }
1197 }
1198
1199 impl Transcriber for FakeTranscriber {
1200 fn validate_language(
1201 &self,
1202 language: &Language,
1203 _cancel: &CancelToken,
1204 ) -> Result<(), TranscriptionError> {
1205 if self.reject_language {
1206 return Err(TranscriptionError::UnsupportedLanguage {
1207 backend: "fake",
1208 language: language.as_str().to_owned(),
1209 });
1210 }
1211 Ok(())
1212 }
1213
1214 fn transcribe_with(
1215 &self,
1216 audio: TranscriptionAudio<'_>,
1217 _language: &Language,
1218 _cancel: &CancelToken,
1219 ) -> Result<Hypothesis, TranscriptionError> {
1220 self.inputs
1221 .lock()
1222 .unwrap_or_else(|poisoned| poisoned.into_inner())
1223 .push(audio.kind());
1224 Ok(Hypothesis::new(self.response))
1225 }
1226 }
1227
1228 struct Chunked;
1230
1231 struct HelloStream;
1232
1233 impl ChunkStream for HelloStream {
1234 fn chunk_samples(&self) -> usize {
1235 2
1236 }
1237
1238 fn feed(&mut self, chunk: &[f32]) -> Result<Emitted, TranscriptionError> {
1239 Ok(Emitted {
1240 words: Hypothesis::new(match chunk.iter().any(|sample| *sample != 0.0) {
1241 true => "hello",
1242 false => "",
1243 }),
1244 joins_previous: false,
1245 })
1246 }
1247 }
1248
1249 impl Transcriber for Chunked {
1250 fn validate_language(
1251 &self,
1252 _language: &Language,
1253 _cancel: &CancelToken,
1254 ) -> Result<(), TranscriptionError> {
1255 Ok(())
1256 }
1257
1258 fn transcribe_with(
1259 &self,
1260 _audio: TranscriptionAudio<'_>,
1261 _language: &Language,
1262 _cancel: &CancelToken,
1263 ) -> Result<Hypothesis, TranscriptionError> {
1264 unreachable!("a native stream never decodes a bounded input")
1265 }
1266
1267 fn open_stream(
1268 &self,
1269 _language: &Language,
1270 _cancel: &CancelToken,
1271 ) -> Result<Option<Box<dyn ChunkStream>>, TranscriptionError> {
1272 Ok(Some(Box::new(HelloStream)))
1273 }
1274 }
1275
1276 #[test]
1277 fn a_boxed_or_shared_transcriber_still_opens_its_stream() {
1278 let (_directory, mut boxed_recording) = recorded(&[1, 2, 3, 4]);
1279 let (_directory, mut shared_recording) = recorded(&[1, 2, 3, 4]);
1280 let boxed: Box<dyn Transcriber> = Box::new(Chunked);
1281 let shared = Arc::new(Chunked);
1282
1283 let from_boxed = Transcription::stream(Stream::Native, boxed)
1284 .transcribe(&mut boxed_recording, &Language::Automatic)
1285 .expect("the boxed stream opens");
1286 let from_shared = Transcription::stream(Stream::Native, shared)
1287 .transcribe(&mut shared_recording, &Language::Automatic)
1288 .expect("the shared stream opens");
1289
1290 assert_eq!(from_boxed.as_str(), "hello hello");
1291 assert_eq!(from_shared.as_str(), "hello hello");
1292 }
1293
1294 #[test]
1295 fn completion_is_available_before_pending_final_segments_and_preview_clear() {
1296 struct Ending(std::sync::atomic::AtomicBool);
1297 impl Transcriber for Ending {
1298 fn validate_language(
1299 &self,
1300 _: &Language,
1301 _: &CancelToken,
1302 ) -> Result<(), TranscriptionError> {
1303 Ok(())
1304 }
1305 fn transcribe_with(
1306 &self,
1307 _: TranscriptionAudio<'_>,
1308 _: &Language,
1309 _: &CancelToken,
1310 ) -> Result<Hypothesis, TranscriptionError> {
1311 Ok(Hypothesis::new(
1312 if self.0.swap(true, std::sync::atomic::Ordering::Relaxed) {
1313 "one two three"
1314 } else {
1315 "one two"
1316 },
1317 ))
1318 }
1319 }
1320 let (mut recording, mut capture) = crate::audio::Recording::open();
1321 let transcription = Transcription::agreement(
1322 Cadence::from_millis(100, 20_000),
1323 Ending(std::sync::atomic::AtomicBool::new(false)),
1324 );
1325 let mut operation = transcription.start(&Language::Automatic).expect("starts");
1326 capture.push(&[0.5; 1600]);
1327 assert!(
1328 matches!(operation.next_update(&mut recording).expect("preview"), Some(TranscriptUpdate::Provisional(text)) if text == "one two")
1329 );
1330 capture.push(&[0.5; 1600]);
1331 capture.finish();
1332 recording.fill().expect("capture ended");
1333 assert!(
1334 matches!(operation.next_update(&mut recording).expect("final"), Some(TranscriptUpdate::Finalized(segment)) if segment.as_str() == "one two")
1335 );
1336 assert_eq!(
1337 operation.completed().expect("already complete").as_str(),
1338 "one two three"
1339 );
1340 assert!(
1341 matches!(operation.next_update(&mut recording).expect("tail"), Some(TranscriptUpdate::Finalized(segment)) if segment.as_str() == "three")
1342 );
1343 assert!(
1344 matches!(operation.next_update(&mut recording).expect("clear"), Some(TranscriptUpdate::Provisional(text)) if text.is_empty())
1345 );
1346 assert!(
1347 operation
1348 .next_update(&mut recording)
1349 .expect("ended")
1350 .is_none()
1351 );
1352 }
1353
1354 #[test]
1355 fn native_and_whole_updates_are_finalized_only() {
1356 fn check<T: Transcriber, M: Mode>(transcription: Transcription<T, M>, expected: &str) {
1357 let (_directory, mut recording) = recorded(&[1, 2, 3, 4]);
1358 let mut operation = transcription.start(&Language::Automatic).expect("starts");
1359 let mut segments = Vec::new();
1360 while let Some(update) = operation.next_update(&mut recording).expect("reads") {
1361 let TranscriptUpdate::Finalized(segment) = update else {
1362 panic!("only agreement exposes provisional words");
1363 };
1364 segments.push(segment.into_text());
1365 }
1366 assert_eq!(segments.join(" "), expected);
1367 assert_eq!(operation.completed().expect("completed").as_str(), expected);
1368 assert_eq!(
1369 operation.finish(&mut recording).expect("finished").as_str(),
1370 expected
1371 );
1372 }
1373 check(
1374 Transcription::stream(Stream::Native, Chunked),
1375 "hello hello",
1376 );
1377 check(
1378 Transcription::complete_samples(FakeTranscriber::answering("hello there")),
1379 "hello there",
1380 );
1381 }
1382
1383 #[test]
1384 fn streaming_natively_without_a_stream_is_an_error() {
1385 let transcription =
1386 Transcription::stream(Stream::Native, FakeTranscriber::answering("hello"));
1387
1388 let outcome = transcription.start(&Language::Automatic).map(|_| ());
1389
1390 assert!(matches!(outcome, Err(TranscriptionError::NoNativeStream)));
1391 }
1392
1393 #[test]
1394 fn complete_samples_invokes_the_engine_once() {
1395 let (_directory, mut recording) = recorded(&[1, 2, 3]);
1396 let transcriber = Arc::new(FakeTranscriber::answering("hello there"));
1397 let transcription = Transcription::complete_samples(Arc::clone(&transcriber));
1398
1399 let transcript = transcription
1400 .transcribe(&mut recording, &Language::Automatic)
1401 .expect("the fake returns speech");
1402
1403 assert_eq!(transcript.as_str(), "hello there");
1404 assert_eq!(transcriber.inputs(), [AudioInputKind::Samples]);
1405 }
1406
1407 #[cfg(not(target_family = "wasm"))]
1408 #[test]
1409 fn complete_wav_preserves_the_file_input_boundary() {
1410 let (_directory, mut recording) = recorded(&[1, 2, 3]);
1411 let transcriber = Arc::new(FakeTranscriber::answering("hello there"));
1412 let transcription = Transcription::complete_wav(Arc::clone(&transcriber));
1413
1414 transcription
1415 .transcribe(&mut recording, &Language::Automatic)
1416 .expect("the fake returns speech");
1417
1418 assert_eq!(transcriber.inputs(), [AudioInputKind::Wav]);
1419 assert!(recording.samples().is_empty());
1420 }
1421
1422 #[test]
1423 fn an_empty_hypothesis_becomes_terminal_no_speech() {
1424 let (_directory, mut recording) = recorded(&[1, 2, 3]);
1425 let transcription = Transcription::complete_samples(FakeTranscriber::answering(" "));
1426
1427 let outcome = transcription.transcribe(&mut recording, &Language::Automatic);
1428
1429 assert!(matches!(outcome, Err(TranscriptionError::NoSpeech)));
1430 }
1431
1432 #[test]
1433 fn cancellation_prevents_complete_inference() {
1434 let (_directory, mut recording) = recorded(&[1, 2, 3]);
1435 let transcriber = Arc::new(FakeTranscriber::answering("unreachable"));
1436 let transcription = Transcription::complete_samples(Arc::clone(&transcriber));
1437 let cancel = CancelToken::new();
1438 cancel.cancel();
1439
1440 let outcome = transcription.transcribe_with(&mut recording, &Language::Automatic, &cancel);
1441
1442 assert!(matches!(outcome, Err(TranscriptionError::Cancelled)));
1443 assert!(transcriber.inputs().is_empty());
1444 }
1445
1446 #[test]
1447 fn language_rejection_prevents_complete_inference() {
1448 let (_directory, mut recording) = recorded(&[1, 2, 3]);
1449 let mut fake = FakeTranscriber::answering("unreachable");
1450 fake.reject_language = true;
1451 let transcriber = Arc::new(fake);
1452 let transcription = Transcription::complete_samples(Arc::clone(&transcriber));
1453
1454 let outcome = transcription.transcribe(&mut recording, &Language::Automatic);
1455
1456 assert!(matches!(
1457 outcome,
1458 Err(TranscriptionError::UnsupportedLanguage { .. })
1459 ));
1460 assert!(transcriber.inputs().is_empty());
1461 }
1462
1463 #[test]
1464 fn sensitive_transcription_types_redact_their_text() {
1465 let secret = "words that must not reach diagnostics";
1466 let hypothesis = Hypothesis::new(secret);
1467 let transcript = Transcript::new(secret).expect("the text is not empty");
1468 let segment =
1469 TranscriptSegment::spanned(&[secret.to_owned()], None).expect("the text is not empty");
1470
1471 assert!(!format!("{hypothesis:?}").contains(secret));
1472 assert!(!format!("{transcript:?}").contains(secret));
1473 assert!(!format!("{segment:?}").contains(secret));
1474 }
1475}