1use super::{
2 coordinator::{self, Control, Session, Subscription},
3 session::{
4 driver::{Driver, FinalTextProcessor},
5 state::failure_result,
6 },
7};
8#[cfg(feature = "backend-dashscope")]
9use crate::DashScopeConfig;
10#[cfg(feature = "backend-openai-realtime")]
11use crate::OpenAiRealtimeConfig;
12#[cfg(feature = "backend-sherpa")]
13use crate::PunctConfig;
14use crate::{
15 AsrError, AudioBuffer, AudioChunk, AudioSpec, BackendCapabilities, EngineConfig, ErrorKind,
16 Segment, SessionOptions, SessionOutcome, SessionResult, SessionView, Update,
17};
18#[cfg(feature = "backend-openai-http")]
19use crate::{HttpMode, HttpResponse, OpenAiHttpConfig};
20#[cfg(all(feature = "backend-sherpa", feature = "vad-silero"))]
21use crate::{OfflineFamily, VadConfig};
22#[cfg(any(
23 feature = "backend-dashscope",
24 feature = "backend-openai-http",
25 feature = "backend-openai-realtime"
26))]
27use std::sync::Mutex;
28use std::sync::{
29 atomic::{AtomicUsize, Ordering},
30 Arc,
31};
32use std::{num::NonZeroUsize, time::Instant};
33
34#[non_exhaustive]
35#[derive(Debug, Clone)]
36pub struct EngineOptions {
37 pub max_active_sessions: NonZeroUsize,
38}
39impl EngineOptions {
40 pub fn new(max_active_sessions: usize) -> Result<Self, AsrError> {
41 let max_active_sessions = NonZeroUsize::new(max_active_sessions)
42 .ok_or_else(|| AsrError::invalid("max_active_sessions must be positive"))?;
43 Ok(Self {
44 max_active_sessions,
45 })
46 }
47}
48impl Default for EngineOptions {
49 fn default() -> Self {
50 Self {
51 max_active_sessions: NonZeroUsize::new(8).unwrap(),
52 }
53 }
54}
55
56pub struct TranscriptSnapshotSink {
73 callback: Box<dyn FnMut(&str) -> bool + Send>,
74}
75
76impl TranscriptSnapshotSink {
77 pub fn new(callback: impl FnMut(&str) -> bool + Send + 'static) -> Self {
78 Self {
79 callback: Box::new(callback),
80 }
81 }
82 pub fn emit(&mut self, text: &str) -> bool {
89 (self.callback)(text)
90 }
91}
92
93impl std::fmt::Debug for TranscriptSnapshotSink {
94 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
95 f.debug_struct("TranscriptSnapshotSink")
96 .finish_non_exhaustive()
97 }
98}
99
100fn load(
101 config: &EngineConfig,
102) -> Result<(Kind, BackendCapabilities, Option<FinalTextProcessor>), AsrError> {
103 super::backends::validate_cloud_config(config)?;
107 #[allow(unused_variables)]
108 match config {
109 #[cfg(feature = "backend-sherpa")]
110 EngineConfig::Streaming(config) => {
111 super::backends::precheck::streaming(config)?;
114 let (recognizer, words, vocabulary) = super::backends::local::load_stream(
115 &config.model_dir,
116 config.bias.as_ref(),
117 config.num_threads,
118 config.provider,
119 )?;
120 let punctuator = load_punctuator(&config.punctuation)?;
121 let supports_session_hints = config.bias.is_some();
122 Ok((
123 Kind::Streaming(recognizer, words, vocabulary),
124 BackendCapabilities {
125 name: "zipformer".into(),
126 audio: AudioSpec { sample_rate: 16000 },
127 realtime_audio: true,
128 partial_results: true,
129 streaming_transcription: true,
130 remote: false,
131 punctuation: punctuator.is_some(),
132 supports_session_hints,
133 },
134 punctuator,
135 ))
136 }
137 #[cfg(all(feature = "backend-sherpa", feature = "vad-silero"))]
138 EngineConfig::Offline(config) => {
139 super::backends::precheck::offline(config)?;
142 super::backends::vad::preflight(&config.vad)?;
146 let (recognizer, words, vocabulary) = super::backends::local::load_offline(
147 &config.model_dir,
148 config.family,
149 config.language.as_deref(),
150 config.transducer_bias.as_ref(),
151 config.prompt_hints.as_ref(),
152 config.num_threads,
153 config.provider,
154 )?;
155 let punctuator = load_punctuator(&config.punctuation)?;
156 let supports_session_hints =
157 config.transducer_bias.is_some() && config.family == OfflineFamily::Transducer;
158 Ok((
159 Kind::Offline(recognizer, config.vad.clone(), words, vocabulary),
160 BackendCapabilities {
161 name: "offline".into(),
162 audio: AudioSpec { sample_rate: 16000 },
163 realtime_audio: false,
164 partial_results: false,
165 streaming_transcription: false,
168 remote: false,
169 punctuation: punctuator.is_some(),
170 supports_session_hints,
171 },
172 punctuator,
173 ))
174 }
175 #[cfg(feature = "backend-dashscope")]
176 EngineConfig::DashScope(cfg) => Ok((
177 Kind::DashScope(cfg.clone()),
178 BackendCapabilities {
179 name: "dashscope".into(),
180 audio: AudioSpec { sample_rate: 16000 },
181 realtime_audio: true,
182 partial_results: true,
183 streaming_transcription: true,
184 remote: true,
185 punctuation: false,
186 supports_session_hints: false,
187 },
188 None,
189 )),
190 #[cfg(feature = "backend-openai-http")]
191 EngineConfig::OpenAiHttp(cfg) => Ok((
192 Kind::Http(cfg.clone()),
193 BackendCapabilities {
196 name: "openai-transcriptions".into(),
197 audio: AudioSpec {
198 sample_rate: cfg.sample_rate,
199 },
200 realtime_audio: false,
201 partial_results: cfg.response == HttpResponse::Sse,
202 streaming_transcription: cfg.response == HttpResponse::Sse,
203 remote: true,
204 punctuation: false,
205 supports_session_hints: false,
206 },
207 None,
208 )),
209 #[cfg(feature = "backend-openai-realtime")]
210 EngineConfig::OpenAiRealtime(cfg) => Ok((
211 Kind::Realtime(cfg.clone()),
212 BackendCapabilities {
213 name: "openai-realtime".into(),
214 audio: AudioSpec { sample_rate: 24000 },
215 realtime_audio: true,
216 partial_results: true,
217 streaming_transcription: true,
218 remote: true,
219 punctuation: false,
220 supports_session_hints: false,
221 },
222 None,
223 )),
224 #[allow(unreachable_patterns)]
225 _ => Err(AsrError::new(
226 ErrorKind::UnsupportedCapability,
227 "configuration",
228 "requested backend is disabled in this build",
229 )),
230 }
231}
232
233#[cfg(feature = "backend-sherpa")]
235fn load_punctuator(config: &Option<PunctConfig>) -> Result<Option<FinalTextProcessor>, AsrError> {
236 #[cfg(feature = "punct-sherpa")]
237 {
238 match config {
239 Some(c) => Ok(Some(super::backends::punct::create(c)?)),
240 None => Ok(None),
241 }
242 }
243 #[cfg(not(feature = "punct-sherpa"))]
244 {
245 if config.is_some() {
246 return Err(AsrError::new(
247 ErrorKind::UnsupportedCapability,
248 "configuration",
249 "punctuation support is disabled in this build",
250 ));
251 }
252 Ok(None)
253 }
254}
255
256fn normalize_session_hints(mut options: SessionOptions) -> SessionOptions {
261 if options
262 .hints
263 .as_ref()
264 .is_some_and(|hints| hints.phrases.is_empty())
265 {
266 options.hints = None;
267 }
268 options
269}
270struct Prepared {
271 config: EngineConfig,
272 capabilities: BackendCapabilities,
273 kind: Kind,
274 punctuator: Option<FinalTextProcessor>,
275 sessions: Arc<AtomicUsize>,
276 max_active_sessions: usize,
277 #[cfg(any(
278 feature = "backend-dashscope",
279 feature = "backend-openai-http",
280 feature = "backend-openai-realtime"
281 ))]
282 network: Mutex<Option<Arc<super::backends::network::Network>>>,
283}
284enum Kind {
285 #[cfg(feature = "backend-sherpa")]
286 Streaming(
287 Arc<sherpa_onnx::OnlineRecognizer>,
288 Option<String>,
289 super::backends::hotwords::HotwordVocabulary,
290 ),
291 #[cfg(all(feature = "backend-sherpa", feature = "vad-silero"))]
292 Offline(
293 Arc<sherpa_onnx::OfflineRecognizer>,
294 VadConfig,
295 Option<String>,
296 super::backends::hotwords::HotwordVocabulary,
297 ),
298 #[cfg(feature = "backend-dashscope")]
299 DashScope(DashScopeConfig),
300 #[cfg(feature = "backend-openai-http")]
301 Http(OpenAiHttpConfig),
302 #[cfg(feature = "backend-openai-realtime")]
303 Realtime(OpenAiRealtimeConfig),
304}
305#[derive(Clone)]
306pub struct Engine {
307 prepared: Arc<Prepared>,
308}
309impl std::fmt::Debug for Engine {
310 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
311 f.debug_struct("Engine")
312 .field("config", &self.prepared.config)
313 .field("capabilities", &self.prepared.capabilities)
314 .finish()
315 }
316}
317impl Engine {
318 pub fn prepare(config: EngineConfig, options: EngineOptions) -> Result<Self, AsrError> {
319 let (kind, capabilities, punctuator) =
320 std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| load(&config)))
321 .unwrap_or_else(|_| Err(AsrError::backend("model preparation panicked")))?;
322 Ok(Self {
323 prepared: Arc::new(Prepared {
324 config,
325 capabilities,
326 kind,
327 punctuator,
328 sessions: Arc::new(AtomicUsize::new(0)),
329 max_active_sessions: options.max_active_sessions.get(),
330 #[cfg(any(
331 feature = "backend-dashscope",
332 feature = "backend-openai-http",
333 feature = "backend-openai-realtime"
334 ))]
335 network: Mutex::new(None),
336 }),
337 })
338 }
339
340 #[cfg(all(test, feature = "backend-openai-http"))]
341 pub(crate) fn active_sessions(&self) -> usize {
342 self.prepared.sessions.load(Ordering::Acquire)
343 }
344
345 pub fn config(&self) -> &EngineConfig {
346 &self.prepared.config
347 }
348 pub fn capabilities(&self) -> &BackendCapabilities {
349 &self.prepared.capabilities
350 }
351
352 pub fn start(&self, options: SessionOptions) -> Result<Session, AsrError> {
353 self.start_internal(options, None)
354 }
355
356 fn start_internal(
357 &self,
358 options: SessionOptions,
359 deadline: Option<Instant>,
360 ) -> Result<Session, AsrError> {
361 let options = normalize_session_hints(options);
362 if options.hints.is_some() && !self.prepared.capabilities.supports_session_hints {
363 return Err(AsrError::new(
364 ErrorKind::UnsupportedCapability,
365 "start",
366 "this engine does not support per-session speech hints",
367 ));
368 }
369 #[cfg(feature = "backend-sherpa")]
370 if let Some(hints) = &options.hints {
371 super::backends::hotwords::validate_session_hints(hints)
372 .map_err(|error| AsrError::new(error.kind, "start", error.message))?;
373 if let Some(vocabulary) = self.session_hotword_vocabulary() {
374 vocabulary.validate_session(hints)?;
375 }
376 }
377 let permit = Permit::acquire(&self.prepared.sessions, self.prepared.max_active_sessions)?;
378 let driver = self.driver(&options)?;
379 coordinator::spawn_with_deadline(
380 driver,
381 self.prepared.capabilities.audio,
382 options,
383 self.prepared.punctuator.clone(),
384 permit,
385 deadline,
386 )
387 }
388
389 #[cfg(any(
390 feature = "backend-dashscope",
391 feature = "backend-openai-http",
392 feature = "backend-openai-realtime"
393 ))]
394 fn network(&self) -> Result<Arc<super::backends::network::Network>, AsrError> {
395 init_on_success(
396 &self.prepared.network,
397 super::backends::network::Network::new,
398 )
399 }
400
401 pub fn transcribe(
402 &self,
403 audio: &AudioBuffer,
404 options: SessionOptions,
405 deadline: Instant,
406 ) -> SessionResult {
407 self.transcribe_with(audio, options, deadline, None)
408 }
409
410 pub fn transcribe_streaming(
426 &self,
427 audio: &AudioBuffer,
428 options: SessionOptions,
429 deadline: Instant,
430 progress: TranscriptSnapshotSink,
431 ) -> SessionResult {
432 self.transcribe_with(audio, options, deadline, Some(progress))
433 }
434
435 fn transcribe_with(
436 &self,
437 audio: &AudioBuffer,
438 options: SessionOptions,
439 deadline: Instant,
440 progress: Option<TranscriptSnapshotSink>,
441 ) -> SessionResult {
442 let chunk_frames = match self.validate_transcribe_input(audio, &options, deadline) {
443 Ok(chunk_frames) => chunk_frames,
444 Err(error) => return failure_result(error, SessionOutcome::default()),
445 };
446 if Instant::now() >= deadline {
447 return failure_result(
448 AsrError::new(
449 ErrorKind::Timeout,
450 "transcribe",
451 "transcription deadline expired before session start",
452 ),
453 SessionOutcome::default(),
454 );
455 }
456 let session = match self.start_internal(options, Some(deadline)) {
457 Ok(session) => session,
458 Err(error) => return failure_result(error, SessionOutcome::default()),
459 };
460 let drainer = progress
465 .zip(session.subscribe())
466 .map(|(sink, subscription)| {
467 let cancel = session.control();
468 std::thread::spawn(move || drain_progress(subscription, sink, cancel, deadline))
469 });
470 let input = session.input();
471 let result = 'feed: {
472 for samples in audio.samples.chunks(chunk_frames) {
473 if Instant::now() >= deadline {
474 break 'feed session.finish(deadline);
475 }
476 let chunk = AudioChunk {
477 samples: samples.to_vec(),
478 spec: audio.spec,
479 };
480 if let Err(error) = input.push_wait_validated(chunk, deadline) {
484 break 'feed session.fail_input(error.error);
485 }
486 }
487 session.finish(deadline)
488 };
489 if let Some(handle) = drainer {
493 let _ = handle.join();
494 }
495 result
496 }
497
498 fn validate_transcribe_input(
499 &self,
500 audio: &AudioBuffer,
501 options: &SessionOptions,
502 deadline: Instant,
503 ) -> Result<usize, AsrError> {
504 if Instant::now() >= deadline {
505 return Err(AsrError::new(
506 ErrorKind::Timeout,
507 "transcribe",
508 "transcription deadline already expired",
509 ));
510 }
511 AudioSpec::mono(audio.spec.sample_rate)?;
512 if audio.spec != options.input {
513 return Err(AsrError::new(
514 ErrorKind::InvalidInput,
515 "transcribe",
516 "audio format does not match SessionOptions input",
517 ));
518 }
519 if audio
520 .samples
521 .iter()
522 .any(|sample| !sample.is_finite() || sample.abs() > 1.0)
523 {
524 return Err(AsrError::new(
525 ErrorKind::InvalidInput,
526 "transcribe",
527 "audio must contain finite normalized samples in -1..=1",
528 ));
529 }
530 let geometry = coordinator::session_geometry(options)?;
531 if audio.samples.len() as u128 > u128::from(geometry.max_session_frames) {
532 return Err(AsrError::new(
533 ErrorKind::ResourceLimit,
534 "transcribe",
535 "audio exceeds maximum session duration",
536 ));
537 }
538 self.validate_whole_recording_size(audio)?;
539 Ok((options.input.sample_rate as usize / 10)
540 .max(1)
541 .min(geometry.max_chunk_frames))
542 }
543
544 #[cfg(feature = "backend-openai-http")]
545 #[allow(irrefutable_let_patterns)]
546 fn validate_whole_recording_size(&self, audio: &AudioBuffer) -> Result<(), AsrError> {
547 let Kind::Http(config) = &self.prepared.kind else {
548 return Ok(());
549 };
550 if !matches!(config.mode, HttpMode::WholeRecording) {
551 return Ok(());
552 }
553 let numerator = (audio.samples.len() as u128)
554 .checked_mul(u128::from(config.sample_rate))
555 .ok_or_else(|| {
556 AsrError::new(ErrorKind::ResourceLimit, "upload", "audio is too large")
557 })?;
558 let input_rate = u128::from(audio.spec.sample_rate);
559 let output_frames = numerator
560 .checked_add(input_rate / 2)
561 .map(|value| value / input_rate)
562 .ok_or_else(|| {
563 AsrError::new(ErrorKind::ResourceLimit, "upload", "audio is too large")
564 })?;
565 let capacity =
566 crate::audio::pcm16_wav_sample_capacity(config.max_upload_bytes).unwrap_or(0);
567 if output_frames > capacity as u128 {
568 return Err(AsrError::new(
569 ErrorKind::ResourceLimit,
570 "upload",
571 "audio exceeds configured upload limit",
572 ));
573 }
574 Ok(())
575 }
576
577 #[cfg(not(feature = "backend-openai-http"))]
578 fn validate_whole_recording_size(&self, _: &AudioBuffer) -> Result<(), AsrError> {
579 Ok(())
580 }
581 #[cfg(all(feature = "backend-sherpa", feature = "vad-silero"))]
583 #[allow(unreachable_patterns)]
584 fn session_hotword_vocabulary(&self) -> Option<&super::backends::hotwords::HotwordVocabulary> {
585 match &self.prepared.kind {
586 Kind::Streaming(_, _, vocabulary) | Kind::Offline(_, _, _, vocabulary) => {
587 Some(vocabulary)
588 }
589 _ => None,
590 }
591 }
592 #[cfg(all(feature = "backend-sherpa", not(feature = "vad-silero")))]
593 #[allow(unreachable_patterns)]
594 fn session_hotword_vocabulary(&self) -> Option<&super::backends::hotwords::HotwordVocabulary> {
595 match &self.prepared.kind {
596 Kind::Streaming(_, _, vocabulary) => Some(vocabulary),
597 _ => None,
598 }
599 }
600 #[allow(unreachable_code, unused_variables)]
601 fn driver(&self, options: &SessionOptions) -> Result<Box<dyn Driver>, AsrError> {
602 let driver: Box<dyn Driver> = match &self.prepared.kind {
603 #[cfg(feature = "backend-sherpa")]
604 Kind::Streaming(r, words, _) => Box::new(super::backends::local::Streaming::new(
605 r.clone(),
606 super::backends::hotwords::merge(words.as_ref(), options.hints.as_ref())?,
607 )),
608 #[cfg(all(feature = "backend-sherpa", feature = "vad-silero"))]
609 Kind::Offline(r, vad, words, _) => Box::new(super::backends::local::Offline::new(
610 r.clone(),
611 vad.clone(),
612 super::backends::hotwords::merge(words.as_ref(), options.hints.as_ref())?,
613 )),
614 #[cfg(feature = "backend-dashscope")]
615 Kind::DashScope(cfg) => Box::new(super::backends::dashscope::DashScopeDriver::new(
616 cfg.clone(),
617 self.network()?,
618 )),
619 #[cfg(feature = "backend-openai-http")]
620 Kind::Http(cfg) => Box::new(super::backends::http::HttpDriver::new(
621 cfg.clone(),
622 self.network()?,
623 )),
624 #[cfg(feature = "backend-openai-realtime")]
625 Kind::Realtime(cfg) => Box::new(super::backends::realtime::RealtimeDriver::new(
626 cfg.clone(),
627 self.network()?,
628 )),
629 #[allow(unreachable_patterns)]
630 _ => {
631 return Err(AsrError::new(
632 ErrorKind::UnsupportedCapability,
633 "start",
634 "no enabled backend",
635 ))
636 }
637 };
638 Ok(driver)
639 }
640}
641fn drain_progress(
646 mut subscription: Subscription,
647 mut sink: TranscriptSnapshotSink,
648 control: Control,
649 deadline: Instant,
650) {
651 loop {
652 let remaining = deadline.saturating_duration_since(Instant::now());
653 match subscription.recv_timeout(remaining) {
654 Ok(Update::Phase(_)) => continue,
657 Ok(_) => {
658 if !sink.emit(&emerging_text(&subscription.view())) {
659 control.fail(AsrError::new(
661 ErrorKind::Cancelled,
662 "session",
663 "session cancelled",
664 ));
665 break;
666 }
667 }
668 Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => break,
669 Err(std::sync::mpsc::RecvTimeoutError::Timeout) => {
672 if Instant::now() >= deadline {
673 break;
674 }
675 }
676 }
677 }
678}
679
680fn emerging_text(view: &SessionView) -> String {
685 let mut transcript = view.transcript.clone();
686 for partial in &view.partials {
687 transcript.segments.push(Segment {
688 id: partial.utterance_id.clone(),
689 index: u64::MAX,
690 text: partial.text.clone(),
691 start_seconds: None,
692 end_seconds: None,
693 });
694 }
695 transcript.text()
696}
697
698#[cfg(any(
703 feature = "backend-dashscope",
704 feature = "backend-openai-http",
705 feature = "backend-openai-realtime"
706))]
707fn init_on_success<T>(
708 slot: &Mutex<Option<Arc<T>>>,
709 build: impl FnOnce() -> Result<T, AsrError>,
710) -> Result<Arc<T>, AsrError> {
711 let mut cached = slot
712 .lock()
713 .unwrap_or_else(std::sync::PoisonError::into_inner);
714 if let Some(value) = &*cached {
715 return Ok(value.clone());
716 }
717 let value = Arc::new(build()?);
718 *cached = Some(value.clone());
719 Ok(value)
720}
721struct Permit(Arc<AtomicUsize>);
722impl Permit {
723 fn acquire(count: &Arc<AtomicUsize>, max: usize) -> Result<Self, AsrError> {
724 count
725 .fetch_update(Ordering::AcqRel, Ordering::Acquire, |n| {
726 (n < max).then_some(n + 1)
727 })
728 .map(|_| Self(count.clone()))
729 .map_err(|_| {
730 AsrError::new(
731 ErrorKind::Busy,
732 "executor",
733 "configured concurrency limit reached",
734 )
735 })
736 }
737}
738impl Drop for Permit {
739 fn drop(&mut self) {
740 self.0.fetch_sub(1, Ordering::AcqRel);
741 }
742}
743
744#[cfg(all(test, feature = "backend-openai-http"))]
745mod tests;
746
747#[cfg(all(
748 test,
749 any(
750 feature = "backend-dashscope",
751 feature = "backend-openai-http",
752 feature = "backend-openai-realtime"
753 )
754))]
755mod network_cache_tests;
756
757#[cfg(test)]
758mod hint_normalization_tests;
759
760#[cfg(all(test, feature = "backend-sherpa"))]
761mod sherpa_hint_normalization_tests;