1#![cfg_attr(
2 not(any(
3 feature = "backend-sherpa",
4 feature = "backend-dashscope",
5 feature = "backend-openai-http",
6 feature = "backend-openai-realtime"
7 )),
8 allow(dead_code)
9)]
10use crate::session::{
11 driver::{Driver, FinalTextProcessor},
12 observe,
13 results::CommitPreflight,
14 state::{failure_result, SessionState as State},
15};
16use crate::{
17 AsrError, AudioChunk, AudioSpec, ErrorKind, PartialView, SessionOptions, SessionOutcome,
18 SessionPhase, SessionResult, SessionView, Update,
19};
20use std::{
21 sync::{Arc, Condvar, Mutex},
22 time::{Duration, Instant},
23};
24
25const NANOS_PER_SECOND: u128 = 1_000_000_000;
26
27const MAX_QUEUE_FRAMES: usize = 16 * 1024 * 1024;
33const MAX_QUEUE_CHUNKS: usize = 4096;
34
35fn duration_frames_raw(duration: Duration, rate: u32) -> u128 {
36 let rate = u128::from(rate);
37
38 u128::from(duration.as_secs()) * rate
39 + u128::from(duration.subsec_nanos()) * rate / NANOS_PER_SECOND
40}
41
42pub(crate) fn duration_frames_usize(
43 duration: Duration,
44 rate: u32,
45 field: &str,
46) -> Result<usize, AsrError> {
47 let frames = duration_frames_raw(duration, rate);
48
49 if frames > usize::MAX as u128 {
50 return Err(AsrError::invalid(format!(
51 "{field} is too large for this platform"
52 )));
53 }
54
55 Ok(frames as usize)
56}
57
58fn duration_frames_u64(duration: Duration, rate: u32, field: &str) -> Result<u64, AsrError> {
59 let frames = duration_frames_raw(duration, rate);
60
61 if frames > u64::MAX as u128 {
62 return Err(AsrError::invalid(format!("{field} is too large")));
63 }
64
65 Ok(frames as u64)
66}
67
68pub(crate) struct SessionGeometry {
76 pub(crate) capacity: usize,
78 pub(crate) max_chunk_frames: usize,
80 pub(crate) max_session_frames: u64,
83}
84
85pub(crate) fn session_geometry(options: &SessionOptions) -> Result<SessionGeometry, AsrError> {
86 let rate = options.input.sample_rate;
87 let capacity = duration_frames_usize(options.queue_duration, rate, "queue_duration")?;
88 let max_chunk_frames =
89 duration_frames_usize(options.max_chunk_duration, rate, "max_chunk_duration")?;
90 let max_session_frames = duration_frames_u64(options.max_duration, rate, "max_duration")?;
91
92 if capacity == 0 {
93 return Err(AsrError::invalid(
94 "queue_duration is shorter than one input frame",
95 ));
96 }
97
98 if max_chunk_frames == 0 {
99 return Err(AsrError::invalid(
100 "max_chunk_duration is shorter than one input frame",
101 ));
102 }
103
104 if max_session_frames == 0 {
105 return Err(AsrError::invalid(
106 "max_duration is shorter than one input frame",
107 ));
108 }
109
110 Ok(SessionGeometry {
111 capacity,
112 max_chunk_frames,
113 max_session_frames,
114 })
115}
116
117pub(crate) struct Shared {
130 pub(crate) state: Mutex<State>,
131 pub(crate) changed: Condvar,
132 pub(crate) options: SessionOptions,
133
134 capacity: usize,
135 max_chunk_frames: usize,
136 max_session_frames: u64,
137
138 #[cfg(any(
139 feature = "backend-dashscope",
140 feature = "backend-openai-http",
141 feature = "backend-openai-realtime"
142 ))]
143 pub(crate) wake: tokio::sync::Notify,
144}
145#[derive(Clone)]
146pub(crate) struct Control(pub(crate) Arc<Shared>);
147impl Control {
148 pub(crate) fn signal(&self) {
149 self.0.changed.notify_all();
150 #[cfg(any(
151 feature = "backend-dashscope",
152 feature = "backend-openai-http",
153 feature = "backend-openai-realtime"
154 ))]
155 self.0.wake.notify_one();
156 }
157 pub(crate) fn check(&self) -> Result<(), AsrError> {
158 let s = self.0.state.lock().unwrap();
159 if let Some(result) = &s.lifecycle.outcome {
160 return Err(match result {
161 Err(e) => e.error.clone(),
162 Ok(_) => AsrError::new(
163 ErrorKind::InputClosed,
164 "session",
165 "session already completed",
166 ),
167 });
168 }
169 if s.lifecycle.deadline.is_some_and(|d| Instant::now() >= d) {
170 return Err(AsrError::new(
171 ErrorKind::Timeout,
172 "session",
173 "session deadline exceeded",
174 ));
175 }
176 Ok(())
177 }
178 #[cfg(any(
179 feature = "backend-dashscope",
180 feature = "backend-openai-http",
181 feature = "backend-openai-realtime"
182 ))]
183 pub(crate) fn deadline(&self) -> Option<Instant> {
184 self.0.state.lock().unwrap().lifecycle.deadline
185 }
186 fn outcome_locked(&self, s: &mut State) -> SessionOutcome {
187 SessionOutcome {
188 transcript: s.results.transcript().clone(),
189 received_frames: s.input.received,
190 processed_frames: s.input.processed,
191 }
192 }
193 fn settle_once_locked(&self, s: &mut State, error: Option<AsrError>) {
194 if s.lifecycle.outcome.is_some() {
195 return;
196 }
197 s.lifecycle.phase = match &error {
198 None => SessionPhase::Completed,
199 Some(e) if e.kind == ErrorKind::Cancelled => SessionPhase::Cancelled,
200 _ => SessionPhase::Failed,
201 };
202 s.results.settle();
205 let outcome = self.outcome_locked(s);
206 s.lifecycle.outcome = Some(match error {
207 Some(error) => failure_result(error, outcome),
208 None => Ok(outcome),
209 });
210 s.lifecycle.accepting = false;
211 s.input.queue.clear();
212 s.input.queued = 0;
213 observe::terminal(s);
214 }
215 pub(crate) fn fail(&self, error: AsrError) {
216 let mut s = self.0.state.lock().unwrap();
217 self.settle_once_locked(&mut s, Some(error));
218 drop(s);
219 self.signal();
220 }
221 fn complete(&self) {
222 let mut s = self.0.state.lock().unwrap();
223 let error = s.results.completion_error();
224 self.settle_once_locked(&mut s, error);
225 drop(s);
226 self.signal();
227 }
228 fn publish_partial_locked(&self, s: &mut State, update: PartialView) {
229 observe::partial(s, &update, self.0.options.event_capacity);
230 }
231 #[cfg_attr(
232 not(any(
233 feature = "backend-sherpa",
234 feature = "backend-dashscope",
235 feature = "backend-openai-http"
236 )),
237 allow(dead_code)
238 )]
239 pub(crate) fn partial(
240 &self,
241 id: impl Into<String>,
242 text: impl Into<String>,
243 ) -> Result<(), AsrError> {
244 self.check()?;
245 let id = id.into();
246 let text = text.into();
247 let mut s = self.0.state.lock().unwrap();
248 if s.lifecycle.outcome.is_some() {
249 return Ok(());
250 }
251 let emit_update = s.observer.subscription_enabled;
252 let Some(update) = s.results.set_partial(id, text, emit_update)? else {
253 return Ok(());
254 };
255 self.publish_partial_locked(&mut s, update);
256 drop(s);
257 self.signal();
258 Ok(())
259 }
260 #[cfg_attr(not(feature = "backend-openai-realtime"), allow(dead_code))]
261 pub(crate) fn append_delta(&self, id: impl Into<String>, delta: &str) -> Result<(), AsrError> {
262 self.check()?;
263 let mut s = self.0.state.lock().unwrap();
264 if s.lifecycle.outcome.is_some() {
265 return Ok(());
266 }
267 let emit_update = s.observer.subscription_enabled;
268 let Some(update) = s.results.append_delta(id.into(), delta, emit_update)? else {
269 return Ok(());
270 };
271 self.publish_partial_locked(&mut s, update);
272 drop(s);
273 self.signal();
274 Ok(())
275 }
276 #[cfg_attr(not(feature = "backend-openai-realtime"), allow(dead_code))]
277 pub(crate) fn register_id(&self, id: &str) -> Result<(), AsrError> {
278 self.check()?;
279 let mut s = self.0.state.lock().unwrap();
280 if s.lifecycle.outcome.is_some() {
281 return Ok(());
282 }
283 s.results.register_id(id)
284 }
285 #[cfg_attr(not(feature = "backend-openai-realtime"), allow(dead_code))]
286 pub(crate) fn preflight_unindexed(&self, id: &str) -> Result<bool, AsrError> {
287 self.check()?;
288 let s = self.0.state.lock().unwrap();
289 if s.lifecycle.outcome.is_some() {
290 return Ok(false);
291 }
292 s.results.preflight_unindexed(id)
293 }
294 #[cfg_attr(not(feature = "backend-openai-realtime"), allow(dead_code))]
295 pub(crate) fn complete_unindexed(
296 &self,
297 id: impl Into<String>,
298 text: impl Into<String>,
299 ) -> Result<(), AsrError> {
300 self.check()?;
301 let id = id.into();
302 let mut s = self.0.state.lock().unwrap();
303 if s.lifecycle.outcome.is_some() {
304 return Ok(());
305 }
306 let removed_partial = s.results.has_partial(&id);
307 if s.results.complete_unindexed(id.clone(), text.into())? && removed_partial {
308 observe::partial_removed_without_segment(&mut s, &id);
309 }
310 drop(s);
311 self.signal();
312 Ok(())
313 }
314 #[cfg_attr(not(feature = "backend-openai-realtime"), allow(dead_code))]
315 pub(crate) fn index_unindexed(
316 &self,
317 index: u64,
318 id: &str,
319 start: Option<f64>,
320 end: Option<f64>,
321 ) -> Result<(), AsrError> {
322 self.check()?;
323 let mut s = self.0.state.lock().unwrap();
324 if s.lifecycle.outcome.is_some() {
325 return Ok(());
326 }
327 let update = s.results.index_unindexed(index, id, start, end)?;
328 observe::final_segments(
329 &mut s,
330 id,
331 update.removed_partial,
332 &update.segments,
333 self.0.options.event_capacity,
334 );
335 drop(s);
336 self.signal();
337 Ok(())
338 }
339 pub(crate) fn commit(
340 &self,
341 index: u64,
342 id: impl Into<String>,
343 text: impl Into<String>,
344 start: Option<f64>,
345 end: Option<f64>,
346 ) -> Result<(), AsrError> {
347 self.check()?;
348 let id = id.into();
349 let text = text.into();
350 let mut s = self.0.state.lock().unwrap();
351 if s.lifecycle.outcome.is_some() {
352 return Ok(());
353 }
354 let update = s.results.commit(index, id.clone(), text, start, end)?;
355 observe::final_segments(
356 &mut s,
357 &id,
358 update.removed_partial,
359 &update.segments,
360 self.0.options.event_capacity,
361 );
362 drop(s);
363 self.signal();
364 Ok(())
365 }
366
367 pub(crate) fn preflight_commit(
368 &self,
369 index: u64,
370 id: &str,
371 ) -> Result<CommitPreflight, AsrError> {
372 self.check()?;
373 let s = self.0.state.lock().unwrap();
374 if s.lifecycle.outcome.is_some() {
375 return Ok(CommitPreflight::Duplicate);
376 }
377 s.results.preflight_commit(index, id)
378 }
379}
380
381pub struct Session {
382 control: Control,
383 input: AudioInput,
386}
387impl Session {
388 pub fn input(&self) -> AudioInput {
389 self.input.clone()
390 }
391 pub fn subscribe(&self) -> Option<Subscription> {
412 let mut state = self.control.0.state.lock().unwrap();
413 if state.observer.subscription_taken {
414 return None;
415 }
416 state.observer.subscription_taken = true;
417 observe::enable(&mut state);
418 drop(state);
419 self.control.signal();
420 Some(Subscription {
421 control: self.control.clone(),
422 })
423 }
424 pub fn cancel(&self) {
425 self.control.fail(AsrError::new(
426 ErrorKind::Cancelled,
427 "session",
428 "session cancelled",
429 ));
430 }
431 pub fn finish(&self, deadline: Instant) -> SessionResult {
432 self.request_finish(deadline);
433 let mut s = self.control.0.state.lock().unwrap();
434 loop {
435 if let Some(result) = &s.lifecycle.outcome {
436 return result.clone();
437 }
438 let deadline = s.lifecycle.deadline.unwrap_or(deadline);
444 let now = Instant::now();
445 if now >= deadline {
446 self.control.settle_once_locked(
447 &mut s,
448 Some(AsrError::new(
449 ErrorKind::Timeout,
450 "finish",
451 "session finish deadline exceeded",
452 )),
453 );
454 self.control.signal();
455 continue;
456 }
457 s = self
458 .control
459 .0
460 .changed
461 .wait_timeout(s, deadline - now)
462 .unwrap()
463 .0;
464 }
465 }
466 fn request_finish(&self, deadline: Instant) {
467 let mut s = self.control.0.state.lock().unwrap();
468 if s.lifecycle.outcome.is_none() {
469 s.lifecycle.accepting = false;
470 s.lifecycle.finishing = true;
471 s.lifecycle.deadline = Some(
472 s.lifecycle
473 .deadline
474 .map_or(deadline, |current| current.min(deadline)),
475 );
476 s.lifecycle.phase = SessionPhase::Finishing;
477 observe::phase(&mut s, self.control.0.options.event_capacity);
478 }
479 drop(s);
480 self.control.signal();
481 }
482 pub(crate) fn fail_input(&self, error: AsrError) -> SessionResult {
483 let mut state = self.control.0.state.lock().unwrap();
484 self.control.settle_once_locked(&mut state, Some(error));
485 let result = state.lifecycle.outcome.as_ref().unwrap().clone();
486 drop(state);
487 self.control.signal();
488 result
489 }
490 pub(crate) fn control(&self) -> Control {
494 self.control.clone()
495 }
496}
497impl Drop for Session {
498 fn drop(&mut self) {
499 self.cancel();
500 }
501}
502
503#[derive(Debug)]
504pub struct PushError {
505 pub error: AsrError,
506 pub chunk: AudioChunk,
507}
508impl std::fmt::Display for PushError {
509 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
510 self.error.fmt(f)
511 }
512}
513impl std::error::Error for PushError {
514 fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
515 Some(&self.error)
516 }
517}
518#[derive(Clone)]
519pub struct AudioInput {
520 control: Control,
521}
522impl AudioInput {
523 pub fn try_push(&self, chunk: AudioChunk) -> Result<(), PushError> {
524 self.try_push_inner(chunk, None, false)
525 }
526 pub(crate) fn push_wait_validated(
533 &self,
534 chunk: AudioChunk,
535 deadline: Instant,
536 ) -> Result<(), PushError> {
537 self.push_wait_inner(chunk, deadline, true)
538 }
539 fn try_push_inner(
540 &self,
541 chunk: AudioChunk,
542 receive_deadline: Option<Instant>,
543 samples_validated: bool,
544 ) -> Result<(), PushError> {
545 let options = &self.control.0.options;
546 let error = if chunk.spec != options.input {
547 Some(AsrError::new(
548 ErrorKind::InvalidInput,
549 "input",
550 "input sample rate changed",
551 ))
552 } else if chunk.samples.len() > self.control.0.max_chunk_frames {
553 Some(AsrError::new(
554 ErrorKind::InvalidInput,
555 "input",
556 "audio chunk exceeds maximum duration",
557 ))
558 } else if !samples_validated
559 && chunk
560 .samples
561 .iter()
562 .any(|s| !s.is_finite() || s.abs() > 1.0)
563 {
564 Some(AsrError::new(
565 ErrorKind::InvalidInput,
566 "input",
567 "audio must contain finite normalized samples in -1..=1",
568 ))
569 } else {
570 None
571 };
572 if let Some(error) = error {
573 return Err(PushError { error, chunk });
574 }
575 let mut s = self.control.0.state.lock().unwrap();
576 let error = if !s.lifecycle.accepting {
577 Some(AsrError::new(
578 ErrorKind::InputClosed,
579 "input",
580 "session input is closed",
581 ))
582 } else if receive_deadline.is_some_and(|deadline| Instant::now() >= deadline) {
583 Some(AsrError::new(
584 ErrorKind::Timeout,
585 "input",
586 "waiting for audio capacity timed out",
587 ))
588 } else if s.input.received.saturating_add(chunk.samples.len() as u64)
589 > self.control.0.max_session_frames
590 {
591 Some(AsrError::new(
592 ErrorKind::ResourceLimit,
593 "input",
594 "maximum session audio duration exceeded",
595 ))
596 } else {
597 None
598 };
599 if let Some(error) = error {
600 return Err(PushError { error, chunk });
601 }
602 if chunk.samples.is_empty() {
606 return Ok(());
607 }
608 if s.input.queue.len() >= MAX_QUEUE_CHUNKS
609 || chunk.samples.len() > self.control.0.capacity - s.input.queued
610 {
611 return Err(PushError {
612 error: AsrError::new(ErrorKind::WouldBlock, "input", "audio queue is full"),
613 chunk,
614 });
615 }
616 s.input.received += chunk.samples.len() as u64;
617 s.input.queued += chunk.samples.len();
618 s.input.queue.push_back(chunk);
619 drop(s);
620 self.control.signal();
621 Ok(())
622 }
623 pub fn push_wait(&self, chunk: AudioChunk, deadline: Instant) -> Result<(), PushError> {
624 self.push_wait_inner(chunk, deadline, false)
625 }
626 fn push_wait_inner(
627 &self,
628 mut chunk: AudioChunk,
629 deadline: Instant,
630 samples_validated: bool,
631 ) -> Result<(), PushError> {
632 loop {
633 match self.try_push_inner(chunk, Some(deadline), samples_validated) {
634 Ok(()) => return Ok(()),
635 Err(e) if e.error.kind == ErrorKind::WouldBlock => {
636 chunk = e.chunk;
637 }
638 Err(e) => return Err(e),
639 }
640 let s = self.control.0.state.lock().unwrap();
641 if !s.lifecycle.accepting || Instant::now() >= deadline {
642 drop(s);
643 continue;
644 }
645 if s.lifecycle.accepting
647 && (s.input.queue.len() >= MAX_QUEUE_CHUNKS
648 || chunk.samples.len() > self.control.0.capacity - s.input.queued)
649 {
650 let _ = self
651 .control
652 .0
653 .changed
654 .wait_timeout(s, deadline.saturating_duration_since(Instant::now()))
655 .unwrap();
656 }
657 }
658 }
659 #[cfg(feature = "capture-cpal")]
660 pub(crate) fn fail(&self, error: AsrError) {
661 self.control.fail(error);
662 }
663}
664pub struct Subscription {
665 control: Control,
666}
667impl Subscription {
668 pub fn view(&self) -> SessionView {
673 observe::current_view(&self.control.0.state.lock().unwrap())
674 }
675
676 pub fn recv(&mut self) -> Option<Update> {
677 let mut state = self.control.0.state.lock().unwrap();
678 loop {
679 match observe::take_next(&mut state) {
680 observe::Delivery::Update(update) => return Some(update),
681 observe::Delivery::Disconnected => return None,
682 observe::Delivery::Pending => {
683 state = self.control.0.changed.wait(state).unwrap();
684 }
685 }
686 }
687 }
688
689 pub fn recv_timeout(
690 &mut self,
691 timeout: Duration,
692 ) -> Result<Update, std::sync::mpsc::RecvTimeoutError> {
693 let deadline = Instant::now().checked_add(timeout);
696 let mut state = self.control.0.state.lock().unwrap();
697 loop {
698 match observe::take_next(&mut state) {
699 observe::Delivery::Update(update) => return Ok(update),
700 observe::Delivery::Disconnected => {
701 return Err(std::sync::mpsc::RecvTimeoutError::Disconnected)
702 }
703 observe::Delivery::Pending => {}
704 }
705 if deadline.is_some_and(|deadline| Instant::now() >= deadline) {
706 return Err(std::sync::mpsc::RecvTimeoutError::Timeout);
707 }
708 state = match deadline {
709 Some(deadline) => {
710 self.control
711 .0
712 .changed
713 .wait_timeout(state, deadline.saturating_duration_since(Instant::now()))
714 .unwrap()
715 .0
716 }
717 None => self.control.0.changed.wait(state).unwrap(),
718 };
719 }
720 }
721}
722impl Drop for Subscription {
723 fn drop(&mut self) {
724 let mut state = self.control.0.state.lock().unwrap();
725 observe::disable(&mut state);
726 }
727}
728#[cfg(test)]
729pub(crate) fn spawn(
730 driver: Box<dyn Driver>,
731 audio: AudioSpec,
732 options: SessionOptions,
733 final_text_processor: Option<FinalTextProcessor>,
734 permit: impl Send + 'static,
735) -> Result<Session, AsrError> {
736 spawn_with_deadline(driver, audio, options, final_text_processor, permit, None)
737}
738
739pub(crate) fn spawn_with_deadline(
740 driver: Box<dyn Driver>,
741 audio: AudioSpec,
742 options: SessionOptions,
743 final_text_processor: Option<FinalTextProcessor>,
744 permit: impl Send + 'static,
745 deadline: Option<Instant>,
746) -> Result<Session, AsrError> {
747 AudioSpec::mono(options.input.sample_rate)?;
748 AudioSpec::mono(audio.sample_rate)?;
749
750 if options.event_capacity == 0 {
751 return Err(AsrError::invalid(
752 "event_capacity must be greater than zero",
753 ));
754 }
755
756 if options.max_transcript_bytes == 0 {
757 return Err(AsrError::invalid(
758 "max_transcript_bytes must be greater than zero",
759 ));
760 }
761
762 if options.queue_duration.is_zero() {
763 return Err(AsrError::invalid(
764 "queue_duration must be greater than zero",
765 ));
766 }
767
768 if options.max_chunk_duration.is_zero() {
769 return Err(AsrError::invalid(
770 "max_chunk_duration must be greater than zero",
771 ));
772 }
773
774 if options.max_duration.is_zero() {
775 return Err(AsrError::invalid("max_duration must be greater than zero"));
776 }
777
778 if options.max_duration > Duration::from_secs(24 * 3600) {
779 return Err(AsrError::invalid("max_duration must not exceed 24 hours"));
780 }
781
782 if options.max_chunk_duration > options.queue_duration {
783 return Err(AsrError::invalid(
784 "max_chunk_duration must not exceed queue_duration",
785 ));
786 }
787
788 let geometry = session_geometry(&options)?;
789 let SessionGeometry {
790 capacity,
791 max_chunk_frames,
792 max_session_frames,
793 } = geometry;
794
795 if capacity > MAX_QUEUE_FRAMES {
796 return Err(AsrError::invalid(format!(
797 "queue_duration requires {capacity} frames, exceeding the \
798 configured safety limit of {MAX_QUEUE_FRAMES} frames"
799 )));
800 }
801
802 let mut state = State::new(options.max_transcript_bytes);
803 state.lifecycle.deadline = deadline;
804 let control = Control(Arc::new(Shared {
805 state: Mutex::new(state),
806 changed: Condvar::new(),
807 options,
808 capacity,
809 max_chunk_frames,
810 max_session_frames,
811 #[cfg(any(
812 feature = "backend-dashscope",
813 feature = "backend-openai-http",
814 feature = "backend-openai-realtime"
815 ))]
816 wake: tokio::sync::Notify::new(),
817 }));
818 let worker = control.clone();
819 std::thread::Builder::new()
820 .name("asr-core-session".into())
821 .spawn(move || {
822 let _permit = permit;
823 let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
824 crate::session::worker::run(driver, audio, &worker, final_text_processor)
825 }));
826 match result {
827 Ok(Ok(())) => worker.complete(),
828 Ok(Err(e)) => worker.fail(e),
829 Err(_) => worker.fail(AsrError::backend("backend worker panicked")),
830 }
831 })
832 .map_err(|e| AsrError::new(ErrorKind::Io, "start", e.to_string()))?;
833 let input = AudioInput {
834 control: control.clone(),
835 };
836 Ok(Session { control, input })
837}
838
839#[cfg(test)]
840mod tests;