1use serde::{Deserialize, Serialize};
47use serde_json::json;
48use std::net::SocketAddr;
49use std::sync::Arc;
50use std::time::{Duration, Instant};
51
52use axum::extract::{
53 multipart::MultipartError, DefaultBodyLimit, FromRequest, Multipart, Request, State,
54};
55use axum::http::StatusCode;
56use axum::response::sse::{Event, KeepAlive, Sse};
57use axum::response::{IntoResponse, Response};
58use axum::routing::{get, post};
59use axum::{Json, Router};
60use futures::StreamExt;
61
62use crate::{
63 AsrError, AudioBuffer, ErrorKind, SessionFailure, SessionOptions, SessionOutcome,
64 SessionResult, TranscriptSnapshotSink,
65};
66
67pub mod delta;
68use delta::TextDeltaBuilder;
69
70pub trait Transcribe: Send + Sync {
76 fn transcribe(
77 &self,
78 buffer: &AudioBuffer,
79 options: SessionOptions,
80 deadline: Instant,
81 ) -> SessionResult;
82
83 fn supports_streaming(&self) -> bool {
90 false
91 }
92
93 fn transcribe_streaming(
100 &self,
101 buffer: &AudioBuffer,
102 options: SessionOptions,
103 deadline: Instant,
104 progress: TranscriptSnapshotSink,
105 ) -> SessionResult {
106 let _ = (buffer, options, deadline, progress);
107 Err(Box::new(SessionFailure {
108 error: AsrError::new(
109 ErrorKind::UnsupportedCapability,
110 "serve",
111 "this engine wrapper does not expose streaming progress",
112 ),
113 outcome: SessionOutcome::default(),
114 }))
115 }
116}
117
118impl Transcribe for crate::Engine {
119 fn transcribe(
120 &self,
121 buffer: &AudioBuffer,
122 options: SessionOptions,
123 deadline: Instant,
124 ) -> SessionResult {
125 crate::Engine::transcribe(self, buffer, options, deadline)
126 }
127
128 fn supports_streaming(&self) -> bool {
129 self.capabilities().streaming_transcription
133 }
134
135 fn transcribe_streaming(
136 &self,
137 buffer: &AudioBuffer,
138 options: SessionOptions,
139 deadline: Instant,
140 progress: TranscriptSnapshotSink,
141 ) -> SessionResult {
142 crate::Engine::transcribe_streaming(self, buffer, options, deadline, progress)
143 }
144}
145
146fn default_bind() -> String {
147 "127.0.0.1:8080".to_string()
148}
149
150fn default_max_body_bytes() -> usize {
151 25 * 1024 * 1024
152}
153
154fn default_max_audio_secs() -> u64 {
155 3600
156}
157
158fn default_model_id() -> String {
159 "asr-core".to_string()
160}
161
162fn default_family() -> String {
163 "unknown".to_string()
164}
165
166#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
172#[serde(deny_unknown_fields)]
173pub struct ServeOptions {
174 #[serde(default = "default_bind")]
176 pub bind: String,
177 #[serde(default = "default_max_body_bytes")]
182 pub max_body_bytes: usize,
183 #[serde(default = "default_max_audio_secs")]
187 pub max_audio_secs: u64,
188 pub timeout_secs: Option<u64>,
192 pub language: Option<String>,
194 #[serde(default = "default_model_id")]
196 pub model_id: String,
197 #[serde(default = "default_family")]
199 pub family: String,
200}
201
202impl Default for ServeOptions {
203 fn default() -> Self {
204 Self {
205 bind: default_bind(),
206 max_body_bytes: default_max_body_bytes(),
207 max_audio_secs: default_max_audio_secs(),
208 timeout_secs: None,
209 language: None,
210 model_id: default_model_id(),
211 family: default_family(),
212 }
213 }
214}
215
216pub fn router(engine: Arc<dyn Transcribe>, options: ServeOptions) -> Router {
220 let max_body_bytes = options.max_body_bytes;
221 let state = Arc::new(AppState { engine, options });
222 Router::new()
223 .route("/v1/audio/transcriptions", post(transcriptions))
224 .route("/v1/models", get(models_list))
225 .route("/health", get(health))
226 .layer(DefaultBodyLimit::max(max_body_bytes))
227 .with_state(state)
228}
229
230struct AppState {
231 engine: Arc<dyn Transcribe>,
232 options: ServeOptions,
233}
234
235async fn transcriptions(State(state): State<Arc<AppState>>, req: Request) -> Response {
236 let mut multipart = match Multipart::from_request(req, &state).await {
243 Ok(multipart) => multipart,
244 Err(error) => {
245 return error_response(
246 StatusCode::BAD_REQUEST,
247 format!("invalid multipart request: {error}"),
248 )
249 }
250 };
251 let mut file: Option<axum::body::Bytes> = None;
252 let mut language: Option<String> = None;
253 let mut response_format = "json".to_string();
254 let mut stream = false;
255 loop {
256 let field = match multipart.next_field().await {
257 Ok(Some(field)) => field,
258 Ok(None) => break,
259 Err(error) => return multipart_error(&error),
260 };
261 match field.name() {
262 Some("file") => match field.bytes().await {
263 Ok(bytes) => file = Some(bytes),
264 Err(error) => return multipart_error(&error),
265 },
266 Some("language") => match field.text().await {
267 Ok(text) => language = Some(text).filter(|s| !s.trim().is_empty()),
268 Err(error) => return multipart_error(&error),
269 },
270 Some("response_format") => match field.text().await {
271 Ok(text) => response_format = text,
272 Err(error) => return multipart_error(&error),
273 },
274 Some("stream") => match field.text().await {
275 Ok(text) => stream = matches!(text.trim(), "true" | "1"),
276 Err(error) => return multipart_error(&error),
277 },
278 _ => {}
282 }
283 }
284 let Some(bytes) = file else {
285 return error_response(
286 StatusCode::BAD_REQUEST,
287 "multipart field 'file' is required",
288 );
289 };
290 if response_format != "json" && response_format != "text" {
291 return error_response(
292 StatusCode::BAD_REQUEST,
293 format!(
294 "unsupported response_format '{response_format}'; only 'json' and \
295 'text' are supported"
296 ),
297 );
298 }
299 if stream && response_format != "json" {
300 return error_response(
301 StatusCode::BAD_REQUEST,
302 "streaming requires response_format 'json' (SSE data frames carry JSON \
303 events); drop 'stream' or use response_format 'json'",
304 );
305 }
306 if let Some(requested) = &language {
307 match &state.options.language {
308 Some(pinned) if pinned == requested => {}
309 Some(pinned) => {
310 return error_response(
311 StatusCode::BAD_REQUEST,
312 format!(
313 "this server pins language '{pinned}'; the engine cannot \
314 switch languages at runtime — restart it with language \
315 '{requested}'"
316 ),
317 );
318 }
319 None => {
320 return error_response(
321 StatusCode::BAD_REQUEST,
322 "this server was started without a pinned language; the engine \
323 cannot switch languages at runtime — restart it with language \
324 '<lang>'",
325 );
326 }
327 }
328 }
329 let engine = state.engine.clone();
330 let options = state.options.clone();
331 let buffer = match tokio::task::spawn_blocking({
335 let options = options.clone();
336 move || decode_upload(&bytes, options.max_audio_secs)
337 })
338 .await
339 {
340 Ok(Ok(buffer)) => buffer,
341 Ok(Err(error)) => {
342 return error_response(
343 StatusCode::BAD_REQUEST,
344 format!("audio decoding failed: {error}"),
345 )
346 }
347 Err(join) => {
348 return error_response(
349 StatusCode::INTERNAL_SERVER_ERROR,
350 format!("decoding task failed: {join}"),
351 )
352 }
353 };
354 let (session_options, budget) = session_plan(&buffer, &options);
355 let deadline = Instant::now() + budget;
356 if stream {
357 if !state.engine.supports_streaming() {
361 return error_response(
362 StatusCode::BAD_REQUEST,
363 "streaming responses are not supported by this engine (no incremental \
364 endpoint); drop 'stream' and read the json body",
365 );
366 }
367 return streaming_transcription(state.engine.clone(), buffer, session_options, deadline);
368 }
369 let outcome = tokio::time::timeout(
376 budget,
377 tokio::task::spawn_blocking(move || {
378 engine
379 .transcribe(&buffer, session_options, deadline)
380 .map(|outcome| outcome.transcript.text())
381 .map_err(|failure| engine_failure(&failure))
382 }),
383 )
384 .await;
385 match outcome {
386 Ok(Ok(Ok(text))) if response_format == "text" => text.into_response(),
387 Ok(Ok(Ok(text))) => Json(json!({ "text": text })).into_response(),
388 Ok(Ok(Err(failure))) => failure_response(failure),
389 Ok(Err(join)) => error_response(
390 StatusCode::INTERNAL_SERVER_ERROR,
391 format!("transcription task failed: {join}"),
392 ),
393 Err(_elapsed) => error_response(
394 StatusCode::GATEWAY_TIMEOUT,
395 format!("transcription exceeded the request budget ({budget:?})"),
396 ),
397 }
398}
399
400fn streaming_transcription(
416 engine: Arc<dyn Transcribe>,
417 buffer: AudioBuffer,
418 session_options: SessionOptions,
419 deadline: Instant,
420) -> Response {
421 let (progress_tx, progress_rx) = tokio::sync::watch::channel(String::new());
422 let sink = TranscriptSnapshotSink::new(move |text| {
423 progress_tx.send_replace(text.to_string());
424 !progress_tx.is_closed()
425 });
426 let handle = tokio::task::spawn_blocking(move || {
427 engine
428 .transcribe_streaming(&buffer, session_options, deadline, sink)
429 .map(|outcome| outcome.transcript.text())
430 .map_err(|failure| engine_failure(&failure))
431 });
432 let events = futures::stream::unfold(
436 (Some(handle), progress_rx, TextDeltaBuilder::new()),
437 |(handle, mut progress, mut deltas)| async move {
438 let mut handle = handle;
439 handle.as_ref()?;
441 let event = loop {
442 if progress.changed().await.is_err() {
447 match handle.take() {
448 Some(handle) => break terminal_event(handle).await,
449 None => return None,
450 }
451 }
452 let snapshot = progress.borrow_and_update().clone();
453 let Some(delta) = deltas.update(&snapshot) else {
457 continue;
458 };
459 break sse_event("transcript.text.delta", json!({ "delta": delta }));
460 };
461 Some((event, (handle, progress, deltas)))
463 },
464 );
465 Sse::new(events.map(Ok::<_, std::convert::Infallible>))
466 .keep_alive(KeepAlive::default())
467 .into_response()
468}
469
470async fn terminal_event(handle: tokio::task::JoinHandle<Result<String, RequestFailure>>) -> Event {
473 match handle.await {
474 Ok(Ok(text)) => sse_event("transcript.text.done", json!({ "text": text })),
475 Ok(Err(failure)) => sse_event("error", json!({ "message": failure_message(&failure) })),
476 Err(join) => sse_event(
477 "error",
478 json!({ "message": format!("transcription task failed: {join}") }),
479 ),
480 }
481}
482
483fn failure_message(failure: &RequestFailure) -> String {
488 match failure {
489 RequestFailure::BadRequest(message)
490 | RequestFailure::Busy(message)
491 | RequestFailure::Timeout(message)
492 | RequestFailure::Engine(message) => message.clone(),
493 }
494}
495
496fn sse_event(name: &str, payload: serde_json::Value) -> Event {
497 let mut payload = payload;
502 if let Some(object) = payload.as_object_mut() {
503 object.insert("type".to_string(), json!(name));
504 }
505 Event::default().event(name).data(payload.to_string())
506}
507
508enum RequestFailure {
510 BadRequest(String),
511 Busy(String),
512 Timeout(String),
513 Engine(String),
514}
515
516fn failure_response(failure: RequestFailure) -> Response {
517 match failure {
518 RequestFailure::BadRequest(message) => error_response(StatusCode::BAD_REQUEST, message),
519 RequestFailure::Busy(message) => error_response(StatusCode::SERVICE_UNAVAILABLE, message),
520 RequestFailure::Timeout(message) => error_response(StatusCode::GATEWAY_TIMEOUT, message),
521 RequestFailure::Engine(message) => {
522 error_response(StatusCode::INTERNAL_SERVER_ERROR, message)
523 }
524 }
525}
526
527fn session_plan(buffer: &AudioBuffer, options: &ServeOptions) -> (SessionOptions, Duration) {
533 let audio_secs = buffer.samples.len() as u64 / u64::from(buffer.spec.sample_rate.max(1));
534 let mut session_options = SessionOptions::new(buffer.spec);
535 session_options.max_duration = Duration::from_secs(audio_secs + 60);
536 session_options.max_transcript_bytes = 2 * 1024 * 1024;
539 let budget = options.timeout_secs.map_or_else(
540 || Duration::from_secs((audio_secs * 3).max(120)),
541 Duration::from_secs,
542 );
543 (session_options, budget)
544}
545
546fn engine_failure(failure: &SessionFailure) -> RequestFailure {
550 let message = format!("transcription failed: {}", failure.error);
551 match failure.error.kind {
552 ErrorKind::Busy => RequestFailure::Busy(
553 "the server is at its concurrent-session limit; retry later".to_string(),
554 ),
555 ErrorKind::Timeout => RequestFailure::Timeout(message),
556 ErrorKind::InvalidInput => RequestFailure::BadRequest(message),
557 _ => RequestFailure::Engine(message),
558 }
559}
560
561fn decode_upload(bytes: &[u8], max_audio_secs: u64) -> Result<AudioBuffer, AsrError> {
569 if is_webm(bytes) {
570 return Err(AsrError::new(
571 ErrorKind::InvalidInput,
572 "serve",
573 "webm audio uses opus, which cannot be decoded here; transcode to \
574 wav, flac or mp3 first",
575 ));
576 }
577 let buffer = crate::audio::decode_audio(bytes)?;
578 if buffer.samples.is_empty() {
579 return Err(AsrError::new(
580 ErrorKind::InvalidInput,
581 "serve",
582 "no audio could be decoded from the input",
583 ));
584 }
585 let max_samples = max_audio_secs.saturating_mul(48_000);
586 if buffer.samples.len() as u64 > max_samples {
587 return Err(AsrError::new(
588 ErrorKind::InvalidInput,
589 "serve",
590 format!(
591 "decoded audio exceeds the sample budget ({max_samples} samples); \
592 the input is too long for offline decoding"
593 ),
594 ));
595 }
596 Ok(buffer)
597}
598
599fn is_webm(bytes: &[u8]) -> bool {
608 if !bytes.starts_with(&[0x1A, 0x45, 0xDF, 0xA3]) {
609 return false;
610 }
611 let head = &bytes[..bytes.len().min(128)];
612 head.windows(7)
613 .any(|w| w == [0x42, 0x82, 0x84, b'w', b'e', b'b', b'm'])
614}
615
616async fn health(State(state): State<Arc<AppState>>) -> Response {
617 Json(json!({
618 "status": "ok",
619 "model": state.options.model_id,
620 "family": state.options.family,
621 }))
622 .into_response()
623}
624
625async fn models_list(State(state): State<Arc<AppState>>) -> Response {
626 Json(json!({
627 "object": "list",
628 "data": [{
629 "id": state.options.model_id,
630 "object": "model",
631 "created": 0,
632 "owned_by": "asr-core",
633 }],
634 }))
635 .into_response()
636}
637
638fn error_response(status: StatusCode, message: impl std::fmt::Display) -> Response {
643 let kind = if status.is_client_error() {
644 "invalid_request_error"
645 } else {
646 "server_error"
647 };
648 (
649 status,
650 Json(json!({ "error": { "message": message.to_string(), "type": kind } })),
651 )
652 .into_response()
653}
654
655fn multipart_error(error: &MultipartError) -> Response {
658 let status = error.status();
659 let message = if status == StatusCode::PAYLOAD_TOO_LARGE {
660 format!("request body exceeds the server's size limit (max_body_bytes): {error}")
661 } else {
662 format!("invalid multipart request: {error}")
663 };
664 error_response(status, message)
665}
666
667async fn shutdown_signal() {
672 let ctrl_c = async {
673 let _ = tokio::signal::ctrl_c().await;
674 };
675 #[cfg(unix)]
676 let sigterm = async {
677 use tokio::signal::unix::{signal, SignalKind};
678 match signal(SignalKind::terminate()) {
679 Ok(mut term) => {
680 term.recv().await;
681 }
682 Err(_) => std::future::pending::<()>().await,
686 }
687 };
688 #[cfg(not(unix))]
689 let sigterm = std::future::pending::<()>();
690 tokio::select! {
691 _ = ctrl_c => {},
692 _ = sigterm => {},
693 }
694}
695
696pub async fn run(engine: Arc<dyn Transcribe>, options: ServeOptions) -> Result<(), AsrError> {
701 let address: SocketAddr = options.bind.parse().map_err(|_| {
702 AsrError::new(
703 ErrorKind::InvalidInput,
704 "serve",
705 format!("invalid bind address '{}'", options.bind),
706 )
707 })?;
708 let app = router(engine, options);
709 let listener = tokio::net::TcpListener::bind(address).await.map_err(|e| {
710 AsrError::new(
711 ErrorKind::Io,
712 "serve",
713 format!("cannot bind {address}: {e}"),
714 )
715 })?;
716 axum::serve(listener, app)
717 .with_graceful_shutdown(shutdown_signal())
718 .await
719 .map_err(|e| AsrError::new(ErrorKind::Io, "serve", format!("server error: {e}")))?;
720 Ok(())
721}
722
723#[cfg(test)]
724mod tests;