use serde::{Deserialize, Serialize};
use serde_json::json;
use std::net::SocketAddr;
use std::sync::Arc;
use std::time::{Duration, Instant};
use axum::extract::{
multipart::MultipartError, DefaultBodyLimit, FromRequest, Multipart, Request, State,
};
use axum::http::StatusCode;
use axum::response::sse::{Event, KeepAlive, Sse};
use axum::response::{IntoResponse, Response};
use axum::routing::{get, post};
use axum::{Json, Router};
use futures::StreamExt;
use crate::{
AsrError, AudioBuffer, ErrorKind, SessionFailure, SessionOptions, SessionOutcome,
SessionResult, TranscriptSnapshotSink,
};
pub mod delta;
use delta::TextDeltaBuilder;
pub trait Transcribe: Send + Sync {
fn transcribe(
&self,
buffer: &AudioBuffer,
options: SessionOptions,
deadline: Instant,
) -> SessionResult;
fn supports_streaming(&self) -> bool {
false
}
fn transcribe_streaming(
&self,
buffer: &AudioBuffer,
options: SessionOptions,
deadline: Instant,
progress: TranscriptSnapshotSink,
) -> SessionResult {
let _ = (buffer, options, deadline, progress);
Err(Box::new(SessionFailure {
error: AsrError::new(
ErrorKind::UnsupportedCapability,
"serve",
"this engine wrapper does not expose streaming progress",
),
outcome: SessionOutcome::default(),
}))
}
}
impl Transcribe for crate::Engine {
fn transcribe(
&self,
buffer: &AudioBuffer,
options: SessionOptions,
deadline: Instant,
) -> SessionResult {
crate::Engine::transcribe(self, buffer, options, deadline)
}
fn supports_streaming(&self) -> bool {
self.capabilities().streaming_transcription
}
fn transcribe_streaming(
&self,
buffer: &AudioBuffer,
options: SessionOptions,
deadline: Instant,
progress: TranscriptSnapshotSink,
) -> SessionResult {
crate::Engine::transcribe_streaming(self, buffer, options, deadline, progress)
}
}
fn default_bind() -> String {
"127.0.0.1:8080".to_string()
}
fn default_max_body_bytes() -> usize {
25 * 1024 * 1024
}
fn default_max_audio_secs() -> u64 {
3600
}
fn default_model_id() -> String {
"asr-core".to_string()
}
fn default_family() -> String {
"unknown".to_string()
}
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
#[serde(deny_unknown_fields)]
pub struct ServeOptions {
#[serde(default = "default_bind")]
pub bind: String,
#[serde(default = "default_max_body_bytes")]
pub max_body_bytes: usize,
#[serde(default = "default_max_audio_secs")]
pub max_audio_secs: u64,
pub timeout_secs: Option<u64>,
pub language: Option<String>,
#[serde(default = "default_model_id")]
pub model_id: String,
#[serde(default = "default_family")]
pub family: String,
}
impl Default for ServeOptions {
fn default() -> Self {
Self {
bind: default_bind(),
max_body_bytes: default_max_body_bytes(),
max_audio_secs: default_max_audio_secs(),
timeout_secs: None,
language: None,
model_id: default_model_id(),
family: default_family(),
}
}
}
pub fn router(engine: Arc<dyn Transcribe>, options: ServeOptions) -> Router {
let max_body_bytes = options.max_body_bytes;
let state = Arc::new(AppState { engine, options });
Router::new()
.route("/v1/audio/transcriptions", post(transcriptions))
.route("/v1/models", get(models_list))
.route("/health", get(health))
.layer(DefaultBodyLimit::max(max_body_bytes))
.with_state(state)
}
struct AppState {
engine: Arc<dyn Transcribe>,
options: ServeOptions,
}
async fn transcriptions(State(state): State<Arc<AppState>>, req: Request) -> Response {
let mut multipart = match Multipart::from_request(req, &state).await {
Ok(multipart) => multipart,
Err(error) => {
return error_response(
StatusCode::BAD_REQUEST,
format!("invalid multipart request: {error}"),
)
}
};
let mut file: Option<axum::body::Bytes> = None;
let mut language: Option<String> = None;
let mut response_format = "json".to_string();
let mut stream = false;
loop {
let field = match multipart.next_field().await {
Ok(Some(field)) => field,
Ok(None) => break,
Err(error) => return multipart_error(&error),
};
match field.name() {
Some("file") => match field.bytes().await {
Ok(bytes) => file = Some(bytes),
Err(error) => return multipart_error(&error),
},
Some("language") => match field.text().await {
Ok(text) => language = Some(text).filter(|s| !s.trim().is_empty()),
Err(error) => return multipart_error(&error),
},
Some("response_format") => match field.text().await {
Ok(text) => response_format = text,
Err(error) => return multipart_error(&error),
},
Some("stream") => match field.text().await {
Ok(text) => stream = matches!(text.trim(), "true" | "1"),
Err(error) => return multipart_error(&error),
},
_ => {}
}
}
let Some(bytes) = file else {
return error_response(
StatusCode::BAD_REQUEST,
"multipart field 'file' is required",
);
};
if response_format != "json" && response_format != "text" {
return error_response(
StatusCode::BAD_REQUEST,
format!(
"unsupported response_format '{response_format}'; only 'json' and \
'text' are supported"
),
);
}
if stream && response_format != "json" {
return error_response(
StatusCode::BAD_REQUEST,
"streaming requires response_format 'json' (SSE data frames carry JSON \
events); drop 'stream' or use response_format 'json'",
);
}
if let Some(requested) = &language {
match &state.options.language {
Some(pinned) if pinned == requested => {}
Some(pinned) => {
return error_response(
StatusCode::BAD_REQUEST,
format!(
"this server pins language '{pinned}'; the engine cannot \
switch languages at runtime — restart it with language \
'{requested}'"
),
);
}
None => {
return error_response(
StatusCode::BAD_REQUEST,
"this server was started without a pinned language; the engine \
cannot switch languages at runtime — restart it with language \
'<lang>'",
);
}
}
}
let engine = state.engine.clone();
let options = state.options.clone();
let buffer = match tokio::task::spawn_blocking({
let options = options.clone();
move || decode_upload(&bytes, options.max_audio_secs)
})
.await
{
Ok(Ok(buffer)) => buffer,
Ok(Err(error)) => {
return error_response(
StatusCode::BAD_REQUEST,
format!("audio decoding failed: {error}"),
)
}
Err(join) => {
return error_response(
StatusCode::INTERNAL_SERVER_ERROR,
format!("decoding task failed: {join}"),
)
}
};
let (session_options, budget) = session_plan(&buffer, &options);
let deadline = Instant::now() + budget;
if stream {
if !state.engine.supports_streaming() {
return error_response(
StatusCode::BAD_REQUEST,
"streaming responses are not supported by this engine (no incremental \
endpoint); drop 'stream' and read the json body",
);
}
return streaming_transcription(state.engine.clone(), buffer, session_options, deadline);
}
let outcome = tokio::time::timeout(
budget,
tokio::task::spawn_blocking(move || {
engine
.transcribe(&buffer, session_options, deadline)
.map(|outcome| outcome.transcript.text())
.map_err(|failure| engine_failure(&failure))
}),
)
.await;
match outcome {
Ok(Ok(Ok(text))) if response_format == "text" => text.into_response(),
Ok(Ok(Ok(text))) => Json(json!({ "text": text })).into_response(),
Ok(Ok(Err(failure))) => failure_response(failure),
Ok(Err(join)) => error_response(
StatusCode::INTERNAL_SERVER_ERROR,
format!("transcription task failed: {join}"),
),
Err(_elapsed) => error_response(
StatusCode::GATEWAY_TIMEOUT,
format!("transcription exceeded the request budget ({budget:?})"),
),
}
}
fn streaming_transcription(
engine: Arc<dyn Transcribe>,
buffer: AudioBuffer,
session_options: SessionOptions,
deadline: Instant,
) -> Response {
let (progress_tx, progress_rx) = tokio::sync::watch::channel(String::new());
let sink = TranscriptSnapshotSink::new(move |text| {
progress_tx.send_replace(text.to_string());
!progress_tx.is_closed()
});
let handle = tokio::task::spawn_blocking(move || {
engine
.transcribe_streaming(&buffer, session_options, deadline, sink)
.map(|outcome| outcome.transcript.text())
.map_err(|failure| engine_failure(&failure))
});
let events = futures::stream::unfold(
(Some(handle), progress_rx, TextDeltaBuilder::new()),
|(handle, mut progress, mut deltas)| async move {
let mut handle = handle;
handle.as_ref()?;
let event = loop {
if progress.changed().await.is_err() {
match handle.take() {
Some(handle) => break terminal_event(handle).await,
None => return None,
}
}
let snapshot = progress.borrow_and_update().clone();
let Some(delta) = deltas.update(&snapshot) else {
continue;
};
break sse_event("transcript.text.delta", json!({ "delta": delta }));
};
Some((event, (handle, progress, deltas)))
},
);
Sse::new(events.map(Ok::<_, std::convert::Infallible>))
.keep_alive(KeepAlive::default())
.into_response()
}
async fn terminal_event(handle: tokio::task::JoinHandle<Result<String, RequestFailure>>) -> Event {
match handle.await {
Ok(Ok(text)) => sse_event("transcript.text.done", json!({ "text": text })),
Ok(Err(failure)) => sse_event("error", json!({ "message": failure_message(&failure) })),
Err(join) => sse_event(
"error",
json!({ "message": format!("transcription task failed: {join}") }),
),
}
}
fn failure_message(failure: &RequestFailure) -> String {
match failure {
RequestFailure::BadRequest(message)
| RequestFailure::Busy(message)
| RequestFailure::Timeout(message)
| RequestFailure::Engine(message) => message.clone(),
}
}
fn sse_event(name: &str, payload: serde_json::Value) -> Event {
let mut payload = payload;
if let Some(object) = payload.as_object_mut() {
object.insert("type".to_string(), json!(name));
}
Event::default().event(name).data(payload.to_string())
}
enum RequestFailure {
BadRequest(String),
Busy(String),
Timeout(String),
Engine(String),
}
fn failure_response(failure: RequestFailure) -> Response {
match failure {
RequestFailure::BadRequest(message) => error_response(StatusCode::BAD_REQUEST, message),
RequestFailure::Busy(message) => error_response(StatusCode::SERVICE_UNAVAILABLE, message),
RequestFailure::Timeout(message) => error_response(StatusCode::GATEWAY_TIMEOUT, message),
RequestFailure::Engine(message) => {
error_response(StatusCode::INTERNAL_SERVER_ERROR, message)
}
}
}
fn session_plan(buffer: &AudioBuffer, options: &ServeOptions) -> (SessionOptions, Duration) {
let audio_secs = buffer.samples.len() as u64 / u64::from(buffer.spec.sample_rate.max(1));
let mut session_options = SessionOptions::new(buffer.spec);
session_options.max_duration = Duration::from_secs(audio_secs + 60);
session_options.max_transcript_bytes = 2 * 1024 * 1024;
let budget = options.timeout_secs.map_or_else(
|| Duration::from_secs((audio_secs * 3).max(120)),
Duration::from_secs,
);
(session_options, budget)
}
fn engine_failure(failure: &SessionFailure) -> RequestFailure {
let message = format!("transcription failed: {}", failure.error);
match failure.error.kind {
ErrorKind::Busy => RequestFailure::Busy(
"the server is at its concurrent-session limit; retry later".to_string(),
),
ErrorKind::Timeout => RequestFailure::Timeout(message),
ErrorKind::InvalidInput => RequestFailure::BadRequest(message),
_ => RequestFailure::Engine(message),
}
}
fn decode_upload(bytes: &[u8], max_audio_secs: u64) -> Result<AudioBuffer, AsrError> {
if is_webm(bytes) {
return Err(AsrError::new(
ErrorKind::InvalidInput,
"serve",
"webm audio uses opus, which cannot be decoded here; transcode to \
wav, flac or mp3 first",
));
}
let buffer = crate::audio::decode_audio(bytes)?;
if buffer.samples.is_empty() {
return Err(AsrError::new(
ErrorKind::InvalidInput,
"serve",
"no audio could be decoded from the input",
));
}
let max_samples = max_audio_secs.saturating_mul(48_000);
if buffer.samples.len() as u64 > max_samples {
return Err(AsrError::new(
ErrorKind::InvalidInput,
"serve",
format!(
"decoded audio exceeds the sample budget ({max_samples} samples); \
the input is too long for offline decoding"
),
));
}
Ok(buffer)
}
fn is_webm(bytes: &[u8]) -> bool {
if !bytes.starts_with(&[0x1A, 0x45, 0xDF, 0xA3]) {
return false;
}
let head = &bytes[..bytes.len().min(128)];
head.windows(7)
.any(|w| w == [0x42, 0x82, 0x84, b'w', b'e', b'b', b'm'])
}
async fn health(State(state): State<Arc<AppState>>) -> Response {
Json(json!({
"status": "ok",
"model": state.options.model_id,
"family": state.options.family,
}))
.into_response()
}
async fn models_list(State(state): State<Arc<AppState>>) -> Response {
Json(json!({
"object": "list",
"data": [{
"id": state.options.model_id,
"object": "model",
"created": 0,
"owned_by": "asr-core",
}],
}))
.into_response()
}
fn error_response(status: StatusCode, message: impl std::fmt::Display) -> Response {
let kind = if status.is_client_error() {
"invalid_request_error"
} else {
"server_error"
};
(
status,
Json(json!({ "error": { "message": message.to_string(), "type": kind } })),
)
.into_response()
}
fn multipart_error(error: &MultipartError) -> Response {
let status = error.status();
let message = if status == StatusCode::PAYLOAD_TOO_LARGE {
format!("request body exceeds the server's size limit (max_body_bytes): {error}")
} else {
format!("invalid multipart request: {error}")
};
error_response(status, message)
}
async fn shutdown_signal() {
let ctrl_c = async {
let _ = tokio::signal::ctrl_c().await;
};
#[cfg(unix)]
let sigterm = async {
use tokio::signal::unix::{signal, SignalKind};
match signal(SignalKind::terminate()) {
Ok(mut term) => {
term.recv().await;
}
Err(_) => std::future::pending::<()>().await,
}
};
#[cfg(not(unix))]
let sigterm = std::future::pending::<()>();
tokio::select! {
_ = ctrl_c => {},
_ = sigterm => {},
}
}
pub async fn run(engine: Arc<dyn Transcribe>, options: ServeOptions) -> Result<(), AsrError> {
let address: SocketAddr = options.bind.parse().map_err(|_| {
AsrError::new(
ErrorKind::InvalidInput,
"serve",
format!("invalid bind address '{}'", options.bind),
)
})?;
let app = router(engine, options);
let listener = tokio::net::TcpListener::bind(address).await.map_err(|e| {
AsrError::new(
ErrorKind::Io,
"serve",
format!("cannot bind {address}: {e}"),
)
})?;
axum::serve(listener, app)
.with_graceful_shutdown(shutdown_signal())
.await
.map_err(|e| AsrError::new(ErrorKind::Io, "serve", format!("server error: {e}")))?;
Ok(())
}
#[cfg(test)]
mod tests;