use std::{
sync::{
atomic::{AtomicUsize, Ordering},
Arc, Condvar, Mutex,
},
time::{Duration, Instant},
};
use crate::{AsrError, Engine, EngineConfig, EngineOptions, ErrorKind};
#[derive(Debug, Clone)]
pub struct EngineSnapshot {
pub generation: u64,
pub engine: Engine,
}
#[derive(Debug, Clone)]
pub struct ReloadStatus {
pub generation: u64,
pub loading: bool,
pub ready: bool,
pub error: Option<AsrError>,
pub available_generation: Option<u64>,
}
struct State {
generation: u64,
request: Option<(EngineConfig, Instant)>,
loading: bool,
available: Option<EngineSnapshot>,
error: Option<AsrError>,
worker_error: Option<AsrError>,
shutdown: bool,
}
struct Inner {
managers: AtomicUsize,
state: Mutex<State>,
changed: Condvar,
}
pub struct EngineManager {
inner: Arc<Inner>,
}
impl Clone for EngineManager {
fn clone(&self) -> Self {
self.inner.managers.fetch_add(1, Ordering::Relaxed);
Self {
inner: Arc::clone(&self.inner),
}
}
}
impl Drop for EngineManager {
fn drop(&mut self) {
if self.inner.managers.fetch_sub(1, Ordering::AcqRel) == 1 {
let mut s = self.inner.state.lock().unwrap();
s.shutdown = true;
self.inner.changed.notify_all();
}
}
}
impl EngineManager {
pub fn new(options: EngineOptions) -> Self {
Self::with_loader(move |config| Engine::prepare(config, options.clone()))
}
fn with_loader(
loader: impl Fn(EngineConfig) -> Result<Engine, AsrError> + Send + 'static,
) -> Self {
let inner = Arc::new(Inner {
managers: AtomicUsize::new(1),
state: Mutex::new(State {
generation: 0,
request: None,
loading: false,
available: None,
error: None,
worker_error: None,
shutdown: false,
}),
changed: Condvar::new(),
});
let spawn = std::thread::Builder::new()
.name("asr-core-loader".into())
.spawn({
let inner = Arc::clone(&inner);
move || loop {
let mut s = inner.state.lock().unwrap();
loop {
if s.shutdown {
return;
}
match s.request.as_ref() {
Some((_, due)) if *due <= Instant::now() => break,
None => s = inner.changed.wait(s).unwrap(),
Some((_, due)) => {
let delay = due.saturating_duration_since(Instant::now());
s = inner.changed.wait_timeout(s, delay).unwrap().0;
}
}
}
let (config, _) = s.request.take().unwrap();
let generation = s.generation;
drop(s);
let result =
std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| loader(config)))
.unwrap_or_else(|_| {
Err(AsrError::new(
ErrorKind::Backend,
"load",
"model loader panicked",
))
});
let mut s = inner.state.lock().unwrap();
let mut retired = None;
if generation == s.generation {
s.loading = false;
match result {
Ok(engine) => {
retired =
s.available.replace(EngineSnapshot { generation, engine });
s.error = None;
}
Err(e) => s.error = Some(e),
}
}
drop(s);
inner.changed.notify_all();
drop(retired);
}
});
if let Err(e) = spawn {
let mut s = inner.state.lock().unwrap();
let error = AsrError::new(ErrorKind::Io, "load", e.to_string());
s.error = Some(error.clone());
s.worker_error = Some(error);
}
Self { inner }
}
pub fn request_reload(&self, config: EngineConfig) -> u64 {
let mut s = self.inner.state.lock().unwrap();
s.generation += 1;
s.loading = s.worker_error.is_none();
s.error = s.worker_error.clone();
s.request = Some((config, Instant::now() + Duration::from_millis(400)));
let generation = s.generation;
drop(s);
self.inner.changed.notify_all();
generation
}
pub fn get(&self) -> Option<EngineSnapshot> {
let s = self.inner.state.lock().unwrap();
s.available
.as_ref()
.filter(|a| a.generation == s.generation && s.error.is_none())
.cloned()
}
pub fn last_available(&self) -> Option<EngineSnapshot> {
self.inner.state.lock().unwrap().available.clone()
}
pub fn status(&self) -> ReloadStatus {
let s = self.inner.state.lock().unwrap();
ReloadStatus {
generation: s.generation,
loading: s.loading,
ready: s
.available
.as_ref()
.is_some_and(|a| a.generation == s.generation)
&& s.error.is_none(),
error: s.error.clone(),
available_generation: s.available.as_ref().map(|a| a.generation),
}
}
pub fn wait(&self, generation: u64, deadline: Instant) -> Result<EngineSnapshot, AsrError> {
let mut s = self.inner.state.lock().unwrap();
loop {
if generation != s.generation {
return Err(AsrError::new(
ErrorKind::Cancelled,
"load",
"reload was superseded",
));
}
if let Some(error) = &s.error {
return Err(error.clone());
}
if let Some(snapshot) = &s.available {
if snapshot.generation == generation {
return Ok(snapshot.clone());
}
}
if Instant::now() >= deadline {
return Err(AsrError::new(
ErrorKind::Timeout,
"load",
"waiting for reload timed out",
));
}
s = self
.inner
.changed
.wait_timeout(s, deadline.saturating_duration_since(Instant::now()))
.unwrap()
.0;
}
}
}
#[cfg(all(test, feature = "backend-dashscope"))]
mod tests;
#[cfg(test)]
mod lifecycle_tests;