use std::collections::VecDeque;
use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, Condvar, Mutex, MutexGuard, TryLockError};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use anyhow::Result;
use crate::residency::allocator_bytes;
const WAIT_SAMPLE_CAP: usize = 512;
static BACKGROUND_LOADS: Mutex<()> = Mutex::new(());
fn allocator_live() -> u64 {
allocator_bytes().live
}
fn now_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
fn lock_or_recover<T>(m: &Mutex<T>) -> MutexGuard<'_, T> {
m.lock().unwrap_or_else(|e| e.into_inner())
}
#[derive(Debug, Clone, Copy, Default)]
pub(crate) struct PoolStats {
pub loaded_sessions: usize,
pub in_flight: usize,
pub total_checkouts: u64,
pub wait_p50: Duration,
pub wait_max: Duration,
}
#[derive(Default)]
struct PoolMetrics {
in_flight: AtomicUsize,
total_checkouts: AtomicU64,
wait_max_ns: AtomicU64,
wait_samples: Mutex<VecDeque<u64>>,
}
impl PoolMetrics {
fn record_wait(&self, ns: u64) {
self.wait_max_ns.fetch_max(ns, Ordering::Relaxed);
let mut samples = lock_or_recover(&self.wait_samples);
if samples.len() >= WAIT_SAMPLE_CAP {
samples.pop_front();
}
samples.push_back(ns);
}
fn p50_ns(&self) -> u64 {
let mut samples: Vec<u64> = lock_or_recover(&self.wait_samples)
.iter()
.copied()
.collect();
if samples.is_empty() {
return 0;
}
samples.sort_unstable();
samples[samples.len() / 2]
}
}
pub(crate) struct SessionPool<T> {
kind: &'static str,
sessions: Vec<Mutex<Option<T>>>,
loaded: Vec<AtomicBool>,
last_used_secs: Vec<AtomicU64>,
resident: Vec<AtomicU64>,
first_run_measured: Vec<AtomicBool>,
free: Mutex<VecDeque<usize>>,
free_cv: Condvar,
metrics: PoolMetrics,
warming: AtomicBool,
busy: fn(Duration, usize) -> anyhow::Error,
meter: fn() -> u64,
}
struct Checkout<'a, T> {
pool: &'a SessionPool<T>,
idx: usize,
}
impl<T> Drop for Checkout<'_, T> {
fn drop(&mut self) {
self.pool.metrics.in_flight.fetch_sub(1, Ordering::Relaxed);
lock_or_recover(&self.pool.free).push_back(self.idx);
self.pool.free_cv.notify_all();
}
}
struct Warming<'a>(&'a AtomicBool);
impl Drop for Warming<'_> {
fn drop(&mut self) {
self.0.store(false, Ordering::Release);
}
}
impl<T: Send + 'static> SessionPool<T> {
pub(crate) fn new(
kind: &'static str,
n: usize,
busy: fn(Duration, usize) -> anyhow::Error,
) -> Self {
Self::with_meter(kind, n, busy, allocator_live)
}
pub(crate) fn with_meter(
kind: &'static str,
n: usize,
busy: fn(Duration, usize) -> anyhow::Error,
meter: fn() -> u64,
) -> Self {
Self {
kind,
sessions: (0..n).map(|_| Mutex::new(None)).collect(),
loaded: (0..n).map(|_| AtomicBool::new(false)).collect(),
last_used_secs: (0..n).map(|_| AtomicU64::new(0)).collect(),
resident: (0..n).map(|_| AtomicU64::new(0)).collect(),
first_run_measured: (0..n).map(|_| AtomicBool::new(false)).collect(),
free: Mutex::new((0..n).collect()),
free_cv: Condvar::new(),
metrics: PoolMetrics::default(),
warming: AtomicBool::new(false),
busy,
meter,
}
}
pub(crate) fn install(&self, idx: usize, load: impl FnOnce() -> Result<T>) -> Result<()> {
let mut slot = lock_or_recover(&self.sessions[idx]);
self.fill(idx, &mut slot, load)?;
self.last_used_secs[idx].store(now_secs(), Ordering::Relaxed);
Ok(())
}
pub(crate) fn slot_resident_bytes(&self) -> Vec<u64> {
self.resident
.iter()
.map(|r| r.load(Ordering::Relaxed))
.collect()
}
pub(crate) fn len(&self) -> usize {
self.sessions.len()
}
pub(crate) fn stats(&self) -> PoolStats {
PoolStats {
loaded_sessions: self
.loaded
.iter()
.filter(|l| l.load(Ordering::Relaxed))
.count(),
in_flight: self.metrics.in_flight.load(Ordering::Relaxed),
total_checkouts: self.metrics.total_checkouts.load(Ordering::Relaxed),
wait_p50: Duration::from_nanos(self.metrics.p50_ns()),
wait_max: Duration::from_nanos(self.metrics.wait_max_ns.load(Ordering::Relaxed)),
}
}
pub(crate) fn checkout<R>(
&self,
wait: Duration,
load: impl FnOnce() -> Result<T>,
with_session: impl FnOnce(&mut T) -> Result<R>,
) -> Result<R> {
let start = Instant::now();
let guard = lock_or_recover(&self.free);
let (mut guard, _) = self
.free_cv
.wait_timeout_while(guard, wait, |q| q.is_empty())
.unwrap_or_else(|e| e.into_inner());
let Some(idx) = guard.pop_front() else {
return Err((self.busy)(start.elapsed(), self.len()));
};
drop(guard);
let _checkout = self.checked_out(idx, start);
let mut slot = lock_or_recover(&self.sessions[idx]);
if slot.is_none() {
tracing::info!("reloading {} model after idle drop (slot {idx})", self.kind);
self.fill(idx, &mut slot, load)?;
}
self.last_used_secs[idx].store(now_secs(), Ordering::Relaxed);
self.run(
idx,
slot.as_mut().expect("session loaded above"),
with_session,
)
}
pub(crate) fn checkout_warm<R>(
self: &Arc<Self>,
wait: Duration,
load: impl FnOnce() -> Result<T> + Send + 'static,
with_session: impl FnOnce(&mut T) -> Result<R>,
) -> Result<R> {
let start = Instant::now();
let guard = lock_or_recover(&self.free);
if self.needs_reload(&guard) {
self.spawn_reload(load);
}
let (mut guard, _) = self
.free_cv
.wait_timeout_while(guard, wait, |q| self.warm_position(q).is_none())
.unwrap_or_else(|e| e.into_inner());
let Some(idx) = self.warm_position(&guard).and_then(|pos| guard.remove(pos)) else {
return Err((self.busy)(start.elapsed(), self.len()));
};
drop(guard);
let _checkout = self.checked_out(idx, start);
let mut slot = lock_or_recover(&self.sessions[idx]);
let Some(session) = slot.as_mut() else {
self.loaded[idx].store(false, Ordering::Relaxed);
return Err((self.busy)(start.elapsed(), self.len()));
};
self.last_used_secs[idx].store(now_secs(), Ordering::Relaxed);
self.run(idx, session, with_session)
}
pub(crate) fn drop_if_idle(&self, idle_secs: u64) {
let now = now_secs();
for (idx, last) in self.last_used_secs.iter().enumerate() {
if now.saturating_sub(last.load(Ordering::Relaxed)) < idle_secs {
continue;
}
let mut slot = match self.sessions[idx].try_lock() {
Ok(slot) => slot,
Err(TryLockError::Poisoned(poisoned)) => poisoned.into_inner(),
Err(TryLockError::WouldBlock) => continue,
};
if slot.is_some() {
*slot = None;
self.loaded[idx].store(false, Ordering::Relaxed);
self.resident[idx].store(0, Ordering::Relaxed);
self.first_run_measured[idx].store(false, Ordering::Relaxed);
tracing::info!(
"{} session {idx} dropped after {idle_secs}s idle",
self.kind
);
}
}
}
fn fill(
&self,
idx: usize,
slot: &mut Option<T>,
load: impl FnOnce() -> Result<T>,
) -> Result<()> {
let before = (self.meter)();
*slot = Some(load()?);
let loaded = (self.meter)().saturating_sub(before);
self.resident[idx].store(loaded, Ordering::Relaxed);
self.first_run_measured[idx].store(false, Ordering::Relaxed);
self.loaded[idx].store(true, Ordering::Relaxed);
Ok(())
}
fn run<R>(
&self,
idx: usize,
session: &mut T,
with_session: impl FnOnce(&mut T) -> Result<R>,
) -> Result<R> {
if self.first_run_measured[idx].load(Ordering::Relaxed) {
return with_session(session);
}
let before = (self.meter)();
let result = with_session(session);
if result.is_ok() {
let growth = (self.meter)().saturating_sub(before);
self.resident[idx].fetch_add(growth, Ordering::Relaxed);
self.first_run_measured[idx].store(true, Ordering::Relaxed);
}
result
}
fn needs_reload(&self, free: &VecDeque<usize>) -> bool {
self.warm_position(free).is_none() && self.loaded.iter().any(|l| !l.load(Ordering::Relaxed))
}
fn warm_position(&self, free: &VecDeque<usize>) -> Option<usize> {
free.iter()
.position(|&idx| self.loaded[idx].load(Ordering::Relaxed))
}
fn checked_out(&self, idx: usize, waiting_since: Instant) -> Checkout<'_, T> {
self.metrics
.record_wait(waiting_since.elapsed().as_nanos() as u64);
self.metrics.total_checkouts.fetch_add(1, Ordering::Relaxed);
self.metrics.in_flight.fetch_add(1, Ordering::Relaxed);
Checkout { pool: self, idx }
}
fn spawn_reload(self: &Arc<Self>, load: impl FnOnce() -> Result<T> + Send + 'static) {
if self.warming.swap(true, Ordering::AcqRel) {
return;
}
let pool = Arc::clone(self);
let spawned = std::thread::Builder::new()
.name(format!("{}-reload", self.kind))
.spawn(move || {
let _one_at_a_time = lock_or_recover(&BACKGROUND_LOADS);
pool.warm_one(load, Warming(&pool.warming));
});
if let Err(e) = spawned {
self.warming.store(false, Ordering::Release);
tracing::warn!("cannot start a {} reload thread: {e}", self.kind);
}
}
pub(crate) fn warm_if_cold(
self: &Arc<Self>,
load: impl FnOnce() -> Result<T> + Send + 'static,
) {
if self.needs_reload(&lock_or_recover(&self.free)) {
self.spawn_reload(load);
}
}
fn warm_one(&self, load: impl FnOnce() -> Result<T>, warming: Warming<'_>) {
let mut free = lock_or_recover(&self.free);
let Some(pos) = free
.iter()
.position(|&idx| !self.loaded[idx].load(Ordering::Relaxed))
else {
return;
};
let idx = free.remove(pos).expect("position is in range");
drop(free);
self.metrics.in_flight.fetch_add(1, Ordering::Relaxed);
let _checkout = Checkout { pool: self, idx };
let _warming = warming;
let mut slot = lock_or_recover(&self.sessions[idx]);
if slot.is_some() {
self.loaded[idx].store(true, Ordering::Relaxed);
return;
}
match self.fill(idx, &mut slot, load) {
Ok(()) => {
self.last_used_secs[idx].store(now_secs(), Ordering::Relaxed);
tracing::info!("{} session {idx} reloaded in the background", self.kind);
}
Err(e) => tracing::warn!(
"reloading {} session {idx} in the background failed: {e:#}",
self.kind
),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
fn busy(waited: Duration, pool_size: usize) -> anyhow::Error {
anyhow::anyhow!("busy after {waited:?} (pool size {pool_size})")
}
fn is_busy(err: &anyhow::Error) -> bool {
err.to_string().starts_with("busy after")
}
fn counted(
calls: &Arc<AtomicUsize>,
delay: Duration,
) -> impl FnOnce() -> Result<u32> + Send + use<> {
let calls = Arc::clone(calls);
move || {
calls.fetch_add(1, Ordering::SeqCst);
std::thread::sleep(delay);
Ok(7)
}
}
fn warm_within(
pool: &Arc<SessionPool<u32>>,
wait: Duration,
calls: &Arc<AtomicUsize>,
) -> Result<u32> {
pool.checkout_warm(wait, counted(calls, Duration::from_millis(200)), |s| Ok(*s))
}
#[test]
fn a_checkout_within_a_short_wait_gives_up_when_no_session_is_free() {
let pool = SessionPool::<u32>::new("test", 1, busy);
lock_or_recover(&pool.free).clear();
let started = Instant::now();
let err = pool
.checkout(
Duration::from_millis(50),
|| unreachable!("no slot is free to load"),
|_| Ok(()),
)
.unwrap_err();
assert!(is_busy(&err), "{err:#}");
assert!(started.elapsed() < Duration::from_secs(1));
}
#[test]
fn a_plain_checkout_still_loads_a_cold_slot_inline() {
let pool = SessionPool::new("test", 1, busy);
let got = pool
.checkout(Duration::from_secs(1), || Ok(5_u32), |s| Ok(*s))
.unwrap();
assert_eq!(got, 5);
assert!(pool.loaded[0].load(Ordering::Relaxed));
}
#[test]
fn a_warm_checkout_never_loads_inline_and_warms_a_cold_slot_in_the_background() {
let pool = Arc::new(SessionPool::new("test", 1, busy));
let calls = Arc::new(AtomicUsize::new(0));
let started = Instant::now();
let err = warm_within(&pool, Duration::from_millis(20), &calls).unwrap_err();
assert!(is_busy(&err), "{err:#}");
assert!(
started.elapsed() < Duration::from_millis(150),
"the cold miss waited {:?}, so it sat through the 200 ms load",
started.elapsed()
);
let warmed = warm_within(&pool, Duration::from_secs(5), &calls).unwrap();
assert_eq!(warmed, 7);
assert_eq!(calls.load(Ordering::SeqCst), 1, "one background load");
}
#[test]
fn a_warm_checkout_takes_a_loaded_session_without_loading() {
let pool = Arc::new(SessionPool::new("test", 2, busy));
pool.install(1, || Ok(3)).unwrap();
let calls = Arc::new(AtomicUsize::new(0));
assert_eq!(warm_within(&pool, Duration::ZERO, &calls).unwrap(), 3);
assert_eq!(calls.load(Ordering::SeqCst), 0);
}
#[test]
fn a_reload_that_panics_does_not_block_later_reloads() {
let pool = Arc::new(SessionPool::<u32>::new("test", 1, busy));
pool.warm_if_cold(|| panic!("model file is corrupt"));
let calls = Arc::new(AtomicUsize::new(0));
let deadline = Instant::now() + Duration::from_secs(5);
let warmed = loop {
match warm_within(&pool, Duration::from_millis(50), &calls) {
Ok(warmed) => break warmed,
Err(_) if Instant::now() < deadline => continue,
Err(e) => panic!("never warmed: {e:#}"),
}
};
assert_eq!(warmed, 7);
}
#[test]
fn warming_a_cold_pool_needs_no_checkout() {
let pool = Arc::new(SessionPool::<u32>::new("test", 2, busy));
let calls = Arc::new(AtomicUsize::new(0));
pool.warm_if_cold(counted(&calls, Duration::ZERO));
let give_up = Instant::now() + Duration::from_secs(5);
while !pool.loaded.iter().any(|l| l.load(Ordering::SeqCst)) {
assert!(Instant::now() < give_up, "warm_if_cold loaded nothing");
std::thread::sleep(Duration::from_millis(5));
}
while lock_or_recover(&pool.free).len() < 2 {
assert!(
Instant::now() < give_up,
"the reloaded slot never came back"
);
std::thread::sleep(Duration::from_millis(5));
}
assert_eq!(calls.load(Ordering::SeqCst), 1);
pool.warm_if_cold(counted(&calls, Duration::from_millis(200)));
assert!(
!pool.warming.load(Ordering::SeqCst),
"one free loaded session is enough, but a second reload started"
);
}
static METER: AtomicU64 = AtomicU64::new(0);
fn meter() -> u64 {
METER.load(Ordering::SeqCst)
}
#[test]
fn a_background_reload_and_a_first_warm_run_count_as_resident() {
const LOAD: u64 = 64 << 20;
const FIRST_RUN: u64 = 16 << 20;
let pool = Arc::new(SessionPool::<u32>::with_meter("test", 1, busy, meter));
pool.warm_if_cold(|| {
METER.fetch_add(LOAD, Ordering::SeqCst);
Ok(1)
});
let give_up = Instant::now() + Duration::from_secs(5);
while lock_or_recover(&pool.free).is_empty() || !pool.loaded[0].load(Ordering::SeqCst) {
assert!(
Instant::now() < give_up,
"the background reload never finished"
);
std::thread::sleep(Duration::from_millis(5));
}
assert_eq!(pool.slot_resident_bytes(), vec![LOAD]);
let grow = |_: &mut u32| {
METER.fetch_add(FIRST_RUN, Ordering::SeqCst);
Ok(())
};
pool.checkout_warm(Duration::from_secs(1), || unreachable!(), grow)
.unwrap();
assert_eq!(pool.slot_resident_bytes(), vec![LOAD + FIRST_RUN]);
pool.checkout_warm(Duration::from_secs(1), || unreachable!(), grow)
.unwrap();
assert_eq!(
pool.slot_resident_bytes(),
vec![LOAD + FIRST_RUN],
"only the first run is measured"
);
pool.drop_if_idle(0);
assert_eq!(pool.slot_resident_bytes(), vec![0]);
}
#[test]
fn an_idle_drop_frees_a_slot_whose_run_panicked() {
let pool = SessionPool::<u32>::new("test", 1, busy);
pool.install(0, || Ok(3)).unwrap();
let poisoned = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let _slot = pool.sessions[0].lock().unwrap();
panic!("a run panicked holding the slot");
}));
assert!(poisoned.is_err());
pool.drop_if_idle(0);
assert!(!pool.loaded[0].load(Ordering::SeqCst));
assert!(lock_or_recover(&pool.sessions[0]).is_none());
}
#[test]
fn an_idle_drop_turns_a_warm_checkout_into_a_miss() {
let pool = Arc::new(SessionPool::new("test", 1, busy));
pool.install(0, || Ok(3)).unwrap();
pool.drop_if_idle(0);
let calls = Arc::new(AtomicUsize::new(0));
assert!(warm_within(&pool, Duration::ZERO, &calls).is_err());
assert_eq!(
warm_within(&pool, Duration::from_secs(5), &calls).unwrap(),
7
);
}
}