use std::collections::HashMap;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex, Weak};
use std::time::{Duration, Instant};
use tokio_util::sync::CancellationToken;
use crate::model::{IsolationBootDiagnostics, IsolationPoolStatus, ProviderError};
use super::config::{ResolvedLaunch, VmConfigKey};
use super::session::PreparedVm;
struct IdleVm {
key: VmConfigKey,
vm: PreparedVm,
idle_since: Instant,
}
struct PoolState {
idle: Vec<IdleVm>,
in_flight: u64,
in_flight_by_key: HashMap<VmConfigKey, u64>,
live: u64,
target: u64,
per_vm_memory_mb: u64,
max_pool_memory_mb: u64,
idle_drain: Duration,
draining: bool,
shutting_down: bool,
last_boot: Option<IsolationBootDiagnostics>,
demand: Vec<(Instant, u64)>,
demand_window: Duration,
}
struct PoolInner {
state: Mutex<PoolState>,
checked_out: AtomicU64,
}
#[derive(Clone)]
pub struct NativeVmPool {
inner: Arc<PoolInner>,
}
pub struct PoolCheckout {
pub vm: PreparedVm,
release: Option<PoolRelease>,
}
pub struct PoolRelease {
callback: Option<Box<dyn FnOnce() + Send + Sync>>,
}
impl PoolRelease {
fn new(callback: Box<dyn FnOnce() + Send + Sync>) -> Self {
Self {
callback: Some(callback),
}
}
}
impl Drop for PoolRelease {
fn drop(&mut self) {
if let Some(callback) = self.callback.take() {
callback();
}
}
}
impl PoolCheckout {
pub fn take_release(&mut self) -> Option<PoolRelease> {
self.release.take()
}
}
impl NativeVmPool {
pub fn new() -> Self {
let pool = Self {
inner: Arc::new(PoolInner {
state: Mutex::new(PoolState {
idle: Vec::new(),
in_flight: 0,
in_flight_by_key: HashMap::new(),
live: 0,
target: 1,
per_vm_memory_mb: 1024,
max_pool_memory_mb: 8192,
idle_drain: Duration::from_secs(60),
draining: false,
shutting_down: false,
last_boot: None,
demand: Vec::new(),
demand_window: Duration::from_secs(60),
}),
checked_out: AtomicU64::new(0),
}),
};
pool.spawn_idle_reaper();
pool
}
pub fn configure(
&self,
target: Option<u64>,
per_vm_memory_mb: u64,
max_pool_memory_mb: Option<u64>,
idle_drain_seconds: Option<u64>,
) {
let mut state = self.inner.state.lock().unwrap();
state.per_vm_memory_mb = per_vm_memory_mb.max(1);
if let Some(target) = target {
state.target = target.max(1);
}
if let Some(maximum) = max_pool_memory_mb {
state.max_pool_memory_mb = maximum.max(state.per_vm_memory_mb);
}
if let Some(seconds) = idle_drain_seconds {
state.idle_drain = Duration::from_secs(seconds.max(1));
}
state.target = state.target.min(memory_capacity(&state));
}
pub async fn warmup(
&self,
launch: ResolvedLaunch,
timeout: Duration,
) -> Result<(), ProviderError> {
let deadline = Instant::now() + timeout;
let key = launch.key();
loop {
let ready = {
let mut state = self.inner.state.lock().unwrap();
let checked_out = self.inner.checked_out.load(Ordering::SeqCst);
let target = effective_target(&mut state, Instant::now(), checked_out);
state.idle.iter().filter(|idle| idle.key == key).count() as u64 >= target
};
if ready {
return Ok(());
}
if Instant::now() >= deadline {
return Err(ProviderError::new(
"VM_POOL_WARMUP_TIMEOUT",
format!("VM pool did not warm within {} ms", timeout.as_millis()),
));
}
let vm = self
.boot_reserved(&launch, &CancellationToken::new())
.await?;
let mut vm = Some(vm);
let discard = {
let mut state = self.inner.state.lock().unwrap();
if state.draining || state.shutting_down {
state.live = state.live.saturating_sub(1);
true
} else {
state.idle.push(IdleVm {
key: key.clone(),
vm: vm.take().unwrap(),
idle_since: Instant::now(),
});
false
}
};
if discard {
vm.unwrap().stop().await?;
}
}
}
pub async fn checkout(
&self,
launch: &ResolvedLaunch,
cancellation: &CancellationToken,
) -> Result<PoolCheckout, ProviderError> {
let key = launch.key();
let idle = {
let mut state = self.inner.state.lock().unwrap();
if state.draining || state.shutting_down {
return Err(ProviderError::new(
"VM_POOL_DRAINING",
"VM pool is not accepting checkouts",
));
}
let checked_out = self.inner.checked_out.fetch_add(1, Ordering::SeqCst) + 1;
record_demand(&mut state, Instant::now(), checked_out);
state
.idle
.iter()
.position(|idle| idle.key == key)
.map(|index| state.idle.swap_remove(index).vm)
};
let vm = match idle {
Some(vm) => vm,
None => match self.boot_reserved(launch, cancellation).await {
Ok(vm) => vm,
Err(error) => {
decrement_checked_out(&self.inner);
return Err(error);
}
},
};
let weak = Arc::downgrade(&self.inner);
let refill_launch = launch.clone();
self.schedule_refill(launch.clone());
Ok(PoolCheckout {
vm,
release: Some(PoolRelease::new(Box::new(move || {
release_checkout(&weak, refill_launch)
}))),
})
}
pub async fn drain(&self) -> Result<(), ProviderError> {
let idle = {
let mut state = self.inner.state.lock().unwrap();
state.draining = true;
let idle = std::mem::take(&mut state.idle);
state.live = state.live.saturating_sub(idle.len() as u64);
idle
};
for idle in idle {
idle.vm.stop().await?;
}
loop {
let done = {
let state = self.inner.state.lock().unwrap();
state.in_flight == 0
};
if done {
break;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
self.inner.state.lock().unwrap().draining = false;
Ok(())
}
pub fn status(&self) -> IsolationPoolStatus {
let mut state = self.inner.state.lock().unwrap();
let checked_out = self.inner.checked_out.load(Ordering::SeqCst);
let target = effective_target(&mut state, Instant::now(), checked_out);
IsolationPoolStatus {
idle: state.idle.len() as u64,
waiters: 0,
in_flight: state.in_flight,
live: state.live,
checked_out,
target,
per_vm_memory_mb: state.per_vm_memory_mb,
max_pool_memory_mb: state.max_pool_memory_mb,
draining: state.draining,
shutting_down: state.shutting_down,
last_boot: state.last_boot.clone(),
}
}
pub fn idle_drain_seconds(&self) -> f64 {
self.inner.state.lock().unwrap().idle_drain.as_secs_f64()
}
async fn boot_reserved(
&self,
launch: &ResolvedLaunch,
cancellation: &CancellationToken,
) -> Result<PreparedVm, ProviderError> {
self.reserve_boot(&launch.key())?;
self.boot_after_reservation(launch, cancellation).await
}
fn reserve_boot(&self, key: &VmConfigKey) -> Result<(), ProviderError> {
{
let mut state = self.inner.state.lock().unwrap();
if state.live >= memory_capacity(&state) {
return Err(ProviderError::new(
"VM_POOL_MEMORY_LIMIT",
format!(
"VM pool memory ceiling {} MiB cannot admit another {} MiB VM",
state.max_pool_memory_mb, state.per_vm_memory_mb
),
));
}
state.in_flight += 1;
*state.in_flight_by_key.entry(key.clone()).or_default() += 1;
state.live += 1;
}
Ok(())
}
async fn boot_after_reservation(
&self,
launch: &ResolvedLaunch,
cancellation: &CancellationToken,
) -> Result<PreparedVm, ProviderError> {
let key = launch.key();
let result = PreparedVm::boot(launch, cancellation).await;
let mut state = self.inner.state.lock().unwrap();
state.in_flight = state.in_flight.saturating_sub(1);
if let Some(count) = state.in_flight_by_key.get_mut(&key) {
*count = count.saturating_sub(1);
if *count == 0 {
state.in_flight_by_key.remove(&key);
}
}
match &result {
Ok(vm) => state.last_boot = Some(vm.diagnostics.clone()),
Err(_) => state.live = state.live.saturating_sub(1),
}
result
}
fn schedule_refill(&self, launch: ResolvedLaunch) {
let should_spawn = {
let mut state = self.inner.state.lock().unwrap();
let key = launch.key();
let matching = state.idle.iter().filter(|idle| idle.key == key).count() as u64;
let matching_in_flight = state.in_flight_by_key.get(&key).copied().unwrap_or(0);
let checked_out = self.inner.checked_out.load(Ordering::SeqCst);
let target = effective_target(&mut state, Instant::now(), checked_out);
let should_spawn = !state.draining
&& !state.shutting_down
&& matching + matching_in_flight < target
&& state.live < memory_capacity(&state);
if should_spawn {
state.in_flight += 1;
*state.in_flight_by_key.entry(key).or_default() += 1;
state.live += 1;
}
should_spawn
};
if !should_spawn {
return;
}
let pool = self.clone();
tokio::spawn(async move {
let key = launch.key();
if let Ok(vm) = pool
.boot_after_reservation(&launch, &CancellationToken::new())
.await
{
let mut vm = Some(vm);
let discard = {
let mut state = pool.inner.state.lock().unwrap();
if state.draining || state.shutting_down {
state.live = state.live.saturating_sub(1);
true
} else {
state.idle.push(IdleVm {
key,
vm: vm.take().unwrap(),
idle_since: Instant::now(),
});
false
}
};
if discard {
let _ = vm.unwrap().stop().await;
}
}
});
}
fn spawn_idle_reaper(&self) {
let pool = self.clone();
tokio::spawn(async move {
loop {
tokio::time::sleep(Duration::from_secs(1)).await;
let expired = {
let mut state = pool.inner.state.lock().unwrap();
if state.shutting_down || state.idle.is_empty() {
Vec::new()
} else {
let deadline = Instant::now()
.checked_sub(state.idle_drain)
.unwrap_or_else(Instant::now);
let mut expired = Vec::new();
let mut index = 0;
while index < state.idle.len() {
if state.idle[index].idle_since <= deadline {
expired.push(state.idle.swap_remove(index));
} else {
index += 1;
}
}
state.live = state.live.saturating_sub(expired.len() as u64);
expired
}
};
for idle in expired {
let _ = idle.vm.stop().await;
}
}
});
}
}
fn memory_capacity(state: &PoolState) -> u64 {
(state.max_pool_memory_mb / state.per_vm_memory_mb.max(1)).max(1)
}
fn effective_target(state: &mut PoolState, now: Instant, checked_out: u64) -> u64 {
let demand_window = state.demand_window;
state
.demand
.retain(|(observed_at, _)| now.saturating_duration_since(*observed_at) <= demand_window);
let recent_peak = state
.demand
.iter()
.map(|(_, peak)| *peak)
.max()
.unwrap_or(0)
.max(checked_out);
state
.target
.min(recent_peak.saturating_add(1).max(1))
.min(memory_capacity(state))
}
fn record_demand(state: &mut PoolState, now: Instant, checked_out: u64) {
let _ = effective_target(state, now, checked_out);
state.demand.push((now, checked_out));
}
fn decrement_checked_out(inner: &PoolInner) {
let _ = inner
.checked_out
.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |value| {
Some(value.saturating_sub(1))
});
}
fn release_checkout(inner: &Weak<PoolInner>, launch: ResolvedLaunch) {
let Some(inner) = inner.upgrade() else {
return;
};
decrement_checked_out(&inner);
{
let mut state = inner.state.lock().unwrap();
state.live = state.live.saturating_sub(1);
}
NativeVmPool { inner }.schedule_refill(launch);
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::AtomicU64;
fn state(target: u64, per_vm_memory_mb: u64, max_pool_memory_mb: u64) -> PoolState {
PoolState {
idle: Vec::new(),
in_flight: 0,
in_flight_by_key: HashMap::new(),
live: 0,
target,
per_vm_memory_mb,
max_pool_memory_mb,
idle_drain: Duration::from_secs(60),
draining: false,
shutting_down: false,
last_boot: None,
demand: Vec::new(),
demand_window: Duration::from_secs(60),
}
}
#[test]
fn effective_target_starts_at_one_without_demand() {
let mut state = state(4, 256, 2048);
assert_eq!(effective_target(&mut state, Instant::now(), 0), 1);
}
#[test]
fn effective_target_tracks_recent_peak_plus_one() {
let now = Instant::now();
let mut state = state(4, 256, 2048);
state.demand.push((now, 2));
assert_eq!(effective_target(&mut state, now, 2), 3);
}
#[test]
fn effective_target_obeys_memory_ceiling() {
let now = Instant::now();
let mut state = state(8, 1024, 2048);
state.demand.push((now, 7));
assert_eq!(effective_target(&mut state, now, 7), 2);
}
#[test]
fn effective_target_discards_expired_demand() {
let now = Instant::now();
let mut state = state(4, 256, 2048);
state.demand.push((now - Duration::from_secs(61), 3));
assert_eq!(effective_target(&mut state, now, 0), 1);
assert!(state.demand.is_empty());
}
#[test]
fn pool_release_runs_exactly_once_when_dropped() {
let calls = Arc::new(AtomicU64::new(0));
let recorded = Arc::clone(&calls);
let release = PoolRelease::new(Box::new(move || {
recorded.fetch_add(1, Ordering::SeqCst);
}));
drop(release);
assert_eq!(calls.load(Ordering::SeqCst), 1);
}
}