use std::sync::{Arc, Mutex, MutexGuard};
use query_planning::error::{QueryError, QueryErrorCode, Result};
use crate::runtime::gpu::DeviceLedger;
const OPERATION: &str = "query_engine.runtime.cache.residency";
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum CacheResidencyState {
#[cfg(test)]
AttemptOwned,
Evictable,
Pinned,
Destroyed,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum CacheResidencyChargeLocation {
Evictable,
Pinned,
Released,
}
#[derive(Debug)]
struct CacheResidencyInnerState {
state: CacheResidencyState,
charge_location: CacheResidencyChargeLocation,
pin_count: usize,
eviction_in_progress: bool,
}
struct CacheResidencyInner {
ledger: DeviceLedger,
bytes: u64,
state: Mutex<CacheResidencyInnerState>,
}
impl CacheResidencyInner {
fn lock(&self, action: &'static str) -> Result<MutexGuard<'_, CacheResidencyInnerState>> {
self.state.lock().map_err(|_error| {
QueryError::new(QueryErrorCode::Internal, OPERATION, "resident-cache state mutex was poisoned")
.with_fact("action", action)
.with_fact("reason", "cache_residency_mutex_poisoned")
})
}
}
#[must_use = "dropping a cache charge transaction rolls its reservation back; commit it only after retention succeeds"]
pub(crate) struct CacheResidencyChargeTransaction {
ledger: DeviceLedger,
bytes: u64,
charged: bool,
}
impl CacheResidencyChargeTransaction {
#[cfg(test)]
pub(crate) const fn state(&self) -> CacheResidencyState {
CacheResidencyState::AttemptOwned
}
#[cfg(all(test, feature = "cugraph"))]
#[doc(hidden)]
pub const fn bytes(&self) -> u64 {
self.bytes
}
pub(crate) fn shrink_to(&mut self, retained_bytes: u64) -> Result<bool> {
if retained_bytes > self.bytes {
return Ok(false);
}
let released_bytes = self.bytes - retained_bytes;
self.ledger.release_cache_residency_precharge_excess(released_bytes)?;
self.bytes = retained_bytes;
Ok(true)
}
pub(crate) fn commit_evictable(mut self) -> CacheResidency {
self.charged = false;
CacheResidency {
inner: Arc::new(CacheResidencyInner {
ledger: self.ledger.clone(),
bytes: self.bytes,
state: Mutex::new(CacheResidencyInnerState {
state: CacheResidencyState::Evictable,
charge_location: CacheResidencyChargeLocation::Evictable,
pin_count: 0,
eviction_in_progress: false,
}),
}),
}
}
pub(crate) fn rollback(mut self) -> Result<()> {
self.rollback_inner()
}
#[cfg(feature = "cugraph")]
pub(crate) fn retain_charge_after_unsafe_teardown(mut self) {
self.charged = false;
}
#[cfg(feature = "cugraph")]
pub(crate) fn release_after_proven_empty(mut self) -> Result<()> {
if !self.charged {
return Ok(());
}
let result = self.ledger.release_cache_residency_charge_after_destroy(self.bytes);
self.charged = false;
result
}
fn rollback_inner(&mut self) -> Result<()> {
if !self.charged {
return Ok(());
}
self.ledger.release_cache_residency_charge_after_destroy(self.bytes)?;
self.charged = false;
Ok(())
}
}
impl Drop for CacheResidencyChargeTransaction {
fn drop(&mut self) {
if let Err(error) = self.rollback_inner() {
eprintln!("[query-engine] warning: failed to roll back resident-cache charge transaction: {error}");
}
}
}
pub(crate) struct CacheResidency {
inner: Arc<CacheResidencyInner>,
}
impl CacheResidency {
pub(crate) fn begin_attempt_owned(
ledger: DeviceLedger,
bytes: u64,
) -> Result<Option<CacheResidencyChargeTransaction>> {
if !ledger.try_reserve_evictable_bytes(bytes)? {
return Ok(None);
}
Ok(Some(CacheResidencyChargeTransaction { ledger, bytes, charged: true }))
}
pub(crate) fn state(&self) -> Result<CacheResidencyState> {
Ok(self.inner.lock("state")?.state)
}
pub(crate) fn bytes(&self) -> u64 {
self.inner.bytes
}
pub(crate) fn try_pin(&self) -> Result<Option<CacheResidencyPin>> {
let mut state = self.inner.lock("try_pin")?;
match state.state {
CacheResidencyState::Evictable if !state.eviction_in_progress => {
self.inner.ledger.pin_cache_residency_charge(self.inner.bytes)?;
state.state = CacheResidencyState::Pinned;
state.charge_location = CacheResidencyChargeLocation::Pinned;
state.pin_count = 1;
Ok(Some(CacheResidencyPin { inner: Some(Arc::clone(&self.inner)) }))
},
CacheResidencyState::Pinned => {
state.pin_count = state.pin_count.checked_add(1).ok_or_else(|| {
QueryError::new(QueryErrorCode::Internal, OPERATION, "resident-cache pin count overflowed")
.with_fact("reason", "cache_residency_pin_count_overflow")
})?;
Ok(Some(CacheResidencyPin { inner: Some(Arc::clone(&self.inner)) }))
},
#[cfg(test)]
CacheResidencyState::AttemptOwned => Ok(None),
CacheResidencyState::Destroyed | CacheResidencyState::Evictable => Ok(None),
}
}
pub(crate) fn begin_eviction(&self) -> Result<Option<CacheResidencyDestruction>> {
let mut state = self.inner.lock("begin_eviction")?;
if state.state != CacheResidencyState::Evictable || state.eviction_in_progress {
return Ok(None);
}
debug_assert_eq!(state.charge_location, CacheResidencyChargeLocation::Evictable);
state.eviction_in_progress = true;
Ok(Some(CacheResidencyDestruction { inner: Some(Arc::clone(&self.inner)) }))
}
}
#[cfg(feature = "cugraph")]
#[must_use = "transient cache-domain work must release its ledger charge only after physical destruction"]
#[cfg(test)]
pub(crate) struct CacheExecutionReservation {
residency: CacheResidency,
pin: Option<CacheResidencyPin>,
}
#[cfg(feature = "cugraph")]
#[cfg(test)]
impl CacheExecutionReservation {
#[cfg(test)]
pub(crate) fn begin(ledger: DeviceLedger, bytes: u64) -> Result<Option<Self>> {
let Some(transaction) = CacheResidency::begin_attempt_owned(ledger, bytes)? else {
return Ok(None);
};
let residency = transaction.commit_evictable();
let pin = residency.try_pin()?.ok_or_else(|| {
QueryError::new(
QueryErrorCode::Internal,
OPERATION,
"fresh transient cache-domain reservation could not become pinned",
)
.with_fact("reason", "cache_execution_reservation_pin_refused")
})?;
Ok(Some(Self { residency, pin: Some(pin) }))
}
#[cfg(test)]
pub(crate) fn release_after_destroy(mut self) -> Result<()> {
drop(self.pin.take());
let destruction = self.residency.begin_eviction()?.ok_or_else(|| {
QueryError::new(
QueryErrorCode::Internal,
OPERATION,
"transient cache-domain reservation was not evictable after destruction",
)
.with_fact("reason", "cache_execution_reservation_not_evictable")
})?;
destruction.commit_after_destroy()
}
}
#[must_use = "dropping a cache pin returns the entry to evictable residency after the final lease"]
pub(crate) struct CacheResidencyPin {
inner: Option<Arc<CacheResidencyInner>>,
}
impl Drop for CacheResidencyPin {
fn drop(&mut self) {
let Some(inner) = self.inner.take() else {
return;
};
if let Err(error) = release_pin(&inner) {
eprintln!("[query-engine] warning: failed to release resident-cache pin: {error}");
}
}
}
fn release_pin(inner: &CacheResidencyInner) -> Result<()> {
let mut state = inner.lock("release_pin")?;
if state.state != CacheResidencyState::Pinned || state.pin_count == 0 {
return Err(QueryError::new(
QueryErrorCode::Internal,
OPERATION,
"resident-cache pin lease does not match pinned entry state",
)
.with_fact("reason", "cache_residency_pin_state_mismatch"));
}
if state.pin_count > 1 {
state.pin_count -= 1;
return Ok(());
}
inner.ledger.unpin_cache_residency_charge(inner.bytes)?;
state.pin_count = 0;
state.state = CacheResidencyState::Evictable;
state.charge_location = CacheResidencyChargeLocation::Evictable;
Ok(())
}
#[must_use = "a destruction token retains the ledger charge until destruction is committed or explicitly aborted"]
pub(crate) struct CacheResidencyDestruction {
inner: Option<Arc<CacheResidencyInner>>,
}
impl CacheResidencyDestruction {
pub(crate) fn commit_after_destroy(mut self) -> Result<()> {
let Some(inner) = self.inner.take() else {
return Ok(());
};
let mut state = inner.lock("commit_after_destroy")?;
if state.state != CacheResidencyState::Evictable || !state.eviction_in_progress {
return Err(QueryError::new(
QueryErrorCode::Internal,
OPERATION,
"resident-cache destruction commit does not match an evictable entry",
)
.with_fact("reason", "cache_residency_destroy_state_mismatch"));
}
inner.ledger.release_cache_residency_charge_after_destroy(inner.bytes)?;
state.state = CacheResidencyState::Destroyed;
state.charge_location = CacheResidencyChargeLocation::Released;
state.eviction_in_progress = false;
Ok(())
}
pub(crate) fn retain_charge_after_failed_destruction(mut self) -> Result<()> {
let Some(inner) = self.inner.take() else {
return Ok(());
};
let mut state = inner.lock("retain_charge_after_failed_destruction")?;
if state.state != CacheResidencyState::Evictable || !state.eviction_in_progress {
return Err(QueryError::new(
QueryErrorCode::Internal,
OPERATION,
"resident-cache failed destruction does not match an evictable entry",
)
.with_fact("reason", "cache_residency_failed_destroy_state_mismatch"));
}
state.state = CacheResidencyState::Destroyed;
state.eviction_in_progress = false;
Ok(())
}
}
impl Drop for CacheResidencyDestruction {
fn drop(&mut self) {
let Some(inner) = self.inner.take() else {
return;
};
if let Err(error) = abort_eviction(&inner) {
eprintln!("[query-engine] warning: failed to abort resident-cache destruction: {error}");
}
}
}
fn abort_eviction(inner: &CacheResidencyInner) -> Result<()> {
let mut state = inner.lock("abort_eviction")?;
if state.state != CacheResidencyState::Evictable || !state.eviction_in_progress {
return Err(QueryError::new(
QueryErrorCode::Internal,
OPERATION,
"resident-cache eviction abort does not match an evictable entry",
)
.with_fact("reason", "cache_residency_abort_state_mismatch"));
}
state.eviction_in_progress = false;
Ok(())
}
#[cfg(test)]
mod tests {
use std::{
sync::{
Arc, Barrier, Mutex,
atomic::{AtomicBool, Ordering},
},
thread,
};
use super::*;
use crate::{
observability::AdmissionRequestId,
runtime::gpu::{DeviceAdmissionHealth, DeviceCapabilities, DeviceResourceEnvelope, DeviceResourceProfile},
};
fn ledger(cache_cap_bytes: u64) -> DeviceLedger {
let profile = DeviceResourceProfile::resolve(DeviceResourceEnvelope {
device_ordinal: 0,
configured_limit_bytes: 10_000,
headroom_bytes: 0,
backend_reserve_bytes: 0,
default_query_limit_bytes: 100,
max_active_attempts: 1,
source_work_per_attempt: 1,
stream_pool_size: 2,
worker_job_capacity: 2,
device_capabilities: DeviceCapabilities::default(),
cache_cap_bytes,
})
.expect("cache test profile must resolve");
DeviceLedger::new(profile)
}
#[test]
fn pin_refuses_eviction_and_destruction_releases_only_after_commit() {
let ledger = ledger(500);
let transaction = CacheResidency::begin_attempt_owned(ledger.clone(), 200).unwrap().expect("charge fits");
assert_eq!(transaction.state(), CacheResidencyState::AttemptOwned);
assert_eq!(ledger.snapshot().unwrap().cache_evictable_resident_bytes, 200);
let residency = transaction.commit_evictable();
assert_eq!(residency.state().unwrap(), CacheResidencyState::Evictable);
let first_pin = residency.try_pin().unwrap().expect("evictable entry pins");
let second_pin = residency.try_pin().unwrap().expect("pinned entry accepts another lease");
let pinned = ledger.snapshot().unwrap();
assert_eq!(pinned.cache_evictable_resident_bytes, 0);
assert_eq!(pinned.cache_pinned_in_use_bytes, 200);
assert!(residency.begin_eviction().unwrap().is_none(), "pinned entry must not evict");
assert_eq!(ledger.snapshot().unwrap(), pinned, "refusal leaves its charge unchanged");
drop(first_pin);
assert_eq!(residency.state().unwrap(), CacheResidencyState::Pinned);
drop(second_pin);
assert_eq!(residency.state().unwrap(), CacheResidencyState::Evictable);
let evictable = ledger.snapshot().unwrap();
assert_eq!(evictable.cache_evictable_resident_bytes, 200);
assert_eq!(evictable.cache_pinned_in_use_bytes, 0);
let destruction = residency.begin_eviction().unwrap().expect("un-pinned entry evicts");
assert_eq!(residency.state().unwrap(), CacheResidencyState::Evictable);
assert_eq!(ledger.snapshot().unwrap().cache_evictable_resident_bytes, 200);
destruction.commit_after_destroy().unwrap();
assert_eq!(residency.state().unwrap(), CacheResidencyState::Destroyed);
let destroyed = ledger.snapshot().unwrap();
assert_eq!(destroyed.cache_evictable_resident_bytes, 0);
assert_eq!(destroyed.cache_pinned_in_use_bytes, 0);
}
#[test]
fn dropped_charge_transaction_rolls_back_exactly_once() {
let ledger = ledger(500);
let before = ledger.snapshot().unwrap().sequence;
let transaction = CacheResidency::begin_attempt_owned(ledger.clone(), 150).unwrap().expect("charge fits");
assert_eq!(ledger.snapshot().unwrap().cache_evictable_resident_bytes, 150);
drop(transaction);
let after = ledger.snapshot().unwrap();
assert_eq!(after.cache_evictable_resident_bytes, 0);
assert_eq!(after.cache_pinned_in_use_bytes, 0);
assert_eq!(after.sequence, before + 2, "one reserve and one rollback transition only");
}
#[test]
fn quarantined_resident_entry_keeps_its_charge_and_closes_admission() {
let ledger = ledger(500);
let transaction = CacheResidency::begin_attempt_owned(ledger.clone(), 200).unwrap().expect("charge fits");
let residency = transaction.commit_evictable();
let destruction = residency.begin_eviction().unwrap().expect("evictable entry begins destruction");
destruction.retain_charge_after_failed_destruction().unwrap();
ledger.close_admission("resident-cache teardown quarantined").unwrap();
assert_eq!(residency.state().unwrap(), CacheResidencyState::Destroyed);
let snapshot = ledger.snapshot().unwrap();
assert_eq!(snapshot.cache_evictable_resident_bytes, 200, "unsafe destruction must retain the sole charge");
assert_eq!(snapshot.cache_pinned_in_use_bytes, 0);
assert!(matches!(snapshot.health, DeviceAdmissionHealth::Closed { .. }));
assert!(ledger.try_admit(AdmissionRequestId::new(1), Some(1),).unwrap().is_none());
}
#[test]
fn concurrent_insert_and_evict_never_undercounts_true_residency() {
const WORKERS: usize = 4;
const ROUNDS: usize = 80;
const BYTES: u64 = 37;
let ledger = Arc::new(ledger(2_000));
let physically_resident_bytes = Arc::new(Mutex::new(0u64));
let running = Arc::new(AtomicBool::new(true));
let start = Arc::new(Barrier::new(WORKERS + 2));
let observer_ledger = Arc::clone(&ledger);
let observer_physical = Arc::clone(&physically_resident_bytes);
let observer_running = Arc::clone(&running);
let observer_start = Arc::clone(&start);
let observer = thread::spawn(move || {
observer_start.wait();
while observer_running.load(Ordering::Acquire) {
let physical = observer_physical.lock().unwrap();
let true_residency = *physical;
let snapshot = observer_ledger.snapshot().unwrap();
drop(physical);
let ledger_residency = snapshot.cache_evictable_resident_bytes + snapshot.cache_pinned_in_use_bytes;
assert!(
ledger_residency >= true_residency,
"ledger undercounted physical residency: ledger={ledger_residency}, physical={true_residency}"
);
thread::yield_now();
}
});
let workers = (0..WORKERS)
.map(|worker| {
let ledger = Arc::clone(&ledger);
let physical = Arc::clone(&physically_resident_bytes);
let start = Arc::clone(&start);
thread::spawn(move || {
start.wait();
for round in 0..ROUNDS {
let transaction = loop {
if let Some(transaction) =
CacheResidency::begin_attempt_owned((*ledger).clone(), BYTES).unwrap()
{
break transaction;
}
thread::yield_now();
};
let residency = transaction.commit_evictable();
{
let mut bytes = physical.lock().unwrap();
*bytes += BYTES;
}
if (worker + round) % 2 == 0 {
let pin = residency.try_pin().unwrap().expect("fresh entry pins");
assert!(residency.begin_eviction().unwrap().is_none());
drop(pin);
}
let destruction = residency.begin_eviction().unwrap().expect("entry becomes evictable");
let mut bytes = physical.lock().unwrap();
*bytes -= BYTES;
destruction.commit_after_destroy().unwrap();
}
})
})
.collect::<Vec<_>>();
start.wait();
for worker in workers {
worker.join().expect("residency worker must not panic");
}
running.store(false, Ordering::Release);
observer.join().expect("residency observer must not panic");
assert_eq!(*physically_resident_bytes.lock().unwrap(), 0);
let snapshot = ledger.snapshot().unwrap();
assert_eq!(snapshot.cache_evictable_resident_bytes, 0);
assert_eq!(snapshot.cache_pinned_in_use_bytes, 0);
}
#[cfg(feature = "cugraph")]
#[test]
fn transient_cache_execution_reservation_is_pinned_until_physical_cleanup() {
let ledger = ledger(500);
let reservation = CacheExecutionReservation::begin(ledger.clone(), 200)
.unwrap()
.expect("the transient cache reservation fits");
let during_execution = ledger.snapshot().unwrap();
assert_eq!(during_execution.cache_evictable_resident_bytes, 0);
assert_eq!(during_execution.cache_pinned_in_use_bytes, 200);
reservation.release_after_destroy().unwrap();
let after_cleanup = ledger.snapshot().unwrap();
assert_eq!(after_cleanup.cache_evictable_resident_bytes, 0);
assert_eq!(after_cleanup.cache_pinned_in_use_bytes, 0);
}
#[cfg(feature = "cugraph")]
#[test]
fn dropped_transient_cache_execution_reservation_retains_charge_fail_closed() {
let ledger = ledger(500);
let reservation = CacheExecutionReservation::begin(ledger.clone(), 200)
.unwrap()
.expect("the transient cache reservation fits");
drop(reservation);
let stranded = ledger.snapshot().unwrap();
assert_eq!(stranded.cache_evictable_resident_bytes, 200);
assert_eq!(stranded.cache_pinned_in_use_bytes, 0);
}
}