use std::collections::VecDeque;
use std::fmt;
use std::task::Waker;
use std::time::{Duration, Instant};
use crate::runtime::sync::sync::atomic::{AtomicU64, Ordering};
use crate::runtime::sync::sync::{Arc, MutexGuard};
use super::{invariant_violation, CarrierCount, PoolInner};
const RESERVATION_CLOSED: u64 = 1 << 63;
const DIRECT_OUTSTANDING_MASK: u64 = !RESERVATION_CLOSED;
const RESERVATION_DRAIN_ARMED: u64 = 1;
const RESERVATION_REPAYMENT_EPOCH: u64 = 2;
mod coverage;
use coverage::CoverageSnapshot;
pub(super) use coverage::{CoverageDebit, CoverageState, MAX_PACKED_CARRIERS};
mod waiter;
pub(crate) use waiter::ReserveFuture;
pub(super) use waiter::{ReservationPoll, WaitSlot, WaitState, Waiter};
pub(super) struct ReservationDrainSignal {
state: AtomicU64,
}
impl ReservationDrainSignal {
pub(super) fn new() -> Self {
Self {
state: AtomicU64::new(0),
}
}
pub(super) fn arm(&self) {
self.state
.fetch_or(RESERVATION_DRAIN_ARMED, Ordering::AcqRel);
}
pub(super) fn publish_waiter_state(&self, waiters_present: bool) {
if waiters_present {
self.arm();
} else {
self.state
.fetch_and(!RESERVATION_DRAIN_ARMED, Ordering::AcqRel);
}
}
pub(super) fn repayment_requires_drain(&self) -> bool {
self.state
.fetch_xor(RESERVATION_REPAYMENT_EPOCH, Ordering::AcqRel)
& RESERVATION_DRAIN_ARMED
!= 0
}
pub(super) fn is_armed(&self) -> bool {
self.state.load(Ordering::Acquire) & RESERVATION_DRAIN_ARMED != 0
}
}
pub(super) struct AdmissionState {
pub(super) ledger: AdmissionLedger,
queue: ReservationQueue,
}
impl AdmissionState {
pub(super) fn new(configured_capacity: CarrierCount) -> Self {
Self {
ledger: AdmissionLedger {
configured_capacity,
prepared_capacity: CarrierCount::ZERO,
active_planned_demand: CarrierCount::ZERO,
},
queue: ReservationQueue::new(Instant::now()),
}
}
pub(super) fn waiters_is_empty(&self) -> bool {
self.queue.is_empty()
}
pub(super) fn waiter_count(&self) -> usize {
self.queue.len()
}
pub(super) fn front_waiter(&self) -> Option<&Waiter> {
self.queue.front()
}
pub(super) fn enqueue_waiter(
&mut self,
waiter: Waiter,
now: Instant,
) -> Result<bool, std::collections::TryReserveError> {
self.queue.enqueue(waiter, now)
}
pub(super) fn remove_waiter(&mut self, slot: &Arc<WaitSlot>, now: Instant) -> bool {
self.queue.remove(slot, now)
}
pub(super) fn pop_front_waiter(&mut self, now: Instant) -> (Option<Waiter>, bool) {
self.queue.pop_front(now)
}
pub(super) fn take_queue_diagnostics(
&mut self,
now: Instant,
) -> ReservationQueueDiagnosticSample {
self.queue.take_diagnostics(now)
}
pub(super) fn reservation_enqueues_total(&self) -> u64 {
self.queue.reservation_enqueues_total
}
#[cfg(test)]
pub(super) fn set_reservation_enqueues_total_for_test(&mut self, total: u64) {
self.queue.reservation_enqueues_total = total;
}
}
struct ReservationQueue {
waiters: VecDeque<Waiter>,
diagnostics: ReservationQueueDiagnostics,
reservation_enqueues_total: u64,
}
impl ReservationQueue {
fn new(now: Instant) -> Self {
Self {
waiters: VecDeque::new(),
diagnostics: ReservationQueueDiagnostics::new(now),
reservation_enqueues_total: 0,
}
}
fn is_empty(&self) -> bool {
self.waiters.is_empty()
}
fn len(&self) -> usize {
self.waiters.len()
}
fn front(&self) -> Option<&Waiter> {
self.waiters.front()
}
fn enqueue(
&mut self,
waiter: Waiter,
now: Instant,
) -> Result<bool, std::collections::TryReserveError> {
self.waiters.try_reserve(1)?;
self.waiters.push_back(waiter);
self.reservation_enqueues_total = self.reservation_enqueues_total.saturating_add(1);
Ok(matches!(
self.diagnostics.record_depth(self.waiters.len(), now),
Some(ReservationQueueTransition::BecameNonempty)
))
}
fn remove(&mut self, slot: &Arc<WaitSlot>, now: Instant) -> bool {
let Some(index) = self
.waiters
.iter()
.position(|waiter| Arc::ptr_eq(&waiter.slot, slot))
else {
return false;
};
self.waiters.remove(index);
matches!(
self.diagnostics.record_depth(self.waiters.len(), now),
Some(ReservationQueueTransition::BecameEmpty)
)
}
fn pop_front(&mut self, now: Instant) -> (Option<Waiter>, bool) {
let waiter = self.waiters.pop_front();
let cleared = waiter.is_some()
&& matches!(
self.diagnostics.record_depth(self.waiters.len(), now),
Some(ReservationQueueTransition::BecameEmpty)
);
(waiter, cleared)
}
fn take_diagnostics(&mut self, now: Instant) -> ReservationQueueDiagnosticSample {
self.diagnostics.take(self.waiters.len(), now)
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
enum ReservationQueueTransition {
BecameNonempty,
BecameEmpty,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(super) struct ReservationQueueDiagnosticSample {
pub(super) current_depth: usize,
pub(super) peak_depth: usize,
pub(super) nonempty_duration: Duration,
pub(super) continuous_nonempty_duration: Option<Duration>,
}
struct ReservationQueueDiagnostics {
current_depth: usize,
peak_depth: usize,
nonempty_duration: Duration,
became_nonempty_at: Option<Instant>,
last_observed_at: Instant,
}
impl ReservationQueueDiagnostics {
fn new(now: Instant) -> Self {
Self {
current_depth: 0,
peak_depth: 0,
nonempty_duration: Duration::ZERO,
became_nonempty_at: None,
last_observed_at: now,
}
}
fn record_depth(&mut self, depth: usize, now: Instant) -> Option<ReservationQueueTransition> {
self.accrue(now);
let transition = match (self.current_depth == 0, depth == 0) {
(true, false) => {
self.became_nonempty_at = Some(now);
Some(ReservationQueueTransition::BecameNonempty)
}
(false, true) => {
self.became_nonempty_at = None;
Some(ReservationQueueTransition::BecameEmpty)
}
_ => None,
};
self.current_depth = depth;
self.peak_depth = self.peak_depth.max(depth);
transition
}
fn take(&mut self, depth: usize, now: Instant) -> ReservationQueueDiagnosticSample {
let _ = self.record_depth(depth, now);
let sample = ReservationQueueDiagnosticSample {
current_depth: depth,
peak_depth: self.peak_depth,
nonempty_duration: self.nonempty_duration,
continuous_nonempty_duration: self
.became_nonempty_at
.map(|started| now.saturating_duration_since(started)),
};
self.peak_depth = depth;
self.nonempty_duration = Duration::ZERO;
sample
}
fn accrue(&mut self, now: Instant) {
let elapsed = now.saturating_duration_since(self.last_observed_at);
if self.current_depth != 0 {
self.nonempty_duration = self.nonempty_duration.saturating_add(elapsed);
}
self.last_observed_at = now;
}
}
pub(super) struct AdmissionGuard<'a> {
pub(super) inner: MutexGuard<'a, AdmissionState>,
}
impl<'a> AdmissionGuard<'a> {
pub(super) fn new(inner: MutexGuard<'a, AdmissionState>) -> Self {
Self { inner }
}
pub(super) fn prepared_capacity(&self) -> CarrierCount {
self.inner.ledger.prepared_capacity
}
pub(super) fn prepared_capacity_mut(&mut self) -> &mut CarrierCount {
&mut self.inner.ledger.prepared_capacity
}
pub(super) fn can_grant(&self, coverage: CoverageSnapshot, envelope: CarrierCount) -> bool {
self.inner.ledger.can_grant(coverage, envelope)
}
pub(super) fn validate_envelope(&self, envelope: CarrierCount) -> Result<(), ReserveError> {
self.inner.ledger.validate_envelope(envelope)
}
pub(super) fn grant_target(
&self,
coverage: CoverageSnapshot,
envelope: CarrierCount,
) -> Result<CarrierCount, ReserveError> {
self.inner.ledger.grant_target(coverage, envelope)
}
pub(super) fn commit_grant(
&mut self,
coverage: &CoverageState,
envelope: CarrierCount,
) -> Result<(), ReserveError> {
self.inner.ledger.commit_grant(coverage, envelope)
}
pub(super) fn close_envelope(
&mut self,
coverage: &CoverageState,
envelope: CarrierCount,
direct_outstanding: CarrierCount,
) {
self.inner
.ledger
.close_envelope(coverage, envelope, direct_outstanding);
}
pub(super) fn maximum_uncovered(&self) -> CarrierCount {
MAX_PACKED_CARRIERS
.checked_sub(self.inner.ledger.active_planned_demand)
.unwrap_or_else(|| {
invariant_violation("active planned demand exceeds packed admission")
})
}
pub(super) fn acquisition_floor(
&self,
coverage: &CoverageState,
) -> Result<CarrierCount, ReserveError> {
self.inner.ledger.admission_used(coverage.snapshot())
}
pub(super) fn rollback_acquisition(
&mut self,
coverage: &CoverageState,
count: CarrierCount,
debit: CoverageDebit,
) {
coverage.rollback_debit(count, debit, self.inner.ledger.active_planned_demand);
self.inner.ledger.assert_invariants(coverage.snapshot());
}
}
pub(super) struct AdmissionLedger {
pub(super) configured_capacity: CarrierCount,
pub(super) prepared_capacity: CarrierCount,
pub(super) active_planned_demand: CarrierCount,
}
impl AdmissionLedger {
pub(super) fn admission_used(
&self,
coverage: CoverageSnapshot,
) -> Result<CarrierCount, ReserveError> {
self.active_planned_demand
.checked_add(coverage.uncovered)
.filter(|count| *count <= MAX_PACKED_CARRIERS)
.ok_or(ReserveError::CapacityOverflow)
}
fn can_grant(&self, coverage: CoverageSnapshot, envelope: CarrierCount) -> bool {
self.admission_used(coverage)
.ok()
.and_then(|used| used.checked_add(envelope))
.is_some_and(|next| next <= self.configured_capacity)
}
fn validate_envelope(&self, envelope: CarrierCount) -> Result<(), ReserveError> {
if envelope > self.configured_capacity {
return Err(ReserveError::ExceedsCapacity);
}
Ok(())
}
fn grant_target(
&self,
coverage: CoverageSnapshot,
envelope: CarrierCount,
) -> Result<CarrierCount, ReserveError> {
self.admission_used(coverage)?
.checked_add(envelope)
.filter(|target| *target <= MAX_PACKED_CARRIERS)
.ok_or(ReserveError::CapacityOverflow)
}
fn commit_grant(
&mut self,
coverage: &CoverageState,
envelope: CarrierCount,
) -> Result<(), ReserveError> {
let snapshot = coverage.snapshot();
let next_active = self
.active_planned_demand
.checked_add(envelope)
.ok_or(ReserveError::CapacityOverflow)?;
let next_admission_used = next_active
.checked_add(snapshot.uncovered)
.filter(|total| *total <= MAX_PACKED_CARRIERS)
.ok_or(ReserveError::CapacityOverflow)?;
if self.prepared_capacity < next_admission_used {
invariant_violation(
"reservation grant did not prepare its admission floor before publication",
);
}
coverage
.add_coverage(envelope)
.map_err(|_| ReserveError::CapacityOverflow)?;
self.active_planned_demand = next_active;
self.assert_invariants(coverage.snapshot());
Ok(())
}
fn close_envelope(
&mut self,
coverage: &CoverageState,
envelope: CarrierCount,
direct_outstanding: CarrierCount,
) {
let Some(remaining_active) = self.active_planned_demand.checked_sub(envelope) else {
invariant_violation("reservation close exceeds active planned demand");
};
self.active_planned_demand = remaining_active;
coverage.remove_coverage(envelope, direct_outstanding, self.active_planned_demand);
self.assert_invariants(coverage.snapshot());
}
pub(super) fn assert_invariants(&self, coverage: CoverageSnapshot) {
let admission_used = self
.admission_used(coverage)
.unwrap_or_else(|_| invariant_violation("completed admission exceeds packed capacity"));
if self.prepared_capacity < admission_used {
invariant_violation("prepared capacity does not cover admission used");
}
if coverage.available > self.active_planned_demand {
invariant_violation("available coverage exceeds active planned demand");
}
}
}
#[non_exhaustive]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(crate) enum ReserveError {
InvalidSize,
ExceedsCapacity,
PhysicalPreparationFailed,
MetadataAllocationFailed,
CapacityOverflow,
}
impl fmt::Display for ReserveError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::InvalidSize => f.write_str("reservation size must be nonzero"),
Self::ExceedsCapacity => {
f.write_str("reservation exceeds configured buffer-pool capacity")
}
Self::PhysicalPreparationFailed => {
f.write_str("physical buffer-pool preparation failed")
}
Self::MetadataAllocationFailed => f.write_str("buffer-pool metadata allocation failed"),
Self::CapacityOverflow => {
f.write_str("reservation exceeds buffer-pool accounting capacity")
}
}
}
}
impl std::error::Error for ReserveError {}
pub(crate) struct Reservation {
state: Option<Arc<ReservationState>>,
}
pub(super) struct ReservationState {
pool: Arc<PoolInner>,
envelope: CarrierCount,
owner_state: ReservationOwnerState,
}
impl Reservation {
pub(super) fn new(pool: Arc<PoolInner>, envelope: CarrierCount) -> Self {
Self {
state: Some(Arc::new(ReservationState {
pool,
envelope,
owner_state: ReservationOwnerState::new(),
})),
}
}
pub(crate) fn close_acquisition(mut self) {
self.close();
}
fn close(&mut self) {
if let Some(state) = self.state.take() {
state.close_acquisition();
}
}
pub(super) fn acquisition_state(&self) -> Option<&Arc<ReservationState>> {
self.state.as_ref()
}
}
impl fmt::Debug for Reservation {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("Reservation")
.field("open", &self.state.is_some())
.finish_non_exhaustive()
}
}
impl Drop for Reservation {
fn drop(&mut self) {
self.close();
}
}
impl ReservationState {
pub(super) fn pool(&self) -> &Arc<PoolInner> {
&self.pool
}
pub(super) fn belongs_to(&self, pool: &Arc<PoolInner>) -> bool {
Arc::ptr_eq(&self.pool, pool)
}
pub(super) fn try_debit(&self, count: CarrierCount) -> Result<(), DirectDebitError> {
self.owner_state.try_debit(self.envelope, count)
}
pub(super) fn precheck_debit(&self, count: CarrierCount) -> Result<(), DirectDebitError> {
self.owner_state.precheck_debit(self.envelope, count)
}
pub(super) fn release(&self, count: CarrierCount) {
self.owner_state.release(count);
}
#[cfg(test)]
pub(super) fn test_owner_state(&self) -> (bool, CarrierCount) {
let snapshot = self.owner_state.snapshot();
(snapshot.closed, snapshot.direct_outstanding)
}
fn close_acquisition(&self) {
let drained = {
let mut admission = AdmissionGuard::new(self.pool.admission.lock());
let Some(direct_outstanding) = self.owner_state.close() else {
return;
};
admission.close_envelope(&self.pool.coverage, self.envelope, direct_outstanding);
PoolInner::drain_fifo_locked(&self.pool, &mut admission, false)
};
if let Some(sample) = drained.queue_sample {
self.pool.log_reservation_queue_transition(sample);
}
wake_all(drained.wakers);
}
}
pub(super) fn wake_all(wakers: Vec<Waker>) {
for waker in wakers {
waker.wake();
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(super) enum DirectDebitError {
Closed,
CapacityExceeded,
CapacityOverflow,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
struct ReservationOwnerSnapshot {
closed: bool,
direct_outstanding: CarrierCount,
}
struct ReservationOwnerState {
packed: AtomicU64,
}
impl ReservationOwnerState {
fn new() -> Self {
Self {
packed: AtomicU64::new(0),
}
}
fn precheck_debit(
&self,
envelope: CarrierCount,
count: CarrierCount,
) -> Result<(), DirectDebitError> {
let envelope =
u64::try_from(envelope.get()).map_err(|_| DirectDebitError::CapacityOverflow)?;
let count = u64::try_from(count.get()).map_err(|_| DirectDebitError::CapacityOverflow)?;
if envelope > DIRECT_OUTSTANDING_MASK || count > DIRECT_OUTSTANDING_MASK {
return Err(DirectDebitError::CapacityOverflow);
}
let current = self.packed.load(Ordering::Acquire);
if current & RESERVATION_CLOSED != 0 {
return Err(DirectDebitError::Closed);
}
(current & DIRECT_OUTSTANDING_MASK)
.checked_add(count)
.filter(|next| *next <= envelope)
.map(|_| ())
.ok_or(DirectDebitError::CapacityExceeded)
}
fn try_debit(
&self,
envelope: CarrierCount,
count: CarrierCount,
) -> Result<(), DirectDebitError> {
let envelope =
u64::try_from(envelope.get()).map_err(|_| DirectDebitError::CapacityOverflow)?;
let count = u64::try_from(count.get()).map_err(|_| DirectDebitError::CapacityOverflow)?;
if envelope > DIRECT_OUTSTANDING_MASK || count > DIRECT_OUTSTANDING_MASK {
return Err(DirectDebitError::CapacityOverflow);
}
#[allow(deprecated)]
let result = self
.packed
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| {
if current & RESERVATION_CLOSED != 0 {
return None;
}
(current & DIRECT_OUTSTANDING_MASK)
.checked_add(count)
.filter(|next| *next <= envelope)
});
match result {
Ok(_) => Ok(()),
Err(current) if current & RESERVATION_CLOSED != 0 => Err(DirectDebitError::Closed),
Err(_) => Err(DirectDebitError::CapacityExceeded),
}
}
fn close(&self) -> Option<CarrierCount> {
let previous = self.packed.fetch_or(RESERVATION_CLOSED, Ordering::AcqRel);
if previous & RESERVATION_CLOSED != 0 {
return None;
}
Some(CarrierCount::new(
usize::try_from(previous & DIRECT_OUTSTANDING_MASK)
.unwrap_or_else(|_| invariant_violation("direct owner count does not fit usize")),
))
}
fn release(&self, count: CarrierCount) -> bool {
let count = u64::try_from(count.get()).unwrap_or_else(|_| {
invariant_violation("direct release count does not fit owner state")
});
#[allow(deprecated)]
let previous = self
.packed
.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| {
let outstanding = current & DIRECT_OUTSTANDING_MASK;
outstanding
.checked_sub(count)
.map(|next| (current & RESERVATION_CLOSED) | next)
})
.unwrap_or_else(|_| {
invariant_violation("direct release exceeds outstanding ownership")
});
previous & RESERVATION_CLOSED == 0
}
fn snapshot(&self) -> ReservationOwnerSnapshot {
let packed = self.packed.load(Ordering::Acquire);
ReservationOwnerSnapshot {
closed: packed & RESERVATION_CLOSED != 0,
direct_outstanding: CarrierCount::new(
usize::try_from(packed & DIRECT_OUTSTANDING_MASK).unwrap_or_else(|_| {
invariant_violation("direct owner count does not fit usize")
}),
),
}
}
}
#[cfg(all(test, not(s3_tm_loom)))]
mod tests {
use super::super::test_util::test_pool;
use super::super::virtual_memory::VirtualMemoryOperation;
use super::super::BufferPool;
use super::*;
#[test]
fn test_queue_diagnostics_report_depth_peak_and_nonempty_durations() {
let start = Instant::now();
let mut diagnostics = ReservationQueueDiagnostics::new(start);
assert_eq!(
diagnostics.record_depth(1, start + Duration::from_millis(10)),
Some(ReservationQueueTransition::BecameNonempty)
);
assert_eq!(
diagnostics.record_depth(4, start + Duration::from_millis(20)),
None
);
assert_eq!(
diagnostics.take(4, start + Duration::from_millis(30)),
ReservationQueueDiagnosticSample {
current_depth: 4,
peak_depth: 4,
nonempty_duration: Duration::from_millis(20),
continuous_nonempty_duration: Some(Duration::from_millis(20)),
}
);
assert_eq!(
diagnostics.record_depth(0, start + Duration::from_millis(50)),
Some(ReservationQueueTransition::BecameEmpty)
);
assert_eq!(
diagnostics.take(0, start + Duration::from_millis(60)),
ReservationQueueDiagnosticSample {
current_depth: 0,
peak_depth: 4,
nonempty_duration: Duration::from_millis(20),
continuous_nonempty_duration: None,
}
);
}
#[test]
fn test_immediate_grant_prepares_before_publication() {
let (pool, carrier_size) = test_pool(2, 4);
let reservation = pool.try_reserve(carrier_size).unwrap().unwrap();
{
let admission = pool.inner.admission.lock();
let coverage = pool.inner.coverage.snapshot();
assert_eq!(admission.ledger.prepared_capacity, CarrierCount::new(2));
assert_eq!(admission.ledger.active_planned_demand, CarrierCount::new(1));
assert_eq!(coverage.available, CarrierCount::new(1));
assert_eq!(coverage.uncovered, CarrierCount::ZERO);
}
drop(reservation);
let admission = pool.inner.admission.lock();
assert_eq!(admission.ledger.active_planned_demand, CarrierCount::ZERO);
assert_eq!(admission.ledger.prepared_capacity, CarrierCount::new(2));
assert_eq!(
pool.inner.coverage.snapshot(),
CoverageSnapshot {
available: CarrierCount::ZERO,
uncovered: CarrierCount::ZERO,
}
);
}
#[test]
fn test_grant_cannot_publish_before_preparing_its_floor() {
let mut admission = AdmissionState::new(CarrierCount::new(1));
let coverage = CoverageState::new();
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
admission
.ledger
.commit_grant(&coverage, CarrierCount::new(1))
.unwrap();
}));
assert!(result.is_err());
assert_eq!(admission.ledger.active_planned_demand, CarrierCount::ZERO);
assert_eq!(
coverage.snapshot(),
CoverageSnapshot {
available: CarrierCount::ZERO,
uncovered: CarrierCount::ZERO,
}
);
}
#[test]
fn test_normal_grant_respects_configured_capacity() {
let (pool, carrier_size) = test_pool(2, 2);
let reservation = pool.try_reserve(carrier_size * 2).unwrap().unwrap();
assert!(pool.try_reserve(carrier_size).unwrap().is_none());
assert_eq!(pool.inner.arena.diagnostics().blocks_prepared, 1);
drop(reservation);
}
#[test]
fn test_reservation_larger_than_configured_capacity_is_rejected() {
let (pool, carrier_size) = test_pool(2, 1);
assert!(matches!(
pool.try_reserve(carrier_size * 2),
Err(ReserveError::ExceedsCapacity)
));
let admission = pool.inner.admission.lock();
assert_eq!(admission.ledger.active_planned_demand, CarrierCount::ZERO);
assert_eq!(admission.ledger.prepared_capacity, CarrierCount::ZERO);
assert_eq!(
pool.inner.coverage.snapshot(),
CoverageSnapshot {
available: CarrierCount::ZERO,
uncovered: CarrierCount::ZERO,
}
);
}
#[test]
fn test_retained_ownership_applies_reservation_backpressure() {
let (pool, carrier_size) = test_pool(1, 1);
let reservation = pool
.try_reserve(carrier_size)
.unwrap()
.expect("initial reservation");
let retained = pool.acquire(&reservation, carrier_size).unwrap();
reservation.close_acquisition();
for _ in 0..4 {
assert!(pool.try_reserve(carrier_size).unwrap().is_none());
let metrics = pool.metrics();
assert_eq!(metrics.active_planned_demand_bytes(), 0);
assert_eq!(metrics.admitted_bytes(), carrier_size as u64);
assert_eq!(metrics.in_use_bytes(), carrier_size as u64);
assert_eq!(metrics.admission_overage_bytes(), 0);
}
drop(retained);
let next = pool
.try_reserve(carrier_size)
.unwrap()
.expect("returned ownership restores admission");
drop(next);
let audit = pool.audit_quiescent();
assert_eq!(audit.active_planned_demand, CarrierCount::ZERO);
assert_eq!(audit.available_coverage, CarrierCount::ZERO);
assert_eq!(audit.uncovered_charges, CarrierCount::ZERO);
assert_eq!(audit.charged_capacity, CarrierCount::ZERO);
assert_eq!(audit.live_carriers, CarrierCount::ZERO);
assert_eq!(audit.queued_reservations, 0);
assert_eq!(audit.cleanup_pending_blocks, 0);
assert_eq!(audit.prepared_capacity, CarrierCount::new(1));
}
#[test]
fn test_preparation_failure_publishes_no_grant() {
let (pool, carrier_size) = test_pool(2, 4);
let first = pool.inner.arena.reserve_slot().unwrap();
let second = pool.inner.arena.reserve_slot().unwrap();
second.inject_failure_once(VirtualMemoryOperation::Prepare);
assert!(matches!(
pool.try_reserve(carrier_size * 3),
Err(ReserveError::PhysicalPreparationFailed)
));
{
let admission = pool.inner.admission.lock();
assert_eq!(admission.ledger.prepared_capacity, CarrierCount::new(2));
assert_eq!(admission.ledger.active_planned_demand, CarrierCount::ZERO);
assert_eq!(
pool.inner.coverage.snapshot(),
CoverageSnapshot {
available: CarrierCount::ZERO,
uncovered: CarrierCount::ZERO,
}
);
}
let reservation = pool.try_reserve(carrier_size * 3).unwrap().unwrap();
assert_eq!(
pool.inner.admission.lock().ledger.prepared_capacity,
CarrierCount::new(4)
);
drop(reservation);
drop(first);
}
#[test]
fn test_close_reclassifies_occupied_coverage() {
let (pool, carrier_size) = test_pool(2, 2);
let reservation = pool.try_reserve(carrier_size * 2).unwrap().unwrap();
assert!(pool
.inner
.coverage
.try_debit_covered(CarrierCount::new(1))
.unwrap());
reservation.close_acquisition();
assert_eq!(
pool.inner.coverage.snapshot(),
CoverageSnapshot {
available: CarrierCount::ZERO,
uncovered: CarrierCount::new(1),
}
);
assert_eq!(
pool.inner.admission.lock().ledger.active_planned_demand,
CarrierCount::ZERO
);
pool.inner.coverage.release(CarrierCount::new(1));
}
#[test]
fn test_invalid_reservation_sizes_fail_before_preparation() {
let (pool, carrier_size) = test_pool(1, 1);
assert!(matches!(
pool.try_reserve(0),
Err(ReserveError::InvalidSize)
));
if let Some(bytes) = (u32::MAX as usize)
.checked_add(1)
.and_then(|carriers| carriers.checked_mul(carrier_size))
{
assert!(matches!(
pool.try_reserve(bytes),
Err(ReserveError::CapacityOverflow)
));
}
assert_eq!(pool.inner.arena.diagnostics().blocks_prepared, 0);
}
#[test]
fn test_pool_ownership_spine_is_send_and_sync() {
fn assert_send_sync<T: Send + Sync>() {}
assert_send_sync::<BufferPool>();
assert_send_sync::<Reservation>();
assert_send_sync::<ReservationState>();
}
#[test]
fn test_close_is_idempotent() {
let state = ReservationOwnerState::new();
assert_eq!(state.close(), Some(CarrierCount::ZERO));
assert_eq!(state.close(), None);
assert_eq!(
state.snapshot(),
ReservationOwnerSnapshot {
closed: true,
direct_outstanding: CarrierCount::ZERO,
}
);
}
#[test]
fn test_direct_debit_cannot_exceed_envelope() {
let state = ReservationOwnerState::new();
state
.try_debit(CarrierCount::new(3), CarrierCount::new(2))
.unwrap();
assert_eq!(
state.try_debit(CarrierCount::new(3), CarrierCount::new(2)),
Err(DirectDebitError::CapacityExceeded)
);
assert_eq!(state.snapshot().direct_outstanding, CarrierCount::new(2));
}
#[test]
fn test_release_after_close_does_not_reopen_reservation() {
let state = ReservationOwnerState::new();
state
.try_debit(CarrierCount::new(2), CarrierCount::new(2))
.unwrap();
assert_eq!(state.close(), Some(CarrierCount::new(2)));
assert!(!state.release(CarrierCount::new(2)));
assert_eq!(
state.snapshot(),
ReservationOwnerSnapshot {
closed: true,
direct_outstanding: CarrierCount::ZERO,
}
);
assert_eq!(
state.try_debit(CarrierCount::new(2), CarrierCount::new(1)),
Err(DirectDebitError::Closed)
);
}
#[test]
fn test_invalid_release_leaves_owner_state_unchanged() {
let state = ReservationOwnerState::new();
let result = std::panic::catch_unwind(|| state.release(CarrierCount::new(1)));
assert!(result.is_err());
assert_eq!(
state.snapshot(),
ReservationOwnerSnapshot {
closed: false,
direct_outstanding: CarrierCount::ZERO,
}
);
}
}
#[cfg(all(test, s3_tm_loom))]
mod loom_tests {
use super::super::test_util::test_single_carrier_pool as test_pool;
use crate::runtime::sync::sync::Arc;
use crate::runtime::sync::thread;
use super::*;
#[test]
fn test_concurrent_immediate_grants_respect_normal_capacity() {
loom::model(|| {
let (pool, carrier_size) = test_pool(1);
let first_pool = pool.clone();
let first = thread::spawn(move || first_pool.try_reserve(carrier_size).unwrap());
let second_pool = pool.clone();
let second = thread::spawn(move || second_pool.try_reserve(carrier_size).unwrap());
let first = first.join().unwrap();
let second = second.join().unwrap();
assert_eq!(
usize::from(first.is_some()) + usize::from(second.is_some()),
1
);
drop(first);
drop(second);
assert_eq!(
pool.inner.admission.lock().ledger.active_planned_demand,
CarrierCount::ZERO
);
});
}
#[test]
fn test_concurrent_oversized_requests_do_not_enter_admission() {
loom::model(|| {
let (pool, carrier_size) = test_pool(1);
let first_pool = pool.clone();
let first = thread::spawn(move || first_pool.try_reserve(carrier_size * 2));
let second_pool = pool.clone();
let second = thread::spawn(move || second_pool.try_reserve(carrier_size * 2));
let first = first.join().unwrap();
let second = second.join().unwrap();
assert!(matches!(first, Err(ReserveError::ExceedsCapacity)));
assert!(matches!(second, Err(ReserveError::ExceedsCapacity)));
let admission = pool.inner.admission.lock();
assert_eq!(admission.ledger.active_planned_demand, CarrierCount::ZERO);
assert_eq!(admission.ledger.prepared_capacity, CarrierCount::ZERO);
assert!(admission.waiters_is_empty());
});
}
#[test]
fn test_direct_debit_racing_close_has_one_linearization() {
loom::model(|| {
let state = Arc::new(ReservationOwnerState::new());
let debiting = Arc::clone(&state);
let debit = thread::spawn(move || {
debiting.try_debit(CarrierCount::new(1), CarrierCount::new(1))
});
let closing = Arc::clone(&state);
let close = thread::spawn(move || closing.close());
let debit = debit.join().unwrap();
let closed_outstanding = close.join().unwrap();
let snapshot = state.snapshot();
assert!(snapshot.closed);
match debit {
Ok(()) => {
assert_eq!(closed_outstanding, Some(CarrierCount::new(1)));
assert_eq!(snapshot.direct_outstanding, CarrierCount::new(1));
}
Err(DirectDebitError::Closed) => {
assert_eq!(closed_outstanding, Some(CarrierCount::ZERO));
assert_eq!(snapshot.direct_outstanding, CarrierCount::ZERO);
}
Err(error) => panic!("unexpected debit error: {error:?}"),
}
});
}
#[test]
fn test_concurrent_direct_debits_consume_authority_once() {
loom::model(|| {
let state = Arc::new(ReservationOwnerState::new());
let first_state = Arc::clone(&state);
let first = thread::spawn(move || {
first_state.try_debit(CarrierCount::new(1), CarrierCount::new(1))
});
let second_state = Arc::clone(&state);
let second = thread::spawn(move || {
second_state.try_debit(CarrierCount::new(1), CarrierCount::new(1))
});
let successes = usize::from(first.join().unwrap().is_ok())
+ usize::from(second.join().unwrap().is_ok());
assert_eq!(successes, 1);
assert_eq!(state.snapshot().direct_outstanding, CarrierCount::new(1));
});
}
#[test]
fn test_release_racing_close_cannot_reopen_reservation() {
loom::model(|| {
let state = Arc::new(ReservationOwnerState::new());
state
.try_debit(CarrierCount::new(1), CarrierCount::new(1))
.unwrap();
let releasing = Arc::clone(&state);
let release = thread::spawn(move || releasing.release(CarrierCount::new(1)));
let closing = Arc::clone(&state);
let close = thread::spawn(move || closing.close());
release.join().unwrap();
close.join().unwrap();
assert_eq!(
state.snapshot(),
ReservationOwnerSnapshot {
closed: true,
direct_outstanding: CarrierCount::ZERO,
}
);
});
}
}