use std::task::Waker;
use crate::config::MemoryDiagnosticsConfig;
#[cfg(not(all(test, s3_tm_loom)))]
use crate::runtime::sync::sync::atomic::{AtomicUsize, Ordering};
use crate::runtime::sync::sync::{Arc, Mutex};
mod acquisition;
mod admission;
mod arena;
#[cfg(s3_tm_bench)]
pub(crate) mod bench_util;
mod block;
mod config;
#[cfg(not(all(test, s3_tm_loom)))]
mod diagnostics;
mod geometry;
mod maintenance;
mod metrics;
mod pooled_buf;
mod segmented_bytes;
#[cfg(any(test, s3_tm_fuzz))]
mod test_util;
#[cfg(all(test, not(s3_tm_loom)))]
mod tests;
mod virtual_memory;
use crate::config::MemoryBudgetConfig;
use acquisition::acquire_count;
pub(crate) use acquisition::AcquireError;
use admission::{
wake_all, AdmissionGuard, AdmissionState, CoverageDebit, CoverageState, ReservationDrainSignal,
ReservationPoll, WaitSlot, WaitState, Waiter, MAX_PACKED_CARRIERS,
};
pub(crate) use admission::{Reservation, ReserveError, ReserveFuture};
use arena::{Arena, ArenaError, ArenaOptions};
use block::BlockError;
pub(crate) use config::auto_capacity_bytes;
use config::PoolConfig;
pub use config::{BufferPoolBuildError, BufferPoolBuilder};
use geometry::{GeometryError, PoolGeometry};
use maintenance::MaintenanceCoordinator;
pub use metrics::MemoryMetrics;
use metrics::{MemoryDiagnostics, MemoryMetricState};
use pooled_buf::GrowthAuthority;
pub(crate) use pooled_buf::PooledBufMut;
pub use segmented_bytes::SegmentedBytes;
#[cfg(s3_tm_fuzz)]
pub(crate) fn run_fuzz_input(data: &[u8]) {
let _ = test_util::model_harness::run_fuzz_input(data);
}
#[cfg(s3_tm_fuzz)]
pub(crate) fn run_placement_fuzz_input(data: &[u8]) {
let _ = test_util::model_harness::run_placement_fuzz_input(data);
}
#[cfg(all(test, not(s3_tm_loom)))]
pub(crate) use test_util::test_pool;
#[cfg(test)]
use test_util::TestHooks;
#[derive(Clone)]
pub struct BufferPool {
inner: Arc<PoolInner>,
}
impl std::fmt::Debug for BufferPool {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("BufferPool").finish_non_exhaustive()
}
}
impl BufferPool {
pub fn builder() -> BufferPoolBuilder {
BufferPoolBuilder::default()
}
pub(crate) fn carrier_size(&self) -> usize {
self.inner.geometry.carrier_size()
}
pub(crate) fn from_capacity(
capacity: MemoryBudgetConfig,
detected_memory: Option<usize>,
diagnostics: MemoryDiagnosticsConfig,
) -> Result<Self, BufferPoolBuildError> {
let resolved = PoolConfig::resolve(capacity, detected_memory)?;
let arena_options = ArenaOptions::new(
resolved.optimistic_scan_words,
diagnostics.enable_detailed_counters(),
);
Ok(Self::from_parts_with_arena_options(
resolved.geometry,
resolved.configured_capacity,
diagnostics,
arena_options,
)
.unwrap_or_else(|_| invariant_violation("validated pool configuration was rejected")))
}
fn from_parts_with_arena_options(
geometry: PoolGeometry,
configured_capacity: CarrierCount,
diagnostics: MemoryDiagnosticsConfig,
arena_options: ArenaOptions,
) -> Result<Self, ArenaError> {
if configured_capacity == CarrierCount::ZERO {
invariant_violation("configured capacity must be nonzero");
}
if configured_capacity > MAX_PACKED_CARRIERS {
invariant_violation("configured capacity must fit packed accounting");
}
let configured_capacity_bytes = configured_capacity
.get()
.checked_mul(geometry.carrier_size())
.unwrap_or_else(|| invariant_violation("configured byte capacity overflowed"));
let optimistic_scan_words = arena_options.optimistic_scan_words();
let inner = Arc::new(PoolInner::new(
geometry,
configured_capacity,
diagnostics,
arena_options,
)?);
inner.maintenance.start_periodic_diagnostics(&inner);
tracing::debug!(
target: crate::telemetry::TARGET_MEMORY,
configured_capacity_bytes,
page_size_bytes = geometry.page_size(),
carrier_size_bytes = geometry.carrier_size(),
block_size_bytes = geometry.block_size(),
optimistic_scan_words,
diagnostic_detail = diagnostics.detail_level(),
diagnostic_snapshot_interval_ms =
diagnostics.snapshot_interval().map(|interval| interval.as_millis()).unwrap_or(0),
"constructed buffer pool"
);
Ok(Self { inner })
}
pub(crate) fn try_reserve(&self, bytes: usize) -> Result<Option<Reservation>, ReserveError> {
let envelope = self.reservation_envelope(bytes)?;
PoolInner::try_reserve_count(&self.inner, envelope)
}
pub(crate) fn reserve(&self, bytes: usize) -> ReserveFuture {
ReserveFuture::new(self.clone(), bytes)
}
pub fn metrics(&self) -> MemoryMetrics {
self.inner.memory_metrics()
}
pub(crate) fn diagnostics(&self) -> MemoryDiagnostics {
MemoryDiagnostics::from_samples(
self.inner.arena.diagnostics(),
self.inner.arena.sample_lifecycle(),
self.inner.maintenance.diagnostics(),
)
}
pub(crate) fn record_managed_activity(&self) {
self.inner.maintenance.record_activity();
}
pub(crate) fn record_global_idle(&self) {
self.inner.maintenance.record_idle(&self.inner);
}
pub(crate) fn acquire(
&self,
reservation: &Reservation,
min_bytes: usize,
) -> Result<PooledBufMut, AcquireError> {
let count = self.acquisition_count(min_bytes)?;
let direct = reservation
.acquisition_state()
.ok_or(AcquireError::ReservationClosed)?;
if !direct.belongs_to(&self.inner) {
return Err(AcquireError::ForeignReservation);
}
let direct = Arc::clone(direct);
let guards = acquire_count(&self.inner, Some(Arc::clone(&direct)), count)?;
PooledBufMut::try_new(GrowthAuthority::reserved(direct), guards)
}
pub(crate) fn acquire_unreserved(
&self,
min_bytes: usize,
) -> Result<PooledBufMut, AcquireError> {
let count = self.acquisition_count(min_bytes)?;
let guards = acquire_count(&self.inner, None, count)?;
PooledBufMut::try_new(GrowthAuthority::unreserved(Arc::clone(&self.inner)), guards)
}
fn reservation_envelope(&self, bytes: usize) -> Result<CarrierCount, ReserveError> {
let envelope = self
.inner
.geometry
.carriers_for_bytes(bytes)
.map_err(map_reservation_geometry_error)?;
if envelope > MAX_PACKED_CARRIERS {
return Err(ReserveError::CapacityOverflow);
}
Ok(envelope)
}
fn acquisition_count(&self, min_bytes: usize) -> Result<CarrierCount, AcquireError> {
self.inner
.geometry
.carriers_for_bytes(min_bytes)
.map_err(|error| match error {
GeometryError::ZeroByteRequest => AcquireError::InvalidSize,
_ => AcquireError::CapacityOverflow,
})
}
}
struct PoolInner {
geometry: PoolGeometry,
admission: Mutex<AdmissionState>,
reservation_drain: ReservationDrainSignal,
#[cfg(not(all(test, s3_tm_loom)))]
active_owner_returns: AtomicUsize,
coverage: CoverageState,
arena: Arena,
maintenance: MaintenanceCoordinator,
#[cfg(test)]
test_hooks: TestHooks,
}
#[derive(Clone, Copy)]
struct ReservationQueueSample {
event: ReservationQueueEvent,
queued_reservations: usize,
reservation_enqueues_total: u64,
admission_used: CarrierCount,
configured_capacity: CarrierCount,
}
#[derive(Clone, Copy)]
enum ReservationQueueEvent {
BecameNonempty { requested: CarrierCount },
BecameEmpty,
}
struct ReservationDrain {
wakers: Vec<Waker>,
queue_sample: Option<ReservationQueueSample>,
}
impl PoolInner {
fn new(
geometry: PoolGeometry,
configured_capacity: CarrierCount,
diagnostics: MemoryDiagnosticsConfig,
arena_options: ArenaOptions,
) -> Result<Self, ArenaError> {
Ok(Self {
geometry,
admission: Mutex::new(AdmissionState::new(configured_capacity)),
reservation_drain: ReservationDrainSignal::new(),
#[cfg(not(all(test, s3_tm_loom)))]
active_owner_returns: AtomicUsize::new(0),
coverage: CoverageState::new(),
arena: Arena::new(geometry, arena_options)?,
maintenance: MaintenanceCoordinator::new(
configured_capacity,
geometry,
diagnostics.snapshot_interval(),
),
#[cfg(test)]
test_hooks: TestHooks::new(),
})
}
fn memory_metrics(&self) -> MemoryMetrics {
let admission = self.admission.lock();
self.memory_metrics_locked(&admission)
}
fn diagnostic_memory_sample(
&self,
now: std::time::Instant,
) -> (MemoryMetrics, admission::ReservationQueueDiagnosticSample) {
let mut admission = self.admission.lock();
let queue = admission.take_queue_diagnostics(now);
let metrics = self.memory_metrics_locked(&admission);
(metrics, queue)
}
fn memory_metrics_locked(&self, admission: &AdmissionState) -> MemoryMetrics {
let coverage = self.coverage.snapshot();
admission.ledger.assert_invariants(coverage);
let admission_used = admission
.ledger
.admission_used(coverage)
.unwrap_or_else(|_| invariant_violation("completed admission exceeds packed capacity"));
let covered_charges = admission
.ledger
.active_planned_demand
.checked_sub(coverage.available)
.unwrap_or_else(|| {
invariant_violation("available coverage exceeds active planned demand")
});
let charged_capacity = covered_charges
.checked_add(coverage.uncovered)
.unwrap_or_else(|| invariant_violation("completed charges exceed packed capacity"));
MemoryMetrics::from_carriers(
self.geometry.carrier_size(),
MemoryMetricState {
configured_capacity: admission.ledger.configured_capacity,
admission_used,
active_planned_demand: admission.ledger.active_planned_demand,
charged_capacity,
prepared_capacity: admission.ledger.prepared_capacity,
queued_reservations: admission.waiter_count(),
reservation_enqueues_total: admission.reservation_enqueues_total(),
},
)
}
fn try_debit_covered(&self, count: CarrierCount) -> Result<bool, ReserveError> {
self.coverage
.try_debit_covered(count)
.map_err(|_| ReserveError::CapacityOverflow)
}
fn debit_and_prepare_locked(
pool: &Arc<Self>,
admission: &mut AdmissionGuard<'_>,
count: CarrierCount,
) -> Result<CoverageDebit, ReserveError> {
let debit = pool
.coverage
.debit(count, admission.maximum_uncovered())
.map_err(|_| ReserveError::CapacityOverflow)?;
let floor = match admission.acquisition_floor(&pool.coverage) {
Ok(floor) => floor,
Err(error) => {
admission.rollback_acquisition(&pool.coverage, count, debit);
return Err(error);
}
};
if let Err(error) = pool.arena.prepare_to(admission, floor) {
admission.rollback_acquisition(&pool.coverage, count, debit);
Self::request_cleanup_after_arena_error(pool, &error);
return Err(map_preparation_error(error));
}
Ok(debit)
}
fn try_reserve_count(
pool: &Arc<Self>,
envelope: CarrierCount,
) -> Result<Option<Reservation>, ReserveError> {
if envelope == CarrierCount::ZERO {
return Err(ReserveError::InvalidSize);
}
if envelope > MAX_PACKED_CARRIERS {
return Err(ReserveError::CapacityOverflow);
}
let mut admission = AdmissionGuard::new(pool.admission.lock());
admission.validate_envelope(envelope)?;
if !admission.inner.waiters_is_empty() {
return Ok(None);
}
let coverage = pool.coverage.snapshot();
if !admission.can_grant(coverage, envelope) {
return Ok(None);
}
Self::prepare_and_grant_locked(pool, &mut admission, envelope).map(Some)
}
fn reserve_or_enqueue(
pool: &Arc<Self>,
envelope: CarrierCount,
waker: Waker,
) -> Result<ReservationPoll, ReserveError> {
#[cfg(test)]
if let Some(error) = pool.test_hooks.take_reservation_failure() {
return Err(error);
}
if envelope == CarrierCount::ZERO {
return Err(ReserveError::InvalidSize);
}
if envelope > MAX_PACKED_CARRIERS {
return Err(ReserveError::CapacityOverflow);
}
let mut admission = AdmissionGuard::new(pool.admission.lock());
admission.validate_envelope(envelope)?;
pool.reservation_drain.arm();
let coverage = pool.coverage.snapshot();
if admission.inner.waiters_is_empty() && admission.can_grant(coverage, envelope) {
let result = Self::prepare_and_grant_locked(pool, &mut admission, envelope)
.map(ReservationPoll::Ready);
pool.reservation_drain
.publish_waiter_state(!admission.inner.waiters_is_empty());
return result;
}
let slot = Arc::new(WaitSlot::new(waker));
#[cfg(test)]
if pool.test_hooks.take_reservation_queue_allocation_failure() {
pool.reservation_drain
.publish_waiter_state(!admission.inner.waiters_is_empty());
return Err(ReserveError::MetadataAllocationFailed);
}
let queue_became_nonempty = match admission.inner.enqueue_waiter(
Waiter {
envelope,
slot: Arc::clone(&slot),
},
std::time::Instant::now(),
) {
Ok(queue_became_nonempty) => queue_became_nonempty,
Err(_) => {
pool.reservation_drain
.publish_waiter_state(!admission.inner.waiters_is_empty());
return Err(ReserveError::MetadataAllocationFailed);
}
};
pool.reservation_drain
.publish_waiter_state(!admission.inner.waiters_is_empty());
let queue_sample = queue_became_nonempty
.then(|| {
Self::reservation_queue_sample(
&admission,
&pool.coverage,
ReservationQueueEvent::BecameNonempty {
requested: envelope,
},
)
})
.flatten();
drop(admission);
if let Some(sample) = queue_sample {
pool.log_reservation_queue_transition(sample);
}
Ok(ReservationPoll::Queued(slot))
}
fn cancel_waiter(pool: &Arc<Self>, slot: &Arc<WaitSlot>) -> Vec<Waker> {
let mut admission = AdmissionGuard::new(pool.admission.lock());
let queue_became_empty = admission
.inner
.remove_waiter(slot, std::time::Instant::now());
let drained = Self::drain_fifo_locked(pool, &mut admission, queue_became_empty);
drop(admission);
if let Some(sample) = drained.queue_sample {
pool.log_reservation_queue_transition(sample);
}
drained.wakers
}
fn prepare_and_grant_locked(
pool: &Arc<Self>,
admission: &mut AdmissionGuard<'_>,
envelope: CarrierCount,
) -> Result<Reservation, ReserveError> {
let coverage = pool.coverage.snapshot();
let target = admission.grant_target(coverage, envelope)?;
if let Err(error) = pool.arena.prepare_to(admission, target) {
Self::request_cleanup_after_arena_error(pool, &error);
return Err(map_preparation_error(error));
}
admission.commit_grant(&pool.coverage, envelope)?;
Ok(Reservation::new(Arc::clone(pool), envelope))
}
fn drain_fifo_locked(
pool: &Arc<Self>,
admission: &mut AdmissionGuard<'_>,
mut queue_became_empty: bool,
) -> ReservationDrain {
let mut wakers = Vec::new();
while let Some(front) = admission.inner.front_waiter() {
let envelope = front.envelope;
let slot = Arc::clone(&front.slot);
let mut slot_state = slot.state.lock();
match &*slot_state {
WaitState::Taken => {
let (_, cleared) = admission.inner.pop_front_waiter(std::time::Instant::now());
queue_became_empty |= cleared;
continue;
}
WaitState::Queued { .. } => {}
WaitState::Granted(_) | WaitState::Failed(_) => {
invariant_violation("terminal reservation result remained linked in the FIFO");
}
}
let coverage = pool.coverage.snapshot();
if !admission.can_grant(coverage, envelope) {
break;
}
let result = Self::prepare_and_grant_locked(pool, admission, envelope);
let previous = std::mem::replace(&mut *slot_state, WaitState::Taken);
let WaitState::Queued { waker } = previous else {
invariant_violation("reservation head changed while its slot lock was held");
};
*slot_state = match result {
Ok(reservation) => WaitState::Granted(reservation),
Err(error) => WaitState::Failed(error),
};
let (removed, cleared) = admission.inner.pop_front_waiter(std::time::Instant::now());
queue_became_empty |= cleared;
let Some(removed) = removed else {
invariant_violation("reservation head disappeared while admission was held");
};
if !Arc::ptr_eq(&removed.slot, &slot) {
invariant_violation("FIFO head changed while admission was held");
}
wakers.push(waker);
}
pool.reservation_drain
.publish_waiter_state(!admission.inner.waiters_is_empty());
let queue_sample = queue_became_empty
.then(|| {
Self::reservation_queue_sample(
admission,
&pool.coverage,
ReservationQueueEvent::BecameEmpty,
)
})
.flatten();
ReservationDrain {
wakers,
queue_sample,
}
}
fn release_acquisition_charges(pool: &Arc<Self>, count: CarrierCount) {
let returned = pool.coverage.release(count);
Self::drain_after_coverage_return(pool, returned.uncovered_removed);
}
fn rollback_acquisition_charges(pool: &Arc<Self>, count: CarrierCount, debit: CoverageDebit) {
let drained = {
let mut admission = AdmissionGuard::new(pool.admission.lock());
let returned = pool.coverage.rollback_debit(
count,
debit,
admission.inner.ledger.active_planned_demand,
);
let drained = (returned.uncovered_removed != CarrierCount::ZERO)
.then(|| Self::drain_fifo_locked(pool, &mut admission, false));
admission
.inner
.ledger
.assert_invariants(pool.coverage.snapshot());
drained
};
if let Some(drained) = drained {
if let Some(sample) = drained.queue_sample {
pool.log_reservation_queue_transition(sample);
}
wake_all(drained.wakers);
}
}
fn drain_after_coverage_return(pool: &Arc<Self>, uncovered_removed: CarrierCount) {
if uncovered_removed == CarrierCount::ZERO
|| !pool.reservation_drain.repayment_requires_drain()
{
return;
}
#[cfg(test)]
pool.test_hooks.record_return_admission_entry();
let drained = {
let mut admission = AdmissionGuard::new(pool.admission.lock());
let drained = Self::drain_fifo_locked(pool, &mut admission, false);
admission
.inner
.ledger
.assert_invariants(pool.coverage.snapshot());
drained
};
if let Some(sample) = drained.queue_sample {
pool.log_reservation_queue_transition(sample);
}
wake_all(drained.wakers);
}
fn begin_owner_return(pool: &Arc<Self>) -> OwnerReturnPermit {
#[cfg(not(all(test, s3_tm_loom)))]
let previous = pool.active_owner_returns.fetch_add(1, Ordering::AcqRel);
#[cfg(not(all(test, s3_tm_loom)))]
let overlapping = previous != 0;
#[cfg(all(test, s3_tm_loom))]
let overlapping = false;
OwnerReturnPermit {
pool: Arc::clone(pool),
batch: overlapping || pool.reservation_drain.is_armed(),
}
}
fn request_cleanup_after_arena_error(pool: &Arc<Self>, error: &ArenaError) {
if !error.cleanup_required() {
return;
}
tracing::warn!(
target: crate::telemetry::TARGET_MEMORY,
error = %error,
"buffer-pool preparation failed; mapping cleanup was scheduled"
);
pool.maintenance.request_cleanup(pool);
}
fn reservation_queue_sample(
admission: &AdmissionGuard<'_>,
coverage: &CoverageState,
event: ReservationQueueEvent,
) -> Option<ReservationQueueSample> {
let admission_used = admission
.inner
.ledger
.admission_used(coverage.snapshot())
.ok()?;
Some(ReservationQueueSample {
event,
queued_reservations: admission.inner.waiter_count(),
reservation_enqueues_total: admission.inner.reservation_enqueues_total(),
admission_used,
configured_capacity: admission.inner.ledger.configured_capacity,
})
}
fn log_reservation_queue_transition(&self, sample: ReservationQueueSample) {
if !tracing::enabled!(
target: crate::telemetry::TARGET_MEMORY,
tracing::Level::DEBUG
) {
return;
}
match sample.event {
ReservationQueueEvent::BecameNonempty { requested } => tracing::debug!(
target: crate::telemetry::TARGET_MEMORY,
requested_bytes = self.carriers_to_bytes(requested),
queued_reservations = sample.queued_reservations,
reservation_enqueues_total = sample.reservation_enqueues_total,
admission_used_bytes = self.carriers_to_bytes(sample.admission_used),
configured_capacity_bytes = self.carriers_to_bytes(sample.configured_capacity),
"buffer-pool reservation queue became non-empty"
),
ReservationQueueEvent::BecameEmpty => tracing::debug!(
target: crate::telemetry::TARGET_MEMORY,
queued_reservations = sample.queued_reservations,
reservation_enqueues_total = sample.reservation_enqueues_total,
admission_used_bytes = self.carriers_to_bytes(sample.admission_used),
configured_capacity_bytes = self.carriers_to_bytes(sample.configured_capacity),
"buffer-pool reservation queue became empty"
),
}
}
fn carriers_to_bytes(&self, count: CarrierCount) -> u64 {
let count = u64::try_from(count.get()).unwrap_or(u64::MAX);
let carrier_size = u64::try_from(self.geometry.carrier_size()).unwrap_or(u64::MAX);
count.saturating_mul(carrier_size)
}
}
pub(super) struct OwnerReturnPermit {
pool: Arc<PoolInner>,
batch: bool,
}
impl OwnerReturnPermit {
pub(super) fn should_batch(&self) -> bool {
self.batch
}
}
impl Drop for OwnerReturnPermit {
fn drop(&mut self) {
#[cfg(not(all(test, s3_tm_loom)))]
{
let previous = self
.pool
.active_owner_returns
.fetch_sub(1, Ordering::AcqRel);
if previous == 0 {
invariant_violation("owner-return permit underflowed");
}
}
}
}
impl Drop for PoolInner {
fn drop(&mut self) {
self.maintenance.shutdown();
}
}
fn map_reservation_geometry_error(error: GeometryError) -> ReserveError {
match error {
GeometryError::ZeroByteRequest => ReserveError::InvalidSize,
_ => invariant_violation("checked geometry rejected byte-to-carrier conversion"),
}
}
fn map_preparation_error(error: ArenaError) -> ReserveError {
match error {
ArenaError::Block(BlockError::PreparedCapacityOverflow)
| ArenaError::SlotIdExhausted
| ArenaError::RegistryCapacityOverflow
| ArenaError::AddressOverflow { .. }
| ArenaError::ScanSpaceOverflow { .. } => ReserveError::CapacityOverflow,
ArenaError::Block(BlockError::Allocation(_)) | ArenaError::Allocation(_) => {
ReserveError::MetadataAllocationFailed
}
ArenaError::Block(_)
| ArenaError::InvalidScanBudget
| ArenaError::InvalidClaimCount
| ArenaError::IncompleteClaim { .. }
| ArenaError::AddressOverlap { .. } => ReserveError::PhysicalPreparationFailed,
}
}
#[repr(transparent)]
#[derive(Clone, Copy, Debug, Default, Eq, Ord, PartialEq, PartialOrd)]
struct CarrierCount(usize);
impl CarrierCount {
const ZERO: Self = Self(0);
const fn new(value: usize) -> Self {
Self(value)
}
fn get(self) -> usize {
self.0
}
fn checked_add(self, other: Self) -> Option<Self> {
self.0.checked_add(other.0).map(Self)
}
fn checked_sub(self, other: Self) -> Option<Self> {
self.0.checked_sub(other.0).map(Self)
}
fn try_as_u32(self) -> Option<u32> {
u32::try_from(self.0).ok()
}
}
#[cold]
#[track_caller]
fn invariant_violation(message: &'static str) -> ! {
tracing::error!(
target: crate::telemetry::TARGET_MEMORY,
reason = message,
"buffer-pool invariant violated; aborting"
);
#[cfg(test)]
panic!("buffer-pool invariant violated: {message}");
#[cfg(not(test))]
{
let _ = message;
std::process::abort()
}
}