Skip to main content

aws_s3_transfer/runtime/
buffer_pool.rs

1/*
2 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3 * SPDX-License-Identifier: Apache-2.0
4 */
5
6//! Bounded transfer-memory admission and fixed-size pooled storage.
7//!
8//! [`BufferPool`] grants planned demand in FIFO order, accounts carrier
9//! ownership independently of reservation lifetime, and prepares backing in
10//! whole blocks. Mutable acquisitions retain exclusive carrier ownership;
11//! immutable segmented values may present adjacent carriers as one byte range
12//! while preserving each carrier's independent return boundary.
13//!
14//! Current memory gauges are always available through [`BufferPool::metrics`].
15//! Pressure and lifecycle counters are updated only when exceptional allocator
16//! work occurs. Per-acquisition scan accounting and periodic snapshots are
17//! opt-in so the normal success path does not pay for diagnostic atomics.
18
19use std::task::Waker;
20
21use crate::config::MemoryDiagnosticsConfig;
22#[cfg(not(all(test, s3_tm_loom)))]
23use crate::runtime::sync::sync::atomic::{AtomicUsize, Ordering};
24use crate::runtime::sync::sync::{Arc, Mutex};
25
26mod acquisition;
27mod admission;
28mod arena;
29#[cfg(s3_tm_bench)]
30pub(crate) mod bench_util;
31mod block;
32mod config;
33#[cfg(not(all(test, s3_tm_loom)))]
34mod diagnostics;
35mod geometry;
36mod maintenance;
37mod metrics;
38mod pooled_buf;
39mod segmented_bytes;
40#[cfg(any(test, s3_tm_fuzz))]
41mod test_util;
42#[cfg(all(test, not(s3_tm_loom)))]
43mod tests;
44mod virtual_memory;
45
46use crate::config::MemoryBudgetConfig;
47use acquisition::acquire_count;
48pub(crate) use acquisition::AcquireError;
49use admission::{
50    wake_all, AdmissionGuard, AdmissionState, CoverageDebit, CoverageState, ReservationDrainSignal,
51    ReservationPoll, WaitSlot, WaitState, Waiter, MAX_PACKED_CARRIERS,
52};
53pub(crate) use admission::{Reservation, ReserveError, ReserveFuture};
54use arena::{Arena, ArenaError, ArenaOptions};
55use block::BlockError;
56pub(crate) use config::auto_capacity_bytes;
57use config::PoolConfig;
58pub use config::{BufferPoolBuildError, BufferPoolBuilder};
59use geometry::{GeometryError, PoolGeometry};
60use maintenance::MaintenanceCoordinator;
61pub use metrics::MemoryMetrics;
62use metrics::{MemoryDiagnostics, MemoryMetricState};
63use pooled_buf::GrowthAuthority;
64pub(crate) use pooled_buf::PooledBufMut;
65pub use segmented_bytes::SegmentedBytes;
66
67/// Executes one compact accounting and ownership fuzz sequence.
68///
69/// The target decodes public pool operations and checks every transition
70/// against the shared reference model and quiescent audit.
71#[cfg(s3_tm_fuzz)]
72pub(crate) fn run_fuzz_input(data: &[u8]) {
73    let _ = test_util::model_harness::run_fuzz_input(data);
74}
75
76/// Executes one large-geometry bitmap-placement fuzz sequence.
77///
78/// The target uses the shared reference model plus the independent free-word
79/// oracle to verify contiguous-first placement and legal fragmented fallback.
80#[cfg(s3_tm_fuzz)]
81pub(crate) fn run_placement_fuzz_input(data: &[u8]) {
82    let _ = test_util::model_harness::run_placement_fuzz_input(data);
83}
84
85#[cfg(all(test, not(s3_tm_loom)))]
86pub(crate) use test_util::test_pool;
87#[cfg(test)]
88use test_util::TestHooks;
89
90/// A cloneable handle to one admission, accounting, and storage domain.
91///
92/// Every clone shares the configured capacity, reservation queue, ownership
93/// charges, prepared storage, maintenance state, and metrics. Constructing a
94/// pool is lazy: no virtual ranges or physical memory are prepared until the
95/// first admitted acquisition needs them.
96#[derive(Clone)]
97pub struct BufferPool {
98    inner: Arc<PoolInner>,
99}
100
101impl std::fmt::Debug for BufferPool {
102    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
103        f.debug_struct("BufferPool").finish_non_exhaustive()
104    }
105}
106
107impl BufferPool {
108    /// Returns a builder for an empty shared memory pool.
109    pub fn builder() -> BufferPoolBuilder {
110        BufferPoolBuilder::default()
111    }
112
113    /// Returns the fixed writable allocation unit used by this pool.
114    ///
115    /// Reservations and acquisitions accept arbitrary byte counts and round
116    /// them to this page-aligned unit.
117    pub(crate) fn carrier_size(&self) -> usize {
118        self.inner.geometry.carrier_size()
119    }
120
121    /// Constructs an empty pool from capacity policy and detected memory.
122    ///
123    /// Capacity and geometry are fixed before publication. Construction does
124    /// not reserve virtual ranges, prepare backing, or start maintenance.
125    pub(crate) fn from_capacity(
126        capacity: MemoryBudgetConfig,
127        detected_memory: Option<usize>,
128        diagnostics: MemoryDiagnosticsConfig,
129    ) -> Result<Self, BufferPoolBuildError> {
130        let resolved = PoolConfig::resolve(capacity, detected_memory)?;
131        let arena_options = ArenaOptions::new(
132            resolved.optimistic_scan_words,
133            diagnostics.enable_detailed_counters(),
134        );
135        Ok(Self::from_parts_with_arena_options(
136            resolved.geometry,
137            resolved.configured_capacity,
138            diagnostics,
139            arena_options,
140        )
141        .unwrap_or_else(|_| invariant_violation("validated pool configuration was rejected")))
142    }
143
144    /// Constructs a pool with explicit internal arena policy.
145    fn from_parts_with_arena_options(
146        geometry: PoolGeometry,
147        configured_capacity: CarrierCount,
148        diagnostics: MemoryDiagnosticsConfig,
149        arena_options: ArenaOptions,
150    ) -> Result<Self, ArenaError> {
151        if configured_capacity == CarrierCount::ZERO {
152            invariant_violation("configured capacity must be nonzero");
153        }
154        if configured_capacity > MAX_PACKED_CARRIERS {
155            invariant_violation("configured capacity must fit packed accounting");
156        }
157
158        let configured_capacity_bytes = configured_capacity
159            .get()
160            .checked_mul(geometry.carrier_size())
161            .unwrap_or_else(|| invariant_violation("configured byte capacity overflowed"));
162        let optimistic_scan_words = arena_options.optimistic_scan_words();
163        let inner = Arc::new(PoolInner::new(
164            geometry,
165            configured_capacity,
166            diagnostics,
167            arena_options,
168        )?);
169        inner.maintenance.start_periodic_diagnostics(&inner);
170        tracing::debug!(
171            target: crate::telemetry::TARGET_MEMORY,
172            configured_capacity_bytes,
173            page_size_bytes = geometry.page_size(),
174            carrier_size_bytes = geometry.carrier_size(),
175            block_size_bytes = geometry.block_size(),
176            optimistic_scan_words,
177            diagnostic_detail = diagnostics.detail_level(),
178            diagnostic_snapshot_interval_ms =
179                diagnostics.snapshot_interval().map(|interval| interval.as_millis()).unwrap_or(0),
180            "constructed buffer pool"
181        );
182        Ok(Self { inner })
183    }
184
185    /// Attempts one immediate reservation grant.
186    ///
187    /// `Ok(None)` reports an older FIFO request or current admission pressure.
188    /// A successful grant has already prepared storage through its complete
189    /// admission floor. A request whose carrier-rounded envelope exceeds the
190    /// configured pool capacity returns [`ReserveError::ExceedsCapacity`].
191    pub(crate) fn try_reserve(&self, bytes: usize) -> Result<Option<Reservation>, ReserveError> {
192        let envelope = self.reservation_envelope(bytes)?;
193        PoolInner::try_reserve_count(&self.inner, envelope)
194    }
195
196    /// Creates a lazy, cancellation-safe reservation request.
197    ///
198    /// The first poll either returns an immediate result or enters the
199    /// pool-wide FIFO. Invalid requests and physical preparation failures
200    /// resolve through the future's `ReserveError`. A request larger than the
201    /// configured pool capacity fails without entering the FIFO.
202    pub(crate) fn reserve(&self, bytes: usize) -> ReserveFuture {
203        ReserveFuture::new(self.clone(), bytes)
204    }
205
206    /// Returns one coherent sample of this pool's memory state.
207    pub fn metrics(&self) -> MemoryMetrics {
208        self.inner.memory_metrics()
209    }
210
211    /// Returns operational counters and a scanned lifecycle sample.
212    pub(crate) fn diagnostics(&self) -> MemoryDiagnostics {
213        MemoryDiagnostics::from_samples(
214            self.inner.arena.diagnostics(),
215            self.inner.arena.sample_lifecycle(),
216            self.inner.maintenance.diagnostics(),
217        )
218    }
219
220    /// Invalidates pending idle reclamation when managed work starts.
221    pub(crate) fn record_managed_activity(&self) {
222        self.inner.maintenance.record_activity();
223    }
224
225    /// Arms idle reclamation after the scheduler becomes globally idle.
226    pub(crate) fn record_global_idle(&self) {
227        self.inner.maintenance.record_idle(&self.inner);
228    }
229
230    /// Acquires at least `min_bytes` under one reservation.
231    ///
232    /// Capacity is rounded up to complete carriers. The reservation must
233    /// belong to this pool, remain open, and retain enough direct-acquisition
234    /// authority for the rounded request. Failure exposes no partial carrier
235    /// ownership.
236    pub(crate) fn acquire(
237        &self,
238        reservation: &Reservation,
239        min_bytes: usize,
240    ) -> Result<PooledBufMut, AcquireError> {
241        let count = self.acquisition_count(min_bytes)?;
242        let direct = reservation
243            .acquisition_state()
244            .ok_or(AcquireError::ReservationClosed)?;
245        if !direct.belongs_to(&self.inner) {
246            return Err(AcquireError::ForeignReservation);
247        }
248        let direct = Arc::clone(direct);
249        let guards = acquire_count(&self.inner, Some(Arc::clone(&direct)), count)?;
250        PooledBufMut::try_new(GrowthAuthority::reserved(direct), guards)
251    }
252
253    /// Acquires at least `min_bytes` without reservation-local authority.
254    ///
255    /// Capacity is rounded up to complete carriers. The request still
256    /// participates in aggregate accounting and may prepare physical capacity
257    /// or raise admission above its normal configured ceiling. Failure exposes
258    /// no partial carrier ownership.
259    pub(crate) fn acquire_unreserved(
260        &self,
261        min_bytes: usize,
262    ) -> Result<PooledBufMut, AcquireError> {
263        let count = self.acquisition_count(min_bytes)?;
264        let guards = acquire_count(&self.inner, None, count)?;
265        PooledBufMut::try_new(GrowthAuthority::unreserved(Arc::clone(&self.inner)), guards)
266    }
267
268    /// Converts one public byte request to its checked carrier envelope.
269    fn reservation_envelope(&self, bytes: usize) -> Result<CarrierCount, ReserveError> {
270        let envelope = self
271            .inner
272            .geometry
273            .carriers_for_bytes(bytes)
274            .map_err(map_reservation_geometry_error)?;
275        if envelope > MAX_PACKED_CARRIERS {
276            return Err(ReserveError::CapacityOverflow);
277        }
278        Ok(envelope)
279    }
280
281    /// Converts one acquisition request to its complete carrier count.
282    fn acquisition_count(&self, min_bytes: usize) -> Result<CarrierCount, AcquireError> {
283        self.inner
284            .geometry
285            .carriers_for_bytes(min_bytes)
286            .map_err(|error| match error {
287                GeometryError::ZeroByteRequest => AcquireError::InvalidSize,
288                _ => AcquireError::CapacityOverflow,
289            })
290    }
291}
292
293/// Shared state retained by pool handles, reservations, and carrier owners.
294struct PoolInner {
295    /// Immutable page, block, carrier, and bitmap dimensions.
296    geometry: PoolGeometry,
297    /// Planned-demand policy and prepared-capacity serialization.
298    admission: Mutex<AdmissionState>,
299    /// Repayment ordering and the reservation FIFO's drain requirement.
300    reservation_drain: ReservationDrainSignal,
301    /// Whole-value returns currently publishing physical ownership.
302    #[cfg(not(all(test, s3_tm_loom)))]
303    active_owner_returns: AtomicUsize,
304    /// Aggregate charges updated by carrier acquisition and final return.
305    coverage: CoverageState,
306    /// Stable virtual ranges and physical carrier ownership.
307    arena: Arena,
308    /// Lazy maintenance and optional diagnostic-reporting worker.
309    maintenance: MaintenanceCoordinator,
310    /// Per-pool failure injection and test-only observations.
311    #[cfg(test)]
312    test_hooks: TestHooks,
313}
314
315/// Reservation pressure captured while admission state is coherent.
316#[derive(Clone, Copy)]
317struct ReservationQueueSample {
318    /// Empty-boundary transition and any data specific to it.
319    event: ReservationQueueEvent,
320    /// Requests retained in FIFO order at this transition.
321    queued_reservations: usize,
322    /// Pool-lifetime requests that have entered the FIFO.
323    reservation_enqueues_total: u64,
324    /// Planned demand and uncovered ownership after this enqueue.
325    admission_used: CarrierCount,
326    /// Normal admission ceiling.
327    configured_capacity: CarrierCount,
328}
329
330/// A reservation FIFO crossing its empty boundary.
331#[derive(Clone, Copy)]
332enum ReservationQueueEvent {
333    /// The FIFO became non-empty for this request.
334    BecameNonempty { requested: CarrierCount },
335    /// The FIFO became empty.
336    BecameEmpty,
337}
338
339/// FIFO work and an optional empty-boundary transition captured under admission.
340struct ReservationDrain {
341    wakers: Vec<Waker>,
342    queue_sample: Option<ReservationQueueSample>,
343}
344
345impl PoolInner {
346    /// Constructs an empty pool domain without reserving virtual ranges.
347    fn new(
348        geometry: PoolGeometry,
349        configured_capacity: CarrierCount,
350        diagnostics: MemoryDiagnosticsConfig,
351        arena_options: ArenaOptions,
352    ) -> Result<Self, ArenaError> {
353        Ok(Self {
354            geometry,
355            admission: Mutex::new(AdmissionState::new(configured_capacity)),
356            reservation_drain: ReservationDrainSignal::new(),
357            #[cfg(not(all(test, s3_tm_loom)))]
358            active_owner_returns: AtomicUsize::new(0),
359            coverage: CoverageState::new(),
360            arena: Arena::new(geometry, arena_options)?,
361            maintenance: MaintenanceCoordinator::new(
362                configured_capacity,
363                geometry,
364                diagnostics.snapshot_interval(),
365            ),
366            #[cfg(test)]
367            test_hooks: TestHooks::new(),
368        })
369    }
370
371    /// Returns one coherent admission and aggregate memory sample.
372    fn memory_metrics(&self) -> MemoryMetrics {
373        let admission = self.admission.lock();
374        self.memory_metrics_locked(&admission)
375    }
376
377    /// Takes memory and reservation-queue state at one diagnostic boundary.
378    fn diagnostic_memory_sample(
379        &self,
380        now: std::time::Instant,
381    ) -> (MemoryMetrics, admission::ReservationQueueDiagnosticSample) {
382        let mut admission = self.admission.lock();
383        let queue = admission.take_queue_diagnostics(now);
384        let metrics = self.memory_metrics_locked(&admission);
385        (metrics, queue)
386    }
387
388    /// Builds public memory metrics while admission remains coherent.
389    fn memory_metrics_locked(&self, admission: &AdmissionState) -> MemoryMetrics {
390        let coverage = self.coverage.snapshot();
391        admission.ledger.assert_invariants(coverage);
392
393        let admission_used = admission
394            .ledger
395            .admission_used(coverage)
396            .unwrap_or_else(|_| invariant_violation("completed admission exceeds packed capacity"));
397        let covered_charges = admission
398            .ledger
399            .active_planned_demand
400            .checked_sub(coverage.available)
401            .unwrap_or_else(|| {
402                invariant_violation("available coverage exceeds active planned demand")
403            });
404        let charged_capacity = covered_charges
405            .checked_add(coverage.uncovered)
406            .unwrap_or_else(|| invariant_violation("completed charges exceed packed capacity"));
407
408        MemoryMetrics::from_carriers(
409            self.geometry.carrier_size(),
410            MemoryMetricState {
411                configured_capacity: admission.ledger.configured_capacity,
412                admission_used,
413                active_planned_demand: admission.ledger.active_planned_demand,
414                charged_capacity,
415                prepared_capacity: admission.ledger.prepared_capacity,
416                queued_reservations: admission.waiter_count(),
417                reservation_enqueues_total: admission.reservation_enqueues_total(),
418            },
419        )
420    }
421
422    /// Attempts an aggregate debit without admission serialization.
423    ///
424    /// `Ok(false)` leaves accounting unchanged and requires the caller to use
425    /// the serialized shortfall path.
426    fn try_debit_covered(&self, count: CarrierCount) -> Result<bool, ReserveError> {
427        self.coverage
428            .try_debit_covered(count)
429            .map_err(|_| ReserveError::CapacityOverflow)
430    }
431
432    /// Publishes a shortfall debit and prepares through its admission floor.
433    ///
434    /// Failure reverses the aggregate debit before returning. The caller
435    /// remains responsible for rolling back reservation-local authority.
436    fn debit_and_prepare_locked(
437        pool: &Arc<Self>,
438        admission: &mut AdmissionGuard<'_>,
439        count: CarrierCount,
440    ) -> Result<CoverageDebit, ReserveError> {
441        let debit = pool
442            .coverage
443            .debit(count, admission.maximum_uncovered())
444            .map_err(|_| ReserveError::CapacityOverflow)?;
445
446        let floor = match admission.acquisition_floor(&pool.coverage) {
447            Ok(floor) => floor,
448            Err(error) => {
449                admission.rollback_acquisition(&pool.coverage, count, debit);
450                return Err(error);
451            }
452        };
453        if let Err(error) = pool.arena.prepare_to(admission, floor) {
454            admission.rollback_acquisition(&pool.coverage, count, debit);
455            Self::request_cleanup_after_arena_error(pool, &error);
456            return Err(map_preparation_error(error));
457        }
458        Ok(debit)
459    }
460
461    /// Attempts one immediate grant without bypassing the FIFO.
462    fn try_reserve_count(
463        pool: &Arc<Self>,
464        envelope: CarrierCount,
465    ) -> Result<Option<Reservation>, ReserveError> {
466        if envelope == CarrierCount::ZERO {
467            return Err(ReserveError::InvalidSize);
468        }
469        if envelope > MAX_PACKED_CARRIERS {
470            return Err(ReserveError::CapacityOverflow);
471        }
472
473        let mut admission = AdmissionGuard::new(pool.admission.lock());
474        admission.validate_envelope(envelope)?;
475        if !admission.inner.waiters_is_empty() {
476            return Ok(None);
477        }
478        let coverage = pool.coverage.snapshot();
479        if !admission.can_grant(coverage, envelope) {
480            return Ok(None);
481        }
482
483        Self::prepare_and_grant_locked(pool, &mut admission, envelope).map(Some)
484    }
485
486    /// Grants one first-poll request or links it behind existing work.
487    fn reserve_or_enqueue(
488        pool: &Arc<Self>,
489        envelope: CarrierCount,
490        waker: Waker,
491    ) -> Result<ReservationPoll, ReserveError> {
492        #[cfg(test)]
493        if let Some(error) = pool.test_hooks.take_reservation_failure() {
494            return Err(error);
495        }
496        if envelope == CarrierCount::ZERO {
497            return Err(ReserveError::InvalidSize);
498        }
499        if envelope > MAX_PACKED_CARRIERS {
500            return Err(ReserveError::CapacityOverflow);
501        }
502
503        let mut admission = AdmissionGuard::new(pool.admission.lock());
504        admission.validate_envelope(envelope)?;
505        pool.reservation_drain.arm();
506        let coverage = pool.coverage.snapshot();
507        if admission.inner.waiters_is_empty() && admission.can_grant(coverage, envelope) {
508            let result = Self::prepare_and_grant_locked(pool, &mut admission, envelope)
509                .map(ReservationPoll::Ready);
510            pool.reservation_drain
511                .publish_waiter_state(!admission.inner.waiters_is_empty());
512            return result;
513        }
514
515        let slot = Arc::new(WaitSlot::new(waker));
516        #[cfg(test)]
517        if pool.test_hooks.take_reservation_queue_allocation_failure() {
518            pool.reservation_drain
519                .publish_waiter_state(!admission.inner.waiters_is_empty());
520            return Err(ReserveError::MetadataAllocationFailed);
521        }
522        let queue_became_nonempty = match admission.inner.enqueue_waiter(
523            Waiter {
524                envelope,
525                slot: Arc::clone(&slot),
526            },
527            std::time::Instant::now(),
528        ) {
529            Ok(queue_became_nonempty) => queue_became_nonempty,
530            Err(_) => {
531                pool.reservation_drain
532                    .publish_waiter_state(!admission.inner.waiters_is_empty());
533                return Err(ReserveError::MetadataAllocationFailed);
534            }
535        };
536        pool.reservation_drain
537            .publish_waiter_state(!admission.inner.waiters_is_empty());
538        let queue_sample = queue_became_nonempty
539            .then(|| {
540                Self::reservation_queue_sample(
541                    &admission,
542                    &pool.coverage,
543                    ReservationQueueEvent::BecameNonempty {
544                        requested: envelope,
545                    },
546                )
547            })
548            .flatten();
549        drop(admission);
550        if let Some(sample) = queue_sample {
551            pool.log_reservation_queue_transition(sample);
552        }
553        Ok(ReservationPoll::Queued(slot))
554    }
555
556    /// Removes one cancelled queue link and reconsiders the exposed head.
557    fn cancel_waiter(pool: &Arc<Self>, slot: &Arc<WaitSlot>) -> Vec<Waker> {
558        let mut admission = AdmissionGuard::new(pool.admission.lock());
559        let queue_became_empty = admission
560            .inner
561            .remove_waiter(slot, std::time::Instant::now());
562        let drained = Self::drain_fifo_locked(pool, &mut admission, queue_became_empty);
563        drop(admission);
564        if let Some(sample) = drained.queue_sample {
565            pool.log_reservation_queue_transition(sample);
566        }
567        drained.wakers
568    }
569
570    /// Prepares and publishes one grant while admission is serialized.
571    fn prepare_and_grant_locked(
572        pool: &Arc<Self>,
573        admission: &mut AdmissionGuard<'_>,
574        envelope: CarrierCount,
575    ) -> Result<Reservation, ReserveError> {
576        let coverage = pool.coverage.snapshot();
577        let target = admission.grant_target(coverage, envelope)?;
578        if let Err(error) = pool.arena.prepare_to(admission, target) {
579            Self::request_cleanup_after_arena_error(pool, &error);
580            return Err(map_preparation_error(error));
581        }
582        admission.commit_grant(&pool.coverage, envelope)?;
583        Ok(Reservation::new(Arc::clone(pool), envelope))
584    }
585
586    /// Transfers every eligible FIFO head and returns wakers for post-unlock use.
587    fn drain_fifo_locked(
588        pool: &Arc<Self>,
589        admission: &mut AdmissionGuard<'_>,
590        mut queue_became_empty: bool,
591    ) -> ReservationDrain {
592        let mut wakers = Vec::new();
593        while let Some(front) = admission.inner.front_waiter() {
594            let envelope = front.envelope;
595            let slot = Arc::clone(&front.slot);
596            let mut slot_state = slot.state.lock();
597
598            match &*slot_state {
599                WaitState::Taken => {
600                    let (_, cleared) = admission.inner.pop_front_waiter(std::time::Instant::now());
601                    queue_became_empty |= cleared;
602                    continue;
603                }
604                WaitState::Queued { .. } => {}
605                WaitState::Granted(_) | WaitState::Failed(_) => {
606                    invariant_violation("terminal reservation result remained linked in the FIFO");
607                }
608            }
609
610            let coverage = pool.coverage.snapshot();
611            if !admission.can_grant(coverage, envelope) {
612                break;
613            }
614
615            let result = Self::prepare_and_grant_locked(pool, admission, envelope);
616            let previous = std::mem::replace(&mut *slot_state, WaitState::Taken);
617            let WaitState::Queued { waker } = previous else {
618                invariant_violation("reservation head changed while its slot lock was held");
619            };
620            *slot_state = match result {
621                Ok(reservation) => WaitState::Granted(reservation),
622                Err(error) => WaitState::Failed(error),
623            };
624
625            let (removed, cleared) = admission.inner.pop_front_waiter(std::time::Instant::now());
626            queue_became_empty |= cleared;
627            let Some(removed) = removed else {
628                invariant_violation("reservation head disappeared while admission was held");
629            };
630            if !Arc::ptr_eq(&removed.slot, &slot) {
631                invariant_violation("FIFO head changed while admission was held");
632            }
633            wakers.push(waker);
634        }
635        pool.reservation_drain
636            .publish_waiter_state(!admission.inner.waiters_is_empty());
637        let queue_sample = queue_became_empty
638            .then(|| {
639                Self::reservation_queue_sample(
640                    admission,
641                    &pool.coverage,
642                    ReservationQueueEvent::BecameEmpty,
643                )
644            })
645            .flatten();
646        ReservationDrain {
647            wakers,
648            queue_sample,
649        }
650    }
651
652    /// Retires aggregate charges and reconsiders work exposed by repayment.
653    ///
654    /// Physical ownership and direct provenance are returned before this
655    /// boundary. Coverage-only returns stay lock-free. Repayment of uncovered
656    /// charges enters admission only when a reservation poll may be queued and
657    /// invokes wakers only after unlocking.
658    fn release_acquisition_charges(pool: &Arc<Self>, count: CarrierCount) {
659        let returned = pool.coverage.release(count);
660        Self::drain_after_coverage_return(pool, returned.uncovered_removed);
661    }
662
663    /// Reverses charges that never became carrier owners.
664    fn rollback_acquisition_charges(pool: &Arc<Self>, count: CarrierCount, debit: CoverageDebit) {
665        let drained = {
666            let mut admission = AdmissionGuard::new(pool.admission.lock());
667            let returned = pool.coverage.rollback_debit(
668                count,
669                debit,
670                admission.inner.ledger.active_planned_demand,
671            );
672            let drained = (returned.uncovered_removed != CarrierCount::ZERO)
673                .then(|| Self::drain_fifo_locked(pool, &mut admission, false));
674            admission
675                .inner
676                .ledger
677                .assert_invariants(pool.coverage.snapshot());
678            drained
679        };
680        if let Some(drained) = drained {
681            if let Some(sample) = drained.queue_sample {
682                pool.log_reservation_queue_transition(sample);
683            }
684            wake_all(drained.wakers);
685        }
686    }
687
688    /// Reconsiders reservation admission after uncovered pressure decreases.
689    fn drain_after_coverage_return(pool: &Arc<Self>, uncovered_removed: CarrierCount) {
690        if uncovered_removed == CarrierCount::ZERO
691            || !pool.reservation_drain.repayment_requires_drain()
692        {
693            return;
694        }
695
696        #[cfg(test)]
697        pool.test_hooks.record_return_admission_entry();
698        let drained = {
699            let mut admission = AdmissionGuard::new(pool.admission.lock());
700            let drained = Self::drain_fifo_locked(pool, &mut admission, false);
701            admission
702                .inner
703                .ledger
704                .assert_invariants(pool.coverage.snapshot());
705            drained
706        };
707        if let Some(sample) = drained.queue_sample {
708            pool.log_reservation_queue_transition(sample);
709        }
710        wake_all(drained.wakers);
711    }
712
713    /// Marks one whole-value return and selects its physical publication mode.
714    fn begin_owner_return(pool: &Arc<Self>) -> OwnerReturnPermit {
715        // Overlap detection selects between two equivalent return strategies.
716        // Modeling it multiplies every multi-owner drop in unrelated Loom
717        // tests; the queue-driven batch path retains modeled synchronization.
718        #[cfg(not(all(test, s3_tm_loom)))]
719        let previous = pool.active_owner_returns.fetch_add(1, Ordering::AcqRel);
720        #[cfg(not(all(test, s3_tm_loom)))]
721        let overlapping = previous != 0;
722        #[cfg(all(test, s3_tm_loom))]
723        let overlapping = false;
724        OwnerReturnPermit {
725            pool: Arc::clone(pool),
726            batch: overlapping || pool.reservation_drain.is_armed(),
727        }
728    }
729
730    /// Schedules recovery when block preparation leaves a slot nonclaimable.
731    fn request_cleanup_after_arena_error(pool: &Arc<Self>, error: &ArenaError) {
732        if !error.cleanup_required() {
733            return;
734        }
735        tracing::warn!(
736            target: crate::telemetry::TARGET_MEMORY,
737            error = %error,
738            "buffer-pool preparation failed; mapping cleanup was scheduled"
739        );
740        pool.maintenance.request_cleanup(pool);
741    }
742
743    /// Captures one reservation-queue transition while admission is coherent.
744    fn reservation_queue_sample(
745        admission: &AdmissionGuard<'_>,
746        coverage: &CoverageState,
747        event: ReservationQueueEvent,
748    ) -> Option<ReservationQueueSample> {
749        let admission_used = admission
750            .inner
751            .ledger
752            .admission_used(coverage.snapshot())
753            .ok()?;
754        Some(ReservationQueueSample {
755            event,
756            queued_reservations: admission.inner.waiter_count(),
757            reservation_enqueues_total: admission.inner.reservation_enqueues_total(),
758            admission_used,
759            configured_capacity: admission.inner.ledger.configured_capacity,
760        })
761    }
762
763    /// Logs one reservation FIFO empty-boundary transition.
764    fn log_reservation_queue_transition(&self, sample: ReservationQueueSample) {
765        if !tracing::enabled!(
766            target: crate::telemetry::TARGET_MEMORY,
767            tracing::Level::DEBUG
768        ) {
769            return;
770        }
771
772        match sample.event {
773            ReservationQueueEvent::BecameNonempty { requested } => tracing::debug!(
774                target: crate::telemetry::TARGET_MEMORY,
775                requested_bytes = self.carriers_to_bytes(requested),
776                queued_reservations = sample.queued_reservations,
777                reservation_enqueues_total = sample.reservation_enqueues_total,
778                admission_used_bytes = self.carriers_to_bytes(sample.admission_used),
779                configured_capacity_bytes = self.carriers_to_bytes(sample.configured_capacity),
780                "buffer-pool reservation queue became non-empty"
781            ),
782            ReservationQueueEvent::BecameEmpty => tracing::debug!(
783                target: crate::telemetry::TARGET_MEMORY,
784                queued_reservations = sample.queued_reservations,
785                reservation_enqueues_total = sample.reservation_enqueues_total,
786                admission_used_bytes = self.carriers_to_bytes(sample.admission_used),
787                configured_capacity_bytes = self.carriers_to_bytes(sample.configured_capacity),
788                "buffer-pool reservation queue became empty"
789            ),
790        }
791    }
792
793    /// Converts one checked carrier count to its byte representation.
794    fn carriers_to_bytes(&self, count: CarrierCount) -> u64 {
795        let count = u64::try_from(count.get()).unwrap_or(u64::MAX);
796        let carrier_size = u64::try_from(self.geometry.carrier_size()).unwrap_or(u64::MAX);
797        count.saturating_mul(carrier_size)
798    }
799}
800
801/// One whole-value return participating in pool-local contention detection.
802///
803/// The permit owns the pool independently of any carrier guard. Drop paths can
804/// therefore drain their mutable owner collections while the permit remains
805/// live and still retire the active-return count afterward.
806pub(super) struct OwnerReturnPermit {
807    /// Pool whose active-return count this permit decrements on drop.
808    pool: Arc<PoolInner>,
809    /// Whether this return should use the batched physical path.
810    batch: bool,
811}
812
813impl OwnerReturnPermit {
814    /// Returns whether contention or queued admission warrants batching.
815    pub(super) fn should_batch(&self) -> bool {
816        self.batch
817    }
818}
819
820impl Drop for OwnerReturnPermit {
821    fn drop(&mut self) {
822        #[cfg(not(all(test, s3_tm_loom)))]
823        {
824            let previous = self
825                .pool
826                .active_owner_returns
827                .fetch_sub(1, Ordering::AcqRel);
828            if previous == 0 {
829                invariant_violation("owner-return permit underflowed");
830            }
831        }
832    }
833}
834
835impl Drop for PoolInner {
836    fn drop(&mut self) {
837        self.maintenance.shutdown();
838    }
839}
840
841fn map_reservation_geometry_error(error: GeometryError) -> ReserveError {
842    match error {
843        GeometryError::ZeroByteRequest => ReserveError::InvalidSize,
844        _ => invariant_violation("checked geometry rejected byte-to-carrier conversion"),
845    }
846}
847
848fn map_preparation_error(error: ArenaError) -> ReserveError {
849    match error {
850        ArenaError::Block(BlockError::PreparedCapacityOverflow)
851        | ArenaError::SlotIdExhausted
852        | ArenaError::RegistryCapacityOverflow
853        | ArenaError::AddressOverflow { .. }
854        | ArenaError::ScanSpaceOverflow { .. } => ReserveError::CapacityOverflow,
855        ArenaError::Block(BlockError::Allocation(_)) | ArenaError::Allocation(_) => {
856            ReserveError::MetadataAllocationFailed
857        }
858        ArenaError::Block(_)
859        | ArenaError::InvalidScanBudget
860        | ArenaError::InvalidClaimCount
861        | ArenaError::IncompleteClaim { .. }
862        | ArenaError::AddressOverlap { .. } => ReserveError::PhysicalPreparationFailed,
863    }
864}
865
866/// A count of fixed-size carriers.
867#[repr(transparent)]
868#[derive(Clone, Copy, Debug, Default, Eq, Ord, PartialEq, PartialOrd)]
869struct CarrierCount(usize);
870
871impl CarrierCount {
872    /// No carriers.
873    const ZERO: Self = Self(0);
874
875    /// Wraps a carrier count.
876    const fn new(value: usize) -> Self {
877        Self(value)
878    }
879
880    /// Returns the underlying count.
881    fn get(self) -> usize {
882        self.0
883    }
884
885    /// Adds two counts, returning `None` on overflow.
886    fn checked_add(self, other: Self) -> Option<Self> {
887        self.0.checked_add(other.0).map(Self)
888    }
889
890    /// Subtracts two counts, returning `None` on underflow.
891    fn checked_sub(self, other: Self) -> Option<Self> {
892        self.0.checked_sub(other.0).map(Self)
893    }
894
895    /// Converts this count to the packed accounting lane width.
896    fn try_as_u32(self) -> Option<u32> {
897        u32::try_from(self.0).ok()
898    }
899}
900
901/// Stops execution after an internal buffer-pool invariant fails.
902#[cold]
903#[track_caller]
904fn invariant_violation(message: &'static str) -> ! {
905    tracing::error!(
906        target: crate::telemetry::TARGET_MEMORY,
907        reason = message,
908        "buffer-pool invariant violated; aborting"
909    );
910
911    #[cfg(test)]
912    panic!("buffer-pool invariant violated: {message}");
913
914    #[cfg(not(test))]
915    {
916        let _ = message;
917        std::process::abort()
918    }
919}