1use std::task::Waker;
20
21use crate::config::MemoryDiagnosticsConfig;
22#[cfg(not(all(test, s3_tm_loom)))]
23use crate::runtime::sync::sync::atomic::{AtomicUsize, Ordering};
24use crate::runtime::sync::sync::{Arc, Mutex};
25
26mod acquisition;
27mod admission;
28mod arena;
29#[cfg(s3_tm_bench)]
30pub(crate) mod bench_util;
31mod block;
32mod config;
33#[cfg(not(all(test, s3_tm_loom)))]
34mod diagnostics;
35mod geometry;
36mod maintenance;
37mod metrics;
38mod pooled_buf;
39mod segmented_bytes;
40#[cfg(any(test, s3_tm_fuzz))]
41mod test_util;
42#[cfg(all(test, not(s3_tm_loom)))]
43mod tests;
44mod virtual_memory;
45
46use crate::config::MemoryBudgetConfig;
47use acquisition::acquire_count;
48pub(crate) use acquisition::AcquireError;
49use admission::{
50 wake_all, AdmissionGuard, AdmissionState, CoverageDebit, CoverageState, ReservationDrainSignal,
51 ReservationPoll, WaitSlot, WaitState, Waiter, MAX_PACKED_CARRIERS,
52};
53pub(crate) use admission::{Reservation, ReserveError, ReserveFuture};
54use arena::{Arena, ArenaError, ArenaOptions};
55use block::BlockError;
56pub(crate) use config::auto_capacity_bytes;
57use config::PoolConfig;
58pub use config::{BufferPoolBuildError, BufferPoolBuilder};
59use geometry::{GeometryError, PoolGeometry};
60use maintenance::MaintenanceCoordinator;
61pub use metrics::MemoryMetrics;
62use metrics::{MemoryDiagnostics, MemoryMetricState};
63use pooled_buf::GrowthAuthority;
64pub(crate) use pooled_buf::PooledBufMut;
65pub use segmented_bytes::SegmentedBytes;
66
67#[cfg(s3_tm_fuzz)]
72pub(crate) fn run_fuzz_input(data: &[u8]) {
73 let _ = test_util::model_harness::run_fuzz_input(data);
74}
75
76#[cfg(s3_tm_fuzz)]
81pub(crate) fn run_placement_fuzz_input(data: &[u8]) {
82 let _ = test_util::model_harness::run_placement_fuzz_input(data);
83}
84
85#[cfg(all(test, not(s3_tm_loom)))]
86pub(crate) use test_util::test_pool;
87#[cfg(test)]
88use test_util::TestHooks;
89
90#[derive(Clone)]
97pub struct BufferPool {
98 inner: Arc<PoolInner>,
99}
100
101impl std::fmt::Debug for BufferPool {
102 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
103 f.debug_struct("BufferPool").finish_non_exhaustive()
104 }
105}
106
107impl BufferPool {
108 pub fn builder() -> BufferPoolBuilder {
110 BufferPoolBuilder::default()
111 }
112
113 pub(crate) fn carrier_size(&self) -> usize {
118 self.inner.geometry.carrier_size()
119 }
120
121 pub(crate) fn from_capacity(
126 capacity: MemoryBudgetConfig,
127 detected_memory: Option<usize>,
128 diagnostics: MemoryDiagnosticsConfig,
129 ) -> Result<Self, BufferPoolBuildError> {
130 let resolved = PoolConfig::resolve(capacity, detected_memory)?;
131 let arena_options = ArenaOptions::new(
132 resolved.optimistic_scan_words,
133 diagnostics.enable_detailed_counters(),
134 );
135 Ok(Self::from_parts_with_arena_options(
136 resolved.geometry,
137 resolved.configured_capacity,
138 diagnostics,
139 arena_options,
140 )
141 .unwrap_or_else(|_| invariant_violation("validated pool configuration was rejected")))
142 }
143
144 fn from_parts_with_arena_options(
146 geometry: PoolGeometry,
147 configured_capacity: CarrierCount,
148 diagnostics: MemoryDiagnosticsConfig,
149 arena_options: ArenaOptions,
150 ) -> Result<Self, ArenaError> {
151 if configured_capacity == CarrierCount::ZERO {
152 invariant_violation("configured capacity must be nonzero");
153 }
154 if configured_capacity > MAX_PACKED_CARRIERS {
155 invariant_violation("configured capacity must fit packed accounting");
156 }
157
158 let configured_capacity_bytes = configured_capacity
159 .get()
160 .checked_mul(geometry.carrier_size())
161 .unwrap_or_else(|| invariant_violation("configured byte capacity overflowed"));
162 let optimistic_scan_words = arena_options.optimistic_scan_words();
163 let inner = Arc::new(PoolInner::new(
164 geometry,
165 configured_capacity,
166 diagnostics,
167 arena_options,
168 )?);
169 inner.maintenance.start_periodic_diagnostics(&inner);
170 tracing::debug!(
171 target: crate::telemetry::TARGET_MEMORY,
172 configured_capacity_bytes,
173 page_size_bytes = geometry.page_size(),
174 carrier_size_bytes = geometry.carrier_size(),
175 block_size_bytes = geometry.block_size(),
176 optimistic_scan_words,
177 diagnostic_detail = diagnostics.detail_level(),
178 diagnostic_snapshot_interval_ms =
179 diagnostics.snapshot_interval().map(|interval| interval.as_millis()).unwrap_or(0),
180 "constructed buffer pool"
181 );
182 Ok(Self { inner })
183 }
184
185 pub(crate) fn try_reserve(&self, bytes: usize) -> Result<Option<Reservation>, ReserveError> {
192 let envelope = self.reservation_envelope(bytes)?;
193 PoolInner::try_reserve_count(&self.inner, envelope)
194 }
195
196 pub(crate) fn reserve(&self, bytes: usize) -> ReserveFuture {
203 ReserveFuture::new(self.clone(), bytes)
204 }
205
206 pub fn metrics(&self) -> MemoryMetrics {
208 self.inner.memory_metrics()
209 }
210
211 pub(crate) fn diagnostics(&self) -> MemoryDiagnostics {
213 MemoryDiagnostics::from_samples(
214 self.inner.arena.diagnostics(),
215 self.inner.arena.sample_lifecycle(),
216 self.inner.maintenance.diagnostics(),
217 )
218 }
219
220 pub(crate) fn record_managed_activity(&self) {
222 self.inner.maintenance.record_activity();
223 }
224
225 pub(crate) fn record_global_idle(&self) {
227 self.inner.maintenance.record_idle(&self.inner);
228 }
229
230 pub(crate) fn acquire(
237 &self,
238 reservation: &Reservation,
239 min_bytes: usize,
240 ) -> Result<PooledBufMut, AcquireError> {
241 let count = self.acquisition_count(min_bytes)?;
242 let direct = reservation
243 .acquisition_state()
244 .ok_or(AcquireError::ReservationClosed)?;
245 if !direct.belongs_to(&self.inner) {
246 return Err(AcquireError::ForeignReservation);
247 }
248 let direct = Arc::clone(direct);
249 let guards = acquire_count(&self.inner, Some(Arc::clone(&direct)), count)?;
250 PooledBufMut::try_new(GrowthAuthority::reserved(direct), guards)
251 }
252
253 pub(crate) fn acquire_unreserved(
260 &self,
261 min_bytes: usize,
262 ) -> Result<PooledBufMut, AcquireError> {
263 let count = self.acquisition_count(min_bytes)?;
264 let guards = acquire_count(&self.inner, None, count)?;
265 PooledBufMut::try_new(GrowthAuthority::unreserved(Arc::clone(&self.inner)), guards)
266 }
267
268 fn reservation_envelope(&self, bytes: usize) -> Result<CarrierCount, ReserveError> {
270 let envelope = self
271 .inner
272 .geometry
273 .carriers_for_bytes(bytes)
274 .map_err(map_reservation_geometry_error)?;
275 if envelope > MAX_PACKED_CARRIERS {
276 return Err(ReserveError::CapacityOverflow);
277 }
278 Ok(envelope)
279 }
280
281 fn acquisition_count(&self, min_bytes: usize) -> Result<CarrierCount, AcquireError> {
283 self.inner
284 .geometry
285 .carriers_for_bytes(min_bytes)
286 .map_err(|error| match error {
287 GeometryError::ZeroByteRequest => AcquireError::InvalidSize,
288 _ => AcquireError::CapacityOverflow,
289 })
290 }
291}
292
293struct PoolInner {
295 geometry: PoolGeometry,
297 admission: Mutex<AdmissionState>,
299 reservation_drain: ReservationDrainSignal,
301 #[cfg(not(all(test, s3_tm_loom)))]
303 active_owner_returns: AtomicUsize,
304 coverage: CoverageState,
306 arena: Arena,
308 maintenance: MaintenanceCoordinator,
310 #[cfg(test)]
312 test_hooks: TestHooks,
313}
314
315#[derive(Clone, Copy)]
317struct ReservationQueueSample {
318 event: ReservationQueueEvent,
320 queued_reservations: usize,
322 reservation_enqueues_total: u64,
324 admission_used: CarrierCount,
326 configured_capacity: CarrierCount,
328}
329
330#[derive(Clone, Copy)]
332enum ReservationQueueEvent {
333 BecameNonempty { requested: CarrierCount },
335 BecameEmpty,
337}
338
339struct ReservationDrain {
341 wakers: Vec<Waker>,
342 queue_sample: Option<ReservationQueueSample>,
343}
344
345impl PoolInner {
346 fn new(
348 geometry: PoolGeometry,
349 configured_capacity: CarrierCount,
350 diagnostics: MemoryDiagnosticsConfig,
351 arena_options: ArenaOptions,
352 ) -> Result<Self, ArenaError> {
353 Ok(Self {
354 geometry,
355 admission: Mutex::new(AdmissionState::new(configured_capacity)),
356 reservation_drain: ReservationDrainSignal::new(),
357 #[cfg(not(all(test, s3_tm_loom)))]
358 active_owner_returns: AtomicUsize::new(0),
359 coverage: CoverageState::new(),
360 arena: Arena::new(geometry, arena_options)?,
361 maintenance: MaintenanceCoordinator::new(
362 configured_capacity,
363 geometry,
364 diagnostics.snapshot_interval(),
365 ),
366 #[cfg(test)]
367 test_hooks: TestHooks::new(),
368 })
369 }
370
371 fn memory_metrics(&self) -> MemoryMetrics {
373 let admission = self.admission.lock();
374 self.memory_metrics_locked(&admission)
375 }
376
377 fn diagnostic_memory_sample(
379 &self,
380 now: std::time::Instant,
381 ) -> (MemoryMetrics, admission::ReservationQueueDiagnosticSample) {
382 let mut admission = self.admission.lock();
383 let queue = admission.take_queue_diagnostics(now);
384 let metrics = self.memory_metrics_locked(&admission);
385 (metrics, queue)
386 }
387
388 fn memory_metrics_locked(&self, admission: &AdmissionState) -> MemoryMetrics {
390 let coverage = self.coverage.snapshot();
391 admission.ledger.assert_invariants(coverage);
392
393 let admission_used = admission
394 .ledger
395 .admission_used(coverage)
396 .unwrap_or_else(|_| invariant_violation("completed admission exceeds packed capacity"));
397 let covered_charges = admission
398 .ledger
399 .active_planned_demand
400 .checked_sub(coverage.available)
401 .unwrap_or_else(|| {
402 invariant_violation("available coverage exceeds active planned demand")
403 });
404 let charged_capacity = covered_charges
405 .checked_add(coverage.uncovered)
406 .unwrap_or_else(|| invariant_violation("completed charges exceed packed capacity"));
407
408 MemoryMetrics::from_carriers(
409 self.geometry.carrier_size(),
410 MemoryMetricState {
411 configured_capacity: admission.ledger.configured_capacity,
412 admission_used,
413 active_planned_demand: admission.ledger.active_planned_demand,
414 charged_capacity,
415 prepared_capacity: admission.ledger.prepared_capacity,
416 queued_reservations: admission.waiter_count(),
417 reservation_enqueues_total: admission.reservation_enqueues_total(),
418 },
419 )
420 }
421
422 fn try_debit_covered(&self, count: CarrierCount) -> Result<bool, ReserveError> {
427 self.coverage
428 .try_debit_covered(count)
429 .map_err(|_| ReserveError::CapacityOverflow)
430 }
431
432 fn debit_and_prepare_locked(
437 pool: &Arc<Self>,
438 admission: &mut AdmissionGuard<'_>,
439 count: CarrierCount,
440 ) -> Result<CoverageDebit, ReserveError> {
441 let debit = pool
442 .coverage
443 .debit(count, admission.maximum_uncovered())
444 .map_err(|_| ReserveError::CapacityOverflow)?;
445
446 let floor = match admission.acquisition_floor(&pool.coverage) {
447 Ok(floor) => floor,
448 Err(error) => {
449 admission.rollback_acquisition(&pool.coverage, count, debit);
450 return Err(error);
451 }
452 };
453 if let Err(error) = pool.arena.prepare_to(admission, floor) {
454 admission.rollback_acquisition(&pool.coverage, count, debit);
455 Self::request_cleanup_after_arena_error(pool, &error);
456 return Err(map_preparation_error(error));
457 }
458 Ok(debit)
459 }
460
461 fn try_reserve_count(
463 pool: &Arc<Self>,
464 envelope: CarrierCount,
465 ) -> Result<Option<Reservation>, ReserveError> {
466 if envelope == CarrierCount::ZERO {
467 return Err(ReserveError::InvalidSize);
468 }
469 if envelope > MAX_PACKED_CARRIERS {
470 return Err(ReserveError::CapacityOverflow);
471 }
472
473 let mut admission = AdmissionGuard::new(pool.admission.lock());
474 admission.validate_envelope(envelope)?;
475 if !admission.inner.waiters_is_empty() {
476 return Ok(None);
477 }
478 let coverage = pool.coverage.snapshot();
479 if !admission.can_grant(coverage, envelope) {
480 return Ok(None);
481 }
482
483 Self::prepare_and_grant_locked(pool, &mut admission, envelope).map(Some)
484 }
485
486 fn reserve_or_enqueue(
488 pool: &Arc<Self>,
489 envelope: CarrierCount,
490 waker: Waker,
491 ) -> Result<ReservationPoll, ReserveError> {
492 #[cfg(test)]
493 if let Some(error) = pool.test_hooks.take_reservation_failure() {
494 return Err(error);
495 }
496 if envelope == CarrierCount::ZERO {
497 return Err(ReserveError::InvalidSize);
498 }
499 if envelope > MAX_PACKED_CARRIERS {
500 return Err(ReserveError::CapacityOverflow);
501 }
502
503 let mut admission = AdmissionGuard::new(pool.admission.lock());
504 admission.validate_envelope(envelope)?;
505 pool.reservation_drain.arm();
506 let coverage = pool.coverage.snapshot();
507 if admission.inner.waiters_is_empty() && admission.can_grant(coverage, envelope) {
508 let result = Self::prepare_and_grant_locked(pool, &mut admission, envelope)
509 .map(ReservationPoll::Ready);
510 pool.reservation_drain
511 .publish_waiter_state(!admission.inner.waiters_is_empty());
512 return result;
513 }
514
515 let slot = Arc::new(WaitSlot::new(waker));
516 #[cfg(test)]
517 if pool.test_hooks.take_reservation_queue_allocation_failure() {
518 pool.reservation_drain
519 .publish_waiter_state(!admission.inner.waiters_is_empty());
520 return Err(ReserveError::MetadataAllocationFailed);
521 }
522 let queue_became_nonempty = match admission.inner.enqueue_waiter(
523 Waiter {
524 envelope,
525 slot: Arc::clone(&slot),
526 },
527 std::time::Instant::now(),
528 ) {
529 Ok(queue_became_nonempty) => queue_became_nonempty,
530 Err(_) => {
531 pool.reservation_drain
532 .publish_waiter_state(!admission.inner.waiters_is_empty());
533 return Err(ReserveError::MetadataAllocationFailed);
534 }
535 };
536 pool.reservation_drain
537 .publish_waiter_state(!admission.inner.waiters_is_empty());
538 let queue_sample = queue_became_nonempty
539 .then(|| {
540 Self::reservation_queue_sample(
541 &admission,
542 &pool.coverage,
543 ReservationQueueEvent::BecameNonempty {
544 requested: envelope,
545 },
546 )
547 })
548 .flatten();
549 drop(admission);
550 if let Some(sample) = queue_sample {
551 pool.log_reservation_queue_transition(sample);
552 }
553 Ok(ReservationPoll::Queued(slot))
554 }
555
556 fn cancel_waiter(pool: &Arc<Self>, slot: &Arc<WaitSlot>) -> Vec<Waker> {
558 let mut admission = AdmissionGuard::new(pool.admission.lock());
559 let queue_became_empty = admission
560 .inner
561 .remove_waiter(slot, std::time::Instant::now());
562 let drained = Self::drain_fifo_locked(pool, &mut admission, queue_became_empty);
563 drop(admission);
564 if let Some(sample) = drained.queue_sample {
565 pool.log_reservation_queue_transition(sample);
566 }
567 drained.wakers
568 }
569
570 fn prepare_and_grant_locked(
572 pool: &Arc<Self>,
573 admission: &mut AdmissionGuard<'_>,
574 envelope: CarrierCount,
575 ) -> Result<Reservation, ReserveError> {
576 let coverage = pool.coverage.snapshot();
577 let target = admission.grant_target(coverage, envelope)?;
578 if let Err(error) = pool.arena.prepare_to(admission, target) {
579 Self::request_cleanup_after_arena_error(pool, &error);
580 return Err(map_preparation_error(error));
581 }
582 admission.commit_grant(&pool.coverage, envelope)?;
583 Ok(Reservation::new(Arc::clone(pool), envelope))
584 }
585
586 fn drain_fifo_locked(
588 pool: &Arc<Self>,
589 admission: &mut AdmissionGuard<'_>,
590 mut queue_became_empty: bool,
591 ) -> ReservationDrain {
592 let mut wakers = Vec::new();
593 while let Some(front) = admission.inner.front_waiter() {
594 let envelope = front.envelope;
595 let slot = Arc::clone(&front.slot);
596 let mut slot_state = slot.state.lock();
597
598 match &*slot_state {
599 WaitState::Taken => {
600 let (_, cleared) = admission.inner.pop_front_waiter(std::time::Instant::now());
601 queue_became_empty |= cleared;
602 continue;
603 }
604 WaitState::Queued { .. } => {}
605 WaitState::Granted(_) | WaitState::Failed(_) => {
606 invariant_violation("terminal reservation result remained linked in the FIFO");
607 }
608 }
609
610 let coverage = pool.coverage.snapshot();
611 if !admission.can_grant(coverage, envelope) {
612 break;
613 }
614
615 let result = Self::prepare_and_grant_locked(pool, admission, envelope);
616 let previous = std::mem::replace(&mut *slot_state, WaitState::Taken);
617 let WaitState::Queued { waker } = previous else {
618 invariant_violation("reservation head changed while its slot lock was held");
619 };
620 *slot_state = match result {
621 Ok(reservation) => WaitState::Granted(reservation),
622 Err(error) => WaitState::Failed(error),
623 };
624
625 let (removed, cleared) = admission.inner.pop_front_waiter(std::time::Instant::now());
626 queue_became_empty |= cleared;
627 let Some(removed) = removed else {
628 invariant_violation("reservation head disappeared while admission was held");
629 };
630 if !Arc::ptr_eq(&removed.slot, &slot) {
631 invariant_violation("FIFO head changed while admission was held");
632 }
633 wakers.push(waker);
634 }
635 pool.reservation_drain
636 .publish_waiter_state(!admission.inner.waiters_is_empty());
637 let queue_sample = queue_became_empty
638 .then(|| {
639 Self::reservation_queue_sample(
640 admission,
641 &pool.coverage,
642 ReservationQueueEvent::BecameEmpty,
643 )
644 })
645 .flatten();
646 ReservationDrain {
647 wakers,
648 queue_sample,
649 }
650 }
651
652 fn release_acquisition_charges(pool: &Arc<Self>, count: CarrierCount) {
659 let returned = pool.coverage.release(count);
660 Self::drain_after_coverage_return(pool, returned.uncovered_removed);
661 }
662
663 fn rollback_acquisition_charges(pool: &Arc<Self>, count: CarrierCount, debit: CoverageDebit) {
665 let drained = {
666 let mut admission = AdmissionGuard::new(pool.admission.lock());
667 let returned = pool.coverage.rollback_debit(
668 count,
669 debit,
670 admission.inner.ledger.active_planned_demand,
671 );
672 let drained = (returned.uncovered_removed != CarrierCount::ZERO)
673 .then(|| Self::drain_fifo_locked(pool, &mut admission, false));
674 admission
675 .inner
676 .ledger
677 .assert_invariants(pool.coverage.snapshot());
678 drained
679 };
680 if let Some(drained) = drained {
681 if let Some(sample) = drained.queue_sample {
682 pool.log_reservation_queue_transition(sample);
683 }
684 wake_all(drained.wakers);
685 }
686 }
687
688 fn drain_after_coverage_return(pool: &Arc<Self>, uncovered_removed: CarrierCount) {
690 if uncovered_removed == CarrierCount::ZERO
691 || !pool.reservation_drain.repayment_requires_drain()
692 {
693 return;
694 }
695
696 #[cfg(test)]
697 pool.test_hooks.record_return_admission_entry();
698 let drained = {
699 let mut admission = AdmissionGuard::new(pool.admission.lock());
700 let drained = Self::drain_fifo_locked(pool, &mut admission, false);
701 admission
702 .inner
703 .ledger
704 .assert_invariants(pool.coverage.snapshot());
705 drained
706 };
707 if let Some(sample) = drained.queue_sample {
708 pool.log_reservation_queue_transition(sample);
709 }
710 wake_all(drained.wakers);
711 }
712
713 fn begin_owner_return(pool: &Arc<Self>) -> OwnerReturnPermit {
715 #[cfg(not(all(test, s3_tm_loom)))]
719 let previous = pool.active_owner_returns.fetch_add(1, Ordering::AcqRel);
720 #[cfg(not(all(test, s3_tm_loom)))]
721 let overlapping = previous != 0;
722 #[cfg(all(test, s3_tm_loom))]
723 let overlapping = false;
724 OwnerReturnPermit {
725 pool: Arc::clone(pool),
726 batch: overlapping || pool.reservation_drain.is_armed(),
727 }
728 }
729
730 fn request_cleanup_after_arena_error(pool: &Arc<Self>, error: &ArenaError) {
732 if !error.cleanup_required() {
733 return;
734 }
735 tracing::warn!(
736 target: crate::telemetry::TARGET_MEMORY,
737 error = %error,
738 "buffer-pool preparation failed; mapping cleanup was scheduled"
739 );
740 pool.maintenance.request_cleanup(pool);
741 }
742
743 fn reservation_queue_sample(
745 admission: &AdmissionGuard<'_>,
746 coverage: &CoverageState,
747 event: ReservationQueueEvent,
748 ) -> Option<ReservationQueueSample> {
749 let admission_used = admission
750 .inner
751 .ledger
752 .admission_used(coverage.snapshot())
753 .ok()?;
754 Some(ReservationQueueSample {
755 event,
756 queued_reservations: admission.inner.waiter_count(),
757 reservation_enqueues_total: admission.inner.reservation_enqueues_total(),
758 admission_used,
759 configured_capacity: admission.inner.ledger.configured_capacity,
760 })
761 }
762
763 fn log_reservation_queue_transition(&self, sample: ReservationQueueSample) {
765 if !tracing::enabled!(
766 target: crate::telemetry::TARGET_MEMORY,
767 tracing::Level::DEBUG
768 ) {
769 return;
770 }
771
772 match sample.event {
773 ReservationQueueEvent::BecameNonempty { requested } => tracing::debug!(
774 target: crate::telemetry::TARGET_MEMORY,
775 requested_bytes = self.carriers_to_bytes(requested),
776 queued_reservations = sample.queued_reservations,
777 reservation_enqueues_total = sample.reservation_enqueues_total,
778 admission_used_bytes = self.carriers_to_bytes(sample.admission_used),
779 configured_capacity_bytes = self.carriers_to_bytes(sample.configured_capacity),
780 "buffer-pool reservation queue became non-empty"
781 ),
782 ReservationQueueEvent::BecameEmpty => tracing::debug!(
783 target: crate::telemetry::TARGET_MEMORY,
784 queued_reservations = sample.queued_reservations,
785 reservation_enqueues_total = sample.reservation_enqueues_total,
786 admission_used_bytes = self.carriers_to_bytes(sample.admission_used),
787 configured_capacity_bytes = self.carriers_to_bytes(sample.configured_capacity),
788 "buffer-pool reservation queue became empty"
789 ),
790 }
791 }
792
793 fn carriers_to_bytes(&self, count: CarrierCount) -> u64 {
795 let count = u64::try_from(count.get()).unwrap_or(u64::MAX);
796 let carrier_size = u64::try_from(self.geometry.carrier_size()).unwrap_or(u64::MAX);
797 count.saturating_mul(carrier_size)
798 }
799}
800
801pub(super) struct OwnerReturnPermit {
807 pool: Arc<PoolInner>,
809 batch: bool,
811}
812
813impl OwnerReturnPermit {
814 pub(super) fn should_batch(&self) -> bool {
816 self.batch
817 }
818}
819
820impl Drop for OwnerReturnPermit {
821 fn drop(&mut self) {
822 #[cfg(not(all(test, s3_tm_loom)))]
823 {
824 let previous = self
825 .pool
826 .active_owner_returns
827 .fetch_sub(1, Ordering::AcqRel);
828 if previous == 0 {
829 invariant_violation("owner-return permit underflowed");
830 }
831 }
832 }
833}
834
835impl Drop for PoolInner {
836 fn drop(&mut self) {
837 self.maintenance.shutdown();
838 }
839}
840
841fn map_reservation_geometry_error(error: GeometryError) -> ReserveError {
842 match error {
843 GeometryError::ZeroByteRequest => ReserveError::InvalidSize,
844 _ => invariant_violation("checked geometry rejected byte-to-carrier conversion"),
845 }
846}
847
848fn map_preparation_error(error: ArenaError) -> ReserveError {
849 match error {
850 ArenaError::Block(BlockError::PreparedCapacityOverflow)
851 | ArenaError::SlotIdExhausted
852 | ArenaError::RegistryCapacityOverflow
853 | ArenaError::AddressOverflow { .. }
854 | ArenaError::ScanSpaceOverflow { .. } => ReserveError::CapacityOverflow,
855 ArenaError::Block(BlockError::Allocation(_)) | ArenaError::Allocation(_) => {
856 ReserveError::MetadataAllocationFailed
857 }
858 ArenaError::Block(_)
859 | ArenaError::InvalidScanBudget
860 | ArenaError::InvalidClaimCount
861 | ArenaError::IncompleteClaim { .. }
862 | ArenaError::AddressOverlap { .. } => ReserveError::PhysicalPreparationFailed,
863 }
864}
865
866#[repr(transparent)]
868#[derive(Clone, Copy, Debug, Default, Eq, Ord, PartialEq, PartialOrd)]
869struct CarrierCount(usize);
870
871impl CarrierCount {
872 const ZERO: Self = Self(0);
874
875 const fn new(value: usize) -> Self {
877 Self(value)
878 }
879
880 fn get(self) -> usize {
882 self.0
883 }
884
885 fn checked_add(self, other: Self) -> Option<Self> {
887 self.0.checked_add(other.0).map(Self)
888 }
889
890 fn checked_sub(self, other: Self) -> Option<Self> {
892 self.0.checked_sub(other.0).map(Self)
893 }
894
895 fn try_as_u32(self) -> Option<u32> {
897 u32::try_from(self.0).ok()
898 }
899}
900
901#[cold]
903#[track_caller]
904fn invariant_violation(message: &'static str) -> ! {
905 tracing::error!(
906 target: crate::telemetry::TARGET_MEMORY,
907 reason = message,
908 "buffer-pool invariant violated; aborting"
909 );
910
911 #[cfg(test)]
912 panic!("buffer-pool invariant violated: {message}");
913
914 #[cfg(not(test))]
915 {
916 let _ = message;
917 std::process::abort()
918 }
919}