use std::mem::MaybeUninit;
use std::ptr::NonNull;
use bytes::buf::UninitSlice;
use bytes::{BufMut, Bytes};
use smallvec::SmallVec;
use super::acquisition::{
acquire_count, allocation_failure_injected, AcquireError, CarrierGuard, OwnerReturn,
};
use super::admission::ReservationState;
use super::block::CarrierLocation;
use super::geometry::GeometryError;
use super::segmented_bytes::SegmentedBytesBuilder;
use super::{invariant_violation, CarrierCount, PoolInner, SegmentedBytes};
use crate::runtime::sync::sync::Arc;
const INLINE_CARRIER_RUNS: usize = 4;
type CarrierRuns = SmallVec<[CarrierRun; INLINE_CARRIER_RUNS]>;
pub(crate) struct PooledBufMut {
growth: GrowthAuthority,
runs: CarrierRuns,
write_cursor: BufferCursor,
publish_cursor: BufferCursor,
initialized: usize,
retained_capacity: usize,
}
impl PooledBufMut {
pub(super) fn try_new(
growth: GrowthAuthority,
guards: Vec<Arc<CarrierGuard>>,
) -> Result<Self, AcquireError> {
let (runs, retained_capacity) = build_runs(growth.pool(), guards)?;
let mut buffer = Self {
growth,
runs,
write_cursor: BufferCursor::START,
publish_cursor: BufferCursor::START,
initialized: 0,
retained_capacity,
};
buffer.reset_cursors();
Ok(buffer)
}
pub(crate) fn capacity(&self) -> usize {
self.retained_capacity
}
pub(crate) fn len(&self) -> usize {
self.initialized
}
pub(crate) fn is_empty(&self) -> bool {
self.initialized == 0
}
#[cfg(any(test, s3_tm_fuzz))]
pub(in crate::runtime::buffer_pool) fn test_run_count(&self) -> usize {
self.runs.len()
}
#[cfg(test)]
pub(super) fn test_run_capacity(&self, index: usize) -> usize {
self.runs[index]
.carriers
.iter()
.map(|carrier| carrier.writable.len())
.sum()
}
#[cfg(all(
test,
not(miri),
not(s3_tm_loom),
not(s3_tm_tsan),
any(
target_os = "android",
target_os = "freebsd",
target_os = "linux",
target_os = "macos",
target_os = "windows"
)
))]
fn test_first_reservation_address_range(&self) -> std::ops::Range<usize> {
self.runs[0].carriers[0]
.guard
.as_ref()
.expect("test buffer retained no first-carrier guard")
.slot()
.reservation_address_range()
}
pub(crate) fn initialized_chunk(&self) -> &[u8] {
if self.initialized == 0 {
return &[];
}
let carrier = self
.carrier(self.publish_cursor)
.filter(|carrier| carrier.initialized != 0)
.unwrap_or_else(|| {
invariant_violation("publication cursor does not name initialized bytes")
});
unsafe {
std::slice::from_raw_parts(
carrier.writable.ptr().as_ptr().cast::<u8>(),
carrier.initialized,
)
}
}
pub(crate) fn reserve(&mut self, min_writable: usize) -> Result<(), AcquireError> {
let remaining = self.remaining_mut();
if min_writable <= remaining {
return Ok(());
}
let shortfall = min_writable
.checked_sub(remaining)
.ok_or(AcquireError::CapacityOverflow)?;
let count = self
.growth
.pool()
.geometry
.carriers_for_bytes(shortfall)
.map_err(map_geometry_error)?;
let guards = self.growth.acquire(count)?;
let (staged, added_capacity) = build_runs(self.growth.pool(), guards)?;
self.append_staged(staged, added_capacity)?;
Ok(())
}
pub(crate) fn publish_prefix(&mut self, count: usize) -> Bytes {
self.normalize_publish_cursor();
let available = self.initialized_chunk().len();
assert!(
count != 0 && count <= available,
"published prefix must be nonzero and within initialized_chunk"
);
let cursor = self.publish_cursor;
let (range, guard) =
{
let carrier = self.carrier_mut(cursor).unwrap_or_else(|| {
invariant_violation("publication cursor is outside the buffer")
});
let range = carrier.writable.take_prefix(count);
carrier.initialized = carrier.initialized.checked_sub(count).unwrap_or_else(|| {
invariant_violation("publication exceeds initialized bytes")
});
let guard =
if carrier.writable.is_empty() {
carrier.guard.take().unwrap_or_else(|| {
invariant_violation("exhausted writable carrier lost its guard")
})
} else {
Arc::clone(carrier.guard.as_ref().unwrap_or_else(|| {
invariant_violation("writable carrier lost its guard")
}))
};
(range, guard)
};
self.initialized = self
.initialized
.checked_sub(count)
.unwrap_or_else(|| invariant_violation("buffer initialization underflow"));
self.retained_capacity = self
.retained_capacity
.checked_sub(count)
.unwrap_or_else(|| invariant_violation("buffer capacity underflow"));
self.normalize_publish_cursor();
Bytes::from_owner(PooledWindow::new(guard, range))
}
pub(crate) fn freeze(mut self) -> SegmentedBytes {
let pool = Arc::clone(self.growth.pool());
let mut builder = SegmentedBytesBuilder::for_pool(pool);
for run in &mut self.runs {
for carrier in &mut run.carriers {
if carrier.initialized == 0 {
continue;
}
let initialized = carrier.initialized;
let range = carrier.writable.take_prefix(initialized);
carrier.initialized = 0;
let guard = carrier
.guard
.take()
.unwrap_or_else(|| invariant_violation("initialized carrier lost its guard"));
builder.push_pooled(range.ptr.cast::<u8>(), range.len, guard);
}
}
builder.finish()
}
fn append_staged(
&mut self,
mut staged: CarrierRuns,
added_capacity: usize,
) -> Result<(), AcquireError> {
let retained_capacity = self
.retained_capacity
.checked_add(added_capacity)
.ok_or(AcquireError::CapacityOverflow)?;
if staged.is_empty() {
invariant_violation("successful growth produced no carrier runs");
}
let merge_first = self
.runs
.last()
.zip(staged.first())
.is_some_and(|(current, incoming)| current.can_append_run(incoming));
let appended_runs = staged.len() - usize::from(merge_first);
if allocation_failure_injected(self.growth.pool()) {
return Err(AcquireError::MetadataAllocationFailed);
}
self.runs
.try_reserve(appended_runs)
.map_err(|_| AcquireError::MetadataAllocationFailed)?;
if merge_first {
let incoming = staged.remove(0);
let current = self
.runs
.last_mut()
.unwrap_or_else(|| invariant_violation("merge target disappeared"));
if allocation_failure_injected(self.growth.pool()) {
return Err(AcquireError::MetadataAllocationFailed);
}
current
.carriers
.try_reserve(incoming.carriers.len())
.map_err(|_| AcquireError::MetadataAllocationFailed)?;
current.carriers.extend(incoming.carriers);
}
self.runs.extend(staged);
self.retained_capacity = retained_capacity;
self.reset_write_cursor();
self.normalize_publish_cursor();
Ok(())
}
fn reset_cursors(&mut self) {
self.reset_write_cursor();
self.publish_cursor = self
.first_cursor_matching(|carrier| carrier.initialized != 0)
.unwrap_or_else(|| BufferCursor::end(self.runs.len()));
}
fn reset_write_cursor(&mut self) {
self.write_cursor = self
.first_cursor_matching(|carrier| carrier.remaining_mut() != 0)
.unwrap_or_else(|| BufferCursor::end(self.runs.len()));
}
fn normalize_write_cursor(&mut self) {
self.write_cursor = self
.cursor_matching_from(self.write_cursor, |carrier| carrier.remaining_mut() != 0)
.unwrap_or_else(|| BufferCursor::end(self.runs.len()));
}
fn normalize_publish_cursor(&mut self) {
self.publish_cursor = self
.cursor_matching_from(self.publish_cursor, |carrier| carrier.initialized != 0)
.unwrap_or_else(|| BufferCursor::end(self.runs.len()));
}
fn first_cursor_matching(
&self,
predicate: impl Fn(&WritableCarrier) -> bool,
) -> Option<BufferCursor> {
for (run, entry) in self.runs.iter().enumerate() {
for (carrier, value) in entry.carriers.iter().enumerate() {
if predicate(value) {
return Some(BufferCursor { run, carrier });
}
}
}
None
}
fn cursor_matching_from(
&self,
mut cursor: BufferCursor,
predicate: impl Fn(&WritableCarrier) -> bool,
) -> Option<BufferCursor> {
loop {
let carrier = self.carrier(cursor)?;
if predicate(carrier) {
return Some(cursor);
}
cursor = self.next_cursor(cursor)?;
}
}
fn next_cursor(&self, cursor: BufferCursor) -> Option<BufferCursor> {
let run = self.runs.get(cursor.run)?;
let carrier = cursor.carrier.checked_add(1)?;
if carrier < run.carriers.len() {
return Some(BufferCursor {
run: cursor.run,
carrier,
});
}
let run = cursor.run.checked_add(1)?;
(run < self.runs.len()).then_some(BufferCursor { run, carrier: 0 })
}
fn carrier(&self, cursor: BufferCursor) -> Option<&WritableCarrier> {
self.runs
.get(cursor.run)
.and_then(|run| run.carriers.get(cursor.carrier))
}
fn carrier_mut(&mut self, cursor: BufferCursor) -> Option<&mut WritableCarrier> {
self.runs
.get_mut(cursor.run)
.and_then(|run| run.carriers.get_mut(cursor.carrier))
}
fn advance_initialized(&mut self, mut count: usize) {
assert!(
count <= self.remaining_mut(),
"advanced beyond pooled writable capacity"
);
if count == 0 {
return;
}
self.normalize_write_cursor();
while count != 0 {
let cursor = self.write_cursor;
let publish_was_empty = self.initialized == 0;
let (advanced, exhausted) = {
let carrier = self.carrier_mut(cursor).unwrap_or_else(|| {
invariant_violation("writable cursor is outside the buffer")
});
let advanced = count.min(carrier.remaining_mut());
if advanced == 0 {
invariant_violation("writable cursor names an exhausted carrier");
}
carrier.initialized = carrier
.initialized
.checked_add(advanced)
.unwrap_or_else(|| invariant_violation("carrier initialization overflow"));
(advanced, carrier.remaining_mut() == 0)
};
if publish_was_empty {
self.publish_cursor = cursor;
}
self.initialized = self
.initialized
.checked_add(advanced)
.unwrap_or_else(|| invariant_violation("buffer initialization overflow"));
count -= advanced;
if exhausted {
self.write_cursor = self
.next_cursor(cursor)
.unwrap_or_else(|| BufferCursor::end(self.runs.len()));
self.normalize_write_cursor();
}
}
}
}
impl Drop for PooledBufMut {
fn drop(&mut self) {
let Some(first) = first_retained_guard(&self.runs) else {
return;
};
let mut owner_return = OwnerReturn::begin(first);
if !owner_return.is_batched() {
self.runs.clear();
return;
}
for run in &mut self.runs {
for carrier in &mut run.carriers {
if let Some(guard) = carrier.guard.take() {
owner_return.push(guard);
}
}
}
}
}
fn first_retained_guard(runs: &CarrierRuns) -> Option<&CarrierGuard> {
let mut guards = runs
.iter()
.flat_map(|run| &run.carriers)
.filter_map(|carrier| carrier.guard.as_deref());
let first = guards.next()?;
guards.next().map(|_| first)
}
impl std::fmt::Debug for PooledBufMut {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("PooledBufMut")
.field("initialized", &self.initialized)
.field("retained_capacity", &self.retained_capacity)
.finish_non_exhaustive()
}
}
unsafe impl Send for PooledBufMut {}
unsafe impl BufMut for PooledBufMut {
fn remaining_mut(&self) -> usize {
self.retained_capacity
.checked_sub(self.initialized)
.unwrap_or_else(|| invariant_violation("initialized bytes exceed retained capacity"))
}
fn chunk_mut(&mut self) -> &mut UninitSlice {
self.normalize_write_cursor();
if self.remaining_mut() == 0 {
return unsafe {
UninitSlice::from_raw_parts_mut(NonNull::<u8>::dangling().as_ptr(), 0)
};
}
let cursor = self.write_cursor;
let carrier = self
.carrier_mut(cursor)
.unwrap_or_else(|| invariant_violation("buffer has no writable carrier"));
let remaining = carrier.remaining_mut();
if remaining == 0 {
invariant_violation("writable cursor names an exhausted carrier");
}
unsafe {
UninitSlice::from_raw_parts_mut(
carrier
.writable
.ptr()
.as_ptr()
.add(carrier.initialized)
.cast::<u8>(),
remaining,
)
}
}
unsafe fn advance_mut(&mut self, count: usize) {
self.advance_initialized(count);
}
}
pub(super) enum GrowthAuthority {
Reserved(Arc<ReservationState>),
Unreserved(Arc<PoolInner>),
}
impl GrowthAuthority {
pub(super) fn reserved(state: Arc<ReservationState>) -> Self {
Self::Reserved(state)
}
pub(super) fn unreserved(pool: Arc<PoolInner>) -> Self {
Self::Unreserved(pool)
}
fn pool(&self) -> &Arc<PoolInner> {
match self {
Self::Reserved(state) => state.pool(),
Self::Unreserved(pool) => pool,
}
}
fn acquire(&self, count: CarrierCount) -> Result<Vec<Arc<CarrierGuard>>, AcquireError> {
match self {
Self::Reserved(state) => acquire_count(state.pool(), Some(Arc::clone(state)), count),
Self::Unreserved(pool) => acquire_count(pool, None, count),
}
}
}
struct CarrierRun {
slot_id: u32,
first_carrier: u32,
carriers: Vec<WritableCarrier>,
}
impl CarrierRun {
fn try_new(pool: &PoolInner, guard: Arc<CarrierGuard>) -> Result<Self, AcquireError> {
let location = guard.location();
let mut carriers = Vec::new();
if allocation_failure_injected(pool) {
return Err(AcquireError::MetadataAllocationFailed);
}
carriers
.try_reserve_exact(1)
.map_err(|_| AcquireError::MetadataAllocationFailed)?;
carriers.push(WritableCarrier::new(guard));
Ok(Self {
slot_id: location.slot_id(),
first_carrier: location.carrier_index(),
carriers,
})
}
fn can_append(&self, guard: &CarrierGuard) -> bool {
let run_len = u32::try_from(self.carriers.len())
.unwrap_or_else(|_| invariant_violation("carrier run length exceeds block geometry"));
let next = self
.first_carrier
.checked_add(run_len)
.unwrap_or_else(|| invariant_violation("carrier run end exceeds block geometry"));
guard.location() == CarrierLocation::new(self.slot_id, next)
}
fn can_append_run(&self, incoming: &Self) -> bool {
let run_len = u32::try_from(self.carriers.len())
.unwrap_or_else(|_| invariant_violation("carrier run length exceeds block geometry"));
self.slot_id == incoming.slot_id
&& self.first_carrier.checked_add(run_len) == Some(incoming.first_carrier)
}
fn try_append(
&mut self,
pool: &PoolInner,
guard: Arc<CarrierGuard>,
) -> Result<(), AcquireError> {
if allocation_failure_injected(pool) {
return Err(AcquireError::MetadataAllocationFailed);
}
self.carriers
.try_reserve(1)
.map_err(|_| AcquireError::MetadataAllocationFailed)?;
self.carriers.push(WritableCarrier::new(guard));
Ok(())
}
}
struct WritableCarrier {
guard: Option<Arc<CarrierGuard>>,
writable: ExclusiveRange,
initialized: usize,
}
impl WritableCarrier {
fn new(guard: Arc<CarrierGuard>) -> Self {
let writable = ExclusiveRange::new(guard.ptr(), guard.capacity());
Self {
guard: Some(guard),
writable,
initialized: 0,
}
}
fn remaining_mut(&self) -> usize {
self.writable
.len()
.checked_sub(self.initialized)
.unwrap_or_else(|| invariant_violation("carrier initialization exceeds its range"))
}
}
struct ExclusiveRange {
ptr: NonNull<MaybeUninit<u8>>,
len: usize,
}
impl ExclusiveRange {
fn new(ptr: NonNull<MaybeUninit<u8>>, len: usize) -> Self {
if len == 0 {
invariant_violation("exclusive range must be nonempty");
}
Self { ptr, len }
}
fn ptr(&self) -> NonNull<MaybeUninit<u8>> {
self.ptr
}
fn len(&self) -> usize {
self.len
}
fn is_empty(&self) -> bool {
self.len == 0
}
fn take_prefix(&mut self, count: usize) -> Self {
assert!(count <= self.len, "split exceeds exclusive range");
let prefix = Self {
ptr: self.ptr,
len: count,
};
self.ptr = unsafe { NonNull::new_unchecked(self.ptr.as_ptr().add(count)) };
self.len -= count;
prefix
}
}
#[derive(Clone, Copy)]
#[cfg_attr(test, derive(Debug, PartialEq))]
struct BufferCursor {
run: usize,
carrier: usize,
}
impl BufferCursor {
const START: Self = Self { run: 0, carrier: 0 };
fn end(run: usize) -> Self {
Self { run, carrier: 0 }
}
}
struct PooledWindow {
guard: Arc<CarrierGuard>,
ptr: NonNull<u8>,
len: usize,
}
impl PooledWindow {
fn new(guard: Arc<CarrierGuard>, range: ExclusiveRange) -> Self {
Self {
guard,
ptr: range.ptr.cast::<u8>(),
len: range.len,
}
}
}
impl AsRef<[u8]> for PooledWindow {
fn as_ref(&self) -> &[u8] {
let _guard = &self.guard;
unsafe { std::slice::from_raw_parts(self.ptr.as_ptr(), self.len) }
}
}
unsafe impl Send for PooledWindow {}
fn build_runs(
pool: &PoolInner,
mut guards: Vec<Arc<CarrierGuard>>,
) -> Result<(CarrierRuns, usize), AcquireError> {
if guards.is_empty() {
invariant_violation("complete acquisition contains no carriers");
}
guards.sort_unstable_by_key(|guard| guard.location());
let mut runs = CarrierRuns::new();
let mut capacity = 0usize;
for guard in guards {
capacity = capacity
.checked_add(guard.capacity())
.ok_or(AcquireError::CapacityOverflow)?;
if let Some(run) = runs.last_mut() {
if run.can_append(&guard) {
run.try_append(pool, guard)?;
continue;
}
}
if allocation_failure_injected(pool) {
return Err(AcquireError::MetadataAllocationFailed);
}
runs.try_reserve(1)
.map_err(|_| AcquireError::MetadataAllocationFailed)?;
runs.push(CarrierRun::try_new(pool, guard)?);
}
Ok((runs, capacity))
}
fn map_geometry_error(error: GeometryError) -> AcquireError {
match error {
GeometryError::ZeroByteRequest => AcquireError::InvalidSize,
_ => AcquireError::CapacityOverflow,
}
}
#[cfg(all(test, not(s3_tm_loom)))]
mod tests {
use std::sync::Barrier;
use std::thread;
use bytes::{Buf, BufMut, BytesMut};
use std::task::{Poll, Waker};
use super::super::test_util::{poll_reserve, test_pool, write_pooled};
use super::super::{BufferPool, ReserveFuture};
use super::*;
struct WaitingReservation {
pool: BufferPool,
prepared_capacity_before_return: u64,
future: ReserveFuture,
}
impl WaitingReservation {
fn assert_granted_without_preparing_more(mut self) {
let Poll::Ready(Ok(_reservation)) = poll_reserve(&mut self.future, Waker::noop())
else {
panic!("reservation was not granted after the buffer was released");
};
assert_eq!(
self.pool.metrics().prepared_capacity_bytes(),
self.prepared_capacity_before_return,
"reservation was granted from a partial release and prepared more memory"
);
}
}
fn buffer_with_waiting_reservation() -> (PooledBufMut, WaitingReservation) {
let (pool, carrier_size) = test_pool(4, 6);
let mutable = pool.acquire_unreserved(carrier_size * 4).unwrap();
let prepared_capacity_before_return = pool.metrics().prepared_capacity_bytes();
assert_eq!(prepared_capacity_before_return, (carrier_size * 4) as u64);
let mut future = pool.reserve(carrier_size * 3);
assert!(poll_reserve(&mut future, Waker::noop()).is_pending());
(
mutable,
WaitingReservation {
pool,
prepared_capacity_before_return,
future,
},
)
}
#[test]
fn test_freeze_returns_unused_carriers_before_queued_grant() {
let (mutable, waiting) = buffer_with_waiting_reservation();
let frozen = mutable.freeze();
assert_eq!(frozen.remaining(), 0);
waiting.assert_granted_without_preparing_more();
}
#[test]
fn test_drop_returns_carriers_before_queued_grant() {
let (mutable, waiting) = buffer_with_waiting_reservation();
drop(mutable);
waiting.assert_granted_without_preparing_more();
}
fn assert_buffer_invariants(buffer: &PooledBufMut) {
let mut retained_capacity = 0usize;
let mut initialized = 0usize;
let mut writable = 0usize;
for run in &buffer.runs {
assert!(!run.carriers.is_empty());
for (offset, carrier) in run.carriers.iter().enumerate() {
assert!(carrier.initialized <= carrier.writable.len());
assert_eq!(carrier.guard.is_some(), !carrier.writable.is_empty());
if let Some(guard) = carrier.guard.as_ref() {
let offset = u32::try_from(offset).expect("test run exceeds u32");
let carrier_index = run
.first_carrier
.checked_add(offset)
.expect("test run exceeds block geometry");
assert_eq!(
guard.location(),
CarrierLocation::new(run.slot_id, carrier_index)
);
}
retained_capacity = retained_capacity
.checked_add(carrier.writable.len())
.expect("test capacity overflow");
initialized = initialized
.checked_add(carrier.initialized)
.expect("test initialization overflow");
writable = writable
.checked_add(carrier.remaining_mut())
.expect("test writable capacity overflow");
}
}
assert_eq!(buffer.retained_capacity, retained_capacity);
assert_eq!(buffer.initialized, initialized);
assert_eq!(buffer.remaining_mut(), writable);
assert_eq!(
buffer.publish_cursor,
buffer
.first_cursor_matching(|carrier| carrier.initialized != 0)
.unwrap_or_else(|| BufferCursor::end(buffer.runs.len()))
);
assert_eq!(
buffer.write_cursor,
buffer
.first_cursor_matching(|carrier| carrier.remaining_mut() != 0)
.unwrap_or_else(|| BufferCursor::end(buffer.runs.len()))
);
}
#[test]
fn test_mutable_initialization_crosses_carrier_boundaries() {
let (pool, carrier_size) = test_pool(2, 2);
let mut buffer = pool.acquire_unreserved(carrier_size + 1).unwrap();
let input: Vec<u8> = (0..carrier_size + 1)
.map(|index| (index % 251) as u8)
.collect();
write_pooled(&mut buffer, &input);
assert_eq!(buffer.capacity(), carrier_size * 2);
assert_eq!(buffer.len(), input.len());
assert_eq!(buffer.remaining_mut(), carrier_size - 1);
assert_eq!(buffer.initialized_chunk(), &input[..carrier_size]);
assert_eq!(buffer.chunk_mut().len(), carrier_size - 1);
}
#[test]
fn test_buf_mut_put_slice_crosses_carrier_boundaries() {
let (pool, carrier_size) = test_pool(2, 2);
let mut buffer = pool.acquire_unreserved(carrier_size + 3).unwrap();
let input: Vec<u8> = (0..carrier_size + 3)
.map(|index| (index.wrapping_mul(19) % 251) as u8)
.collect();
buffer.put_slice(&input);
assert_eq!(buffer.len(), input.len());
assert_eq!(buffer.remaining_mut(), carrier_size - 3);
assert_eq!(buffer.initialized_chunk(), &input[..carrier_size]);
let first = buffer.publish_prefix(carrier_size);
assert_eq!(first, input[..carrier_size]);
assert_eq!(buffer.initialized_chunk(), &input[carrier_size..]);
}
#[test]
fn test_buf_mut_zero_and_exact_boundary_advance_select_next_carrier() {
let (pool, carrier_size) = test_pool(2, 2);
let mut buffer = pool.acquire_unreserved(carrier_size * 2).unwrap();
let initial_len = buffer.chunk_mut().len();
unsafe { buffer.advance_mut(0) };
assert_eq!(buffer.chunk_mut().len(), initial_len);
let initialized = vec![0x5a; carrier_size];
buffer.chunk_mut()[..carrier_size].copy_from_slice(&initialized);
unsafe { buffer.advance_mut(carrier_size) };
assert_eq!(buffer.len(), carrier_size);
assert_eq!(buffer.chunk_mut().len(), carrier_size);
assert_eq!(buffer.initialized_chunk(), initialized);
}
#[test]
fn test_incremental_growth_reuses_one_buffers_writable_tail() {
let (pool, carrier_size) = test_pool(4, 3);
let total = carrier_size * 2 + 1;
let reservation = pool
.try_reserve(total)
.unwrap()
.expect("three-carrier reservation");
let mut buffer = pool.acquire(&reservation, 1).unwrap();
let increments = [
carrier_size / 2,
carrier_size / 2,
carrier_size / 2,
carrier_size / 2 + 1,
];
let mut next = 0u8;
for increment in increments {
buffer.reserve(increment).unwrap();
let bytes: Vec<u8> = (0..increment)
.map(|_| {
let value = next;
next = next.wrapping_add(1);
value
})
.collect();
write_pooled(&mut buffer, &bytes);
}
assert_eq!(buffer.len(), total);
assert_eq!(buffer.capacity(), carrier_size * 3);
assert_eq!(buffer.remaining_mut(), carrier_size - 1);
assert_eq!(buffer.test_run_count(), 1);
}
#[test]
fn test_reserve_zero_and_satisfied_tail_do_not_acquire() {
let (pool, carrier_size) = test_pool(2, 2);
let reservation = pool
.try_reserve(carrier_size * 2)
.unwrap()
.expect("reservation");
let mut buffer = pool.acquire(&reservation, 1).unwrap();
write_pooled(&mut buffer, b"tail");
let capacity = buffer.capacity();
let metrics = pool.metrics();
buffer.reserve(0).unwrap();
buffer.reserve(buffer.remaining_mut()).unwrap();
assert_eq!(buffer.capacity(), capacity);
assert_eq!(pool.metrics().in_use_bytes(), metrics.in_use_bytes());
}
#[test]
fn test_closed_reservation_allows_tail_use_but_rejects_growth() {
let (pool, carrier_size) = test_pool(2, 2);
let reservation = pool
.try_reserve(carrier_size * 2)
.unwrap()
.expect("reservation");
let mut buffer = pool.acquire(&reservation, 1).unwrap();
reservation.close_acquisition();
buffer.reserve(carrier_size).unwrap();
let before = (buffer.capacity(), buffer.len(), buffer.remaining_mut());
assert_eq!(
buffer.reserve(carrier_size + 1),
Err(AcquireError::ReservationClosed)
);
assert_eq!(
(buffer.capacity(), buffer.len(), buffer.remaining_mut()),
before
);
}
#[test]
fn test_reservation_exhaustion_preserves_existing_buffer() {
let (pool, carrier_size) = test_pool(1, 1);
let reservation = pool
.try_reserve(carrier_size)
.unwrap()
.expect("reservation");
let mut buffer = pool.acquire(&reservation, 1).unwrap();
let input = vec![0x5a; carrier_size];
write_pooled(&mut buffer, &input);
let before = (buffer.capacity(), buffer.len(), buffer.remaining_mut());
assert_eq!(
buffer.reserve(1),
Err(AcquireError::ReservationCapacityExceeded)
);
assert_eq!(
(buffer.capacity(), buffer.len(), buffer.remaining_mut()),
before
);
assert_eq!(buffer.initialized_chunk(), input.as_slice());
}
#[test]
fn test_growth_metadata_failure_preserves_bytes_cursors_and_authority() {
for boundary in 1.. {
let (pool, carrier_size) = test_pool(2, 2);
let reservation = pool
.try_reserve(carrier_size * 2)
.unwrap()
.expect("reservation");
let mut buffer = pool.acquire(&reservation, 1).unwrap();
write_pooled(&mut buffer, b"abc");
pool.inject_acquisition_allocation_failure(boundary);
let result = buffer.reserve(carrier_size);
if pool.acquisition_allocation_failure_pending() {
assert!(result.is_ok(), "boundary {boundary} was not reached");
break;
}
assert_eq!(result, Err(AcquireError::MetadataAllocationFailed));
assert_eq!(buffer.capacity(), carrier_size);
assert_eq!(buffer.len(), 3);
assert_eq!(buffer.remaining_mut(), carrier_size - 3);
assert_eq!(buffer.initialized_chunk(), b"abc");
assert_eq!(pool.metrics().in_use_bytes(), carrier_size as u64);
buffer
.reserve(carrier_size)
.expect("rollback restored direct authority");
assert_eq!(buffer.capacity(), carrier_size * 2);
}
}
#[test]
fn test_exhausted_buffer_returns_empty_mutable_chunk() {
let (pool, carrier_size) = test_pool(1, 1);
let mut buffer = pool.acquire_unreserved(carrier_size).unwrap();
write_pooled(&mut buffer, &vec![0; carrier_size]);
assert_eq!(buffer.remaining_mut(), 0);
assert_eq!(buffer.chunk_mut().len(), 0);
}
#[test]
fn test_rounding_is_per_buffer_instead_of_per_growth_call() {
let (stream_pool, carrier_size) = test_pool(2, 2);
let increment = carrier_size / 2 + 1;
let total = increment * 3;
assert!(total <= carrier_size * 2);
let stream_reservation = stream_pool
.try_reserve(total)
.unwrap()
.expect("two-carrier stream reservation");
let mut stream = stream_pool.acquire(&stream_reservation, increment).unwrap();
write_pooled(&mut stream, &vec![0; increment]);
for _ in 0..2 {
stream.reserve(increment).unwrap();
write_pooled(&mut stream, &vec![0; increment]);
}
assert_eq!(stream.capacity(), carrier_size * 2);
let (split_pool, _) = test_pool(2, 2);
let split_reservation = split_pool
.try_reserve(total)
.unwrap()
.expect("two-carrier split reservation");
let first = split_pool.acquire(&split_reservation, increment).unwrap();
let second = split_pool.acquire(&split_reservation, increment).unwrap();
assert!(matches!(
split_pool.acquire(&split_reservation, increment),
Err(AcquireError::ReservationCapacityExceeded)
));
drop((first, second));
}
#[test]
fn test_interleaved_publication_growth_and_freeze_preserve_multirun_state() {
let (pool, carrier_size) = test_pool(1, 3);
let first_input: Vec<u8> = (0..carrier_size + 3)
.map(|index| (index.wrapping_mul(17) % 251) as u8)
.collect();
let second_input: Vec<u8> = (0..carrier_size + 2)
.map(|index| (index.wrapping_mul(29) % 251) as u8)
.collect();
let mut buffer = pool.acquire_unreserved(first_input.len()).unwrap();
assert_eq!(buffer.test_run_count(), 2);
assert_buffer_invariants(&buffer);
write_pooled(&mut buffer, &first_input);
assert_buffer_invariants(&buffer);
let first = buffer.publish_prefix(carrier_size);
assert_buffer_invariants(&buffer);
let second_prefix = buffer.publish_prefix(2);
assert_buffer_invariants(&buffer);
buffer.reserve(carrier_size + 2).unwrap();
assert_eq!(buffer.test_run_count(), 3);
assert_buffer_invariants(&buffer);
write_pooled(&mut buffer, &second_input);
assert_buffer_invariants(&buffer);
let second_rest = buffer.publish_prefix(carrier_size - 2);
assert_buffer_invariants(&buffer);
let frozen = buffer.freeze().into_contiguous();
let mut output = BytesMut::with_capacity(first_input.len() + second_input.len());
output.extend_from_slice(&first);
output.extend_from_slice(&second_prefix);
output.extend_from_slice(&second_rest);
output.extend_from_slice(&frozen);
let mut expected = first_input;
expected.extend_from_slice(&second_input);
assert_eq!(output, expected.as_slice());
assert_eq!(pool.metrics().in_use_bytes(), (carrier_size * 3) as u64);
drop(first);
assert_eq!(pool.metrics().in_use_bytes(), (carrier_size * 2) as u64);
drop(second_prefix);
assert_eq!(pool.metrics().in_use_bytes(), (carrier_size * 2) as u64);
drop(second_rest);
assert_eq!(pool.metrics().in_use_bytes(), carrier_size as u64);
drop(frozen);
assert_eq!(pool.metrics().in_use_bytes(), 0);
}
#[test]
fn test_carrier_runs_spill_preserves_mutable_stream() {
let run_count = INLINE_CARRIER_RUNS + 1;
let (pool, carrier_size) = test_pool(1, run_count);
let input: Vec<u8> = (0..carrier_size * run_count)
.map(|index| (index.wrapping_mul(37) % 251) as u8)
.collect();
let mut buffer = pool.acquire_unreserved(input.len()).unwrap();
assert_eq!(buffer.test_run_count(), run_count);
assert!(buffer.runs.spilled());
assert_buffer_invariants(&buffer);
write_pooled(&mut buffer, &input);
assert_buffer_invariants(&buffer);
let frozen = buffer.freeze().into_contiguous();
assert_eq!(frozen, input);
assert_eq!(pool.metrics().in_use_bytes(), 0);
}
#[test]
fn test_dropping_empty_mutable_buffer_returns_every_carrier() {
let (pool, carrier_size) = test_pool(1, 2);
let buffer = pool.acquire_unreserved(carrier_size * 2).unwrap();
assert_eq!(pool.metrics().in_use_bytes(), (carrier_size * 2) as u64);
drop(buffer);
assert_eq!(pool.metrics().in_use_bytes(), 0);
}
#[cfg(target_pointer_width = "64")]
#[test]
fn test_reserve_overflow_preserves_existing_buffer() {
let (pool, carrier_size) = test_pool(1, 1);
let mut buffer = pool.acquire_unreserved(carrier_size).unwrap();
write_pooled(&mut buffer, b"existing");
let before = (buffer.capacity(), buffer.len(), buffer.remaining_mut());
assert_eq!(
buffer.reserve(usize::MAX),
Err(AcquireError::CapacityOverflow)
);
assert_eq!(
(buffer.capacity(), buffer.len(), buffer.remaining_mut()),
before
);
assert_eq!(buffer.initialized_chunk(), b"existing");
assert_buffer_invariants(&buffer);
}
#[test]
fn test_publish_prefix_leaves_a_disjoint_mutable_suffix() {
let (pool, carrier_size) = test_pool(1, 1);
let mut buffer = pool.acquire_unreserved(carrier_size).unwrap();
write_pooled(&mut buffer, b"abcdef");
let first = buffer.publish_prefix(3);
assert_eq!(first, b"abc"[..]);
assert_eq!(buffer.len(), 3);
assert_eq!(buffer.capacity(), carrier_size - 3);
assert_eq!(buffer.initialized_chunk(), b"def");
write_pooled(&mut buffer, b"XYZ");
assert_eq!(first, b"abc"[..]);
assert_eq!(buffer.initialized_chunk(), b"defXYZ");
let second = buffer.publish_prefix(6);
assert_eq!(second, b"defXYZ"[..]);
}
#[test]
fn test_published_prefix_read_races_retained_suffix_write() {
let (pool, carrier_size) = test_pool(1, 1);
let prefix_len = carrier_size.min(64);
let prefix = vec![0x5a; prefix_len];
let mut buffer = pool.acquire_unreserved(carrier_size).unwrap();
write_pooled(&mut buffer, &prefix);
let published = buffer.publish_prefix(prefix_len);
let suffix = vec![0xa5; buffer.remaining_mut()];
let reader_expected = prefix.clone();
let writer_input = suffix.clone();
let start = Arc::new(Barrier::new(2));
let reader_start = Arc::clone(&start);
let reading = thread::spawn(move || {
reader_start.wait();
for _ in 0..8 {
assert_eq!(published, reader_expected);
}
published
});
let writer_start = Arc::clone(&start);
let writing = thread::spawn(move || {
writer_start.wait();
write_pooled(&mut buffer, &writer_input);
buffer
});
let published = reading.join().unwrap();
let buffer = writing.join().unwrap();
assert_eq!(published, prefix);
assert_eq!(buffer.initialized_chunk(), suffix);
let frozen = buffer.freeze();
assert_eq!(frozen.chunk(), suffix);
drop(published);
drop(frozen);
assert_eq!(pool.metrics().in_use_bytes(), 0);
}
#[cfg(all(
not(miri),
any(
target_os = "android",
target_os = "freebsd",
target_os = "linux",
target_os = "macos",
target_os = "windows"
)
))]
mod guard_page_tests {
#[cfg(not(s3_tm_tsan))]
use std::io::Write;
#[cfg(not(s3_tm_tsan))]
use std::process::Command;
#[cfg(all(unix, not(s3_tm_tsan)))]
use std::os::unix::process::ExitStatusExt;
use super::super::super::test_util::{test_guarded_pool, write_pooled};
use super::super::super::CarrierCount;
use super::*;
#[cfg(not(s3_tm_tsan))]
const PROBE_ENV: &str = "AWS_S3_TM_GUARD_PAGE_PROBE";
#[cfg(not(s3_tm_tsan))]
const PROBE_TEST: &str = concat!(
"runtime::buffer_pool::pooled_buf::tests::guard_page_tests::",
"test_guard_pages_reject_out_of_bounds_writes"
);
#[cfg(not(s3_tm_tsan))]
const READY_MARKER: &str = "S3_TM_GUARD_PROBE_READY";
#[cfg(not(s3_tm_tsan))]
const SURVIVED_MARKER: &str = "S3_TM_GUARD_PROBE_SURVIVED";
#[test]
fn test_guarded_blocks_preserve_buffer_lifecycle() {
let (pool, carrier_size) = test_guarded_pool(1, 2);
let input: Vec<u8> = (0..carrier_size * 2)
.map(|index| (index.wrapping_mul(17) % 251) as u8)
.collect();
let mut buffer = pool.acquire_unreserved(input.len()).unwrap();
write_pooled(&mut buffer, &input);
let first = buffer.publish_prefix(carrier_size);
let second = buffer.freeze();
assert_eq!(first, input[..carrier_size]);
assert_eq!(second.chunk(), &input[carrier_size..]);
drop(first);
drop(second);
let audit = pool.audit_quiescent();
assert_eq!(audit.charged_capacity, CarrierCount::ZERO);
assert_eq!(audit.live_carriers, CarrierCount::ZERO);
}
#[cfg(not(s3_tm_tsan))]
#[test]
fn test_guard_pages_reject_out_of_bounds_writes() {
if let Some(probe) = std::env::var_os(PROBE_ENV) {
run_guard_page_probe(
probe
.to_str()
.unwrap_or_else(|| panic!("{PROBE_ENV} is not valid UTF-8")),
);
return;
}
assert_guard_page_fault("underflow");
assert_guard_page_fault("overflow");
}
#[cfg(not(s3_tm_tsan))]
fn assert_guard_page_fault(probe: &str) {
let output = Command::new(std::env::current_exe().expect("current test executable"))
.arg("--exact")
.arg(PROBE_TEST)
.arg("--nocapture")
.env(PROBE_ENV, probe)
.output()
.expect("guard-page child process");
let status = output.status;
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(
!status.success(),
"{probe} unexpectedly remained accessible\n{stderr}"
);
assert!(
stderr.contains(READY_MARKER),
"{probe} failed before reaching the guard write: {status:?}\n{stderr}"
);
assert!(
!stderr.contains(SURVIVED_MARKER),
"{probe} guard write completed: {status:?}\n{stderr}"
);
#[cfg(all(unix, not(s3_tm_asan)))]
{
let signal = status.signal();
assert!(
signal == Some(libc::SIGSEGV) || signal == Some(libc::SIGBUS),
"{probe} failed without a guard-page signal: {status:?}"
);
}
#[cfg(all(target_os = "windows", not(s3_tm_asan)))]
assert_eq!(
status.code(),
Some(0xC000_0005u32 as i32),
"{probe} failed without an access violation: {status:?}\n{stderr}"
);
#[cfg(s3_tm_asan)]
{
#[cfg(unix)]
let native_fault =
matches!(status.signal(), Some(libc::SIGSEGV) | Some(libc::SIGBUS));
#[cfg(not(unix))]
let native_fault = false;
let sanitizer_fault = stderr.contains("AddressSanitizer:DEADLYSIGNAL")
&& (stderr.contains("SEGV") || stderr.contains("BUS"));
assert!(
native_fault || sanitizer_fault,
"{probe} failed without a native fault or ASan report: {status:?}\n{stderr}"
);
}
}
#[cfg(not(s3_tm_tsan))]
fn run_guard_page_probe(probe: &str) {
let (pool, carrier_size) = test_guarded_pool(1, 1);
let mut buffer = pool.acquire_unreserved(carrier_size).unwrap();
let ptr = buffer.chunk_mut().as_mut_ptr();
let reservation = buffer.test_first_reservation_address_range();
let fault = match probe {
"underflow" => ptr.wrapping_sub(1),
"overflow" => unsafe { ptr.add(carrier_size) },
other => panic!("unknown guard-page probe {other}"),
};
assert!(
reservation.contains(&fault.addr()),
"{probe} address is outside the guarded reservation"
);
eprintln!("{READY_MARKER}:{probe}");
std::io::stderr()
.flush()
.expect("flush guard-page readiness marker");
unsafe { fault.write_volatile(0x5a) };
eprintln!("{SURVIVED_MARKER}:{probe}");
}
}
#[test]
fn test_growth_skips_a_fully_published_carrier_entry() {
let (pool, carrier_size) = test_pool(1, 2);
let mut buffer = pool.acquire_unreserved(carrier_size).unwrap();
buffer.put_slice(&vec![0x11; carrier_size]);
let published = buffer.publish_prefix(carrier_size);
assert!(buffer.is_empty());
assert_eq!(buffer.capacity(), 0);
buffer.reserve(1).unwrap();
assert_buffer_invariants(&buffer);
buffer.put_slice(b"next");
let frozen = buffer.freeze();
assert_eq!(published, vec![0x11; carrier_size]);
assert_eq!(frozen.chunk(), b"next");
}
#[test]
fn test_publication_stops_at_each_carrier_boundary() {
let (pool, carrier_size) = test_pool(2, 2);
let mut buffer = pool.acquire_unreserved(carrier_size + 3).unwrap();
let input: Vec<u8> = (0..carrier_size + 3)
.map(|index| (index % 251) as u8)
.collect();
write_pooled(&mut buffer, &input);
assert_eq!(buffer.initialized_chunk().len(), carrier_size);
let first = buffer.publish_prefix(carrier_size);
assert_eq!(first, input[..carrier_size]);
assert_eq!(buffer.initialized_chunk(), &input[carrier_size..]);
assert_eq!(buffer.capacity(), carrier_size);
assert_eq!(pool.metrics().in_use_bytes(), (carrier_size * 2) as u64);
drop(first);
assert_eq!(pool.metrics().in_use_bytes(), carrier_size as u64);
let second = buffer.publish_prefix(3);
drop(buffer);
assert_eq!(pool.metrics().in_use_bytes(), carrier_size as u64);
drop(second);
assert_eq!(pool.metrics().in_use_bytes(), 0);
}
#[test]
fn test_published_clones_and_slices_share_one_carrier_return() {
let (pool, carrier_size) = test_pool(1, 1);
let mut buffer = pool.acquire_unreserved(carrier_size).unwrap();
write_pooled(&mut buffer, b"immutable");
let published = buffer.publish_prefix(9);
let clone = published.clone();
let slice = published.slice(2..7);
drop(buffer);
assert_eq!(pool.metrics().in_use_bytes(), carrier_size as u64);
drop(published);
drop(clone);
assert_eq!(slice, b"mutab"[..]);
assert_eq!(pool.metrics().in_use_bytes(), carrier_size as u64);
drop(slice);
assert_eq!(pool.metrics().in_use_bytes(), 0);
}
#[test]
fn test_published_owner_holds_direct_authority_until_final_drop() {
let (pool, carrier_size) = test_pool(2, 2);
let reservation = pool
.try_reserve(carrier_size * 2)
.unwrap()
.expect("reservation");
let mut buffer = pool.acquire(&reservation, carrier_size).unwrap();
write_pooled(&mut buffer, &vec![0x11; carrier_size]);
let published = buffer.publish_prefix(carrier_size);
buffer.reserve(1).unwrap();
write_pooled(&mut buffer, &vec![0x22; carrier_size]);
assert_eq!(
buffer.reserve(1),
Err(AcquireError::ReservationCapacityExceeded)
);
drop(published);
buffer
.reserve(1)
.expect("final immutable drop restored direct authority");
}
#[test]
fn test_publication_and_freeze_preserve_bytes_across_representative_lengths() {
for carriers in 1..=3 {
let (pool, carrier_size) = test_pool(3, 3);
let lengths = [
1,
carrier_size - 1,
carrier_size,
carrier_size + 1,
carrier_size * carriers - 1,
];
for length in lengths {
if length == 0 || length > carrier_size * 3 {
continue;
}
let input: Vec<u8> = (0..length)
.map(|index| (index.wrapping_mul(17) % 251) as u8)
.collect();
let mut buffer = pool.acquire_unreserved(length).unwrap();
write_pooled(&mut buffer, &input);
let publish_target = length / 2;
let mut published = 0;
let mut views = Vec::new();
while published < publish_target {
let count = (publish_target - published)
.min(buffer.initialized_chunk().len())
.min(97);
views.push(buffer.publish_prefix(count));
published += count;
}
let mut frozen = buffer.freeze();
let mut output = BytesMut::with_capacity(length);
for view in views {
output.extend_from_slice(&view);
}
while frozen.has_remaining() {
let count = frozen.chunk().len();
output.extend_from_slice(frozen.chunk());
frozen.advance(count);
}
assert_eq!(output.as_ref(), input.as_slice(), "length {length}");
assert_eq!(pool.metrics().in_use_bytes(), 0);
}
}
}
#[test]
#[should_panic(expected = "publication cursor does not name initialized bytes")]
fn test_initialized_chunk_rejects_a_stale_publication_cursor() {
let (pool, carrier_size) = test_pool(2, 2);
let mut buffer = pool.acquire_unreserved(carrier_size + 1).unwrap();
write_pooled(&mut buffer, &vec![0x5a; carrier_size + 1]);
let _first = buffer.publish_prefix(carrier_size);
buffer.publish_cursor = BufferCursor::START;
let _ = buffer.initialized_chunk();
}
#[test]
#[should_panic(expected = "published prefix must be nonzero")]
fn test_publish_prefix_rejects_zero() {
let (pool, _) = test_pool(1, 1);
let mut buffer = pool.acquire_unreserved(1).unwrap();
write_pooled(&mut buffer, b"x");
let _ = buffer.publish_prefix(0);
}
#[test]
#[should_panic(expected = "published prefix must be nonzero")]
fn test_publish_prefix_rejects_cross_carrier_count() {
let (pool, carrier_size) = test_pool(2, 2);
let mut buffer = pool.acquire_unreserved(carrier_size + 1).unwrap();
write_pooled(&mut buffer, &vec![0; carrier_size + 1]);
let _ = buffer.publish_prefix(carrier_size + 1);
}
}