use std::future::Future;
use std::io;
use std::pin::Pin;
use std::ptr::NonNull;
use std::task::{Context, Poll};
use bytes::buf::UninitSlice;
use bytes::BufMut;
use crate::memory::{BufferPool, SegmentedBytes};
use crate::runtime::buffer_pool::{PooledBufMut, Reservation, ReserveFuture};
#[derive(Debug)]
pub struct PartBuffer {
pool: BufferPool,
max_len: usize,
completed: SegmentedBytes,
state: PartBufferState,
}
unsafe impl Sync for PartBuffer {}
#[derive(Debug)]
enum PartBufferState {
Empty,
Reserving {
envelope_remaining: usize,
future: ReserveFuture,
},
Writable {
envelope_remaining: usize,
reservation: Option<Reservation>,
buffer: PooledBufMut,
},
}
impl PartBuffer {
pub fn new(pool: &BufferPool, max_len: usize) -> Self {
assert!(max_len != 0, "part buffer length must be nonzero");
Self {
pool: pool.clone(),
max_len,
completed: SegmentedBytes::from(bytes::Bytes::new()),
state: PartBufferState::Empty,
}
}
pub fn poll_acquire(&mut self, cx: &mut Context<'_>, envelope: usize) -> Poll<io::Result<()>> {
loop {
match &mut self.state {
PartBufferState::Writable {
envelope_remaining,
buffer,
..
} if *envelope_remaining != 0 && buffer.remaining_mut() != 0 => {
return Poll::Ready(Ok(()));
}
PartBufferState::Writable {
envelope_remaining, ..
} => {
let continuation = *envelope_remaining;
let tail = self.publish_current();
if continuation == 0 {
let remaining = self.remaining();
if envelope == 0 || envelope > remaining {
return Poll::Ready(Err(invalid_envelope(envelope, remaining)));
}
if let Some(buffer) = tail {
self.state = PartBufferState::Writable {
envelope_remaining: envelope,
reservation: None,
buffer,
};
return Poll::Ready(Ok(()));
}
self.start_reservation(envelope);
} else {
if tail.is_some() {
panic!("exhausted part-buffer storage retained writable capacity");
}
self.start_reservation(continuation);
}
}
PartBufferState::Empty => {
let remaining = self.remaining();
if envelope == 0 || envelope > remaining {
return Poll::Ready(Err(invalid_envelope(envelope, remaining)));
}
self.start_reservation(envelope);
}
PartBufferState::Reserving {
envelope_remaining,
future,
} => {
let reservation = match Pin::new(future).poll(cx) {
Poll::Pending => return Poll::Pending,
Poll::Ready(Ok(reservation)) => reservation,
Poll::Ready(Err(error)) => {
self.state = PartBufferState::Empty;
return Poll::Ready(Err(io::Error::other(error)));
}
};
let envelope_remaining = *envelope_remaining;
match self.pool.acquire(&reservation, envelope_remaining) {
Ok(buffer) => {
self.state = PartBufferState::Writable {
envelope_remaining,
reservation: Some(reservation),
buffer,
};
return Poll::Ready(Ok(()));
}
Err(error) => {
self.state = PartBufferState::Empty;
return Poll::Ready(Err(io::Error::other(error)));
}
}
}
}
}
}
pub fn len(&self) -> usize {
self.completed
.len()
.checked_add(self.current_len())
.expect("part buffer length overflowed")
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
pub fn remaining(&self) -> usize {
self.max_len
.checked_sub(self.len())
.expect("part buffer length exceeded its limit")
}
pub fn writable(&self) -> usize {
match &self.state {
PartBufferState::Writable {
envelope_remaining,
buffer,
..
} => (*envelope_remaining).min(buffer.remaining_mut()),
PartBufferState::Empty | PartBufferState::Reserving { .. } => 0,
}
}
pub fn fill_from_slice(&mut self, src: &[u8]) -> usize {
let mut copied = 0;
while copied < src.len() {
let chunk = self.chunk_mut();
let count = chunk.len().min(src.len() - copied);
if count == 0 {
break;
}
chunk[..count].copy_from_slice(&src[copied..copied + count]);
unsafe {
self.advance_mut(count);
}
copied += count;
}
copied
}
pub fn freeze(mut self) -> SegmentedBytes {
if matches!(self.state, PartBufferState::Writable { .. }) {
self.freeze_current();
}
self.completed
}
fn current_len(&self) -> usize {
match &self.state {
PartBufferState::Writable { buffer, .. } => buffer.len(),
PartBufferState::Empty | PartBufferState::Reserving { .. } => 0,
}
}
fn start_reservation(&mut self, envelope_remaining: usize) {
self.state = PartBufferState::Reserving {
envelope_remaining,
future: self.pool.reserve(envelope_remaining),
};
}
fn publish_current(&mut self) -> Option<PooledBufMut> {
let state = std::mem::replace(&mut self.state, PartBufferState::Empty);
let PartBufferState::Writable {
reservation,
mut buffer,
..
} = state
else {
panic!("part buffer has no writable envelope to publish");
};
while !buffer.is_empty() {
let count = buffer.initialized_chunk().len();
let published = buffer.publish_prefix(count);
self.completed.append_pool_view(&self.pool, published);
}
if let Some(reservation) = reservation {
reservation.close_acquisition();
}
(buffer.remaining_mut() != 0).then_some(buffer)
}
fn freeze_current(&mut self) {
let state = std::mem::replace(&mut self.state, PartBufferState::Empty);
let PartBufferState::Writable {
reservation,
buffer,
..
} = state
else {
panic!("part buffer has no writable envelope to freeze");
};
if let Some(reservation) = reservation {
reservation.close_acquisition();
}
self.completed.append(buffer.freeze());
}
}
impl PartBuffer {
pub(crate) fn chunk_mut(&mut self) -> &mut UninitSlice {
let PartBufferState::Writable {
envelope_remaining,
buffer,
..
} = &mut self.state
else {
return unsafe {
UninitSlice::from_raw_parts_mut(NonNull::<u8>::dangling().as_ptr(), 0)
};
};
let chunk = buffer.chunk_mut();
let exposed = chunk.len().min(*envelope_remaining);
&mut chunk[..exposed]
}
pub(crate) unsafe fn advance_mut(&mut self, count: usize) {
assert!(
count <= self.writable(),
"advanced beyond the current part-buffer envelope"
);
let PartBufferState::Writable {
envelope_remaining,
buffer,
..
} = &mut self.state
else {
assert_eq!(count, 0, "advanced a part buffer without writable storage");
return;
};
unsafe {
buffer.advance_mut(count);
}
*envelope_remaining = envelope_remaining
.checked_sub(count)
.expect("advanced beyond the current part-buffer envelope");
}
}
fn invalid_envelope(envelope: usize, remaining: usize) -> io::Error {
io::Error::new(
io::ErrorKind::InvalidInput,
format!("part buffer envelope must be between 1 and {remaining} bytes, got {envelope}"),
)
}
#[cfg(test)]
mod tests {
use crate::types::ByteSize;
use std::task::Waker;
use bytes::Buf;
use super::*;
use crate::config::MemoryBudgetConfig;
const TEST_POOL_CAPACITY: usize = 1024 * 1024;
fn test_pool() -> BufferPool {
BufferPool::builder()
.memory_budget(MemoryBudgetConfig::Limit(ByteSize::new(
TEST_POOL_CAPACITY as u64,
)))
.build()
.unwrap()
}
fn poll_acquire(buffer: &mut PartBuffer, envelope: usize) -> Poll<io::Result<()>> {
let waker = Waker::noop();
let mut cx = Context::from_waker(waker);
buffer.poll_acquire(&mut cx, envelope)
}
#[test]
fn complete_envelope_freezes_without_copying() {
let pool = test_pool();
let carrier_size = pool.carrier_size();
let mut buffer = PartBuffer::new(&pool, carrier_size);
assert!(poll_acquire(&mut buffer, carrier_size).is_ready());
let ptr = buffer.chunk_mut().as_mut_ptr();
let data = b"payload";
assert_eq!(buffer.fill_from_slice(data), data.len());
assert_eq!(buffer.len(), data.len());
assert_eq!(buffer.remaining(), carrier_size - data.len());
let frozen = buffer.freeze();
assert_eq!(frozen.chunk().as_ptr(), ptr.cast());
assert_eq!(frozen.chunk(), data);
assert_eq!(pool.metrics().in_use_bytes(), carrier_size as u64);
drop(frozen);
assert_eq!(pool.metrics().in_use_bytes(), 0);
}
#[test]
fn full_envelopes_roll_over_without_losing_carrier_tails() {
let pool = test_pool();
let first = b"abc";
let second = b"defg";
let expected = [&first[..], &second[..]].concat();
let mut buffer = PartBuffer::new(&pool, expected.len());
assert!(poll_acquire(&mut buffer, first.len()).is_ready());
assert_eq!(buffer.fill_from_slice(first), first.len());
assert_eq!(buffer.writable(), 0);
assert!(poll_acquire(&mut buffer, second.len()).is_ready());
assert_eq!(buffer.fill_from_slice(second), second.len());
assert_eq!(buffer.len(), expected.len());
assert_eq!(pool.metrics().in_use_bytes(), pool.carrier_size() as u64);
let frozen = buffer.freeze();
let segments = frozen.into_segments();
assert_eq!(segments.len(), 1);
assert_eq!(segments[0], expected);
drop(segments);
assert_eq!(pool.metrics().in_use_bytes(), 0);
}
#[test]
fn envelope_continues_after_exhausting_a_reused_tail() {
let pool = test_pool();
let carrier_size = pool.carrier_size();
let mut buffer = PartBuffer::new(&pool, carrier_size + 3);
let mut expected = Vec::with_capacity(carrier_size + 3);
let head = b"abc";
assert!(poll_acquire(&mut buffer, 3).is_ready());
assert_eq!(buffer.fill_from_slice(head), head.len());
expected.extend_from_slice(head);
assert!(poll_acquire(&mut buffer, carrier_size).is_ready());
let tail_len = buffer.writable();
let tail = vec![b'x'; tail_len];
assert_eq!(buffer.fill_from_slice(&tail), tail.len());
expected.extend_from_slice(&tail);
assert_eq!(buffer.writable(), 0);
assert!(poll_acquire(&mut buffer, 1).is_ready());
assert_eq!(buffer.writable(), 3);
let end = b"end";
assert_eq!(buffer.fill_from_slice(end), end.len());
expected.extend_from_slice(end);
assert_eq!(buffer.freeze().into_contiguous(), expected);
assert_eq!(pool.metrics().in_use_bytes(), 0);
}
#[test]
fn partial_envelope_remains_active_across_acquire_polls() {
let pool = test_pool();
let mut buffer = PartBuffer::new(&pool, 8);
let first = b"abc";
let rest = b"defgh";
assert!(poll_acquire(&mut buffer, 8).is_ready());
assert_eq!(buffer.fill_from_slice(first), first.len());
assert!(poll_acquire(&mut buffer, 2).is_ready());
assert_eq!(buffer.writable(), 5);
assert_eq!(buffer.fill_from_slice(rest), rest.len());
assert_eq!(
buffer.freeze().into_contiguous(),
[&first[..], &rest[..]].concat()
);
}
#[test]
fn pending_reservation_is_retained_and_cancelled_on_drop() {
let pool = test_pool();
let carrier_size = pool.carrier_size();
let holder = pool.try_reserve(TEST_POOL_CAPACITY).unwrap().unwrap();
let held = pool.acquire(&holder, TEST_POOL_CAPACITY).unwrap();
let mut buffer = PartBuffer::new(&pool, carrier_size);
assert!(poll_acquire(&mut buffer, carrier_size).is_pending());
assert_eq!(pool.metrics().waiting_requests(), 1);
drop(buffer);
assert_eq!(pool.metrics().waiting_requests(), 0);
drop(held);
holder.close_acquisition();
}
#[test]
fn pending_reservation_latches_its_first_envelope() {
let pool = test_pool();
let carrier_size = pool.carrier_size();
let holder = pool.try_reserve(TEST_POOL_CAPACITY).unwrap().unwrap();
let held = pool.acquire(&holder, TEST_POOL_CAPACITY).unwrap();
let mut buffer = PartBuffer::new(&pool, carrier_size);
assert!(poll_acquire(&mut buffer, 3).is_pending());
assert!(poll_acquire(&mut buffer, 7).is_pending());
drop(held);
holder.close_acquisition();
assert!(poll_acquire(&mut buffer, 7).is_ready());
assert_eq!(buffer.writable(), 3);
}
#[test]
fn invalid_envelope_does_not_enter_admission() {
let pool = test_pool();
let mut buffer = PartBuffer::new(&pool, 8);
let Poll::Ready(result) = poll_acquire(&mut buffer, 9) else {
panic!("invalid envelope entered admission");
};
let error = result.unwrap_err();
assert_eq!(error.kind(), io::ErrorKind::InvalidInput);
assert_eq!(pool.metrics().waits_total(), 0);
}
#[test]
fn fill_from_slice_copies_up_to_the_envelope_and_returns_the_count() {
let pool = test_pool();
let carrier_size = pool.carrier_size();
let first = 2 * carrier_size + 1;
let max_len = first + 3;
let mut buffer = PartBuffer::new(&pool, max_len);
let source: Vec<u8> = (0..max_len + 5).map(|i| i as u8).collect();
assert!(poll_acquire(&mut buffer, first).is_ready());
assert_eq!(buffer.writable(), first);
assert_eq!(buffer.fill_from_slice(&source), first);
assert_eq!(buffer.len(), first);
assert_eq!(buffer.writable(), 0);
assert_eq!(buffer.fill_from_slice(&source[first..]), 0);
assert_eq!(buffer.len(), first);
assert!(poll_acquire(&mut buffer, 3).is_ready());
assert_eq!(buffer.writable(), 3);
assert_eq!(buffer.fill_from_slice(&source[first..]), 3);
assert_eq!(buffer.remaining(), 0);
assert_eq!(buffer.writable(), 0);
assert_eq!(buffer.freeze().into_contiguous(), source[..max_len]);
assert_eq!(pool.metrics().in_use_bytes(), 0);
}
#[test]
fn writable_is_zero_before_acquire_and_when_full() {
let pool = test_pool();
let holder = pool.try_reserve(TEST_POOL_CAPACITY).unwrap().unwrap();
let mut buffer = PartBuffer::new(&pool, 8);
assert_eq!(buffer.writable(), 0);
assert_eq!(buffer.fill_from_slice(b"abc"), 0);
assert!(buffer.is_empty());
assert!(poll_acquire(&mut buffer, 4).is_pending());
assert_eq!(buffer.writable(), 0);
assert_eq!(buffer.fill_from_slice(b"abc"), 0);
holder.close_acquisition();
assert!(poll_acquire(&mut buffer, 4).is_ready());
assert_eq!(buffer.writable(), 4);
let data = b"abcd";
assert_eq!(buffer.fill_from_slice(data), data.len());
assert_eq!(buffer.writable(), 0);
assert_eq!(buffer.remaining(), 4);
assert_eq!(buffer.fill_from_slice(b"e"), 0);
assert_eq!(buffer.freeze().into_contiguous(), data[..]);
}
#[test]
fn part_buffer_is_send_and_sync() {
fn assert_send_sync<T: Send + Sync>() {}
assert_send_sync::<PartBuffer>();
}
}
#[cfg(all(test, not(s3_tm_loom)))]
mod prop_tests;