use std::io::IoSlice;
use bytes::Buf;
use crate::runtime::buffer_pool::{Reservation, SegmentedBytes};
use crate::runtime::sync::sync::atomic::{AtomicU64, Ordering};
use crate::runtime::sync::sync::Arc;
use super::recv_buffer::{
DrainMode, FillOutcome, PagedRecvBuffer, RecvBufferConsumer, SegmentWrite, SlotHandle,
};
use super::sink::SinkWrite;
use super::ChunkMetadata;
const SEG_SIZE: usize = super::recv_buffer::DEFAULT_SEG_SIZE;
use super::transfer::DownloadTransfer;
#[cfg(not(all(test, s3_tm_loom)))]
use tokio::sync::Notify as WakeNotify;
#[cfg(all(test, s3_tm_loom))]
struct WakeNotify;
#[cfg(all(test, s3_tm_loom))]
impl WakeNotify {
fn new() -> Self {
Self
}
fn notify_one(&self) {}
async fn notified(&self) {}
}
pub(crate) struct DiskWriteCursor<'a> {
write: &'a SegmentWrite<ChunkOutput>,
payload_index: usize,
payload_offset: usize,
remaining: usize,
}
impl<'a> DiskWriteCursor<'a> {
fn new(write: &'a SegmentWrite<ChunkOutput>) -> std::io::Result<(u64, Self)> {
let mut object_offset = None;
let mut expected_offset = None;
let mut remaining = 0usize;
for slot in write.payloads() {
let chunk = slot.get().ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::InvalidData,
"claimed disk-write run contains an unfilled slot",
)
})?;
let chunk_len = chunk.data.remaining();
if chunk_len == 0 {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"claimed disk-write run contains an empty payload",
));
}
if let Some(expected) = expected_offset {
if chunk.offset != expected {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"claimed disk-write run has noncontiguous object offsets",
));
}
} else {
object_offset = Some(chunk.offset);
}
let chunk_len_u64 = u64::try_from(chunk_len).map_err(|_| {
std::io::Error::new(
std::io::ErrorKind::InvalidData,
"disk-write payload length exceeds object-offset representation",
)
})?;
expected_offset = Some(chunk.offset.checked_add(chunk_len_u64).ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::InvalidData,
"disk-write object offset overflow",
)
})?);
remaining = remaining.checked_add(chunk_len).ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::InvalidData,
"disk-write run length overflow",
)
})?;
}
let object_offset = object_offset.ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::InvalidData,
"claimed disk-write run contains no payloads",
)
})?;
Ok((
object_offset,
Self {
write,
payload_index: 0,
payload_offset: 0,
remaining,
},
))
}
fn current(&self) -> &SegmentedBytes {
self.write.payloads()[self.payload_index]
.get()
.map(|chunk| &chunk.data)
.expect("validated disk-write payload disappeared")
}
}
impl Buf for DiskWriteCursor<'_> {
fn remaining(&self) -> usize {
self.remaining
}
fn chunk(&self) -> &[u8] {
if self.remaining == 0 {
return &[];
}
self.current().chunk_from(self.payload_offset)
}
fn chunks_vectored<'a>(&'a self, dst: &mut [IoSlice<'a>]) -> usize {
let mut written = 0;
for (index, slot) in self.write.payloads()[self.payload_index..]
.iter()
.enumerate()
{
if written == dst.len() {
break;
}
let chunk = slot
.get()
.expect("validated disk-write payload disappeared");
let offset = if index == 0 { self.payload_offset } else { 0 };
written += chunk.data.chunks_vectored_from(offset, &mut dst[written..]);
}
written
}
fn advance(&mut self, mut count: usize) {
assert!(
count <= self.remaining,
"advanced beyond download write run"
);
self.remaining -= count;
while count != 0 {
let current_len = self.current().remaining();
let available = current_len
.checked_sub(self.payload_offset)
.expect("disk-write cursor exceeds its payload");
let advanced = count.min(available);
self.payload_offset += advanced;
count -= advanced;
if self.payload_offset == current_len {
self.payload_index += 1;
self.payload_offset = 0;
}
}
}
}
#[derive(Debug)]
enum Mode {
Stream,
Disk {
sink: Box<dyn SinkWrite>,
object_range_start: std::sync::OnceLock<u64>,
written: AtomicU64,
},
}
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
pub(crate) struct Drained {
pub(crate) parts: u64,
pub(crate) bytes: u64,
}
#[derive(Debug)]
pub(crate) struct DrainError {
pub(crate) drained: Drained,
pub(crate) source: std::io::Error,
}
#[derive(Clone)]
pub(crate) struct BodyWriter {
buffer: PagedRecvBuffer<ChunkOutput>,
notify: Arc<WakeNotify>,
mode: Arc<Mode>,
}
impl std::fmt::Debug for BodyWriter {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("BodyWriter")
.field("mode", &self.mode)
.finish_non_exhaustive()
}
}
pub(crate) struct BodySlot {
handle: Option<SlotHandle<ChunkOutput>>,
buffer: PagedRecvBuffer<ChunkOutput>,
notify: Arc<WakeNotify>,
reservation: Option<Reservation>,
}
impl std::fmt::Debug for BodySlot {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("BodySlot")
.field("seq", &self.handle.as_ref().map(|h| h.seq()))
.finish_non_exhaustive()
}
}
impl BodySlot {
pub(crate) fn seq(&self) -> u64 {
self.handle.as_ref().expect("slot already consumed").seq()
}
pub(crate) fn reservation(&self) -> &Reservation {
self.reservation
.as_ref()
.expect("body slot dispatched without a memory reservation")
}
pub(crate) fn attach_reservation(&mut self, reservation: Reservation) {
self.reservation = Some(reservation);
}
pub(crate) fn fill(mut self, chunk: ChunkOutput) -> FillOutcome {
let handle = self.handle.take().expect("slot already consumed");
if let Some(reservation) = self.reservation.take() {
reservation.close_acquisition();
}
let outcome = self.buffer.fill(handle, chunk);
self.notify.notify_one();
outcome
}
}
impl Drop for BodySlot {
fn drop(&mut self) {
if self.handle.is_some() {
self.notify.notify_one();
}
}
}
impl BodyWriter {
pub(crate) fn has_sink(&self) -> bool {
matches!(&*self.mode, Mode::Disk { .. })
}
#[cfg(test)]
pub(crate) fn written(&self) -> u64 {
match &*self.mode {
Mode::Disk { written, .. } => written.load(Ordering::Acquire),
Mode::Stream => 0,
}
}
pub(crate) fn claim(&self) -> BodySlot {
let handle = self.buffer.claim();
BodySlot {
handle: Some(handle),
buffer: self.buffer.clone(),
notify: self.notify.clone(),
reservation: None,
}
}
pub(crate) fn drain(&self, mode: DrainMode) -> Result<Drained, DrainError> {
let mut drained = Drained::default();
if let Mode::Disk {
sink,
object_range_start,
written,
} = &*self.mode
{
let Some(object_range_start) = object_range_start.get().copied() else {
if self.buffer.has_undrained_fills() {
return Err(DrainError {
drained: Drained::default(),
source: std::io::Error::new(
std::io::ErrorKind::InvalidData,
"disk payload arrived before destination preparation",
),
});
}
return Ok(drained);
};
while let Some(sw) = self.buffer.take_drain_run(mode) {
let bytes = match write_run(sink.as_ref(), object_range_start, &sw) {
Ok(bytes) => bytes,
Err(source) => return Err(DrainError { drained, source }),
};
written.fetch_add(bytes, Ordering::Release);
drained.bytes = drained.bytes.saturating_add(bytes);
drained.parts = drained.parts.saturating_add(sw.complete());
}
}
Ok(drained)
}
pub(crate) fn terminal_drain(&self) -> Result<Drained, DrainError> {
if !matches!(&*self.mode, Mode::Disk { .. }) {
return Ok(Drained::default());
}
let drained = self.drain(DrainMode::Eager)?;
tracing::debug!(
target: crate::telemetry::TARGET_TRANSFER_DOWNLOAD,
terminal_parts = drained.parts,
terminal_bytes = drained.bytes,
"terminal drain complete; tail flushed to disk",
);
Ok(drained)
}
pub(crate) fn finalize(&self, expected_download_len: u64) -> Result<Drained, DrainError> {
let drained = self.terminal_drain()?;
if let Mode::Disk { sink, written, .. } = &*self.mode {
let written = written.load(Ordering::Acquire);
if written != expected_download_len {
return Err(DrainError {
drained,
source: std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
format!(
"destination accepted {written} of {expected_download_len} expected bytes"
),
),
});
}
sink.finalize(expected_download_len)
.map_err(|source| DrainError { drained, source })?;
}
Ok(drained)
}
pub(crate) fn has_drainable_resident(&self) -> bool {
matches!(&*self.mode, Mode::Disk { .. }) && self.buffer.has_drainable_prefix()
}
pub(crate) fn flush_resident(&self) -> Result<Drained, DrainError> {
self.drain(DrainMode::Eager)
}
pub(crate) fn sync_drain_batch(&self, window: u64) {
let batch = std::cmp::min(SEG_SIZE as u64, window.max(1)) as usize;
self.buffer.set_drain_batch(batch);
}
pub(crate) fn notify_consumer(&self) {
self.notify.notify_one();
}
pub(crate) fn prepare(
&self,
object_range_start: u64,
expected_download_len: u64,
) -> std::io::Result<()> {
if let Mode::Disk {
sink,
object_range_start: configured_start,
..
} = &*self.mode
{
assert!(
configured_start.set(object_range_start).is_ok(),
"disk destination prepared more than once"
);
sink.prepare(expected_download_len)?;
}
Ok(())
}
}
fn write_run(
sink: &dyn SinkWrite,
object_range_start: u64,
sw: &SegmentWrite<ChunkOutput>,
) -> std::io::Result<u64> {
let (object_offset, mut cursor) = DiskWriteCursor::new(sw)?;
let len = cursor.remaining() as u64;
let file_pos = object_offset
.checked_sub(object_range_start)
.ok_or_else(|| {
std::io::Error::new(
std::io::ErrorKind::InvalidData,
"disk-write payload begins before the requested object range",
)
})?;
sink.write_all_at(&mut cursor, file_pos)?;
Ok(len)
}
pub(crate) struct RecvBodyConsumer {
consumer: RecvBufferConsumer<ChunkOutput>,
notify: Arc<WakeNotify>,
}
impl std::fmt::Debug for RecvBodyConsumer {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RecvBodyConsumer").finish_non_exhaustive()
}
}
impl RecvBodyConsumer {
#[cfg(test)]
pub(crate) fn try_take_next(&mut self) -> Option<ChunkOutput> {
self.consumer.poll_next()
}
}
pub(crate) fn new_recv_body() -> (BodyWriter, RecvBodyConsumer) {
let (buffer, consumer) = PagedRecvBuffer::new_with_segment_size(SEG_SIZE);
let notify = Arc::new(WakeNotify::new());
let writer = BodyWriter {
buffer,
notify: notify.clone(),
mode: Arc::new(Mode::Stream),
};
let slot_consumer = RecvBodyConsumer { consumer, notify };
(writer, slot_consumer)
}
pub(crate) fn new_recv_body_with_disk_mode(
sink: Box<dyn SinkWrite>,
) -> (BodyWriter, RecvBodyConsumer) {
let (buffer, consumer) = PagedRecvBuffer::new_with_segment_size(SEG_SIZE);
let notify = Arc::new(WakeNotify::new());
let resolved_range_start = std::sync::OnceLock::new();
let writer = BodyWriter {
buffer,
notify: notify.clone(),
mode: Arc::new(Mode::Disk {
sink,
object_range_start: resolved_range_start,
written: AtomicU64::new(0),
}),
};
let slot_consumer = RecvBodyConsumer { consumer, notify };
(writer, slot_consumer)
}
#[cfg(test)]
pub(crate) fn new_recv_body_with_sink(
file: std::fs::File,
owns_file: bool,
) -> (BodyWriter, RecvBodyConsumer) {
new_recv_body_with_disk_mode(Box::new(super::sink::FileSink::new(file, owns_file)))
}
pub struct Body {
consumer: RecvBufferConsumer<ChunkOutput>,
notify: Arc<WakeNotify>,
transfer: DownloadTransfer,
}
impl std::fmt::Debug for Body {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Body").finish_non_exhaustive()
}
}
#[derive(Clone)]
#[non_exhaustive]
pub struct ChunkOutput {
pub(crate) seq: u64,
pub(crate) offset: u64,
pub data: SegmentedBytes,
pub metadata: ChunkMetadata,
}
impl std::fmt::Debug for ChunkOutput {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ChunkOutput")
.field("seq", &self.seq)
.field("offset", &self.offset)
.field("data_len", &self.data.remaining())
.field("metadata", &self.metadata)
.finish()
}
}
impl Body {
pub(crate) fn new(consumer: RecvBodyConsumer, transfer: DownloadTransfer) -> Self {
Self {
consumer: consumer.consumer,
notify: consumer.notify,
transfer,
}
}
pub(crate) fn close(&mut self) {
}
pub async fn next(&mut self) -> Option<Result<ChunkOutput, crate::error::Error>> {
loop {
let notified = self.notify.notified();
if let Some(chunk) = self.consumer.poll_next() {
self.transfer.release_stream_occupancy();
return Some(Ok(chunk));
}
if !self.transfer.ctx().is_active() {
if let Some(chunk) = self.consumer.poll_next() {
self.transfer.release_stream_occupancy();
return Some(Ok(chunk));
}
return self.terminal_result();
}
notified.await;
}
}
fn terminal_result(&self) -> Option<Result<ChunkOutput, crate::error::Error>> {
if self.transfer.ctx().is_cancelled() {
Some(Err(crate::error::from_kind(
crate::error::ErrorKind::OperationCancelled,
)("download cancelled")))
} else if self.transfer.ctx().is_failed() {
let kind = self
.transfer
.ctx()
.error_kind()
.unwrap_or(crate::error::ErrorKind::ChildOperationFailed);
Some(Err(crate::error::from_kind(kind)("transfer failed")))
} else {
None }
}
}
#[cfg(test)]
mod tests {
use crate::memory::SegmentedBytes;
use crate::operation::download::transfer::DownloadTransfer;
use crate::types::ByteSize;
use crate::{error, operation::download::body::ChunkOutput};
use bytes::{BufMut, Bytes};
use std::sync::Arc;
use super::{new_recv_body, Body, BodyWriter};
fn chunk_resp(seq: u64, data: SegmentedBytes) -> ChunkOutput {
ChunkOutput {
seq,
offset: 0,
data,
metadata: Default::default(),
}
}
fn test_body() -> (Body, DownloadTransfer, BodyWriter) {
use crate::operation::download::DownloadInput;
use crate::transfer::TransferContext;
use crate::types::BucketType;
let s3_client = aws_sdk_s3::Client::from_conf(
aws_sdk_s3::Config::builder()
.with_test_defaults()
.region(aws_sdk_s3::config::Region::new("us-west-2"))
.build(),
);
let config = crate::Config::builder()
.sdk_client(s3_client)
.build()
.expect("valid config");
let tm = crate::Client::new(config);
let input = DownloadInput::builder()
.bucket("test")
.key("test")
.build()
.unwrap();
let (ctx, _) = TransferContext::new(tm.handle.clone());
let (writer, consumer) = new_recv_body();
let transfer = DownloadTransfer::new(ctx, BucketType::Standard, input, writer.clone());
(Body::new(consumer, transfer.clone()), transfer, writer)
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_body_next() {
let (mut body, transfer, writer) = test_body();
let ctx_clone = transfer.clone();
tokio::spawn(async move {
let s0 = writer.claim();
let s1 = writer.claim();
let s2 = writer.claim();
let seq2 = s2.seq();
s2.fill(chunk_resp(
seq2,
SegmentedBytes::from(Bytes::from("chunk 2")),
));
let seq0 = s0.seq();
s0.fill(chunk_resp(
seq0,
SegmentedBytes::from(Bytes::from("chunk 0")),
));
let seq1 = s1.seq();
s1.fill(chunk_resp(
seq1,
SegmentedBytes::from(Bytes::from("chunk 1")),
));
ctx_clone.ctx().set_completed();
});
let mut received = Vec::new();
while let Some(chunk) = body.next().await {
let chunk = chunk.expect("chunk ok");
let data = String::from_utf8(chunk.data.into_contiguous().to_vec()).unwrap();
received.push(data);
}
let expected: Vec<String> = [0, 1, 2].iter().map(|i| format!("chunk {i}")).collect();
assert_eq!(expected, received);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_body_next_on_failed_transfer() {
let (mut body, transfer, writer) = test_body();
let s0 = writer.claim();
let seq0 = s0.seq();
s0.fill(chunk_resp(
seq0,
SegmentedBytes::from(Bytes::from("chunk 0")),
));
transfer.ctx().set_failed(error::Error::new(
error::ErrorKind::ChildOperationFailed,
"simulated failure",
));
let chunk = body.next().await.unwrap().unwrap();
assert_eq!(chunk.seq, 0);
let err = body.next().await.unwrap().unwrap_err();
assert!(matches!(err.kind(), error::ErrorKind::ChildOperationFailed));
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_body_next_on_cancelled_transfer() {
let (mut body, transfer, _writer) = test_body();
transfer.ctx().set_cancelled();
let err = body.next().await.unwrap().unwrap_err();
assert!(matches!(err.kind(), error::ErrorKind::OperationCancelled));
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_body_next_unblocks_on_cancel() {
let (mut body, transfer, _writer) = test_body();
let waiting = Arc::new(tokio::sync::Notify::new());
let waiting_clone = waiting.clone();
let cancel_transfer = transfer.clone();
let consumer = tokio::spawn(async move {
waiting_clone.notify_one();
body.next().await
});
waiting.notified().await;
tokio::task::yield_now().await;
cancel_transfer.ctx().set_cancelled();
cancel_transfer.writer().notify_consumer();
let result = consumer.await.unwrap();
let err = result.unwrap().unwrap_err();
assert!(matches!(err.kind(), error::ErrorKind::OperationCancelled));
}
use super::{
new_recv_body_with_disk_mode, new_recv_body_with_sink, BodyWriter as Writer,
DiskWriteCursor, DrainMode, Drained, RecvBodyConsumer, Reservation,
};
use crate::operation::download::sink::{SinkFactory, SinkWrite};
use crate::operation::download::test_util::sink::{
Action, Match, Op, ScriptedSinkFactory, WriteScript,
};
use bytes::Buf as _;
use std::collections::BTreeMap;
use std::sync::Mutex as StdMutex;
fn chunk_at(seq: u64, offset: u64, data: &[u8]) -> ChunkOutput {
ChunkOutput {
seq,
offset,
data: SegmentedBytes::from(Bytes::copy_from_slice(data)),
metadata: Default::default(),
}
}
#[test]
fn disk_write_cursor_vectors_and_advances_across_chunk_boundaries() {
let (writer, _consumer, _sink) = new_recv_body_with_capture(0);
writer.claim().fill(chunk_at(0, 0, b"ab"));
writer.claim().fill(chunk_at(1, 2, b"cdef"));
let write = writer
.buffer
.take_drain_run(DrainMode::Eager)
.expect("two filled payloads");
{
let (object_offset, mut cursor) = DiskWriteCursor::new(&write).unwrap();
let mut slices = [std::io::IoSlice::new(&[]); 4];
let count = cursor.chunks_vectored(&mut slices);
assert_eq!(object_offset, 0);
assert_eq!(count, 2);
assert_eq!(slices[0].as_ref(), b"ab");
assert_eq!(slices[1].as_ref(), b"cdef");
cursor.advance(3);
assert_eq!(cursor.remaining(), 3);
assert_eq!(cursor.chunk(), b"def");
}
assert_eq!(write.complete(), 2);
}
#[derive(Debug, Default)]
struct CaptureSink {
writes: StdMutex<BTreeMap<u64, Vec<u8>>>,
}
impl CaptureSink {
fn assembled(&self) -> Vec<u8> {
let writes = self.writes.lock().unwrap();
let mut out = Vec::new();
for (&pos, bytes) in writes.iter() {
assert_eq!(pos as usize, out.len(), "gap or overlap at pos {pos}");
out.extend_from_slice(bytes);
}
out
}
fn write_count(&self) -> usize {
self.writes.lock().unwrap().len()
}
}
impl SinkWrite for CaptureSink {
fn write_all_at(&self, buf: &mut DiskWriteCursor<'_>, pos: u64) -> std::io::Result<()> {
let mut bytes = vec![0u8; buf.remaining()];
buf.copy_to_slice(&mut bytes);
self.writes.lock().unwrap().insert(pos, bytes);
Ok(())
}
}
fn new_recv_body_with_capture(
object_range_start: u64,
) -> (Writer, RecvBodyConsumer, Arc<CaptureSink>) {
let sink = Arc::new(CaptureSink::default());
#[derive(Debug)]
struct Shared(Arc<CaptureSink>);
impl SinkWrite for Shared {
fn write_all_at(&self, buf: &mut DiskWriteCursor<'_>, pos: u64) -> std::io::Result<()> {
self.0.write_all_at(buf, pos)
}
}
let (writer, consumer) = new_recv_body_with_disk_mode(Box::new(Shared(sink.clone())));
writer.prepare(object_range_start, 0).unwrap();
(writer, consumer, sink)
}
fn new_test_recv_body_with_sink(
file: std::fs::File,
object_range_start: u64,
) -> (Writer, RecvBodyConsumer) {
let (writer, consumer) = new_recv_body_with_sink(file, false);
writer.prepare(object_range_start, 0).unwrap();
(writer, consumer)
}
#[test]
fn disk_terminal_drain_before_prepare_is_empty() {
let file = tempfile::tempfile().unwrap();
let (writer, _consumer) = new_recv_body_with_sink(file, false);
assert_eq!(writer.terminal_drain().unwrap(), Drained::default());
}
#[test]
fn disk_terminal_drain_before_prepare_rejects_payload() {
let file = tempfile::tempfile().unwrap();
let (writer, _consumer) = new_recv_body_with_sink(file, false);
writer.claim().fill(chunk_at(0, 0, b"payload"));
let error = writer
.terminal_drain()
.expect_err("unpositioned payload must fail");
assert_eq!(error.source.kind(), std::io::ErrorKind::InvalidData);
assert_eq!(error.drained, Drained::default());
}
#[test]
fn has_drainable_resident_only_on_disk_with_a_filled_run() {
let (stream_writer, _consumer) = new_recv_body();
stream_writer.claim().fill(chunk_at(0, 0, b"streamed"));
assert!(
!stream_writer.has_drainable_resident(),
"stream mode has no drainable-resident run (consumer drives release)"
);
let (disk_writer, _consumer, _sink) = new_recv_body_with_capture(0);
assert!(
!disk_writer.has_drainable_resident(),
"disk mode with nothing filled is not drainable-resident"
);
disk_writer.claim().fill(chunk_at(0, 0, b"resident"));
assert!(
disk_writer.has_drainable_resident(),
"disk mode with a filled run at the cursor is drainable-resident"
);
}
#[test]
fn disk_submits_one_contiguous_run_as_one_positioned_write() {
let (writer, _consumer, sink) = new_recv_body_with_capture(100);
writer.claim().fill(chunk_at(0, 100, b"ab"));
writer.claim().fill(chunk_at(1, 102, b"cdef"));
assert_eq!(
writer.terminal_drain().unwrap(),
Drained { parts: 2, bytes: 6 }
);
assert_eq!(sink.write_count(), 1);
assert_eq!(sink.assembled(), b"abcdef");
}
#[test]
fn disk_rejects_gaps_and_overlaps_within_a_claimed_run() {
for second_offset in [1, 3] {
let (writer, _consumer, sink) = new_recv_body_with_capture(0);
writer.claim().fill(chunk_at(0, 0, b"ab"));
writer.claim().fill(chunk_at(1, second_offset, b"cd"));
let error = writer
.terminal_drain()
.expect_err("offsets are not contiguous");
assert_eq!(error.source.kind(), std::io::ErrorKind::InvalidData);
assert_eq!(sink.write_count(), 0);
}
}
#[test]
fn disk_rejects_an_empty_claimed_payload() {
let (writer, _consumer, sink) = new_recv_body_with_capture(0);
writer.claim().fill(chunk_at(0, 0, b""));
let error = writer.terminal_drain().expect_err("disk payload is empty");
assert_eq!(error.source.kind(), std::io::ErrorKind::InvalidData);
assert_eq!(sink.write_count(), 0);
}
#[test]
fn disk_writes_full_segment() {
use super::SEG_SIZE;
const CHUNK_LEN: usize = 4;
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("out");
let file = std::fs::File::create(&path).unwrap();
let (writer, _consumer) = new_test_recv_body_with_sink(file, 0);
let mut expected = Vec::with_capacity(SEG_SIZE * CHUNK_LEN);
for i in 0..SEG_SIZE as u64 {
let slot = writer.claim();
let data = [i as u8; CHUNK_LEN];
let offset = i * CHUNK_LEN as u64;
expected.extend_from_slice(&data);
slot.fill(chunk_at(i, offset, &data));
}
writer.drain(DrainMode::Batched).unwrap();
assert_eq!(std::fs::read(&path).unwrap(), expected);
}
#[test]
fn disk_eof_partial_tail() {
use super::SEG_SIZE;
const CHUNK_LEN: usize = 2;
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("out");
let file = std::fs::File::create(&path).unwrap();
let (writer, _consumer) = new_test_recv_body_with_sink(file, 0);
let partial = SEG_SIZE / 2;
let mut expected = Vec::with_capacity(partial * CHUNK_LEN);
for i in 0..partial as u64 {
let slot = writer.claim();
let data = [i as u8; CHUNK_LEN];
let offset = i * CHUNK_LEN as u64;
expected.extend_from_slice(&data);
slot.fill(chunk_at(i, offset, &data));
}
writer.finalize(expected.len() as u64).unwrap();
assert_eq!(std::fs::read(&path).unwrap(), expected);
}
#[test]
fn disk_finalize_removes_an_existing_file_tail() {
const ORIGINAL: &[u8] = b"old contents with a stale tail";
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("out");
std::fs::write(&path, ORIGINAL).unwrap();
let file = std::fs::OpenOptions::new()
.read(true)
.write(true)
.open(&path)
.unwrap();
let (writer, _consumer) = new_recv_body_with_sink(file, false);
writer.prepare(0, 3).unwrap();
assert_eq!(
std::fs::metadata(&path).unwrap().len(),
ORIGINAL.len() as u64,
"preparing a caller-owned file must not resize it"
);
writer.claim().fill(chunk_at(0, 0, b"new"));
writer.terminal_drain().unwrap();
assert_eq!(
std::fs::metadata(&path).unwrap().len(),
ORIGINAL.len() as u64,
"terminal cleanup must not establish the successful length"
);
writer.finalize(3).unwrap();
assert_eq!(std::fs::read(&path).unwrap(), b"new");
}
fn scripted_writer_over(
path: &std::path::Path,
script: &Arc<WriteScript>,
) -> (Writer, RecvBodyConsumer) {
let file = std::fs::OpenOptions::new()
.read(true)
.write(true)
.open(path)
.unwrap();
new_recv_body_with_disk_mode(
ScriptedSinkFactory(Arc::clone(script))
.create(file, false)
.unwrap(),
)
}
#[test]
fn failed_drain_returns_the_runs_written_before_it() {
use super::SEG_SIZE;
const RUN_LEN: u64 = 4;
let script = Arc::new(WriteScript::default()).on(
Match::NthWrite(3),
Action::Fail(std::io::ErrorKind::StorageFull),
);
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("out");
std::fs::File::create(&path).unwrap();
let (writer, _consumer) = scripted_writer_over(&path, &script);
writer.prepare(0, 3 * RUN_LEN).unwrap();
let seg = SEG_SIZE as u64;
for seq in 0..=2 * seg {
let slot = writer.claim();
if seq % seg == 0 {
let run = seq / seg;
slot.fill(chunk_at(
seq,
run * RUN_LEN,
&[run as u8 + 1; RUN_LEN as usize],
));
}
}
let error = writer
.drain(DrainMode::Eager)
.expect_err("the third write is scripted to fail");
assert_eq!(error.source.kind(), std::io::ErrorKind::StorageFull);
assert_eq!(error.drained, Drained { parts: 2, bytes: 8 });
assert_eq!(writer.written(), 8);
}
#[test]
fn disk_finalize_propagates_resize_failure() {
let script = Arc::new(WriteScript::default()).on(
Match::Finalize,
Action::Fail(std::io::ErrorKind::PermissionDenied),
);
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("out");
std::fs::write(&path, b"existing").unwrap();
let (writer, _consumer) = scripted_writer_over(&path, &script);
writer.prepare(0, 3).unwrap();
writer.claim().fill(chunk_at(0, 0, b"new"));
writer.drain(DrainMode::Eager).unwrap();
assert_eq!(writer.written(), 3, "the whole object was written");
let error = writer
.finalize(3)
.expect_err("the resize is scripted to fail");
assert_eq!(error.source.kind(), std::io::ErrorKind::PermissionDenied);
let contents = std::fs::read(&path).unwrap();
assert_eq!(contents.len(), 8, "a failed resize keeps the length");
assert_eq!(&contents[..3], b"new");
}
#[test]
fn disk_finalize_rejects_a_short_destination_without_resizing() {
const ORIGINAL: &[u8] = b"previous contents";
const HOLD_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
let script = Arc::new(WriteScript::default()).on(Match::NthWrite(1), Action::Hold);
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("out");
std::fs::write(&path, ORIGINAL).unwrap();
let (writer, _consumer) = scripted_writer_over(&path, &script);
writer.prepare(0, 4).unwrap();
writer.claim().fill(chunk_at(0, 0, b"ab"));
std::thread::scope(|scope| {
let drain = scope.spawn(|| writer.drain(DrainMode::Eager));
let held = script.wait_held(HOLD_TIMEOUT);
assert_eq!(held.op(), Op::Write { pos: 0, len: 2 });
writer.claim().fill(chunk_at(1, 2, b"cd"));
let error = writer
.finalize(4)
.expect_err("two of four bytes reached the destination");
assert_eq!(error.source.kind(), std::io::ErrorKind::UnexpectedEof);
assert!(
error.source.to_string().contains("2 of 4"),
"the error names both byte counts: {}",
error.source
);
let contents = std::fs::read(&path).unwrap();
assert_eq!(
contents.len(),
ORIGINAL.len(),
"a short destination must not be resized"
);
assert_eq!(&contents[..4], b"prcd");
held.release();
drain.join().unwrap().unwrap();
});
}
#[test]
fn disk_finalize_truncates_an_empty_download() {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("out");
std::fs::write(&path, b"existing").unwrap();
let file = std::fs::OpenOptions::new()
.read(true)
.write(true)
.open(&path)
.unwrap();
let (writer, _consumer) = new_recv_body_with_sink(file, false);
writer.prepare(0, 0).unwrap();
assert_eq!(std::fs::metadata(&path).unwrap().len(), 8);
writer.finalize(0).unwrap();
assert!(std::fs::read(&path).unwrap().is_empty());
}
#[test]
fn disk_finalize_drains_full_and_tail() {
use super::SEG_SIZE;
const CHUNK_LEN: usize = 2;
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("out");
let file = std::fs::File::create(&path).unwrap();
let (writer, _consumer) = new_test_recv_body_with_sink(file, 0);
let total = SEG_SIZE + 2;
let mut expected = Vec::with_capacity(total * CHUNK_LEN);
for i in 0..total as u64 {
let slot = writer.claim();
let data = [i as u8; CHUNK_LEN];
let offset = i * CHUNK_LEN as u64;
expected.extend_from_slice(&data);
slot.fill(chunk_at(i, offset, &data));
}
writer.finalize(expected.len() as u64).unwrap();
assert_eq!(std::fs::read(&path).unwrap(), expected);
}
#[test]
fn disk_offset_translation() {
use super::SEG_SIZE;
const CHUNK_LEN: usize = 2;
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("out");
let file = std::fs::File::create(&path).unwrap();
let (writer, _consumer) = new_test_recv_body_with_sink(file, 1000);
let n = SEG_SIZE;
let mut expected = Vec::with_capacity(n * CHUNK_LEN);
for i in 0..n as u64 {
let slot = writer.claim();
let data = [i as u8; CHUNK_LEN];
let offset = 1000 + i * CHUNK_LEN as u64;
expected.extend_from_slice(&data);
slot.fill(chunk_at(i, offset, &data));
}
writer.drain(DrainMode::Batched).unwrap();
assert_eq!(std::fs::read(&path).unwrap(), expected);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn disk_concurrent_fill() {
use super::SEG_SIZE;
const CHUNK_LEN: usize = 4;
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("out");
let file = std::fs::File::create(&path).unwrap();
let (writer, _consumer) = new_test_recv_body_with_sink(file, 0);
let n = SEG_SIZE;
let mut handles = Vec::new();
for _ in 0..n {
let slot = writer.claim();
handles.push(tokio::spawn(async move {
tokio::task::yield_now().await;
let seq = slot.seq();
let offset = seq * CHUNK_LEN as u64;
let data = [seq as u8; CHUNK_LEN];
slot.fill(chunk_at(seq, offset, &data));
}));
}
for h in handles {
h.await.unwrap();
}
writer.finalize((n * CHUNK_LEN) as u64).unwrap();
let contents = std::fs::read(&path).unwrap();
for seq in 0..n as u64 {
let offset = seq as usize * CHUNK_LEN;
assert_eq!(
&contents[offset..offset + CHUNK_LEN],
&[seq as u8; CHUNK_LEN],
"mismatch at seq {seq}"
);
}
}
#[test]
fn disk_drain_releases_read_ahead_occupancy() {
use super::SEG_SIZE;
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("out");
let file = std::fs::File::create(&path).unwrap();
let (writer, _consumer) = new_test_recv_body_with_sink(file, 0);
let total = (2 * SEG_SIZE) as u64;
let mut freed_total = 0u64;
for i in 0..total {
let slot = writer.claim();
let outcome = slot.fill(chunk_at(i, i, &[i as u8]));
if outcome == super::FillOutcome::DrainReady {
freed_total += writer.drain(DrainMode::Batched).unwrap().parts;
}
}
freed_total += writer.terminal_drain().unwrap().parts;
let occupancy = total - freed_total;
assert_eq!(
occupancy, 0,
"all {total} parts drained to disk, but the drains reported only \
{freed_total} freed; issuance cannot reopen past the window"
);
}
#[test]
fn disk_concurrent_out_of_order_drain_reassembles() {
use super::SEG_SIZE;
let parts = 3 * SEG_SIZE + 5;
let part_len = 64usize;
let range_start = 4096u64;
let part_bytes = |seq: usize| -> Vec<u8> {
(0..part_len)
.map(|b| (seq as u8).wrapping_add(b as u8))
.collect()
};
let mut expected = Vec::with_capacity(parts * part_len);
for seq in 0..parts {
expected.extend_from_slice(&part_bytes(seq));
}
let (writer, _consumer, sink) = new_recv_body_with_capture(range_start);
let slots: Vec<_> = (0..parts).map(|_| writer.claim()).collect();
let mut order: Vec<usize> = (0..parts).filter(|i| i % 2 == 1).collect();
order.extend((0..parts).filter(|i| i % 2 == 0));
let mut by_seq: std::collections::HashMap<usize, super::BodySlot> =
slots.into_iter().map(|s| (s.seq() as usize, s)).collect();
let writer = Arc::new(writer);
let freed_total = Arc::new(std::sync::atomic::AtomicU64::new(0));
let n_threads = 4;
let chunks: Vec<Vec<usize>> = (0..n_threads)
.map(|t| order.iter().copied().skip(t).step_by(n_threads).collect())
.collect();
std::thread::scope(|scope| {
for thread_seqs in &chunks {
let mut my_slots = Vec::new();
for &seq in thread_seqs {
my_slots.push((seq, by_seq.remove(&seq).expect("slot for seq")));
}
let writer = writer.clone();
let freed_total = freed_total.clone();
scope.spawn(move || {
for (seq, slot) in my_slots {
let offset = range_start + (seq as u64) * (part_len as u64);
let outcome = slot.fill(chunk_at(seq as u64, offset, &part_bytes(seq)));
if outcome == super::FillOutcome::DrainReady {
let freed = writer.drain(DrainMode::Batched).unwrap().parts;
freed_total.fetch_add(freed, std::sync::atomic::Ordering::Relaxed);
}
}
});
}
});
assert!(by_seq.is_empty(), "every slot dispatched to a thread");
freed_total.fetch_add(
writer.terminal_drain().unwrap().parts,
std::sync::atomic::Ordering::Relaxed,
);
assert_eq!(
freed_total.load(std::sync::atomic::Ordering::Relaxed),
parts as u64,
"all parts drained"
);
assert_eq!(
sink.assembled(),
expected,
"out-of-order concurrent drain must reassemble the object byte-for-byte"
);
}
use crate::config::MemoryBudgetConfig;
use crate::memory::BufferPool;
fn test_reservation_pool() -> (BufferPool, usize) {
let pool = BufferPool::from_capacity(
MemoryBudgetConfig::Limit(ByteSize::new(1024 * 1024)),
None,
crate::config::MemoryDiagnosticsConfig::default(),
)
.expect("test pool");
let carrier_size = pool.carrier_size();
(pool, carrier_size)
}
fn pooled_data(pool: &BufferPool, reservation: &Reservation, data: &[u8]) -> SegmentedBytes {
let mut buffer = pool
.acquire(reservation, data.len())
.expect("reserved pooled buffer");
buffer.put_slice(data);
buffer.freeze()
}
#[test]
fn test_stream_fill_closes_reservation_while_payload_retains_charge() {
let (pool, chunk_bytes) = test_reservation_pool();
let (writer, mut consumer) = new_recv_body();
let reservation = pool
.try_reserve(chunk_bytes)
.unwrap()
.expect("has capacity");
let data = pooled_data(&pool, &reservation, b"hello");
let mut slot = writer.claim();
slot.attach_reservation(reservation);
assert_eq!(
pool.metrics().active_planned_demand_bytes(),
chunk_bytes as u64
);
slot.fill(chunk_resp(0, data));
let metrics = pool.metrics();
assert_eq!(metrics.active_planned_demand_bytes(), 0);
assert_eq!(metrics.in_use_bytes(), chunk_bytes as u64);
assert_eq!(metrics.admitted_bytes(), chunk_bytes as u64);
let chunk = consumer.try_take_next().expect("chunk delivered");
assert_eq!(chunk.data.clone().into_contiguous().as_ref(), b"hello");
assert_eq!(pool.metrics().in_use_bytes(), chunk_bytes as u64);
drop(chunk);
assert_eq!(pool.metrics().in_use_bytes(), 0);
assert_eq!(pool.metrics().admitted_bytes(), 0);
}
#[test]
fn test_stream_charge_holds_until_last_chunk_clone_drops() {
let (pool, chunk_bytes) = test_reservation_pool();
let (writer, mut consumer) = new_recv_body();
let reservation = pool
.try_reserve(chunk_bytes)
.unwrap()
.expect("has capacity");
let data = pooled_data(&pool, &reservation, b"shared");
let mut slot = writer.claim();
slot.attach_reservation(reservation);
slot.fill(chunk_resp(0, data));
let chunk = consumer.try_take_next().expect("chunk delivered");
let clone = chunk.clone();
assert_eq!(pool.metrics().active_planned_demand_bytes(), 0);
assert_eq!(
pool.metrics().in_use_bytes(),
chunk_bytes as u64,
"one physical buffer, charged once"
);
drop(chunk);
assert_eq!(
pool.metrics().in_use_bytes(),
chunk_bytes as u64,
"a surviving clone still holds the resident bytes"
);
drop(clone);
assert_eq!(pool.metrics().in_use_bytes(), 0);
assert_eq!(pool.metrics().admitted_bytes(), 0);
}
#[test]
fn test_disk_payload_charge_returns_on_drain() {
let (pool, chunk_bytes) = test_reservation_pool();
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("out");
let file = std::fs::File::create(&path).unwrap();
let (writer, _consumer) = new_test_recv_body_with_sink(file, 0);
let reservation = pool
.try_reserve(chunk_bytes)
.unwrap()
.expect("has capacity");
let data = pooled_data(&pool, &reservation, b"flushed");
let mut slot = writer.claim();
slot.attach_reservation(reservation);
slot.fill(ChunkOutput {
seq: 0,
offset: 0,
data,
metadata: Default::default(),
});
assert_eq!(pool.metrics().active_planned_demand_bytes(), 0);
assert_eq!(pool.metrics().in_use_bytes(), chunk_bytes as u64);
writer.terminal_drain().unwrap();
assert_eq!(pool.metrics().in_use_bytes(), 0);
assert_eq!(pool.metrics().admitted_bytes(), 0);
assert_eq!(&std::fs::read(&path).unwrap()[..7], b"flushed");
}
}