use parking_lot::Mutex;
use path_clean::PathClean;
use std::borrow::Cow;
use std::collections::{HashMap, HashSet};
use std::fmt;
use std::path::{Component, Path, PathBuf};
use std::pin::Pin;
use std::sync::Arc;
use crate::error::{self, Error, ErrorKind};
use crate::io::walk::S3Walk;
use crate::model::Object;
use crate::operation::download::sink::{FileSinkFactory, SinkFactory};
use crate::operation::download::{Download, DownloadInput, ManagedDownloadHandle};
use crate::operation::download_objects::observability::{
emit_event, emit_terminal_summary, DownloadObjectsEvent, DownloadObjectsStateSnapshot,
};
use crate::telemetry::TARGET_TRANSFER_DOWNLOAD_OBJECTS;
use crate::transfer::CompositePendingReason;
use crate::transfer::{IoRequest, PollWork, Transfer, TransferContext, TransferId, WorkOutcome};
use crate::types::{FailedDownload, FailedTransferPolicy};
const MAX_REAP_PER_POLL: usize = 64;
const MAX_ENTRIES_PER_WALK: usize = 1000;
const WALK_LOW_WATER: usize = MAX_ENTRIES_PER_WALK;
const WALK_FLUSH_CHUNK: usize = 64;
pub(crate) enum DownloadObjectsWork {
AdvanceWalker { walk: Option<Box<S3Walk>> },
JoinChildren { batch: Option<ReapingBatch> },
}
impl DownloadObjectsWork {
fn kind(&self) -> &'static str {
match self {
Self::AdvanceWalker { .. } => "advance_walker",
Self::JoinChildren { .. } => "join_children",
}
}
}
impl std::fmt::Debug for DownloadObjectsWork {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::AdvanceWalker { .. } => f.debug_struct("AdvanceWalker").finish_non_exhaustive(),
Self::JoinChildren { batch } => f
.debug_struct("JoinChildren")
.field("count", &batch.as_ref().map(|b| b.count))
.finish(),
}
}
}
pub(crate) struct ChildTransfer {
pub(crate) handle: ManagedDownloadHandle,
pub(crate) key: String,
}
struct Reservation {
transfer: Arc<DownloadObjectsTransferInner>,
count: usize,
consumed: bool,
}
impl Reservation {
fn consume(mut self, state: &mut State) {
debug_assert!(
!self.consumed,
"Reservation::consume called on already-consumed reservation"
);
debug_assert!(
state.children_reserved >= self.count,
"children_reserved underflow: have {}, releasing {}",
state.children_reserved,
self.count,
);
state.children_reserved -= self.count;
self.consumed = true;
}
}
impl Drop for Reservation {
fn drop(&mut self) {
if !self.consumed && self.count > 0 {
let mut state = self.transfer.state.lock();
state.children_reserved = state.children_reserved.saturating_sub(self.count);
tracing::warn!(
target: crate::telemetry::TARGET_TRANSFER_DOWNLOAD_OBJECTS,
tid = %self.transfer.ctx.id,
count = self.count,
"Reservation dropped unconsumed; children_reserved recovered"
);
}
}
}
pub(crate) struct ReapingBatch {
transfer: Arc<DownloadObjectsTransferInner>,
children: Option<Vec<ChildTransfer>>,
count: usize,
consumed: bool,
}
impl fmt::Debug for ReapingBatch {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("ReapingBatch")
.field("count", &self.count)
.field("consumed", &self.consumed)
.field("children_len", &self.children.as_ref().map(|c| c.len()))
.finish_non_exhaustive()
}
}
impl ReapingBatch {
fn take_children(&mut self) -> Vec<ChildTransfer> {
self.children
.take()
.expect("ReapingBatch::take_children called twice or after consume")
}
fn consume(mut self, state: &mut State) {
debug_assert!(
!self.consumed,
"ReapingBatch::consume called on already-consumed batch"
);
debug_assert!(
state.reaping_in_flight >= self.count,
"reaping_in_flight underflow: have {}, releasing {}",
state.reaping_in_flight,
self.count,
);
state.reaping_in_flight -= self.count;
self.consumed = true;
}
}
impl Drop for ReapingBatch {
fn drop(&mut self) {
if !self.consumed {
let mut state = self.transfer.state.lock();
state.reaping_in_flight = state.reaping_in_flight.saturating_sub(self.count);
tracing::warn!(
target: crate::telemetry::TARGET_TRANSFER_DOWNLOAD_OBJECTS,
tid = %self.transfer.ctx.id,
count = self.count,
"ReapingBatch dropped unconsumed; reaping_in_flight recovered"
);
}
}
}
struct State {
walk: Option<S3Walk>,
validated: bool,
walk_in_flight: bool,
pending_entries: std::collections::VecDeque<Object>,
children: HashMap<TransferId, ChildTransfer>,
children_reserved: usize,
reaping_in_flight: usize,
failed: Vec<FailedDownload>,
successful_downloads: u64,
}
impl State {
fn snapshot(&self) -> DownloadObjectsStateSnapshot {
DownloadObjectsStateSnapshot {
pending_entries: self.pending_entries.len(),
walk_in_flight: self.walk_in_flight,
children: self.children.len(),
children_reserved: self.children_reserved,
reaping_in_flight: self.reaping_in_flight,
successful: self.successful_downloads,
failed: self.failed.len(),
}
}
fn pending_reason(&self, active: bool) -> CompositePendingReason {
let listing = self.walk_in_flight || self.walk.as_ref().is_some_and(|w| !w.is_done());
if !active {
CompositePendingReason::Draining
} else if self.reaping_in_flight > 0 {
CompositePendingReason::Reaping
} else if !self.pending_entries.is_empty() {
CompositePendingReason::ChildCapacity
} else if listing {
CompositePendingReason::Listing
} else {
CompositePendingReason::ChildCompletion
}
}
}
#[derive(Clone)]
pub(crate) struct DownloadObjectsTransfer {
inner: Arc<DownloadObjectsTransferInner>,
}
impl std::fmt::Debug for DownloadObjectsTransfer {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("DownloadObjectsTransfer")
.finish_non_exhaustive()
}
}
struct DownloadObjectsTransferInner {
ctx: TransferContext,
state: Mutex<State>,
bucket: String,
destination: PathBuf,
delimiter: Option<String>,
failure_policy: FailedTransferPolicy,
pipeline_depth: usize,
created_dirs: Mutex<HashSet<PathBuf>>,
#[cfg(test)]
last_summary: Mutex<Option<crate::transfer::CompositeTransferSummary>>,
}
impl DownloadObjectsTransfer {
pub(crate) fn new(
ctx: TransferContext,
input: &crate::operation::download_objects::DownloadObjectsInput,
walk: S3Walk,
pipeline_depth: usize,
) -> Self {
let bucket = input.bucket().unwrap().to_string();
let destination = input.destination().unwrap().clean();
let delimiter = input.delimiter().map(|s| s.to_string());
let failure_policy = input.failure_policy().clone();
Self {
inner: Arc::new(DownloadObjectsTransferInner {
ctx,
state: Mutex::new(State {
walk: Some(walk),
validated: false,
walk_in_flight: false,
pending_entries: std::collections::VecDeque::new(),
children: HashMap::new(),
children_reserved: 0,
reaping_in_flight: 0,
failed: Vec::new(),
successful_downloads: 0,
}),
bucket,
destination,
delimiter,
failure_policy,
pipeline_depth,
created_dirs: Mutex::new(HashSet::new()),
#[cfg(test)]
last_summary: Mutex::new(None),
}),
}
}
pub(crate) fn ctx(&self) -> &TransferContext {
&self.inner.ctx
}
pub(crate) fn successful_downloads(&self) -> u64 {
self.inner.state.lock().successful_downloads
}
pub(crate) fn take_failed(&self) -> Option<Vec<FailedDownload>> {
let mut state = self.inner.state.lock();
if state.failed.is_empty() {
None
} else {
Some(std::mem::take(&mut state.failed))
}
}
#[tracing::instrument(
level = "trace",
target = TARGET_TRANSFER_DOWNLOAD_OBJECTS,
skip_all,
fields(tid = %self.inner.ctx.id),
)]
pub(crate) fn poll_work(&self) -> PollWork {
let mut state = self.inner.state.lock();
if self.inner.ctx.is_active() {
if let Some(work) = self.dispatch_walk(&mut state) {
return work;
}
}
let mut attempted = false;
let mut materialized = false;
if self.inner.ctx.is_active() {
let (to_spawn, reservation) = self.claim_one(&mut state);
if !to_spawn.is_empty() {
attempted = true;
drop(state);
let spawned = self.spawn_children(to_spawn);
state = self.inner.state.lock();
materialized = self.merge_spawned(&mut state, spawned, reservation) > 0;
}
}
let reaped = self.drain_terminal_children(&mut state);
if let Some(batch) = reaped {
return PollWork::Ready {
io: IoRequest {
data: Some(Box::new(DownloadObjectsWork::JoinChildren {
batch: Some(batch),
})),
},
spawned: materialized,
};
}
if attempted {
return PollWork::Spawned;
}
if let Some(result) = self.check_terminal(&state) {
return result;
}
let reason = state.pending_reason(self.inner.ctx.is_active());
self.inner.ctx.set_pending(reason);
PollWork::Pending
}
fn drain_terminal_children(&self, state: &mut State) -> Option<ReapingBatch> {
let terminal_ids: Vec<TransferId> = state
.children
.iter()
.filter(|(_, child)| child.handle.status() != crate::types::TransferStatus::Active)
.map(|(id, _)| *id)
.take(MAX_REAP_PER_POLL)
.collect();
if terminal_ids.is_empty() {
return None;
}
let batch: Vec<ChildTransfer> = terminal_ids
.into_iter()
.filter_map(|id| state.children.remove(&id))
.collect();
tracing::trace!(
target: crate::telemetry::TARGET_TRANSFER_DOWNLOAD_OBJECTS,
tid = %self.inner.ctx.id,
reaping = batch.len(),
remaining_children = state.children.len(),
"download_objects draining terminal children",
);
let count = batch.len();
state.reaping_in_flight += count;
emit_event(
&self.inner.ctx,
DownloadObjectsEvent::ReapScheduled,
state.snapshot(),
);
Some(ReapingBatch {
transfer: self.inner.clone(),
children: Some(batch),
count,
consumed: false,
})
}
fn claim_one(&self, state: &mut State) -> (Vec<Object>, Reservation) {
let active = state
.children
.values()
.filter(|c| c.handle.status() == crate::types::TransferStatus::Active)
.count()
+ state.children_reserved;
let available = self.inner.pipeline_depth.saturating_sub(active);
let count = available.min(1).min(state.pending_entries.len());
let batch: Vec<Object> = state.pending_entries.drain(..count).collect();
state.children_reserved += batch.len();
let reservation = Reservation {
transfer: self.inner.clone(),
count: batch.len(),
consumed: false,
};
(batch, reservation)
}
fn spawn_children(&self, entries: Vec<Object>) -> Vec<(Object, Result<ChildTransfer, Error>)> {
let handle = &self.inner.ctx.handle;
let parent_id = self.inner.ctx.id.id;
tracing::trace!(
target: crate::telemetry::TARGET_TRANSFER_DOWNLOAD_OBJECTS,
tid = %self.inner.ctx.id,
count = entries.len(),
"download_objects spawning children",
);
entries
.into_iter()
.map(|obj| {
let key = obj
.key()
.expect("S3Walk yields objects with keys")
.to_string();
let listed = crate::operation::download::ListedObject::from_listing(&obj);
let result = self.spawn_single_child(handle, &key, parent_id, listed);
(obj, result.map(|h| ChildTransfer { handle: h, key }))
})
.collect()
}
fn prepare_destination(&self) -> Result<(), Error> {
let dest = &self.inner.destination;
match std::fs::metadata(dest) {
Ok(metadata) => crate::operation::validate_target_is_dir(&metadata, dest),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => std::fs::create_dir_all(dest)
.map_err(|e| error::io_at("creating directory", dest, e)),
Err(e) => Err(error::io_at("reading metadata of", dest, e)),
}
}
fn spawn_single_child(
&self,
handle: &Arc<crate::client::Handle>,
key: &str,
parent_id: u64,
listed: Option<crate::operation::download::ListedObject>,
) -> Result<ManagedDownloadHandle, Error> {
let dest_path = local_key_path(
&self.inner.destination,
key,
self.inner.delimiter.as_deref(),
)?;
if let Some(parent) = dest_path.parent() {
let known = self.inner.created_dirs.lock().contains(parent);
if !known {
std::fs::create_dir_all(parent).map_err(|e| {
error::io_at("creating directory", parent, e).with_local_key(key)
})?;
self.inner.created_dirs.lock().insert(parent.to_path_buf());
}
}
let input = DownloadInput::builder()
.bucket(&self.inner.bucket)
.key(key)
.build()
.expect("bucket and key are set");
let (file, temp_path) =
crate::io::fs::create_first_new(crate::operation::download::temp_file_candidates(
&dest_path,
std::iter::repeat_with(|| fastrand::u32(..)),
))
.map_err(|e| {
error::io_at("creating temporary file for", &dest_path, e).with_local_key(key)
})?;
let sink = FileSinkFactory.create(file, true).map_err(|e| {
let _ = std::fs::remove_file(&temp_path);
error::io_at("preparing temporary file for", &dest_path, e).with_local_key(key)
})?;
let inner =
Download::orchestrate_with_sink(handle.clone(), input, sink, Some(parent_id), listed)?;
Ok(ManagedDownloadHandle::new(inner, temp_path, dest_path))
}
fn merge_spawned(
&self,
state: &mut State,
spawned: Vec<(Object, Result<ChildTransfer, Error>)>,
reservation: Reservation,
) -> usize {
let mut inserted = 0usize;
for (obj, result) in spawned {
match result {
Ok(child) => {
let id = child.handle.transfer_id();
tracing::trace!(
target: crate::telemetry::TARGET_TRANSFER_DOWNLOAD_OBJECTS,
tid = %self.inner.ctx.id,
child_id = %id,
key = %child.key,
"spawned child download"
);
state.children.insert(id, child);
inserted += 1;
}
Err(err) => {
let key = obj
.key()
.expect("S3Walk yields objects with keys")
.to_string();
tracing::warn!(
target: crate::telemetry::TARGET_TRANSFER_DOWNLOAD_OBJECTS,
tid = %self.inner.ctx.id,
key = %key,
error = %err,
"failed to spawn child download"
);
let failed_input = DownloadInput::builder()
.bucket(&self.inner.bucket)
.key(&key)
.build()
.unwrap();
state.failed.push(FailedDownload {
input: failed_input,
error: err,
});
if self.inner.failure_policy == FailedTransferPolicy::Abort {
self.inner.ctx.set_failed(child_failed());
self.report_terminal(state.snapshot());
self.inner.ctx.signal_terminal();
}
}
}
}
reservation.consume(state);
inserted
}
fn report_terminal(&self, snapshot: DownloadObjectsStateSnapshot) {
self.inner.ctx.finalize_terminal_metrics();
if !self.inner.ctx.claim_terminal_report() {
return;
}
let _summary = emit_terminal_summary(&self.inner.ctx, snapshot);
#[cfg(test)]
{
*self.inner.last_summary.lock() = _summary;
}
}
#[cfg(test)]
pub(crate) fn test_terminal_summary(
&self,
) -> Option<crate::transfer::CompositeTransferSummary> {
self.inner.last_summary.lock().clone()
}
fn check_terminal(&self, state: &State) -> Option<PollWork> {
if !self.inner.ctx.is_active() {
if state.children.is_empty()
&& state.children_reserved == 0
&& state.reaping_in_flight == 0
&& !state.walk_in_flight
{
tracing::debug!(
target: crate::telemetry::TARGET_TRANSFER_DOWNLOAD_OBJECTS,
tid = %self.inner.ctx.id,
successful = state.successful_downloads,
failed = state.failed.len(),
"download_objects terminal (cancelled/failed), signaling",
);
self.report_terminal(state.snapshot());
self.inner.ctx.signal_terminal();
return Some(PollWork::Done);
}
return None;
}
let walk_done = state.walk.as_ref().is_none_or(|w| w.is_done()) && !state.walk_in_flight;
if walk_done
&& state.pending_entries.is_empty()
&& state.children.is_empty()
&& state.children_reserved == 0
&& state.reaping_in_flight == 0
{
tracing::debug!(
target: crate::telemetry::TARGET_TRANSFER_DOWNLOAD_OBJECTS,
tid = %self.inner.ctx.id,
successful = state.successful_downloads,
failed = state.failed.len(),
"download_objects terminal (walk exhausted), signaling",
);
self.inner.ctx.set_completed();
self.report_terminal(state.snapshot());
self.inner.ctx.signal_terminal();
return Some(PollWork::Done);
}
None
}
fn dispatch_walk(&self, state: &mut State) -> Option<PollWork> {
if state.walk_in_flight {
return None;
}
if state.pending_entries.len() >= WALK_LOW_WATER {
return None;
}
let walk = match state.walk.take() {
Some(w) if !w.is_done() => w,
Some(w) => {
state.walk = Some(w);
return None;
}
None => return None,
};
state.walk_in_flight = true;
emit_event(
&self.inner.ctx,
DownloadObjectsEvent::WalkDispatched,
state.snapshot(),
);
Some(PollWork::ready(IoRequest {
data: Some(Box::new(DownloadObjectsWork::AdvanceWalker {
walk: Some(Box::new(walk)),
})),
}))
}
#[tracing::instrument(
level = "trace",
target = TARGET_TRANSFER_DOWNLOAD_OBJECTS,
skip_all,
fields(tid = %self.inner.ctx.id, work = tracing::field::Empty),
)]
pub(crate) async fn execute(&self, work: &mut IoRequest) -> WorkOutcome {
let data = work
.data
.as_mut()
.expect("work data must be set")
.as_any_mut()
.downcast_mut::<DownloadObjectsWork>()
.expect("work data must be DownloadObjectsWork");
tracing::Span::current().record("work", data.kind());
match data {
DownloadObjectsWork::AdvanceWalker { walk } => {
self.execute_advance_walker(*walk.take().unwrap()).await
}
DownloadObjectsWork::JoinChildren { batch } => {
let batch = batch.take().expect("batch already taken");
self.execute_join_children(batch).await
}
}
}
async fn execute_advance_walker(&self, mut walk: S3Walk) -> WorkOutcome {
let needs_validation = {
let mut state = self.inner.state.lock();
let first = !state.validated;
state.validated = true;
first
};
if needs_validation {
if let Err(err) = self.prepare_destination() {
let mut state = self.inner.state.lock();
state.walk_in_flight = false;
self.inner.ctx.set_failed(err);
self.report_terminal(state.snapshot());
if self.check_terminal(&state).is_some() {
return WorkOutcome::Success { data: None };
}
drop(state);
self.inner.ctx.try_wake();
return WorkOutcome::Success { data: None };
}
}
let mut chunk = Vec::with_capacity(WALK_FLUSH_CHUNK);
let mut discovered = 0usize;
let mut fatal_error = None;
loop {
match walk.next().await {
Some(Ok(obj)) => {
chunk.push(obj);
discovered += 1;
if chunk.len() >= WALK_FLUSH_CHUNK {
let mut state = self.inner.state.lock();
state.pending_entries.extend(chunk.drain(..));
drop(state);
self.inner.ctx.try_wake();
}
if discovered >= MAX_ENTRIES_PER_WALK {
break;
}
}
Some(Err(err)) => {
if err.is_fatal() {
fatal_error = Some(err);
break;
}
tracing::warn!(
target: crate::telemetry::TARGET_TRANSFER_DOWNLOAD_OBJECTS,
tid = %self.inner.ctx.id,
error = %err,
"non-fatal walker error, continuing"
);
}
None => break, }
}
let mut state = self.inner.state.lock();
state.walk_in_flight = false;
if let Some(err) = fatal_error {
tracing::debug!(
target: crate::telemetry::TARGET_TRANSFER_DOWNLOAD_OBJECTS,
tid = %self.inner.ctx.id,
error = %err,
"download_objects fatal walker error",
);
self.inner.ctx.set_failed(err);
self.report_terminal(state.snapshot());
} else {
tracing::trace!(
target: crate::telemetry::TARGET_TRANSFER_DOWNLOAD_OBJECTS,
tid = %self.inner.ctx.id,
discovered,
walk_done = walk.is_done(),
"download_objects walker advanced",
);
state.pending_entries.extend(chunk.drain(..));
if !walk.is_done() {
state.walk = Some(walk);
} else {
emit_event(
&self.inner.ctx,
DownloadObjectsEvent::ListingExhausted,
state.snapshot(),
);
}
}
if self.check_terminal(&state).is_some() {
drop(state);
return WorkOutcome::Success { data: None };
}
drop(state);
self.inner.ctx.try_wake();
WorkOutcome::Success { data: None }
}
async fn execute_join_children(&self, mut batch: ReapingBatch) -> WorkOutcome {
let children = batch.take_children();
for child in children {
let key = child.key;
let metrics = child.handle.metrics();
match child.handle.join().await {
Ok(_output) => {
let mut state = self.inner.state.lock();
state.successful_downloads += 1;
drop(state);
self.inner.ctx.metrics.record_io(&crate::metrics::IoSample {
network_tx: metrics.network_tx,
network_rx: metrics.network_rx,
disk_read: metrics.disk_read,
disk_write: metrics.disk_write,
});
}
Err(err) => {
let mut state = self.inner.state.lock();
let failed_input = DownloadInput::builder()
.bucket(&self.inner.bucket)
.key(&key)
.build()
.unwrap();
state.failed.push(FailedDownload {
input: failed_input,
error: err.with_local_key(key),
});
if self.inner.failure_policy == FailedTransferPolicy::Abort {
self.inner.ctx.set_failed(child_failed());
self.report_terminal(state.snapshot());
self.inner.ctx.signal_terminal();
}
}
}
}
let mut state = self.inner.state.lock();
batch.consume(&mut state);
emit_event(
&self.inner.ctx,
DownloadObjectsEvent::ReapCompleted,
state.snapshot(),
);
if self.check_terminal(&state).is_some() {
drop(state);
return WorkOutcome::Success { data: None };
}
drop(state);
self.inner.ctx.try_wake();
WorkOutcome::Success { data: None }
}
}
impl Transfer for DownloadObjectsTransfer {
fn ctx(&self) -> &TransferContext {
&self.inner.ctx
}
fn poll_work(&self) -> PollWork {
self.poll_work()
}
fn execute<'a>(
&'a self,
work: &'a mut IoRequest,
) -> Pin<Box<dyn std::future::Future<Output = WorkOutcome> + Send + 'a>> {
Box::pin(self.execute(work))
}
fn on_terminal(&self) {
let snapshot = self.inner.state.lock().snapshot();
self.report_terminal(snapshot);
}
}
fn child_failed() -> Error {
Error::new(ErrorKind::ChildOperationFailed, "download_objects aborted")
}
fn replace_delim<'a>(key: &'a str, delimiter: Option<&str>, path_separator: &str) -> Cow<'a, str> {
match delimiter {
Some(delim) if delim != path_separator => {
let replaced = key.replace(delim, path_separator);
Cow::Owned(replaced)
}
_ => Cow::Borrowed(key),
}
}
pub(crate) fn local_key_path(
root_dir: &Path,
key: &str,
delimiter: Option<&str>,
) -> Result<PathBuf, Error> {
let relative_path = replace_delim(key, delimiter, std::path::MAIN_SEPARATOR_STR);
let root_dir = root_dir.clean();
let local_path = root_dir.join(relative_path.as_ref()).clean();
validate_path(&root_dir, &local_path, key)?;
Ok(local_path)
}
fn validate_path(root_dir: &Path, local_path: &Path, key: &str) -> Result<(), Error> {
let rest = if root_dir == Path::new(".") {
Some(local_path)
} else {
local_path.strip_prefix(root_dir).ok()
};
let reason = match rest.map(|rest| rest.components().next()) {
Some(Some(Component::Normal(_))) => return Ok(()),
Some(None | Some(Component::CurDir)) => "resolves to the destination directory itself",
_ => "resolves outside the target destination directory",
};
Err(Error::new(
ErrorKind::InputInvalid,
format!("Unable to download key: '{key}', its relative path {reason}"),
))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::io::walk::{S3WalkContext, S3Walker};
use crate::operation::download::test_util::fixtures::head_rule;
use crate::transfer::TransferContext;
use aws_sdk_s3::operation::get_object::GetObjectOutput;
use aws_sdk_s3::operation::list_objects_v2::ListObjectsV2Output;
use aws_sdk_s3::types::Object;
use aws_smithy_mocks::{mock, mock_client, RuleMode};
use aws_smithy_types::error::display::DisplayErrorContext;
use std::time::Duration;
use tempfile::tempdir;
use tokio::time::timeout;
fn mock_s3_success() -> aws_sdk_s3::Client {
let list = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![
Object::builder().key("a.txt").size(5).build(),
Object::builder().key("b.txt").size(5).build(),
Object::builder().key("c.txt").size(5).build(),
]))
.build()
});
let get = mock!(aws_sdk_s3::Client::get_object).then_output(|| {
GetObjectOutput::builder()
.content_length(5)
.content_range("bytes 0-4/5")
.e_tag("test-etag")
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"hello"))
.build()
});
mock_client!(aws_sdk_s3, RuleMode::MatchAny, &[head_rule(5), list, get])
}
fn mock_s3_get_failure() -> aws_sdk_s3::Client {
let list = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![Object::builder().key("a.txt").size(5).build()]))
.build()
});
let get = mock!(aws_sdk_s3::Client::get_object).then_error(|| {
aws_sdk_s3::operation::get_object::GetObjectError::generic(
aws_sdk_s3::error::ErrorMetadata::builder()
.code("NoSuchKey")
.message("not found")
.build(),
)
});
mock_client!(aws_sdk_s3, RuleMode::MatchAny, &[head_rule(5), list, get])
}
fn mock_s3_list_failure() -> aws_sdk_s3::Client {
let list = mock!(aws_sdk_s3::Client::list_objects_v2).then_error(|| {
aws_sdk_s3::operation::list_objects_v2::ListObjectsV2Error::generic(
aws_sdk_s3::error::ErrorMetadata::builder()
.code("AccessDenied")
.message("access denied")
.build(),
)
});
mock_client!(aws_sdk_s3, RuleMode::MatchAny, &[list])
}
fn setup(
dest: &Path,
policy: FailedTransferPolicy,
s3_client: aws_sdk_s3::Client,
) -> (
DownloadObjectsTransfer,
crate::transfer::StateMachineTerminalReceiver,
) {
setup_with_walker(dest, policy, s3_client, S3Walker::builder().build())
}
fn setup_with_walker(
dest: &Path,
policy: FailedTransferPolicy,
s3_client: aws_sdk_s3::Client,
walker: S3Walker,
) -> (
DownloadObjectsTransfer,
crate::transfer::StateMachineTerminalReceiver,
) {
let config = crate::Config::builder()
.sdk_client(s3_client.clone())
.build()
.expect("valid config");
let handle = crate::client::Handle::test_handle_tokio(config);
let walk = walker.walk(
S3WalkContext::builder()
.sdk_client(s3_client)
.bucket("test-bucket")
.build(),
);
let (ctx, completion_rx) = TransferContext::new(handle);
ctx.handle
.scheduler
.register_empty_group_for_test(ctx.id.id);
let input = crate::operation::download_objects::DownloadObjectsInput::builder()
.bucket("test-bucket")
.destination(dest)
.failure_policy(policy)
.build()
.unwrap();
let transfer = DownloadObjectsTransfer::new(ctx, &input, walk, 1000);
(transfer, completion_rx)
}
async fn drive_transfer(transfer: &DownloadObjectsTransfer) {
loop {
match transfer.poll_work() {
PollWork::Ready { io: mut work, .. } => {
transfer.execute(&mut work).await;
}
PollWork::Spawned => {}
PollWork::Pending => {
tokio::time::sleep(Duration::from_millis(10)).await;
}
PollWork::Done => break,
}
}
}
fn setup_enqueued(
dest: &Path,
policy: FailedTransferPolicy,
s3_client: aws_sdk_s3::Client,
handle: std::sync::Arc<crate::client::Handle>,
) -> (
DownloadObjectsTransfer,
crate::transfer::StateMachineTerminalReceiver,
) {
let walk = S3Walker::builder().build().walk(
S3WalkContext::builder()
.sdk_client(s3_client)
.bucket("test-bucket")
.build(),
);
let (ctx, completion_rx) = TransferContext::new(handle.clone());
let input = crate::operation::download_objects::DownloadObjectsInput::builder()
.bucket("test-bucket")
.destination(dest)
.failure_policy(policy)
.build()
.unwrap();
let transfer = DownloadObjectsTransfer::new(ctx, &input, walk, 1000);
handle
.scheduler
.enqueue_transfer(Box::new(transfer.clone()));
(transfer, completion_rx)
}
#[test]
fn test_local_key_path_delimiters() {
for delim in ["/", "//", "\\", "|", "delim"] {
let key = format!("notes{delim}2021{delim}1.txt");
let actual = local_key_path(Path::new("test"), &key, Some(delim))
.unwrap_or_else(|e| panic!("delim={delim:?}: {e}"));
assert_eq!(
Path::new("test/notes/2021/1.txt"),
actual,
"delim={delim:?}"
);
}
}
#[cfg(target_family = "unix")]
#[test]
fn test_local_key_path_comprehensive() {
type Case = (
&'static str,
Option<&'static str>,
Result<&'static str, &'static str>,
);
let root = Path::new("test");
let cases: &[Case] = &[
("2023/Jan/1.png", None, Ok("test/2023/Jan/1.png")),
("2023-Jan-1.png", Some("-"), Ok("test/2023/Jan/1.png")),
("2023-Jan-.png", Some("-"), Ok("test/2023/Jan/.png")),
(
"many////delims-in-a-row",
Some("/"),
Ok("test/many/delims-in-a-row"),
),
("safe/../escape.bin", None, Ok("test/escape.bin")),
("../2023/Jan/1.png", None, Err("outside the target")),
("/2023/Jan/1.png", None, Err("outside the target")),
(
"foo/../2023/../../Jan/1.png",
None,
Err("outside the target"),
),
("../test-2/object.dat", None, Err("outside the target")),
];
for (key, delim, expected) in cases {
let actual = local_key_path(root, key, *delim);
match expected {
Ok(path) => {
let actual =
actual.unwrap_or_else(|e| panic!("expected Ok for key={key:?}: {e}"));
assert_eq!(Path::new(path), actual, "key={key:?}");
}
Err(_) => {
let err = actual.expect_err(&format!("expected Err for key={key:?}"));
assert_eq!(err.kind(), &ErrorKind::InputInvalid, "key={key:?}");
}
}
}
}
#[cfg(any(unix, windows))]
fn assert_dot_component_destinations(cases: &[(&str, &str)], escapes: &[&str]) {
for &(root, cleaned) in cases {
let root = Path::new(root);
for delim in [None, Some("/")] {
let actual = local_key_path(root, "a/b.txt", delim)
.unwrap_or_else(|e| panic!("root={root:?} delim={delim:?}: {e}"));
assert_eq!(
Path::new(cleaned).join("a/b.txt").clean(),
actual,
"root={root:?} delim={delim:?}"
);
for escape in escapes {
let err = local_key_path(root, escape, delim).expect_err(&format!(
"root={root:?} delim={delim:?} key={escape:?} must be rejected"
));
assert_eq!(err.kind(), &ErrorKind::InputInvalid, "key={escape:?}");
let message = DisplayErrorContext(&err).to_string();
assert!(
message.contains("outside the target destination"),
"root={root:?} key={escape:?}: {message}"
);
}
for key in [".", "a/..", "./"] {
let err = local_key_path(root, key, delim).expect_err(&format!(
"root={root:?} delim={delim:?} key={key:?} maps to the destination"
));
assert_eq!(err.kind(), &ErrorKind::InputInvalid, "key={key:?}");
let message = DisplayErrorContext(&err).to_string();
assert!(
message.contains("destination directory itself"),
"root={root:?} key={key:?}: {message}"
);
}
let actual = local_key_path(root, "photos/", delim)
.unwrap_or_else(|e| panic!("root={root:?} delim={delim:?}: {e}"));
assert_eq!(
Path::new(cleaned).join("photos").clean(),
actual,
"root={root:?} delim={delim:?}"
);
}
}
}
#[cfg(target_family = "unix")]
#[test]
fn test_local_key_path_destination_with_dot_components() {
assert_dot_component_destinations(
&[
("./out", "out"),
("out/../out", "out"),
("out/./", "out"),
("/data/./out", "/data/out"),
("/data/x/../out", "/data/out"),
(".", "."),
("./", "."),
("..", ".."),
("../out", "../out"),
],
&["../b.txt", "a/../../b.txt", "/etc/passwd"],
);
}
#[cfg(windows)]
#[test]
fn test_local_key_path_destination_with_dot_components_windows() {
assert_dot_component_destinations(
&[
(r".\out", "out"),
(r"out\..\out", "out"),
(r"out\.\", "out"),
(r"C:\data\.\out", r"C:\data\out"),
(r"C:\data\x\..\out", r"C:\data\out"),
(".", "."),
(r"..\out", r"..\out"),
],
&[
"../b.txt",
"a/../../b.txt",
"/etc/passwd",
r"\x",
"C:x",
"C:/x",
r"\\server\share\x",
],
);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_happy_path_3_files() {
let dir = tempdir().unwrap();
let (transfer, completion_rx) = setup(
dir.path(),
FailedTransferPolicy::Continue,
mock_s3_success(),
);
timeout(Duration::from_secs(10), async {
drive_transfer(&transfer).await;
let _ = completion_rx.await;
})
.await
.expect("transfer should complete within timeout");
assert_eq!(transfer.successful_downloads(), 3);
assert!(transfer.take_failed().is_none());
assert!(dir.path().join("a.txt").exists());
assert!(dir.path().join("b.txt").exists());
assert!(dir.path().join("c.txt").exists());
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_missing_destination_is_created() {
let dir = tempdir().unwrap();
let dest = dir.path().join("a/b/out");
let (transfer, completion_rx) =
setup(&dest, FailedTransferPolicy::Abort, mock_s3_success());
timeout(Duration::from_secs(10), async {
drive_transfer(&transfer).await;
let _ = completion_rx.await;
})
.await
.expect("transfer should complete within timeout");
assert!(
transfer.ctx().take_error().is_none(),
"a missing destination must not fail the transfer"
);
assert_eq!(transfer.successful_downloads(), 3);
for name in ["a.txt", "b.txt", "c.txt"] {
assert_eq!(std::fs::read(dest.join(name)).unwrap(), b"hello");
}
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_destination_that_is_a_file_is_input_invalid() {
let dir = tempdir().unwrap();
let dest = dir.path().join("out");
std::fs::write(&dest, b"not a directory").unwrap();
let get = mock!(aws_sdk_s3::Client::get_object).then_output(|| {
GetObjectOutput::builder()
.content_length(5)
.content_range("bytes 0-4/5")
.e_tag("test-etag")
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"hello"))
.build()
});
let list = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![Object::builder().key("a.txt").size(5).build()]))
.build()
});
let client = mock_client!(
aws_sdk_s3,
RuleMode::MatchAny,
&[head_rule(5), list, get.clone()]
);
let (transfer, completion_rx) = setup(&dest, FailedTransferPolicy::Continue, client);
timeout(Duration::from_secs(10), async {
drive_transfer(&transfer).await;
let _ = completion_rx.await;
})
.await
.expect("transfer should complete within timeout");
let err = transfer
.ctx()
.take_error()
.expect("a file destination must fail the transfer");
assert_eq!(err.kind(), &ErrorKind::InputInvalid);
assert_eq!(get.num_calls(), 0);
assert_eq!(std::fs::read(&dest).unwrap(), b"not a directory");
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_child_failure_continue() {
let dir = tempdir().unwrap();
let (transfer, completion_rx) = setup(
dir.path(),
FailedTransferPolicy::Continue,
mock_s3_get_failure(),
);
timeout(Duration::from_secs(10), async {
drive_transfer(&transfer).await;
let _ = completion_rx.await;
})
.await
.expect("transfer should complete within timeout");
assert_eq!(transfer.successful_downloads(), 0);
let failed = transfer.take_failed().unwrap();
assert_eq!(failed.len(), 1);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_child_failure_abort() {
let dir = tempdir().unwrap();
let (transfer, completion_rx) = setup(
dir.path(),
FailedTransferPolicy::Abort,
mock_s3_get_failure(),
);
timeout(Duration::from_secs(10), async {
drive_transfer(&transfer).await;
let _ = completion_rx.await;
})
.await
.expect("transfer should complete within timeout");
assert_eq!(transfer.successful_downloads(), 0);
assert!(transfer.inner.ctx.is_failed());
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_fatal_walker_error() {
let dir = tempdir().unwrap();
let (transfer, completion_rx) = setup(
dir.path(),
FailedTransferPolicy::Continue,
mock_s3_list_failure(),
);
timeout(Duration::from_secs(10), async {
drive_transfer(&transfer).await;
let _ = completion_rx.await;
})
.await
.expect("transfer should complete within timeout");
assert!(transfer.inner.ctx.is_failed());
assert_eq!(transfer.successful_downloads(), 0);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_empty_listing() {
let dir = tempdir().unwrap();
let list = mock!(aws_sdk_s3::Client::list_objects_v2)
.then_output(|| ListObjectsV2Output::builder().build());
let client = mock_client!(aws_sdk_s3, RuleMode::MatchAny, &[list]);
let (transfer, completion_rx) = setup(dir.path(), FailedTransferPolicy::Continue, client);
timeout(Duration::from_secs(10), async {
drive_transfer(&transfer).await;
let _ = completion_rx.await;
})
.await
.expect("transfer should complete within timeout");
assert_eq!(transfer.successful_downloads(), 0);
assert!(transfer.take_failed().is_none());
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn pending_reason_follows_parent_state() {
let dir = tempdir().unwrap();
let (transfer, _completion_rx) = setup(
dir.path(),
FailedTransferPolicy::Continue,
mock_s3_success(),
);
let mut state = transfer.inner.state.lock();
assert_eq!(
state.pending_reason(false),
CompositePendingReason::Draining
);
assert_eq!(state.pending_reason(true), CompositePendingReason::Listing);
state.reaping_in_flight = 1;
assert_eq!(state.pending_reason(true), CompositePendingReason::Reaping);
state.reaping_in_flight = 0;
state
.pending_entries
.push_back(crate::model::Object::builder().key("a.txt").build());
assert_eq!(
state.pending_reason(true),
CompositePendingReason::ChildCapacity
);
state.pending_entries.clear();
let walk = state.walk.take();
state.walk_in_flight = true;
assert_eq!(state.pending_reason(true), CompositePendingReason::Listing);
state.walk_in_flight = false;
assert_eq!(
state.pending_reason(true),
CompositePendingReason::ChildCompletion
);
state.walk = walk;
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn terminal_summary_reported_once_on_success() {
let dir = tempdir().unwrap();
let (transfer, completion_rx) = setup(
dir.path(),
FailedTransferPolicy::Continue,
mock_s3_success(),
);
timeout(Duration::from_secs(10), async {
drive_transfer(&transfer).await;
let _ = completion_rx.await;
})
.await
.expect("transfer should complete within timeout");
let summary = transfer
.test_terminal_summary()
.expect("terminal summary reported");
assert_eq!(summary.outcome, crate::transfer::TerminalOutcome::Completed);
assert_eq!(summary.successful, 3);
assert_eq!(summary.failed, 0);
transfer.inner.state.lock().successful_downloads = 99;
Transfer::on_terminal(&transfer);
let summary = transfer
.test_terminal_summary()
.expect("terminal summary reported");
assert_eq!(summary.successful, 3, "summary reported more than once");
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn terminal_summary_reported_once_on_failure() {
let dir = tempdir().unwrap();
let (transfer, completion_rx) = setup(
dir.path(),
FailedTransferPolicy::Continue,
mock_s3_list_failure(),
);
timeout(Duration::from_secs(10), async {
drive_transfer(&transfer).await;
let _ = completion_rx.await;
})
.await
.expect("transfer should complete within timeout");
let summary = transfer
.test_terminal_summary()
.expect("terminal summary reported");
assert_eq!(summary.outcome, crate::transfer::TerminalOutcome::Failed);
assert!(summary.error_kind.is_some());
assert_eq!(summary.successful, 0);
transfer.inner.state.lock().successful_downloads = 99;
Transfer::on_terminal(&transfer);
let summary = transfer
.test_terminal_summary()
.expect("terminal summary reported");
assert_eq!(summary.successful, 0, "summary reported more than once");
}
struct NeverBody;
impl http_body_1x::Body for NeverBody {
type Data = bytes::Bytes;
type Error = std::convert::Infallible;
fn poll_frame(
self: std::pin::Pin<&mut Self>,
_cx: &mut std::task::Context<'_>,
) -> std::task::Poll<Option<Result<http_body_1x::Frame<Self::Data>, Self::Error>>> {
std::task::Poll::Pending
}
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn terminal_summary_reported_on_listing_failure_with_children_in_flight() {
let dir = tempdir().unwrap();
let page1 = mock!(aws_sdk_s3::Client::list_objects_v2)
.match_requests(|r| r.continuation_token().is_none())
.then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(
(0..MAX_ENTRIES_PER_WALK)
.map(|i| Object::builder().key(format!("k{i:04}")).size(5).build())
.collect(),
))
.is_truncated(true)
.next_continuation_token("page-2")
.build()
});
let page2 = mock!(aws_sdk_s3::Client::list_objects_v2)
.match_requests(|r| r.continuation_token() == Some("page-2"))
.then_error(|| {
std::thread::sleep(Duration::from_millis(300));
aws_sdk_s3::operation::list_objects_v2::ListObjectsV2Error::generic(
aws_sdk_s3::error::ErrorMetadata::builder()
.code("AccessDenied")
.message("access denied")
.build(),
)
});
let get = mock!(aws_sdk_s3::Client::get_object).then_output(|| {
GetObjectOutput::builder()
.content_length(5)
.content_range("bytes 0-4/5")
.e_tag("test-etag")
.body(aws_sdk_s3::primitives::ByteStream::new(
aws_smithy_types::body::SdkBody::from_body_1_x(NeverBody),
))
.build()
});
let s3_client = mock_client!(
aws_sdk_s3,
RuleMode::MatchAny,
&[head_rule(5), page1, page2, get]
);
let config = crate::Config::builder()
.sdk_client(s3_client.clone())
.build()
.expect("valid config");
let handle = crate::client::Handle::test_handle_managed(config);
let (transfer, completion_rx) = setup_enqueued(
dir.path(),
FailedTransferPolicy::Continue,
s3_client,
handle,
);
timeout(Duration::from_secs(10), async {
let _ = completion_rx.await;
})
.await
.expect("transfer should end within timeout");
assert!(transfer.ctx().is_failed());
assert!(
!transfer.inner.state.lock().children.is_empty(),
"the parent ended with children still recorded as running"
);
let summary = transfer
.test_terminal_summary()
.expect("terminal summary reported");
assert_eq!(summary.outcome, crate::transfer::TerminalOutcome::Failed);
crate::runtime::test_util::shutdown(&transfer.ctx().handle).await;
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn terminal_summary_reported_once_on_cancel() {
let dir = tempdir().unwrap();
let (transfer, _completion_rx) = setup(
dir.path(),
FailedTransferPolicy::Continue,
mock_s3_success(),
);
transfer.inner.ctx.set_cancelled();
Transfer::on_terminal(&transfer);
let summary = transfer
.test_terminal_summary()
.expect("terminal summary reported");
assert_eq!(summary.outcome, crate::transfer::TerminalOutcome::Cancelled);
assert_eq!(summary.successful, 0);
transfer.inner.state.lock().successful_downloads = 99;
Transfer::on_terminal(&transfer);
let summary = transfer
.test_terminal_summary()
.expect("terminal summary reported");
assert_eq!(summary.successful, 0, "summary reported more than once");
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_path_traversal_rejected_as_failure() {
let dir = tempdir().unwrap();
let list = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![Object::builder()
.key("../../../etc/passwd")
.size(100)
.build()]))
.build()
});
let client = mock_client!(aws_sdk_s3, RuleMode::MatchAny, &[list]);
let (transfer, completion_rx) = setup(dir.path(), FailedTransferPolicy::Continue, client);
timeout(Duration::from_secs(10), async {
drive_transfer(&transfer).await;
let _ = completion_rx.await;
})
.await
.expect("transfer should complete within timeout");
assert_eq!(transfer.successful_downloads(), 0);
let failed = transfer.take_failed().unwrap();
assert_eq!(failed.len(), 1);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_nested_keys_create_directories() {
let dir = tempdir().unwrap();
let list = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![
Object::builder()
.key("photos/2023/jan/img.jpg")
.size(5)
.build(),
Object::builder().key("docs/readme.txt").size(5).build(),
]))
.build()
});
let get = mock!(aws_sdk_s3::Client::get_object).then_output(|| {
GetObjectOutput::builder()
.content_length(5)
.content_range("bytes 0-4/5")
.e_tag("test-etag")
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"hello"))
.build()
});
let client = mock_client!(aws_sdk_s3, RuleMode::MatchAny, &[head_rule(5), list, get]);
let (transfer, completion_rx) = setup(dir.path(), FailedTransferPolicy::Continue, client);
timeout(Duration::from_secs(10), async {
drive_transfer(&transfer).await;
let _ = completion_rx.await;
})
.await
.expect("transfer should complete within timeout");
assert_eq!(transfer.successful_downloads(), 2);
assert!(dir.path().join("photos/2023/jan/img.jpg").exists());
assert!(dir.path().join("docs/readme.txt").exists());
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_parent_directory_failure_names_directory_and_key() {
let dir = tempdir().unwrap();
std::fs::write(dir.path().join("docs"), b"a file, not a directory").unwrap();
let list = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![Object::builder()
.key("docs/readme.txt")
.size(5)
.build()]))
.build()
});
let client = mock_client!(aws_sdk_s3, RuleMode::MatchAny, &[list]);
let (transfer, completion_rx) = setup(dir.path(), FailedTransferPolicy::Continue, client);
timeout(Duration::from_secs(10), async {
drive_transfer(&transfer).await;
let _ = completion_rx.await;
})
.await
.expect("transfer should complete within timeout");
let failed = transfer.take_failed().expect("the key must fail");
let err = failed[0].error();
assert_eq!(err.kind(), &ErrorKind::IOError);
assert_eq!(
err.to_string(),
format!(
"I/O error creating directory \"{}\" (key \"docs/readme.txt\")",
dir.path().join("docs").display()
)
);
assert!(std::error::Error::source(err)
.and_then(|s| s.downcast_ref::<std::io::Error>())
.is_some());
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_key_mapping_to_destination_fails_before_any_file_is_created() {
let dir = tempdir().unwrap();
let dest = dir.path().join("out");
let keys = [".", "./", "a/.."];
let get = mock!(aws_sdk_s3::Client::get_object).then_output(|| {
GetObjectOutput::builder()
.content_length(5)
.content_range("bytes 0-4/5")
.e_tag("test-etag")
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"hello"))
.build()
});
let list = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(move || {
ListObjectsV2Output::builder()
.set_contents(Some(
keys.iter()
.map(|key| Object::builder().key(*key).size(5).build())
.collect(),
))
.build()
});
let client = mock_client!(
aws_sdk_s3,
RuleMode::MatchAny,
&[head_rule(5), list, get.clone()]
);
let (transfer, completion_rx) = setup(&dest, FailedTransferPolicy::Continue, client);
timeout(Duration::from_secs(10), async {
drive_transfer(&transfer).await;
let _ = completion_rx.await;
})
.await
.expect("transfer should complete within timeout");
assert!(transfer.ctx().take_error().is_none());
let failed = transfer.take_failed().expect("the keys must fail");
let mut failed_keys: Vec<_> = failed.iter().map(|f| f.input().key()).collect();
failed_keys.sort();
assert_eq!(failed_keys, [Some("."), Some("./"), Some("a/..")]);
for f in &failed {
assert_eq!(f.error().kind(), &ErrorKind::InputInvalid);
}
assert_eq!(get.num_calls(), 0);
let entries: Vec<_> = std::fs::read_dir(dir.path())
.unwrap()
.map(|e| e.unwrap().file_name())
.collect();
assert_eq!(
entries,
["out"],
"nothing may be created beside the destination"
);
assert_eq!(std::fs::read_dir(&dest).unwrap().count(), 0);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_key_equal_to_prefix_downloads_to_its_last_segment() {
let dir = tempdir().unwrap();
let list = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![Object::builder().key("photos/").size(5).build()]))
.build()
});
let get = mock!(aws_sdk_s3::Client::get_object).then_output(|| {
GetObjectOutput::builder()
.content_length(5)
.content_range("bytes 0-4/5")
.e_tag("test-etag")
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"hello"))
.build()
});
let client = mock_client!(aws_sdk_s3, RuleMode::MatchAny, &[head_rule(5), list, get]);
let (transfer, completion_rx) = setup_with_walker(
dir.path(),
FailedTransferPolicy::Abort,
client,
S3Walker::builder().prefix("photos/").build(),
);
timeout(Duration::from_secs(10), async {
drive_transfer(&transfer).await;
let _ = completion_rx.await;
})
.await
.expect("transfer should complete within timeout");
assert!(transfer.ctx().take_error().is_none());
assert!(transfer.take_failed().is_none());
assert_eq!(transfer.successful_downloads(), 1);
assert_eq!(std::fs::read(dir.path().join("photos")).unwrap(), b"hello");
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_rename_failure_names_path_and_key() {
let dir = tempdir().unwrap();
let blocked = dir.path().join("a.txt");
std::fs::create_dir(&blocked).unwrap();
std::fs::write(blocked.join("inner"), b"kept").unwrap();
let (transfer, completion_rx) = setup(
dir.path(),
FailedTransferPolicy::Continue,
mock_s3_success(),
);
timeout(Duration::from_secs(10), async {
drive_transfer(&transfer).await;
let _ = completion_rx.await;
})
.await
.expect("transfer should complete within timeout");
assert_eq!(transfer.successful_downloads(), 2);
let failed = transfer.take_failed().expect("a.txt must fail");
assert_eq!(failed.len(), 1);
assert_eq!(failed[0].input().key(), Some("a.txt"));
let err = failed[0].error();
assert_eq!(err.kind(), &ErrorKind::IOError);
assert_eq!(
err.to_string(),
format!(
"I/O error renaming temporary file to \"{}\" (key \"a.txt\")",
blocked.display()
)
);
assert!(std::error::Error::source(err)
.and_then(|s| s.downcast_ref::<std::io::Error>())
.is_some());
assert_eq!(std::fs::read(blocked.join("inner")).unwrap(), b"kept");
let mut entries: Vec<_> = std::fs::read_dir(dir.path())
.unwrap()
.map(|e| e.unwrap().file_name())
.collect();
entries.sort();
assert_eq!(
entries,
["a.txt", "b.txt", "c.txt"],
"the failed child's temporary file must be removed"
);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_walker_prefix_keeps_full_key_in_local_path() {
let dir = tempdir().unwrap();
let list = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![
Object::builder().key("backup/2023/a.txt").size(3).build(),
Object::builder().key("backup/2023/b.txt").size(3).build(),
]))
.build()
});
let get = mock!(aws_sdk_s3::Client::get_object).then_output(|| {
GetObjectOutput::builder()
.content_length(3)
.content_range("bytes 0-2/3")
.e_tag("test-etag")
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"abc"))
.build()
});
let client = mock_client!(aws_sdk_s3, RuleMode::MatchAny, &[head_rule(3), list, get]);
let (transfer, completion_rx) = setup_with_walker(
dir.path(),
FailedTransferPolicy::Continue,
client,
S3Walker::builder().prefix("backup/2023/").build(),
);
timeout(Duration::from_secs(10), async {
drive_transfer(&transfer).await;
let _ = completion_rx.await;
})
.await
.expect("transfer should complete within timeout");
assert_eq!(transfer.successful_downloads(), 2);
assert!(transfer.take_failed().is_none());
assert_eq!(
std::fs::read(dir.path().join("backup/2023/a.txt")).unwrap(),
b"abc"
);
assert_eq!(
std::fs::read(dir.path().join("backup/2023/b.txt")).unwrap(),
b"abc"
);
let entries: Vec<_> = std::fs::read_dir(dir.path())
.unwrap()
.map(|e| e.unwrap().file_name())
.collect();
assert_eq!(entries, ["backup"]);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_many_objects() {
let dir = tempdir().unwrap();
let objects: Vec<Object> = (0..20)
.map(|i| {
Object::builder()
.key(format!("file{i:02}.txt"))
.size(2)
.build()
})
.collect();
let list = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(move || {
ListObjectsV2Output::builder()
.set_contents(Some(objects.clone()))
.build()
});
let get = mock!(aws_sdk_s3::Client::get_object).then_output(|| {
GetObjectOutput::builder()
.content_length(2)
.content_range("bytes 0-1/2")
.e_tag("test-etag")
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"ok"))
.build()
});
let client = mock_client!(aws_sdk_s3, RuleMode::MatchAny, &[head_rule(2), list, get]);
let (transfer, completion_rx) = setup(dir.path(), FailedTransferPolicy::Continue, client);
timeout(Duration::from_secs(10), async {
drive_transfer(&transfer).await;
let _ = completion_rx.await;
})
.await
.expect("transfer should complete within timeout");
assert_eq!(transfer.successful_downloads(), 20);
for i in 0..20 {
assert!(dir.path().join(format!("file{i:02}.txt")).exists());
}
assert_eq!(
transfer.ctx().transfer_status(),
crate::types::TransferStatus::Completed
);
let m = transfer.ctx().metrics();
assert_eq!(
m.network_rx, 40,
"child network_rx must aggregate to parent"
);
assert_eq!(
m.disk_write, 40,
"child disk_write must aggregate to parent"
);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_cancellation() {
let dir = tempdir().unwrap();
let objects: Vec<Object> = (0..100)
.map(|i| {
Object::builder()
.key(format!("file{i}.txt"))
.size(5)
.build()
})
.collect();
let list = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(move || {
ListObjectsV2Output::builder()
.set_contents(Some(objects.clone()))
.build()
});
let get = mock!(aws_sdk_s3::Client::get_object).then_output(|| {
GetObjectOutput::builder()
.content_length(5)
.content_range("bytes 0-4/5")
.e_tag("test-etag")
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"hello"))
.build()
});
let client = mock_client!(aws_sdk_s3, RuleMode::MatchAny, &[head_rule(5), list, get]);
let (transfer, _completion_rx) = setup(dir.path(), FailedTransferPolicy::Continue, client);
transfer.inner.ctx.set_cancelled();
timeout(Duration::from_secs(10), async {
drive_transfer(&transfer).await;
})
.await
.expect("cancelled transfer should terminate");
assert!(!transfer.inner.ctx.is_active());
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_happy_path_managed_runtime() {
let dir = tempdir().unwrap();
let config = crate::Config::builder()
.sdk_client(mock_s3_success())
.build()
.expect("valid config");
let handle = crate::client::Handle::test_handle_managed(config);
let (transfer, completion_rx) = setup_enqueued(
dir.path(),
FailedTransferPolicy::Continue,
mock_s3_success(),
handle,
);
timeout(Duration::from_secs(10), async {
let _ = completion_rx.await;
})
.await
.expect("transfer should complete within timeout");
assert_eq!(transfer.successful_downloads(), 3);
assert!(dir.path().join("a.txt").exists());
assert!(dir.path().join("b.txt").exists());
assert!(dir.path().join("c.txt").exists());
crate::runtime::test_util::shutdown(&transfer.ctx().handle).await;
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_many_objects_managed_runtime() {
let dir = tempdir().unwrap();
let objects: Vec<Object> = (0..20)
.map(|i| {
Object::builder()
.key(format!("file{i:02}.txt"))
.size(2)
.build()
})
.collect();
let list = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(move || {
ListObjectsV2Output::builder()
.set_contents(Some(objects.clone()))
.build()
});
let get = mock!(aws_sdk_s3::Client::get_object).then_output(|| {
GetObjectOutput::builder()
.content_length(2)
.content_range("bytes 0-1/2")
.e_tag("test-etag")
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"ok"))
.build()
});
let s3_client = mock_client!(aws_sdk_s3, RuleMode::MatchAny, &[head_rule(2), list, get]);
let config = crate::Config::builder()
.sdk_client(s3_client.clone())
.build()
.expect("valid config");
let handle = crate::client::Handle::test_handle_managed(config);
let (transfer, completion_rx) = setup_enqueued(
dir.path(),
FailedTransferPolicy::Continue,
s3_client,
handle,
);
timeout(Duration::from_secs(10), async {
let _ = completion_rx.await;
})
.await
.expect("transfer should complete within timeout");
assert_eq!(transfer.successful_downloads(), 20);
for i in 0..20 {
assert!(dir.path().join(format!("file{i:02}.txt")).exists());
}
assert_eq!(
transfer.ctx().transfer_status(),
crate::types::TransferStatus::Completed
);
let m = transfer.ctx().metrics();
assert_eq!(
m.network_rx, 40,
"child network_rx must aggregate to parent"
);
assert_eq!(
m.disk_write, 40,
"child disk_write must aggregate to parent"
);
crate::runtime::test_util::shutdown(&transfer.ctx().handle).await;
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_child_failure_abort_managed_runtime_does_not_strand() {
let dir = tempdir().unwrap();
let objects: Vec<Object> = (0..20)
.map(|i| {
Object::builder()
.key(format!("file{i:02}.txt"))
.size(2)
.build()
})
.collect();
let list = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(move || {
ListObjectsV2Output::builder()
.set_contents(Some(objects.clone()))
.build()
});
let get = mock!(aws_sdk_s3::Client::get_object).then_error(|| {
aws_sdk_s3::operation::get_object::GetObjectError::generic(
aws_sdk_s3::error::ErrorMetadata::builder()
.code("NoSuchKey")
.message("not found")
.build(),
)
});
let s3_client = mock_client!(aws_sdk_s3, RuleMode::MatchAny, &[head_rule(2), list, get]);
let config = crate::Config::builder()
.sdk_client(s3_client.clone())
.build()
.expect("valid config");
let handle = crate::client::Handle::test_handle_managed(config);
let (transfer, completion_rx) =
setup_enqueued(dir.path(), FailedTransferPolicy::Abort, s3_client, handle);
timeout(Duration::from_secs(10), async {
let _ = completion_rx.await;
})
.await
.expect("Abort failure must signal parent terminal, not strand join()");
assert!(transfer.ctx().is_failed());
let ctx = transfer.ctx();
timeout(
Duration::from_secs(10),
ctx.handle.scheduler.cancel_transfer(ctx.id).wait_for_idle(),
)
.await
.expect("failed parent and child work did not drain");
crate::runtime::test_util::shutdown(&ctx.handle).await;
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_two_failed_spawns_continue_managed_runtime_does_not_hang() {
let dir = tempdir().unwrap();
let list = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![
Object::builder().key("../../../etc/passwd").size(2).build(),
Object::builder().key("../../../etc/shadow").size(2).build(),
]))
.build()
});
let get = mock!(aws_sdk_s3::Client::get_object).then_output(|| {
GetObjectOutput::builder()
.content_length(2)
.content_range("bytes 0-1/2")
.e_tag("test-etag")
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"ok"))
.build()
});
let s3_client = mock_client!(aws_sdk_s3, RuleMode::MatchAny, &[head_rule(2), list, get]);
let config = crate::Config::builder()
.sdk_client(s3_client.clone())
.build()
.expect("valid config");
let handle = crate::client::Handle::test_handle_managed(config);
let (transfer, completion_rx) = setup_enqueued(
dir.path(),
FailedTransferPolicy::Continue,
s3_client,
handle,
);
timeout(Duration::from_secs(10), async {
let _ = completion_rx.await;
})
.await
.expect("two claimed-but-failed spawns must not strand the parent's completion");
assert_eq!(transfer.successful_downloads(), 0);
assert_eq!(
transfer.take_failed().map(|f| f.len()).unwrap_or(0),
2,
"both failed entries must be recorded"
);
crate::runtime::test_util::shutdown(&transfer.ctx().handle).await;
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_fatal_walker_error_managed_runtime() {
let dir = tempdir().unwrap();
let s3_client = mock_s3_list_failure();
let config = crate::Config::builder()
.sdk_client(s3_client.clone())
.build()
.expect("valid config");
let handle = crate::client::Handle::test_handle_managed(config);
let (transfer, completion_rx) = setup_enqueued(
dir.path(),
FailedTransferPolicy::Continue,
s3_client,
handle,
);
timeout(Duration::from_secs(10), async {
let _ = completion_rx.await;
})
.await
.expect("transfer should complete within timeout");
assert_eq!(transfer.successful_downloads(), 0);
assert!(transfer.ctx().is_failed());
crate::runtime::test_util::shutdown(&transfer.ctx().handle).await;
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_reservation_consume_decrements_counter() {
let dir = tempdir().unwrap();
let (transfer, _rx) = setup(
dir.path(),
FailedTransferPolicy::Continue,
mock_s3_success(),
);
{
let mut state = transfer.inner.state.lock();
state.children_reserved = 5;
}
let reservation = Reservation {
transfer: transfer.inner.clone(),
count: 5,
consumed: false,
};
{
let mut state = transfer.inner.state.lock();
reservation.consume(&mut state);
assert_eq!(0, state.children_reserved, "consume must decrement");
}
let state = transfer.inner.state.lock();
assert_eq!(
0, state.children_reserved,
"Drop after consume must not double-decrement"
);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_reservation_drop_without_consume_recovers_counter() {
let dir = tempdir().unwrap();
let (transfer, _rx) = setup(
dir.path(),
FailedTransferPolicy::Continue,
mock_s3_success(),
);
{
let mut state = transfer.inner.state.lock();
state.children_reserved = 5;
}
{
let _reservation = Reservation {
transfer: transfer.inner.clone(),
count: 5,
consumed: false,
};
}
let state = transfer.inner.state.lock();
assert_eq!(
0, state.children_reserved,
"Drop on unconsumed Reservation must recover the counter"
);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_reservation_drop_underflow_saturates() {
let dir = tempdir().unwrap();
let (transfer, _rx) = setup(
dir.path(),
FailedTransferPolicy::Continue,
mock_s3_success(),
);
{
let _reservation = Reservation {
transfer: transfer.inner.clone(),
count: 5,
consumed: false,
};
}
let state = transfer.inner.state.lock();
assert_eq!(0, state.children_reserved, "Drop must saturate, not panic");
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_reaping_batch_consume_decrements_counter() {
let dir = tempdir().unwrap();
let (transfer, _rx) = setup(
dir.path(),
FailedTransferPolicy::Continue,
mock_s3_success(),
);
{
let mut state = transfer.inner.state.lock();
state.reaping_in_flight = 3;
}
let batch = ReapingBatch {
transfer: transfer.inner.clone(),
children: Some(Vec::new()),
count: 3,
consumed: false,
};
{
let mut state = transfer.inner.state.lock();
batch.consume(&mut state);
assert_eq!(0, state.reaping_in_flight, "consume must decrement");
}
let state = transfer.inner.state.lock();
assert_eq!(
0, state.reaping_in_flight,
"Drop after consume must not double-decrement"
);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_reaping_batch_drop_without_consume_recovers_counter() {
let dir = tempdir().unwrap();
let (transfer, _rx) = setup(
dir.path(),
FailedTransferPolicy::Continue,
mock_s3_success(),
);
{
let mut state = transfer.inner.state.lock();
state.reaping_in_flight = 3;
}
{
let _batch = ReapingBatch {
transfer: transfer.inner.clone(),
children: Some(Vec::new()),
count: 3,
consumed: false,
};
}
let state = transfer.inner.state.lock();
assert_eq!(
0, state.reaping_in_flight,
"Drop on unconsumed ReapingBatch must recover the counter"
);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_reaping_batch_take_children_then_consume() {
let dir = tempdir().unwrap();
let (transfer, _rx) = setup(
dir.path(),
FailedTransferPolicy::Continue,
mock_s3_success(),
);
{
let mut state = transfer.inner.state.lock();
state.reaping_in_flight = 2;
}
let mut batch = ReapingBatch {
transfer: transfer.inner.clone(),
children: Some(Vec::new()),
count: 2,
consumed: false,
};
let _children = batch.take_children();
{
let mut state = transfer.inner.state.lock();
batch.consume(&mut state);
assert_eq!(0, state.reaping_in_flight);
}
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_single_ticket_500_objects_completes() {
let dir = tempdir().unwrap();
let objects: Vec<Object> = (0..500)
.map(|i| {
Object::builder()
.key(format!("obj{i:04}.bin"))
.size(4)
.build()
})
.collect();
let list = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(move || {
ListObjectsV2Output::builder()
.set_contents(Some(objects.clone()))
.build()
});
let get = mock!(aws_sdk_s3::Client::get_object).then_output(|| {
GetObjectOutput::builder()
.content_length(4)
.content_range("bytes 0-3/4")
.e_tag("test-etag")
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"data"))
.build()
});
let client = mock_client!(aws_sdk_s3, RuleMode::MatchAny, &[head_rule(4), list, get]);
let config = crate::Config::builder()
.sdk_client(client.clone())
.build()
.expect("valid config");
let handle = crate::client::Handle::test_handle_managed(config);
let (transfer, completion_rx) =
setup_enqueued(dir.path(), FailedTransferPolicy::Continue, client, handle);
timeout(Duration::from_secs(30), async {
let _ = completion_rx.await;
})
.await
.expect("500-object transfer must complete without stalling");
assert_eq!(transfer.successful_downloads(), 500);
for i in 0..500 {
assert!(
dir.path().join(format!("obj{i:04}.bin")).exists(),
"object {i} must be present on disk"
);
}
crate::runtime::test_util::shutdown(&transfer.ctx().handle).await;
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_backstop_limits_in_flight_children() {
let dir = tempdir().unwrap();
let max_concurrent: usize = 4;
let objects: Vec<Object> = (0..20)
.map(|i| {
Object::builder()
.key(format!("file{i:02}.txt"))
.size(3)
.build()
})
.collect();
let list = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(move || {
ListObjectsV2Output::builder()
.set_contents(Some(objects.clone()))
.build()
});
let get = mock!(aws_sdk_s3::Client::get_object).then_output(|| {
GetObjectOutput::builder()
.content_length(3)
.content_range("bytes 0-2/3")
.e_tag("test-etag")
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"abc"))
.build()
});
let client = mock_client!(aws_sdk_s3, RuleMode::MatchAny, &[head_rule(3), list, get]);
let config = crate::Config::builder()
.sdk_client(client.clone())
.build()
.expect("valid config");
let handle = crate::client::Handle::test_handle_tokio(config);
let walk = S3Walker::builder().build().walk(
S3WalkContext::builder()
.sdk_client(client)
.bucket("test-bucket")
.build(),
);
let (ctx, completion_rx) = TransferContext::new(handle);
ctx.handle
.scheduler
.register_empty_group_for_test(ctx.id.id);
let input = crate::operation::download_objects::DownloadObjectsInput::builder()
.bucket("test-bucket")
.destination(dir.path())
.failure_policy(FailedTransferPolicy::Continue)
.build()
.unwrap();
let transfer = DownloadObjectsTransfer::new(ctx, &input, walk, max_concurrent);
let mut max_observed: usize = 0;
timeout(Duration::from_secs(10), async {
loop {
{
let state = transfer.inner.state.lock();
let active = state
.children
.values()
.filter(|c| c.handle.status() == crate::types::TransferStatus::Active)
.count()
+ state.children_reserved;
if active > max_observed {
max_observed = active;
}
}
match transfer.poll_work() {
PollWork::Ready { io: mut work, .. } => {
transfer.execute(&mut work).await;
}
PollWork::Spawned => {}
PollWork::Pending => {
tokio::time::sleep(Duration::from_millis(5)).await;
}
PollWork::Done => break,
}
}
let _ = completion_rx.await;
})
.await
.expect("transfer should complete within timeout");
assert_eq!(transfer.successful_downloads(), 20);
assert!(
max_observed <= max_concurrent,
"in-flight children ({max_observed}) must not exceed backstop ({max_concurrent})"
);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_reap_is_continuous_not_batched() {
let dir = tempdir().unwrap();
let objects: Vec<Object> = (0..300)
.map(|i| {
Object::builder()
.key(format!("f{i:04}.txt"))
.size(2)
.build()
})
.collect();
let list = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(move || {
ListObjectsV2Output::builder()
.set_contents(Some(objects.clone()))
.build()
});
let get = mock!(aws_sdk_s3::Client::get_object).then_output(|| {
GetObjectOutput::builder()
.content_length(2)
.content_range("bytes 0-1/2")
.e_tag("test-etag")
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"ok"))
.build()
});
let client = mock_client!(aws_sdk_s3, RuleMode::MatchAny, &[head_rule(2), list, get]);
let (transfer, completion_rx) = setup(dir.path(), FailedTransferPolicy::Continue, client);
let mut fused_reap_spawn = 0usize;
let mut max_terminal_backlog = 0usize;
timeout(Duration::from_secs(30), async {
loop {
let terminal_before = {
let state = transfer.inner.state.lock();
state
.children
.values()
.filter(|c| c.handle.status() != crate::types::TransferStatus::Active)
.count()
};
max_terminal_backlog = max_terminal_backlog.max(terminal_before);
match transfer.poll_work() {
PollWork::Ready {
io: mut work,
spawned,
} => {
if spawned {
fused_reap_spawn += 1;
}
transfer.execute(&mut work).await;
}
PollWork::Spawned => {}
PollWork::Pending => {
tokio::time::sleep(Duration::from_millis(5)).await;
}
PollWork::Done => break,
}
}
let _ = completion_rx.await;
})
.await
.expect("transfer should complete within timeout");
assert_eq!(transfer.successful_downloads(), 300);
let _ = (max_terminal_backlog, fused_reap_spawn);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_dispatch_walk_gates_on_key_buffer_not_children() {
let dir = tempdir().unwrap();
{
let (transfer, _rx) = setup(
dir.path(),
FailedTransferPolicy::Continue,
mock_s3_success(),
);
let mut state = transfer.inner.state.lock();
assert!(state.pending_entries.len() < WALK_LOW_WATER);
assert!(
transfer.dispatch_walk(&mut state).is_some(),
"should dispatch a walk when the key buffer is below low-water"
);
}
{
let (transfer, _rx) = setup(
dir.path(),
FailedTransferPolicy::Continue,
mock_s3_success(),
);
let mut state = transfer.inner.state.lock();
for i in 0..WALK_LOW_WATER {
state.pending_entries.push_back(
crate::model::Object::builder()
.key(format!("k{i}"))
.size(1)
.build(),
);
}
assert!(
transfer.dispatch_walk(&mut state).is_none(),
"should not dispatch a walk when the key buffer is at/above low-water"
);
}
{
let (transfer, _rx) = setup(
dir.path(),
FailedTransferPolicy::Continue,
mock_s3_success(),
);
let mut state = transfer.inner.state.lock();
state.children_reserved = transfer.inner.pipeline_depth;
assert!(
transfer.dispatch_walk(&mut state).is_some(),
"listing must proceed even when children are at the in-flight budget"
);
}
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_incremental_delivery_lossless_and_ordered() {
let dir = tempdir().unwrap();
let n = 2 * WALK_FLUSH_CHUNK + 22; let objects: Vec<Object> = (0..n)
.map(|i| Object::builder().key(format!("k{i:05}")).size(1).build())
.collect();
let expected: Vec<String> = objects
.iter()
.map(|o| o.key().unwrap().to_string())
.collect();
let list = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(move || {
ListObjectsV2Output::builder()
.set_contents(Some(objects.clone()))
.build()
});
let get = mock!(aws_sdk_s3::Client::get_object).then_output(|| {
GetObjectOutput::builder()
.content_length(1)
.content_range("bytes 0-0/1")
.e_tag("test-etag")
.body(aws_sdk_s3::primitives::ByteStream::from_static(b"x"))
.build()
});
let client = mock_client!(aws_sdk_s3, RuleMode::MatchAny, &[head_rule(1), list, get]);
let (transfer, _rx) = setup(dir.path(), FailedTransferPolicy::Continue, client);
let mut work = {
let mut state = transfer.inner.state.lock();
match transfer.dispatch_walk(&mut state) {
Some(PollWork::Ready { io, .. }) => io,
other => panic!("expected an AdvanceWalker work item, got {other:?}"),
}
};
transfer.execute(&mut work).await;
let delivered: Vec<String> = {
let state = transfer.inner.state.lock();
state
.pending_entries
.iter()
.map(|o| o.key().unwrap().to_string())
.collect()
};
assert_eq!(
delivered.len(),
n,
"all discovered keys must be delivered exactly once (no drop/dup across chunks)"
);
assert_eq!(
delivered, expected,
"keys must be delivered in listing order"
);
}
}