use std::collections::VecDeque;
use std::sync::Arc;
use super::error::{WalkError, WalkErrorKind};
use crate::model::Object;
use crate::types::BucketType;
type FilterFn = Arc<dyn Fn(&Object) -> bool + Send + Sync>;
#[derive(Debug)]
enum S3WalkServiceClient {
TransferManager(crate::Client),
Provided(aws_sdk_s3::Client),
}
impl S3WalkServiceClient {
fn sdk_client(&self) -> &aws_sdk_s3::Client {
match self {
Self::TransferManager(client) => &client.handle.s3_client,
Self::Provided(client) => client,
}
}
}
pub(crate) struct ListPageResult {
pub(crate) objects: Vec<Object>,
pub(crate) common_prefixes: Vec<String>,
pub(crate) next_token: Option<String>,
}
#[derive(Clone)]
pub struct S3Walker {
prefix: Option<String>,
delimiter: Option<String>,
expected_bucket_owner: Option<String>,
request_payer: Option<crate::model::RequestPayer>,
filter: Option<FilterFn>,
start_after: Option<String>,
continuation_token: Option<String>,
page_size: Option<i32>,
}
impl std::fmt::Debug for S3Walker {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("S3Walker")
.field("prefix", &self.prefix)
.field("delimiter", &self.delimiter)
.field("expected_bucket_owner", &self.expected_bucket_owner)
.field("request_payer", &self.request_payer)
.field("start_after", &self.start_after)
.field("continuation_token", &self.continuation_token)
.field("page_size", &self.page_size)
.finish()
}
}
impl Default for S3Walker {
fn default() -> Self {
S3Walker::builder().build()
}
}
fn exclude_s3_folder_markers(obj: &Object) -> bool {
let dominated_by_slash = obj.key().unwrap_or("").ends_with('/');
let is_zero_byte = obj.size().unwrap_or(1) == 0;
!(dominated_by_slash && is_zero_byte)
}
impl S3Walker {
#[must_use]
pub fn builder() -> S3WalkerBuilder {
S3WalkerBuilder::default()
}
#[must_use]
pub fn walk(self, ctx: S3WalkContext) -> S3Walk {
if ctx.bucket.kind() == BucketType::Express {
tracing::warn!(
target: crate::telemetry::TARGET_TRANSFER,
bucket = %ctx.bucket.name(),
kind = ?ctx.bucket.kind(),
"directory bucket detected; listing results may not be lexicographically sorted"
);
}
let initial_prefix = self.prefix.clone().unwrap_or_default();
tracing::debug!(
target: crate::telemetry::TARGET_TRANSFER,
bucket = %ctx.bucket.name(),
kind = ?ctx.bucket.kind(),
prefix = %initial_prefix,
delimiter = ?self.delimiter,
"s3 walk started",
);
let mut pending_prefixes = VecDeque::new();
pending_prefixes.push_back(initial_prefix);
S3Walk {
config: self,
client: ctx.client,
bucket: ctx.bucket,
pending_prefixes,
ready_objects: VecDeque::new(),
current_prefix: None,
next_token: None,
done: false,
initial_first_page_pending: true,
}
}
}
#[derive(Default)]
pub struct S3WalkerBuilder {
prefix: Option<String>,
delimiter: Option<String>,
expected_bucket_owner: Option<String>,
request_payer: Option<crate::model::RequestPayer>,
filter: Option<FilterFn>,
start_after: Option<String>,
continuation_token: Option<String>,
page_size: Option<i32>,
}
impl std::fmt::Debug for S3WalkerBuilder {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("S3WalkerBuilder")
.field("prefix", &self.prefix)
.field("delimiter", &self.delimiter)
.finish()
}
}
impl S3WalkerBuilder {
#[must_use]
pub fn prefix(mut self, prefix: impl Into<String>) -> Self {
self.prefix = Some(prefix.into());
self
}
#[must_use]
pub fn delimiter(mut self, delimiter: impl Into<String>) -> Self {
self.delimiter = Some(delimiter.into());
self
}
#[must_use]
pub fn expected_bucket_owner(mut self, owner: impl Into<String>) -> Self {
self.expected_bucket_owner = Some(owner.into());
self
}
#[must_use]
pub fn request_payer(mut self, payer: crate::model::RequestPayer) -> Self {
self.request_payer = Some(payer);
self
}
#[must_use]
pub fn filter(mut self, f: impl Fn(&Object) -> bool + Send + Sync + 'static) -> Self {
self.filter = Some(Arc::new(f));
self
}
#[must_use]
pub fn start_after(mut self, key: impl Into<String>) -> Self {
self.start_after = Some(key.into());
self
}
#[must_use]
pub fn continuation_token(mut self, token: impl Into<String>) -> Self {
self.continuation_token = Some(token.into());
self
}
#[must_use]
pub fn page_size(mut self, n: i32) -> Self {
self.page_size = Some(n);
self
}
#[must_use]
pub fn build(self) -> S3Walker {
S3Walker {
prefix: self.prefix,
delimiter: self.delimiter,
expected_bucket_owner: self.expected_bucket_owner,
request_payer: self.request_payer,
filter: self.filter,
start_after: self.start_after,
continuation_token: self.continuation_token,
page_size: self.page_size,
}
}
}
pub struct S3WalkContext {
client: S3WalkServiceClient,
bucket: crate::types::Bucket,
}
impl std::fmt::Debug for S3WalkContext {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("S3WalkContext")
.field("bucket", &self.bucket)
.finish()
}
}
impl S3WalkContext {
#[must_use]
pub fn builder() -> S3WalkContextBuilder {
S3WalkContextBuilder {
client: None,
bucket: None,
}
}
}
#[derive(Debug, Default)]
pub struct S3WalkContextBuilder {
client: Option<S3WalkServiceClient>,
bucket: Option<String>,
}
impl S3WalkContextBuilder {
#[must_use]
pub fn client(mut self, client: &crate::Client) -> Self {
self.client = Some(S3WalkServiceClient::TransferManager(client.clone()));
self
}
#[cfg(feature = "sdk-v1")]
#[must_use]
pub fn sdk_v1_client(self, client: aws_sdk_s3::Client) -> Self {
self.sdk_client(client)
}
pub(crate) fn sdk_client(mut self, client: aws_sdk_s3::Client) -> Self {
self.client = Some(S3WalkServiceClient::Provided(client));
self
}
#[must_use]
pub fn bucket(mut self, bucket: impl Into<String>) -> Self {
self.bucket = Some(bucket.into());
self
}
#[must_use]
pub fn build(self) -> S3WalkContext {
let bucket_name = self.bucket.expect("required field `bucket` should be set");
S3WalkContext {
client: self.client.expect("required field `client` should be set"),
bucket: crate::types::Bucket::new(bucket_name),
}
}
}
pub struct S3Walk {
config: S3Walker,
client: S3WalkServiceClient,
bucket: crate::types::Bucket,
pending_prefixes: VecDeque<String>,
ready_objects: VecDeque<Object>,
current_prefix: Option<String>,
next_token: Option<String>,
done: bool,
initial_first_page_pending: bool,
}
impl std::fmt::Debug for S3Walk {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("S3Walk")
.field("bucket", &self.bucket)
.field("done", &self.done)
.finish()
}
}
impl S3Walk {
pub async fn next(&mut self) -> Option<Result<Object, WalkError>> {
loop {
if let Some(obj) = self.ready_objects.pop_front() {
return Some(Ok(obj));
}
if self.done {
return None;
}
if let Some(prefix) = &self.current_prefix {
let prefix = prefix.clone();
let token = self.next_token.take();
let first_page = self.initial_first_page_pending;
match self.list_page(&prefix, token.as_deref(), first_page).await {
Ok(page) => {
self.initial_first_page_pending = false;
tracing::trace!(
target: crate::telemetry::TARGET_TRANSFER,
%prefix,
objects = page.objects.len(),
common_prefixes = page.common_prefixes.len(),
has_next = page.next_token.is_some(),
"page listed",
);
self.ready_objects.extend(page.objects);
for cp in page.common_prefixes {
self.pending_prefixes.push_back(cp);
}
if page.next_token.is_some() {
self.next_token = page.next_token;
self.current_prefix = Some(prefix);
} else {
self.current_prefix = None;
}
}
Err(e) => {
self.done = true;
return Some(Err(e));
}
}
continue;
}
match self.pending_prefixes.pop_front() {
Some(prefix) => {
self.current_prefix = Some(prefix);
}
None => {
self.done = true;
return None;
}
}
}
}
pub fn is_done(&self) -> bool {
self.done
}
pub fn is_directory_bucket(&self) -> bool {
self.bucket.kind() == BucketType::Express
}
async fn list_page(
&self,
prefix: &str,
token: Option<&str>,
first_page_of_initial_prefix: bool,
) -> Result<ListPageResult, WalkError> {
let mut req = self
.client
.sdk_client()
.list_objects_v2()
.bucket(self.bucket.name())
.prefix(prefix);
let effective_token = if first_page_of_initial_prefix {
self.config.continuation_token.as_deref().or(token)
} else {
token
};
if let Some(t) = effective_token {
req = req.continuation_token(t);
}
if first_page_of_initial_prefix {
if let Some(start_after) = &self.config.start_after {
if effective_token.is_none() {
req = req.start_after(start_after);
}
}
}
if let Some(delimiter) = &self.config.delimiter {
req = req.delimiter(delimiter);
}
if let Some(owner) = &self.config.expected_bucket_owner {
req = req.expected_bucket_owner(owner);
}
if let Some(payer) = &self.config.request_payer {
req = req.request_payer(crate::sdk_v1::request_payer_to_sdk(payer));
}
if let Some(page_size) = self.config.page_size {
req = req.max_keys(page_size);
}
let output = req
.send()
.await
.map_err(|e| WalkError::new(None, WalkErrorKind::Service, Box::new(e)))?;
let mut objects: Vec<Object> = output
.contents
.unwrap_or_default()
.into_iter()
.map(|object| crate::sdk_v1::object_from_sdk(&object))
.collect();
objects.retain(|obj| {
exclude_s3_folder_markers(obj) && self.config.filter.as_ref().is_none_or(|f| f(obj))
});
let common_prefixes = output
.common_prefixes
.unwrap_or_default()
.into_iter()
.filter_map(|cp| cp.prefix)
.collect();
Ok(ListPageResult {
objects,
common_prefixes,
next_token: output.next_continuation_token,
})
}
}
#[cfg(test)]
mod tests {
use super::*;
use aws_sdk_s3::operation::list_objects_v2::ListObjectsV2Output;
use aws_sdk_s3::types::{CommonPrefix, Object};
use aws_smithy_mocks::{mock, mock_client, RuleMode};
fn walker() -> S3WalkerBuilder {
S3Walker::builder()
}
fn s3ctx(client: aws_sdk_s3::Client, bucket: impl Into<String>) -> S3WalkContext {
let builder = S3WalkContext::builder();
#[cfg(feature = "sdk-v1")]
let builder = builder.sdk_v1_client(client);
#[cfg(not(feature = "sdk-v1"))]
let builder = builder.sdk_client(client);
builder.bucket(bucket).build()
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn standalone_walk_retains_transfer_manager_until_dropped() {
let list = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.contents(Object::builder().key("object").build())
.build()
});
let sdk = mock_client!(aws_sdk_s3, RuleMode::Sequential, &[&list]);
let tm = crate::Client::new(
crate::Config::builder()
.sdk_client(sdk)
.build()
.expect("valid config"),
);
let handle = Arc::downgrade(&tm.handle);
let context = S3WalkContext::builder()
.client(&tm)
.bucket("bucket")
.build();
drop(tm);
assert!(handle.upgrade().is_some());
let mut walk = S3Walker::default().walk(context);
assert_eq!(walk.next().await.unwrap().unwrap().key(), Some("object"));
assert!(walk.next().await.is_none());
assert!(handle.upgrade().is_some());
drop(walk);
assert!(handle.upgrade().is_none());
assert_eq!(list.num_calls(), 1);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn shared_config_loopback_endpoint_uses_path_style_without_sdk_settings() {
use aws_sdk_s3::config::{Credentials, SharedCredentialsProvider};
use aws_smithy_http_client::test_util::capture_request;
let (http_client, captured) = capture_request(None);
let shared = aws_types::SdkConfig::builder()
.behavior_version(aws_config::BehaviorVersion::latest())
.region(aws_config::Region::new("us-east-1"))
.credentials_provider(SharedCredentialsProvider::new(Credentials::for_tests()))
.endpoint_url("http://127.0.0.1:12345")
.http_client(http_client)
.build();
let tm = crate::Client::new(
crate::Config::builder()
.s3_config(crate::config::S3ClientConfig::new(&shared))
.build()
.expect("valid config"),
);
let context = S3WalkContext::builder()
.client(&tm)
.bucket("bucket")
.build();
let mut walk = S3Walker::default().walk(context);
assert!(walk.next().await.unwrap().is_err());
let request = captured.expect_request();
assert_eq!(
request.uri().to_string(),
"http://127.0.0.1:12345/bucket/?list-type=2&prefix="
);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_list_single_page() {
let rule = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![
Object::builder().key("a.txt").build(),
Object::builder().key("b.txt").build(),
Object::builder().key("c.txt").build(),
]))
.build()
});
let client = mock_client!(aws_sdk_s3, RuleMode::Sequential, &[&rule]);
let mut walk = walker()
.prefix("prefix/")
.build()
.walk(s3ctx(client, "test-bucket"));
let mut keys = Vec::new();
while let Some(result) = walk.next().await {
keys.push(result.unwrap().key.unwrap());
}
assert_eq!(keys, vec!["a.txt", "b.txt", "c.txt"]);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_list_with_pagination() {
let page1 = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.is_truncated(true)
.next_continuation_token("token1")
.set_contents(Some(vec![
Object::builder().key("a.txt").build(),
Object::builder().key("b.txt").build(),
]))
.build()
});
let page2 = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![Object::builder().key("c.txt").build()]))
.build()
});
let client = mock_client!(aws_sdk_s3, RuleMode::Sequential, &[&page1, &page2]);
let mut walk = walker().build().walk(s3ctx(client, "test-bucket"));
let mut keys = Vec::new();
while let Some(result) = walk.next().await {
keys.push(result.unwrap().key.unwrap());
}
assert_eq!(keys, vec!["a.txt", "b.txt", "c.txt"]);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_list_with_filter() {
let rule = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![
Object::builder().key("file.txt").build(),
Object::builder().key("folder/").build(),
Object::builder().key("other.txt").build(),
]))
.build()
});
let client = mock_client!(aws_sdk_s3, RuleMode::Sequential, &[&rule]);
let mut walk = walker()
.filter(|obj: &crate::model::Object| !obj.key().unwrap_or("").ends_with('/'))
.build()
.walk(s3ctx(client, "test-bucket"));
let mut keys = Vec::new();
while let Some(result) = walk.next().await {
let object: crate::model::Object = result.unwrap();
keys.push(object.key.unwrap());
}
assert_eq!(keys, vec!["file.txt", "other.txt"]);
}
async fn walk_marker_page(builder: S3WalkerBuilder) -> Vec<String> {
let rule = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![
Object::builder().key("dir/").size(0).build(),
Object::builder().key("dir/file.txt").size(4).build(),
Object::builder().key("data/").size(4).build(),
]))
.build()
});
let client = mock_client!(aws_sdk_s3, RuleMode::Sequential, &[&rule]);
let mut walk = builder.build().walk(s3ctx(client, "test-bucket"));
let mut keys = Vec::new();
while let Some(result) = walk.next().await {
keys.push(result.unwrap().key.unwrap());
}
keys
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_folder_markers_skipped_without_filter() {
assert_eq!(
walk_marker_page(walker()).await,
vec!["dir/file.txt", "data/"]
);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_folder_markers_skipped_with_filter() {
let seen = Arc::new(std::sync::Mutex::new(Vec::new()));
let recorder = seen.clone();
let builder = walker().filter(move |obj: &crate::model::Object| {
recorder
.lock()
.unwrap()
.push(obj.key().unwrap_or_default().to_owned());
true
});
assert_eq!(
walk_marker_page(builder).await,
vec!["dir/file.txt", "data/"]
);
assert_eq!(*seen.lock().unwrap(), vec!["dir/file.txt", "data/"]);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_list_empty_prefix() {
let rule = mock!(aws_sdk_s3::Client::list_objects_v2)
.then_output(|| ListObjectsV2Output::builder().build());
let client = mock_client!(aws_sdk_s3, RuleMode::Sequential, &[&rule]);
let mut walk = walker()
.prefix("empty/")
.build()
.walk(s3ctx(client, "test-bucket"));
assert!(walk.next().await.is_none());
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_list_with_common_prefixes() {
let root_page = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![Object::builder().key("root.txt").build()]))
.set_common_prefixes(Some(vec![
CommonPrefix::builder().prefix("sub1/").build(),
CommonPrefix::builder().prefix("sub2/").build(),
]))
.build()
});
let sub1_page = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![Object::builder().key("sub1/a.txt").build()]))
.build()
});
let sub2_page = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![Object::builder().key("sub2/b.txt").build()]))
.build()
});
let client = mock_client!(
aws_sdk_s3,
RuleMode::Sequential,
&[&root_page, &sub1_page, &sub2_page]
);
let mut walk = walker()
.delimiter("/")
.build()
.walk(s3ctx(client, "test-bucket"));
let mut keys = Vec::new();
while let Some(result) = walk.next().await {
keys.push(result.unwrap().key.unwrap());
}
assert_eq!(keys, vec!["root.txt", "sub1/a.txt", "sub2/b.txt"]);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_list_service_error_terminates_walk() {
use aws_sdk_s3::operation::list_objects_v2::ListObjectsV2Error;
use aws_smithy_types::error::metadata::ErrorMetadata;
let rule = mock!(aws_sdk_s3::Client::list_objects_v2).then_error(|| {
ListObjectsV2Error::generic(ErrorMetadata::builder().code("NoSuchBucket").build())
});
let client = mock_client!(aws_sdk_s3, RuleMode::Sequential, &[&rule]);
let mut walk = walker().build().walk(s3ctx(client, "missing-bucket"));
let first = walk.next().await;
assert!(matches!(first, Some(Err(_))), "expected service error");
assert!(walk.next().await.is_none());
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_list_pagination_across_common_prefix() {
let root_page = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![Object::builder().key("root.txt").build()]))
.set_common_prefixes(Some(vec![CommonPrefix::builder().prefix("sub/").build()]))
.build()
});
let sub_page1 = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.is_truncated(true)
.next_continuation_token("tok")
.set_contents(Some(vec![
Object::builder().key("sub/a.txt").build(),
Object::builder().key("sub/b.txt").build(),
]))
.build()
});
let sub_page2 = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![Object::builder().key("sub/c.txt").build()]))
.build()
});
let client = mock_client!(
aws_sdk_s3,
RuleMode::Sequential,
&[&root_page, &sub_page1, &sub_page2]
);
let mut walk = walker()
.delimiter("/")
.build()
.walk(s3ctx(client, "test-bucket"));
let mut keys = Vec::new();
while let Some(result) = walk.next().await {
keys.push(result.unwrap().key.unwrap());
}
assert_eq!(
keys,
vec!["root.txt", "sub/a.txt", "sub/b.txt", "sub/c.txt"]
);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_list_delimiter_without_common_prefixes() {
let rule = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![
Object::builder().key("a.txt").build(),
Object::builder().key("b.txt").build(),
]))
.build()
});
let client = mock_client!(aws_sdk_s3, RuleMode::Sequential, &[&rule]);
let mut walk = walker()
.delimiter("/")
.build()
.walk(s3ctx(client, "test-bucket"));
let mut keys = Vec::new();
while let Some(result) = walk.next().await {
keys.push(result.unwrap().key.unwrap());
}
assert_eq!(keys, vec!["a.txt", "b.txt"]);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_list_no_delimiter_no_recursion() {
let rule = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![
Object::builder().key("a.txt").build(),
Object::builder().key("sub/b.txt").build(),
Object::builder().key("sub/c.txt").build(),
]))
.build()
});
let client = mock_client!(aws_sdk_s3, RuleMode::Sequential, &[&rule]);
let mut walk = walker().build().walk(s3ctx(client, "test-bucket"));
let mut keys = Vec::new();
while let Some(result) = walk.next().await {
keys.push(result.unwrap().key.unwrap());
}
assert_eq!(keys, vec!["a.txt", "sub/b.txt", "sub/c.txt"]);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_s3walker_start_after() {
let rule = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![
Object::builder().key("c.txt").build(),
Object::builder().key("d.txt").build(),
]))
.build()
});
let client = mock_client!(aws_sdk_s3, RuleMode::Sequential, &[&rule]);
let mut walk = walker()
.start_after("b.txt")
.build()
.walk(s3ctx(client, "test-bucket"));
let mut keys = Vec::new();
while let Some(result) = walk.next().await {
keys.push(result.unwrap().key.unwrap());
}
assert_eq!(keys, vec!["c.txt", "d.txt"]);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_s3walker_page_size() {
let rule = mock!(aws_sdk_s3::Client::list_objects_v2)
.match_requests(|req| req.max_keys() == Some(1))
.then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![Object::builder().key("only.txt").build()]))
.build()
});
let client = mock_client!(aws_sdk_s3, RuleMode::Sequential, &[&rule]);
let mut walk = walker()
.page_size(1)
.build()
.walk(s3ctx(client, "test-bucket"));
let mut keys = Vec::new();
while let Some(result) = walk.next().await {
keys.push(result.unwrap().key.unwrap());
}
assert_eq!(keys, vec!["only.txt"]);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_s3walker_expected_bucket_owner() {
let rule = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![Object::builder().key("a.txt").build()]))
.build()
});
let client = mock_client!(aws_sdk_s3, RuleMode::Sequential, &[&rule]);
let mut walk = walker()
.expected_bucket_owner("123456789012")
.build()
.walk(s3ctx(client, "test-bucket"));
let mut keys = Vec::new();
while let Some(result) = walk.next().await {
keys.push(result.unwrap().key.unwrap());
}
assert_eq!(keys, vec!["a.txt"]);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_s3walker_is_directory_bucket() {
let rule = mock!(aws_sdk_s3::Client::list_objects_v2)
.then_output(|| ListObjectsV2Output::builder().build());
let client = mock_client!(aws_sdk_s3, RuleMode::Sequential, &[&rule, &rule]);
let walk_express = walker()
.build()
.walk(s3ctx(client.clone(), "my-bucket--usw2-az1--x-s3"));
assert!(walk_express.is_directory_bucket());
let walk_standard = walker().build().walk(s3ctx(client, "my-normal-bucket"));
assert!(!walk_standard.is_directory_bucket());
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_s3walker_continuation_token_only_on_first_page() {
let first = mock!(aws_sdk_s3::Client::list_objects_v2)
.match_requests(|req| req.continuation_token() == Some("resume-here"))
.then_output(|| {
ListObjectsV2Output::builder()
.is_truncated(true)
.next_continuation_token("server-next")
.set_contents(Some(vec![Object::builder().key("a.txt").build()]))
.build()
});
let second = mock!(aws_sdk_s3::Client::list_objects_v2)
.match_requests(|req| req.continuation_token() == Some("server-next"))
.then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![Object::builder().key("b.txt").build()]))
.build()
});
let client = mock_client!(aws_sdk_s3, RuleMode::Sequential, &[&first, &second]);
let mut walk = walker()
.continuation_token("resume-here")
.build()
.walk(s3ctx(client, "test-bucket"));
let mut keys = Vec::new();
while let Some(result) = walk.next().await {
keys.push(result.unwrap().key.unwrap());
}
assert_eq!(keys, vec!["a.txt", "b.txt"]);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_s3walker_prefix_propagates() {
let rule = mock!(aws_sdk_s3::Client::list_objects_v2)
.match_requests(|req| req.prefix() == Some("my-prefix/"))
.then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![Object::builder().key("my-prefix/a.txt").build()]))
.build()
});
let client = mock_client!(aws_sdk_s3, RuleMode::Sequential, &[&rule]);
let mut walk = walker()
.prefix("my-prefix/")
.build()
.walk(s3ctx(client, "test-bucket"));
let mut keys = Vec::new();
while let Some(result) = walk.next().await {
keys.push(result.unwrap().key.unwrap());
}
assert_eq!(keys, vec!["my-prefix/a.txt"]);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_s3walker_request_payer_propagates() {
let rule = mock!(aws_sdk_s3::Client::list_objects_v2)
.match_requests(|req| {
req.request_payer() == Some(&aws_sdk_s3::types::RequestPayer::Requester)
})
.then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![Object::builder().key("a.txt").build()]))
.build()
});
let client = mock_client!(aws_sdk_s3, RuleMode::Sequential, &[&rule]);
let mut walk = walker()
.request_payer(crate::model::RequestPayer::Requester)
.build()
.walk(s3ctx(client, "test-bucket"));
let mut keys = Vec::new();
while let Some(result) = walk.next().await {
keys.push(result.unwrap().key.unwrap());
}
assert_eq!(keys, vec!["a.txt"]);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_s3walker_delimiter_propagates_on_recursed_prefixes() {
let root = mock!(aws_sdk_s3::Client::list_objects_v2)
.match_requests(|req| req.delimiter() == Some("/") && req.prefix() == Some(""))
.then_output(|| {
ListObjectsV2Output::builder()
.set_common_prefixes(Some(vec![CommonPrefix::builder().prefix("a/").build()]))
.build()
});
let a_page = mock!(aws_sdk_s3::Client::list_objects_v2)
.match_requests(|req| req.delimiter() == Some("/") && req.prefix() == Some("a/"))
.then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![Object::builder().key("a/file.txt").build()]))
.set_common_prefixes(Some(vec![CommonPrefix::builder().prefix("a/b/").build()]))
.build()
});
let ab_page = mock!(aws_sdk_s3::Client::list_objects_v2)
.match_requests(|req| req.delimiter() == Some("/") && req.prefix() == Some("a/b/"))
.then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![Object::builder().key("a/b/file.txt").build()]))
.set_common_prefixes(Some(vec![CommonPrefix::builder()
.prefix("a/b/c/")
.build()]))
.build()
});
let abc_page = mock!(aws_sdk_s3::Client::list_objects_v2)
.match_requests(|req| req.delimiter() == Some("/") && req.prefix() == Some("a/b/c/"))
.then_output(|| {
ListObjectsV2Output::builder()
.set_contents(Some(vec![Object::builder().key("a/b/c/file.txt").build()]))
.build()
});
let client = mock_client!(
aws_sdk_s3,
RuleMode::Sequential,
&[&root, &a_page, &ab_page, &abc_page]
);
let mut walk = walker()
.delimiter("/")
.build()
.walk(s3ctx(client, "test-bucket"));
let mut keys = Vec::new();
while let Some(result) = walk.next().await {
keys.push(result.unwrap().key.unwrap());
}
assert_eq!(keys, vec!["a/file.txt", "a/b/file.txt", "a/b/c/file.txt"]);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_s3walker_empty_contents_none() {
let rule = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.build()
});
let client = mock_client!(aws_sdk_s3, RuleMode::Sequential, &[&rule]);
let mut walk = walker().build().walk(s3ctx(client, "test-bucket"));
let mut count = 0;
while let Some(result) = walk.next().await {
result.unwrap();
count += 1;
}
assert_eq!(count, 0, "None contents should yield zero objects");
}
#[test]
fn test_s3walker_is_send_sync() {
fn assert_send_sync<T: Send + Sync>() {}
assert_send_sync::<S3Walker>();
assert_send_sync::<S3Walk>();
assert_send_sync::<S3WalkContext>();
assert_send_sync::<S3WalkerBuilder>();
assert_send_sync::<S3WalkContextBuilder>();
}
}