use aws_sdk_s3::operation::get_object::GetObjectOutput;
use aws_smithy_observability::unit_of_work::checksums::ChecksumCapture;
use aws_smithy_observability::unit_of_work::Capture;
use aws_smithy_types::body::SdkBody;
use aws_smithy_types::byte_stream::ByteStream;
use std::ops::RangeInclusive;
use std::str::FromStr;
use std::{cmp, mem};
use tracing::Instrument;
use super::checksum::{stored::StoredChecksum, ValidationPlan};
use super::observability::{DownloadRequestKind, DownloadRequestMeasurement};
use super::transfer::DownloadTransfer;
use super::DownloadInput;
use super::{ChunkMetadata, ObjectMetadata};
use crate::error;
use crate::http::header::{self, ByteRange};
use crate::sdk_v1::{
copy_download_input_fields_to_get_object as copy_fields_to_get_object_request,
copy_download_input_fields_to_head_object as copy_fields_to_head_object_request,
};
#[derive(Debug, Clone, PartialEq)]
enum ObjectDiscoveryStrategy {
HeadObject,
RangedGet(Option<RangeInclusive<u64>>),
}
#[derive(Debug)]
pub(super) struct ObjectDiscovery {
pub(super) remaining: Option<RangeInclusive<u64>>,
pub(super) object_range_start: u64,
pub(super) chunk_meta: Option<ChunkMetadata>,
pub(super) object_meta: ObjectMetadata,
pub(super) initial_chunk: Option<InitialChunk>,
pub(super) effective_part_size: u64,
pub(super) validation_plan: ValidationPlan,
}
#[derive(Debug)]
pub(super) struct InitialChunk {
pub(super) body: ByteStream,
pub(super) checksum: ChecksumCapture,
pub(super) expected_len: usize,
pub(super) request_metrics: Option<DownloadRequestMeasurement>,
}
fn parts_from_etag(etag: &str) -> Option<u32> {
let trimmed = etag.trim_matches('"');
let (_, n) = trimmed.rsplit_once('-')?;
n.parse::<u32>().ok().filter(|&n| n > 1)
}
impl ObjectDiscoveryStrategy {
fn from_request(
input: &DownloadInput,
validation_enabled: bool,
) -> Result<ObjectDiscoveryStrategy, crate::error::Error> {
let strategy = match input.range() {
Some(h) => {
let byte_range = header::Range::from_str(h)?.0;
match byte_range {
ByteRange::Inclusive(start, end) => {
ObjectDiscoveryStrategy::RangedGet(Some(start..=end))
}
_ => ObjectDiscoveryStrategy::HeadObject,
}
}
None if validation_enabled => ObjectDiscoveryStrategy::HeadObject,
None => ObjectDiscoveryStrategy::RangedGet(None),
};
Ok(strategy)
}
}
pub(super) async fn discover_obj(
transfer: &DownloadTransfer,
input: &DownloadInput,
validation_enabled: bool,
) -> Result<ObjectDiscovery, crate::error::Error> {
let configured_part_size = transfer.ctx().handle.part_size_bytes();
let whole_object_validation = validation_enabled && input.range().is_none();
let strategy = ObjectDiscoveryStrategy::from_request(input, validation_enabled)?;
tracing::trace!(
target: crate::telemetry::TARGET_TRANSFER_DOWNLOAD,
?strategy,
validation_enabled,
"download discovery strategy selected",
);
let listed = transfer.listed_object().filter(|listed| {
whole_object_validation
&& input.if_match().is_none()
&& listed.size != 0
&& listed.size <= configured_part_size
&& parts_from_etag(&listed.e_tag).is_none()
});
let mut listed_get = false;
let mut discovery = match (strategy, listed) {
(ObjectDiscoveryStrategy::HeadObject, Some(listed)) => {
match discover_obj_with_get(transfer, input, None, Some(&listed.e_tag))
.instrument(tracing::trace_span!(
target: crate::telemetry::TARGET_TRANSFER_DOWNLOAD,
"send-listed-get-for-discovery",
))
.await
{
Ok(discovery) => {
listed_get = true;
Ok(discovery)
}
Err(error) if error.http_status() == Some(412) => {
discover_obj_with_head(transfer, input)
.instrument(tracing::trace_span!(
target: crate::telemetry::TARGET_TRANSFER_DOWNLOAD,
"send-head-object-for-discovery",
))
.await
}
Err(error) => Err(error),
}
}
(ObjectDiscoveryStrategy::HeadObject, None) => {
discover_obj_with_head(transfer, input)
.instrument(tracing::trace_span!(
target: crate::telemetry::TARGET_TRANSFER_DOWNLOAD,
"send-head-object-for-discovery",
))
.await
}
(ObjectDiscoveryStrategy::RangedGet(range), _) => {
discover_obj_with_get(transfer, input, range, None)
.instrument(tracing::trace_span!(
target: crate::telemetry::TARGET_TRANSFER_DOWNLOAD,
"send-ranged-get-for-discovery",
))
.await
}
}?;
discovery.effective_part_size = configured_part_size;
let reason;
if listed_get {
reason = "listed object fits one range";
discovery.validation_plan = ValidationPlan::PerResponse;
} else if whole_object_validation {
let object_len = discovery
.object_meta
.content_length
.and_then(|length| u64::try_from(length).ok())
.ok_or_else(|| error::discovery_failed("HeadObject has invalid content-length"))?;
let stored = StoredChecksum::from_metadata(&discovery.object_meta)?;
let composite = matches!(stored, StoredChecksum::Composite);
reason = match &stored {
StoredChecksum::FullObjectCrc { .. } => "stored full-object CRC",
StoredChecksum::Composite => "stored composite checksum",
StoredChecksum::Ordered => "ordered object hashing is unavailable",
StoredChecksum::Unavailable => "usable stored checksum is unavailable",
};
discovery.validation_plan = stored.into_plan(object_len);
let multipart = discovery
.object_meta
.e_tag
.as_deref()
.and_then(parts_from_etag)
.is_some();
if object_len != 0 && composite && multipart && !transfer.ctx().handle.user_set_part_size()
{
let mut layout = discover_obj_with_get_first_part(
transfer,
input,
discovery.object_meta.e_tag.as_deref(),
)
.await?;
validate_layout_response(&discovery.object_meta, &layout.object_meta)?;
let stored_part_size = layout
.initial_chunk
.as_ref()
.ok_or_else(|| error::discovery_failed("nonempty object has an empty first part"))?
.expected_len as u64;
layout.effective_part_size = stored_part_size;
let parts_count = layout.object_meta.parts_count;
layout.object_meta = discovery.object_meta;
layout.object_meta.parts_count = parts_count;
layout.validation_plan = discovery.validation_plan;
discovery = layout;
}
} else {
discovery.validation_plan = if validation_enabled {
reason = "caller range";
ValidationPlan::PerResponse
} else {
reason = "validation disabled";
ValidationPlan::Disabled
};
}
tracing::trace!(
target: crate::telemetry::TARGET_TRANSFER_DOWNLOAD,
remaining = ?discovery.remaining,
initial_chunk = discovery.initial_chunk.is_some(),
effective_part_size = discovery.effective_part_size,
"discovered object",
);
discovery
.validation_plan
.log(&discovery.object_meta, reason);
Ok(discovery)
}
async fn discover_obj_with_get_first_part(
transfer: &DownloadTransfer,
input: &DownloadInput,
etag: Option<&str>,
) -> Result<ObjectDiscovery, error::Error> {
let mut req_metrics = transfer.start_request(DownloadRequestKind::DiscoveryPart);
let retry_classify = crate::retry::classify_discovery_retry;
let result = crate::retry::retry(req_metrics.metrics_mut(), retry_classify, |_allow_hedge| {
let mut builder =
copy_fields_to_get_object_request(input, transfer.ctx().s3_client().get_object());
if let Some(etag) = etag {
builder = builder.if_match(etag);
}
let checksum = ChecksumCapture::new();
let req = builder
.set_range(None)
.set_part_number(Some(1))
.customize()
.interceptor(Capture::new().with(&checksum))
.config_override(transfer.ctx().bucket_partition_override(input.bucket()));
async move {
req.send()
.await
.map(|resp| (resp, checksum))
.map_err(|e| crate::retry::GuardError::Inner(error::service_error(e)))
}
})
.instrument(tracing::trace_span!(
target: crate::telemetry::TARGET_TRANSFER_DOWNLOAD,
"discover-get-first-part",
tid = %transfer.ctx().id
))
.await;
let (resp, checksum) = result?;
let discovery = first_chunk_response_handler(resp, checksum, None, None)?;
Ok(attach_request_measurement(discovery, req_metrics))
}
async fn discover_obj_with_head(
transfer: &DownloadTransfer,
input: &DownloadInput,
) -> Result<ObjectDiscovery, crate::error::Error> {
let mut req_metrics = transfer.start_request(DownloadRequestKind::DiscoveryHead);
let retry_classify = crate::retry::classify_discovery_retry;
let result = crate::retry::retry(req_metrics.metrics_mut(), retry_classify, |_allow_hedge| {
let mut builder =
copy_fields_to_head_object_request(input, transfer.ctx().s3_client().head_object());
if input.range().is_none() {
builder = builder.checksum_mode(aws_sdk_s3::types::ChecksumMode::Enabled);
}
let req = builder
.customize()
.config_override(transfer.ctx().bucket_partition_override(input.bucket()));
async move {
req.send()
.await
.map_err(|e| crate::retry::GuardError::Inner(error::head_object_error(e)))
}
})
.instrument(tracing::trace_span!(
target: crate::telemetry::TARGET_TRANSFER_DOWNLOAD,
"discover-head",
tid = %transfer.ctx().id
))
.await;
req_metrics.finish();
let resp = result?;
let object_meta = crate::sdk_v1::object_metadata_from_head_object(&resp);
let remaining = if input.range().is_some() {
Some(validate_head_response_range(input, &object_meta)?)
} else {
validate_whole_object_head(&object_meta)?
};
let object_range_start = remaining.as_ref().map_or(0, |range| *range.start());
Ok(ObjectDiscovery {
remaining,
object_range_start,
chunk_meta: None,
object_meta,
initial_chunk: None,
effective_part_size: 0,
validation_plan: ValidationPlan::PerResponse,
})
}
fn validate_whole_object_head(
meta: &ObjectMetadata,
) -> Result<Option<RangeInclusive<u64>>, error::Error> {
let length = meta
.content_length
.and_then(|length| u64::try_from(length).ok())
.ok_or_else(|| error::discovery_failed("HeadObject has invalid content-length"))?;
if meta.content_range.is_some() {
return Err(unusable_response(
"HeadObject",
meta,
"whole-object HeadObject has unexpected content-range",
));
}
if length != 0 && meta.e_tag.as_deref().is_none_or(str::is_empty) {
return Err(error::discovery_failed(
"nonempty HeadObject response missing ETag",
));
}
Ok(length.checked_sub(1).map(|end| 0..=end))
}
fn validate_layout_response(
head: &ObjectMetadata,
part: &ObjectMetadata,
) -> Result<(), error::Error> {
let response = validate_response_content_range("GetObject", part)?;
if *response.range.start() != 0
|| Some(response.total) != head.content_length.and_then(|len| u64::try_from(len).ok())
|| part.e_tag != head.e_tag
{
return Err(error::discovery_failed(
"partNumber response disagrees with HeadObject",
));
}
Ok(())
}
fn validate_head_response_range(
input: &DownloadInput,
object_meta: &ObjectMetadata,
) -> Result<RangeInclusive<u64>, error::Error> {
let requested = input
.range()
.ok_or_else(|| error::discovery_failed("ranged HeadObject request missing range"))
.and_then(header::Range::from_str)?
.0;
Ok(validate_ranged_response("HeadObject", &requested, object_meta)?.range)
}
fn unusable_response(
operation: &'static str,
meta: &ObjectMetadata,
message: &str,
) -> error::Error {
error::invalid_service_response_ids(
operation,
meta.request_id(),
meta.extended_request_id(),
message,
)
}
#[derive(Debug, Clone, PartialEq)]
struct ValidatedContentRange {
range: RangeInclusive<u64>,
total: u64,
}
fn validate_response_content_range(
operation: &'static str,
object_meta: &ObjectMetadata,
) -> Result<ValidatedContentRange, error::Error> {
let unusable = |message| unusable_response(operation, object_meta, message);
let content_range = object_meta
.content_range
.as_deref()
.ok_or_else(|| unusable("ranged response missing content-range"))?;
let response = object_meta
.range_from_content_range()
.ok_or_else(|| unusable("ranged response has invalid content-range"))?;
let total = crate::http::header::parse_content_range_total(content_range)
.filter(|total| *total > 0)
.ok_or_else(|| unusable("ranged response has invalid object length"))?;
let start = *response.start();
let end = *response.end();
if start > end || end >= total {
return Err(unusable("ranged response exceeds the object"));
}
let response_len = end - start + 1;
let content_length = object_meta
.content_length
.and_then(|length| u64::try_from(length).ok())
.ok_or_else(|| unusable("ranged response has invalid content-length"))?;
if content_length != response_len {
return Err(unusable(
"ranged response length disagrees with content-range",
));
}
Ok(ValidatedContentRange {
range: response,
total,
})
}
fn validate_ranged_response(
operation: &'static str,
requested: &ByteRange,
object_meta: &ObjectMetadata,
) -> Result<ValidatedContentRange, error::Error> {
let validated = validate_response_content_range(operation, object_meta)?;
let start = *validated.range.start();
let end = *validated.range.end();
let response_len = end - start + 1;
let matches_request = match *requested {
ByteRange::Inclusive(requested_start, requested_end) => {
start == requested_start && end == requested_end.min(validated.total - 1)
}
ByteRange::AllFrom(requested_start) => {
start == requested_start && end == validated.total - 1
}
ByteRange::Last(requested_len) => {
end == validated.total - 1 && response_len == requested_len.min(validated.total)
}
};
if !matches_request {
return Err(unusable_response(
operation,
object_meta,
"ranged response does not match the requested range",
));
}
Ok(validated)
}
async fn discover_obj_with_get(
transfer: &DownloadTransfer,
input: &DownloadInput,
range_from_user: Option<RangeInclusive<u64>>,
if_match: Option<&str>,
) -> Result<ObjectDiscovery, error::Error> {
let target_part_size = transfer.ctx().handle.part_size_bytes();
let byte_range = match range_from_user.as_ref() {
Some(r) => ByteRange::Inclusive(
*r.start(),
cmp::min(*r.start() + target_part_size - 1, *r.end()),
),
None => ByteRange::Inclusive(0, target_part_size - 1),
};
let mut req_metrics = transfer.start_request(DownloadRequestKind::DiscoveryRange);
let retry_classify = crate::retry::classify_discovery_retry;
let result = crate::retry::retry(req_metrics.metrics_mut(), retry_classify, |_allow_hedge| {
let mut builder =
copy_fields_to_get_object_request(input, transfer.ctx().s3_client().get_object());
if let Some(if_match) = if_match {
builder = builder.if_match(if_match);
}
let checksum = ChecksumCapture::new();
let req = builder
.range(header::Range::bytes(byte_range.clone()))
.customize()
.interceptor(Capture::new().with(&checksum))
.config_override(transfer.ctx().bucket_partition_override(input.bucket()));
async move {
req.send()
.await
.map(|resp| (resp, checksum))
.map_err(|e| crate::retry::GuardError::Inner(error::service_error(e)))
}
})
.instrument(tracing::trace_span!(
target: crate::telemetry::TARGET_TRANSFER_DOWNLOAD,
"discover-ranged-get",
tid = %transfer.ctx().id
))
.await;
match result {
Err(error) if error.code() == Some("InvalidRange") && range_from_user.is_none() => {
req_metrics.finish();
discover_obj_with_get_first_part(transfer, input, None).await
}
Err(error) => {
req_metrics.finish();
Err(error)
}
Ok((response, checksum)) => {
let discovery = first_chunk_response_handler(
response,
checksum,
range_from_user,
Some(&byte_range),
)?;
Ok(attach_request_measurement(discovery, req_metrics))
}
}
}
fn attach_request_measurement(
mut discovery: ObjectDiscovery,
mut req_metrics: DownloadRequestMeasurement,
) -> ObjectDiscovery {
match discovery.initial_chunk.as_mut() {
Some(initial) => {
req_metrics.pause();
initial.request_metrics = Some(req_metrics);
}
None => {
req_metrics.finish();
}
}
discovery
}
fn first_chunk_response_handler(
mut resp: GetObjectOutput,
checksum: ChecksumCapture,
range_from_user: Option<RangeInclusive<u64>>,
requested_range: Option<&ByteRange>,
) -> Result<ObjectDiscovery, error::Error> {
let empty_stream = ByteStream::new(SdkBody::empty());
let body = mem::replace(&mut resp.body, empty_stream);
let object_meta = crate::sdk_v1::object_metadata_from_get_object(&resp);
let chunk_meta = crate::sdk_v1::chunk_metadata_from_get_object(&resp);
let chunk_content_len = chunk_meta
.content_length
.ok_or_else(|| error::discovery_failed("response missing content-length"))
.and_then(|length| {
u64::try_from(length)
.map_err(|_| error::discovery_failed("response has negative content-length"))
})?;
let validated_range =
match requested_range {
Some(requested) if object_meta.content_range.is_some() => Some(
validate_ranged_response("GetObject", requested, &object_meta)?,
),
Some(_) if range_from_user.is_none() => None,
Some(_) => {
return Err(unusable_response(
"GetObject",
&object_meta,
"ranged response missing content-range",
));
}
None if chunk_content_len == 0 => None,
None if object_meta.content_range.is_some() => {
let validated = validate_response_content_range("GetObject", &object_meta)?;
if *validated.range.start() != 0 {
return Err(unusable_response(
"GetObject",
&object_meta,
"partNumber response does not start at byte zero",
));
}
Some(validated)
}
None => {
return Err(unusable_response(
"GetObject",
&object_meta,
"nonempty partNumber response missing content-range",
));
}
};
let (object_range_start, next_byte, object_size) = match validated_range.as_ref() {
Some(validated) => (
*validated.range.start(),
validated.range.end().checked_add(1),
validated.total,
),
None => (0, Some(chunk_content_len), chunk_content_len),
};
let remaining = object_size.checked_sub(1).and_then(|object_end| {
let start = next_byte?;
let end = range_from_user
.as_ref()
.map_or(object_end, |range| (*range.end()).min(object_end));
(start <= end).then_some(start..=end)
});
let initial_chunk = match chunk_content_len {
0 => None,
length => Some(InitialChunk {
body,
checksum,
expected_len: usize::try_from(length).map_err(|_| {
error::discovery_failed("response content-length exceeds platform representation")
})?,
request_metrics: None,
}),
};
Ok(ObjectDiscovery {
remaining,
object_range_start,
chunk_meta: Some(chunk_meta),
object_meta,
initial_chunk,
effective_part_size: 0,
validation_plan: ValidationPlan::PerResponse,
})
}
#[cfg(test)]
mod tests {
use crate::config::PartSize;
use crate::http::header::ByteRange;
use crate::operation::download::discovery::{
discover_obj, discover_obj_with_head, first_chunk_response_handler,
validate_head_response_range, ObjectDiscoveryStrategy,
};
use crate::operation::download::transfer::DownloadTransfer;
use crate::operation::download::DownloadInput;
use crate::transfer::TransferContext;
use crate::types::BucketType;
use crate::types::ByteSize;
use aws_sdk_s3::operation::get_object::{GetObjectError, GetObjectOutput};
use aws_sdk_s3::operation::head_object::HeadObjectOutput;
use aws_sdk_s3::Client;
use aws_smithy_mocks::{mock, mock_client};
use aws_smithy_observability::unit_of_work::checksums::ChecksumCapture;
use aws_smithy_types::byte_stream::ByteStream;
use aws_smithy_types::error::ErrorMetadata;
use bytes::Buf;
use std::sync::Arc;
fn strategy_from_range(
range: Option<&str>,
validation_enabled: bool,
) -> ObjectDiscoveryStrategy {
let input = DownloadInput::builder()
.bucket("test-bucket")
.key("test-key")
.set_range(range.map(|r| r.to_string()))
.build()
.unwrap();
ObjectDiscoveryStrategy::from_request(&input, validation_enabled).unwrap()
}
fn test_handle(
client: aws_sdk_s3::Client,
target_part_size: u64,
) -> Arc<crate::client::Handle> {
let tm_config = crate::Config::builder()
.sdk_client(client)
.set_target_part_size(PartSize::Target(ByteSize::new(target_part_size)))
.build()
.expect("valid config");
let tm = crate::Client::new(tm_config);
tm.handle.clone()
}
fn test_handle_auto(client: aws_sdk_s3::Client) -> Arc<crate::client::Handle> {
let tm_config = crate::Config::builder()
.sdk_client(client)
.build()
.expect("valid config");
let tm = crate::Client::new(tm_config);
tm.handle.clone()
}
fn test_transfer(
handle: Arc<crate::client::Handle>,
input: &DownloadInput,
) -> DownloadTransfer {
use crate::operation::download::body;
let (writer, _consumer) = body::new_recv_body();
let (ctx, _completion_rx) = TransferContext::new(handle);
DownloadTransfer::new(ctx, BucketType::Standard, input.clone(), writer)
}
#[test]
fn discovery_strategy_uses_resolved_validation_and_caller_bounds() {
for validation_enabled in [false, true] {
let whole_object = if validation_enabled {
ObjectDiscoveryStrategy::HeadObject
} else {
ObjectDiscoveryStrategy::RangedGet(None)
};
assert_eq!(strategy_from_range(None, validation_enabled), whole_object);
assert_eq!(
strategy_from_range(Some("bytes=100-200"), validation_enabled),
ObjectDiscoveryStrategy::RangedGet(Some(100..=200)),
);
for range in ["bytes=100-", "bytes=-500"] {
assert_eq!(
strategy_from_range(Some(range), validation_enabled),
ObjectDiscoveryStrategy::HeadObject,
);
}
}
}
#[test]
fn test_first_chunk_rejects_negative_content_length() {
let response = GetObjectOutput::builder().content_length(-1).build();
let error = first_chunk_response_handler(response, ChecksumCapture::new(), None, None)
.expect_err("negative content-length must not become an unsigned response length");
assert_eq!(
error.kind(),
&crate::error::ErrorKind::ObjectNotDiscoverable
);
}
#[test]
fn test_ranged_get_uses_validated_response_offsets() {
let response = GetObjectOutput::builder()
.content_length(100)
.content_range("bytes 100-199/500")
.build();
let requested_chunk = ByteRange::Inclusive(100, 199);
let discovery = first_chunk_response_handler(
response,
ChecksumCapture::new(),
Some(100..=300),
Some(&requested_chunk),
)
.unwrap();
assert_eq!(100, discovery.object_range_start);
assert_eq!(Some(200..=300), discovery.remaining);
}
#[test]
fn test_ranged_get_rejects_mismatched_response_offsets() {
let response = GetObjectOutput::builder()
.content_length(100)
.content_range("bytes 0-99/500")
.build();
let requested_chunk = ByteRange::Inclusive(100, 199);
let error = first_chunk_response_handler(
response,
ChecksumCapture::new(),
Some(100..=300),
Some(&requested_chunk),
)
.expect_err("response offsets must match the range sent to S3");
assert_eq!(error.kind(), &crate::error::ErrorKind::ServiceError);
assert_eq!(error.operation_name(), Some("GetObject"));
}
#[test]
fn test_full_object_get_accepts_response_without_content_range() {
let response = GetObjectOutput::builder().content_length(5).build();
let requested_chunk = ByteRange::Inclusive(0, 499);
let discovery = first_chunk_response_handler(
response,
ChecksumCapture::new(),
None,
Some(&requested_chunk),
)
.unwrap();
assert_eq!(0, discovery.object_range_start);
assert!(discovery.remaining.is_none());
assert_eq!(5, discovery.initial_chunk.unwrap().expected_len);
}
#[test]
fn test_empty_first_part_ignores_content_range() {
let response = GetObjectOutput::builder()
.content_length(0)
.content_range("bytes */0")
.build();
let discovery =
first_chunk_response_handler(response, ChecksumCapture::new(), None, None).unwrap();
assert_eq!(0, discovery.object_range_start);
assert!(discovery.remaining.is_none());
assert!(discovery.initial_chunk.is_none());
}
#[test]
fn test_user_ranged_get_requires_content_range() {
let response = GetObjectOutput::builder().content_length(100).build();
let requested_chunk = ByteRange::Inclusive(100, 199);
let error = first_chunk_response_handler(
response,
ChecksumCapture::new(),
Some(100..=300),
Some(&requested_chunk),
)
.expect_err("a user range requires absolute response offsets");
assert_eq!(error.kind(), &crate::error::ErrorKind::ServiceError);
assert_eq!(error.operation_name(), Some("GetObject"));
}
#[test]
fn test_nonempty_first_part_requires_valid_content_range() {
for content_range in [None, Some("bytes 5-9/10"), Some("bytes 0-4/*")] {
let response = GetObjectOutput::builder()
.content_length(5)
.set_content_range(content_range.map(str::to_string))
.build();
let error = first_chunk_response_handler(response, ChecksumCapture::new(), None, None)
.expect_err("a nonempty partNumber response requires valid object offsets");
assert_eq!(
error.kind(),
&crate::error::ErrorKind::ServiceError,
"accepted {content_range:?}"
);
assert_eq!(error.operation_name(), Some("GetObject"));
}
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_discover_obj_with_head() {
let head_obj_rule = mock!(Client::head_object).then_output(|| {
HeadObjectOutput::builder()
.content_length(500)
.content_range("bytes 0-499/10485760")
.build()
});
let client = mock_client!(aws_sdk_s3, &[&head_obj_rule]);
let input = DownloadInput::builder()
.bucket("test-bucket")
.key("test-key")
.range("bytes=0-499")
.build()
.unwrap();
let transfer = test_transfer(test_handle(client, ByteSize::mib(5).as_u64()), &input);
let discovery = discover_obj_with_head(&transfer, &input).await.unwrap();
let remaining = discovery.remaining.unwrap();
assert_eq!(500, remaining.clone().count());
assert_eq!(0..=499, remaining);
assert_eq!(0, discovery.object_range_start);
assert_eq!(transfer.ctx().metrics.request_metrics().requests, 1);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_discover_obj_with_head_ranged_keeps_the_offsets() {
let total = ByteSize::mib(10).as_u64();
let head_obj_rule = mock!(Client::head_object).then_output(move || {
HeadObjectOutput::builder()
.content_length(500)
.content_range(format!("bytes {}-{}/{total}", total - 500, total - 1))
.build()
});
let client = mock_client!(aws_sdk_s3, &[&head_obj_rule]);
let input = DownloadInput::builder()
.bucket("test-bucket")
.key("test-key")
.range("bytes=-500")
.build()
.unwrap();
let transfer = test_transfer(test_handle(client, ByteSize::mib(5).as_u64()), &input);
let discovery = discover_obj_with_head(&transfer, &input).await.unwrap();
assert_eq!(
Some((total - 500)..=(total - 1)),
discovery.remaining,
"the range to fetch is the one S3 reported, not its length anchored at 0"
);
assert_eq!(total - 500, discovery.object_range_start);
assert_eq!(
total,
discovery.object_meta.total_object_size(),
"a 206 HEAD reports the range's length, so the object's size comes from the range total"
);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_discover_obj_with_head_ranged_requires_content_range() {
let head_obj_rule = mock!(Client::head_object)
.then_output(|| HeadObjectOutput::builder().content_length(500).build());
let client = mock_client!(aws_sdk_s3, &[&head_obj_rule]);
let input = DownloadInput::builder()
.bucket("test-bucket")
.key("test-key")
.range("bytes=-500")
.build()
.unwrap();
let transfer = test_transfer(test_handle(client, ByteSize::mib(5).as_u64()), &input);
let error = discover_obj_with_head(&transfer, &input)
.await
.expect_err("ranged discovery requires absolute response offsets");
assert_eq!(error.kind(), &crate::error::ErrorKind::ServiceError);
assert_eq!(error.operation_name(), Some("HeadObject"));
}
#[test]
fn test_head_response_range_must_match_the_request() {
for (requested, returned, length) in [
("bytes=200-", "bytes 201-499/500", 299),
("bytes=-100", "bytes 399-499/500", 101),
("bytes=-100", "bytes 400-500/500", 101),
("bytes=-100", "bytes 400-499/500", 99),
("bytes=-100", "bytes 400-499/*", 100),
("bytes=-100", "bytes 400-499/ignored/500", 100),
] {
let input = DownloadInput::builder()
.bucket("test-bucket")
.key("test-key")
.range(requested)
.build()
.unwrap();
let response = HeadObjectOutput::builder()
.content_length(length)
.content_range(returned)
.build();
let object_meta = crate::sdk_v1::object_metadata_from_head_object(&response);
let error = validate_head_response_range(&input, &object_meta)
.expect_err("response range must match the request");
assert_eq!(
error.kind(),
&crate::error::ErrorKind::ServiceError,
"{requested} accepted {returned}"
);
assert_eq!(error.operation_name(), Some("HeadObject"));
}
}
#[test]
fn test_head_response_accepts_suffix_larger_than_object() {
let input = DownloadInput::builder()
.bucket("test-bucket")
.key("test-key")
.range("bytes=-1000")
.build()
.unwrap();
let response = HeadObjectOutput::builder()
.content_length(500)
.content_range("bytes 0-499/500")
.build();
let object_meta = crate::sdk_v1::object_metadata_from_head_object(&response);
let response = validate_head_response_range(&input, &object_meta).unwrap();
assert_eq!(0..=499, response);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_discover_obj_with_head_forwards_request_fields() {
let head_obj_rule = mock!(Client::head_object)
.match_requests(|request| {
request.range() == Some("bytes=-500")
&& request.version_id() == Some("version")
&& request.if_match() == Some("etag")
&& request.sse_customer_algorithm() == Some("AES256")
&& request.sse_customer_key() == Some("secret")
&& request.sse_customer_key_md5() == Some("key-md5")
&& request.request_payer() == Some(&aws_sdk_s3::types::RequestPayer::Requester)
&& request.expected_bucket_owner() == Some("owner")
})
.then_output(|| {
HeadObjectOutput::builder()
.content_length(500)
.content_range("bytes 500-999/1000")
.build()
});
let client = mock_client!(aws_sdk_s3, &[&head_obj_rule]);
let input = DownloadInput::builder()
.bucket("test-bucket")
.key("test-key")
.range("bytes=-500")
.version_id("version")
.if_match("etag")
.sse_customer_algorithm("AES256")
.sse_customer_key("secret")
.sse_customer_key_md5("key-md5")
.request_payer(crate::model::RequestPayer::Requester)
.expected_bucket_owner("owner")
.build()
.unwrap();
let transfer = test_transfer(test_handle(client, ByteSize::mib(5).as_u64()), &input);
let discovery = discover_obj_with_head(&transfer, &input).await.unwrap();
assert_eq!(Some(500..=999), discovery.remaining);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_discover_obj_with_get_full_range() {
let target_part_size = 500;
let bytes = &[0u8; 500];
let get_obj_rule = mock!(Client::get_object)
.match_requests(|r| r.range() == Some("bytes=0-499"))
.then_output(|| {
GetObjectOutput::builder()
.content_length(500)
.content_range("bytes 0-499/700")
.body(ByteStream::from_static(bytes))
.build()
});
let client = mock_client!(aws_sdk_s3, &[&get_obj_rule]);
let request = DownloadInput::builder()
.bucket("test-bucket")
.key("test-key")
.build()
.unwrap();
let transfer = test_transfer(test_handle(client, target_part_size), &request);
let discovery = discover_obj(&transfer, &request, false).await.unwrap();
let remaining = discovery.remaining.unwrap();
assert_eq!(200, remaining.clone().count());
assert_eq!(500..=699, remaining);
let initial = discovery.initial_chunk.expect("initial chunk");
assert_eq!(
transfer.ctx().metrics.request_metrics().requests,
0,
"the ranged request remains open until its lazy body is consumed"
);
let initial_chunk = initial.body.collect().await.expect("valid body");
initial
.request_metrics
.expect("request measurement")
.finish();
assert_eq!(500, initial_chunk.remaining());
assert_eq!(transfer.ctx().metrics.request_metrics().requests, 1);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_discover_obj_with_get_single_part() {
let target_part_size = 500;
let bytes = &[0u8; 400];
let get_obj_rule = mock!(Client::get_object)
.match_requests(|r| r.range() == Some("bytes=0-499"))
.then_output(|| {
GetObjectOutput::builder()
.content_length(400)
.content_range("bytes 0-399/400")
.body(ByteStream::from_static(bytes))
.build()
});
let client = mock_client!(aws_sdk_s3, &[&get_obj_rule]);
let request = DownloadInput::builder()
.bucket("test-bucket")
.key("test-key")
.build()
.unwrap();
let transfer = test_transfer(test_handle(client, target_part_size), &request);
let discovery = discover_obj(&transfer, &request, false).await.unwrap();
assert!(discovery.remaining.is_none());
let initial_chunk = discovery
.initial_chunk
.expect("initial chunk")
.body
.collect()
.await
.expect("valid body");
assert_eq!(400, initial_chunk.remaining());
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_discover_obj_aligns_composite_parts_and_preserves_head_metadata() {
const MIB: u64 = 1024 * 1024;
let stored_part = 16 * MIB;
let total = 36 * MIB;
let head_output = HeadObjectOutput::builder()
.content_length(total as i64)
.e_tag("\"abc-3\"")
.checksum_crc32("AAAAAA==-3")
.checksum_type(aws_sdk_s3::types::ChecksumType::Composite)
.metadata("source", "head")
.version_id("head-version")
.server_side_encryption(aws_sdk_s3::types::ServerSideEncryption::AwsKms)
.ssekms_key_id("head-key")
.content_type("application/octet-stream")
.build();
let mut expected_object_meta =
crate::sdk_v1::object_metadata_from_head_object(&head_output);
expected_object_meta.parts_count = Some(3);
let head = mock!(Client::head_object)
.match_requests(|r| {
r.checksum_mode() == Some(&aws_sdk_s3::types::ChecksumMode::Enabled)
&& r.if_match() == Some("caller-etag")
&& r.version_id() == Some("version")
})
.then_output(move || head_output.clone());
let part = mock!(Client::get_object)
.match_requests(|r| {
r.part_number() == Some(1)
&& r.range().is_none()
&& r.if_match() == Some("\"abc-3\"")
&& r.version_id() == Some("version")
})
.then_output(move || {
GetObjectOutput::builder()
.content_length(stored_part as i64)
.content_range(format!("bytes 0-{}/{}", stored_part - 1, total))
.e_tag("\"abc-3\"")
.checksum_crc32("AQIDBA==")
.parts_count(3)
.metadata("source", "part")
.version_id("part-version")
.server_side_encryption(aws_sdk_s3::types::ServerSideEncryption::Aes256)
.content_type("text/plain")
.cache_control("part-only")
.body(ByteStream::from_static(&[0u8; 8]))
.build()
});
let client = mock_client!(aws_sdk_s3, &[&head, &part]);
let input = DownloadInput::builder()
.bucket("bucket")
.key("key")
.if_match("caller-etag")
.version_id("version")
.build()
.unwrap();
let transfer = test_transfer(test_handle_auto(client), &input);
let discovery = discover_obj(&transfer, &input, true).await.unwrap();
assert_eq!(head.num_calls(), 1);
assert_eq!(part.num_calls(), 1);
assert_eq!(discovery.effective_part_size, stored_part);
assert_eq!(discovery.remaining, Some(stored_part..=total - 1));
assert_eq!(discovery.object_meta, expected_object_meta);
assert_eq!(
discovery.object_meta.version_id.as_deref(),
Some("head-version")
);
assert_eq!(
discovery.object_meta.ssekms_key_id.as_deref(),
Some("head-key")
);
assert_eq!(
discovery.object_meta.server_side_encryption,
Some(crate::model::ServerSideEncryption::AwsKms),
);
assert_eq!(
discovery.object_meta.content_type.as_deref(),
Some("application/octet-stream"),
);
assert_eq!(
discovery
.object_meta
.metadata
.as_ref()
.unwrap()
.get("source")
.map(String::as_str),
Some("head"),
);
assert_eq!(discovery.object_meta.content_range, None);
assert_eq!(discovery.object_meta.parts_count, Some(3));
assert_eq!(discovery.object_meta.cache_control, None);
assert_eq!(discovery.object_meta.content_length, Some(total as i64));
assert_eq!(
discovery.object_meta.checksum_crc32.as_deref(),
Some("AAAAAA==-3")
);
let chunk_meta = discovery.chunk_meta.expect("part metadata");
assert_eq!(chunk_meta.checksum_crc32.as_deref(), Some("AQIDBA=="));
assert_eq!(chunk_meta.content_length, Some(stored_part as i64));
assert_eq!(
chunk_meta.content_range.as_deref(),
Some(format!("bytes 0-{}/{}", stored_part - 1, total).as_str()),
);
assert_eq!(chunk_meta.parts_count, Some(3));
assert_eq!(
chunk_meta
.metadata
.as_ref()
.unwrap()
.get("source")
.map(String::as_str),
Some("part")
);
assert_eq!(chunk_meta.version_id.as_deref(), Some("part-version"));
assert_eq!(
chunk_meta.server_side_encryption,
Some(crate::model::ServerSideEncryption::Aes256)
);
assert_eq!(chunk_meta.cache_control.as_deref(), Some("part-only"));
assert_eq!(
discovery.initial_chunk.unwrap().expected_len,
stored_part as usize
);
assert!(matches!(
discovery.validation_plan,
super::ValidationPlan::PerResponse
));
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_discover_obj_no_align_when_validation_disabled() {
let configured = 500;
let ranged = mock!(Client::get_object)
.match_requests(|r| r.range() == Some("bytes=0-499"))
.then_output(|| {
GetObjectOutput::builder()
.content_length(500)
.content_range("bytes 0-499/700")
.e_tag("\"abc-2\"")
.body(ByteStream::from_static(&[0u8; 500]))
.build()
});
let client = mock_client!(aws_sdk_s3, &[&ranged]);
let request = DownloadInput::builder()
.bucket("test-bucket")
.key("test-key")
.build()
.unwrap();
let transfer = test_transfer(test_handle(client, configured), &request);
let discovery = discover_obj(&transfer, &request, false).await.unwrap();
assert_eq!(discovery.effective_part_size, configured);
assert_eq!(discovery.remaining, Some(500..=699));
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_discover_obj_with_get_partial_range() {
let target_part_size = 100;
let bytes = &[0u8; 100];
let get_obj_rule = mock!(Client::get_object)
.match_requests(|r| r.range() == Some("bytes=200-299"))
.then_output(|| {
GetObjectOutput::builder()
.content_length(100)
.content_range("bytes 200-299/700")
.body(ByteStream::from_static(bytes))
.build()
});
let client = mock_client!(aws_sdk_s3, &[&get_obj_rule]);
let request = DownloadInput::builder()
.bucket("test-bucket")
.key("test-key")
.range("bytes=200-499")
.build()
.unwrap();
let transfer = test_transfer(test_handle(client, target_part_size), &request);
let discovery = discover_obj(&transfer, &request, false).await.unwrap();
let remaining = discovery.remaining.unwrap();
assert_eq!(200, remaining.clone().count());
assert_eq!(300..=499, remaining);
let initial_chunk = discovery
.initial_chunk
.expect("initial chunk")
.body
.collect()
.await
.expect("valid body");
assert_eq!(100, initial_chunk.remaining());
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_discover_obj_with_get_over_range() {
let target_part_size = 100;
let bytes = &[0u8; 50];
let get_obj_rule = mock!(Client::get_object)
.match_requests(|r| r.range() == Some("bytes=200-299"))
.then_output(|| {
GetObjectOutput::builder()
.content_length(50)
.content_range("bytes 200-249/250")
.body(ByteStream::from_static(bytes))
.build()
});
let client = mock_client!(aws_sdk_s3, &[&get_obj_rule]);
let request = DownloadInput::builder()
.bucket("test-bucket")
.key("test-key")
.range("bytes=200-123456")
.build()
.unwrap();
let transfer = test_transfer(test_handle(client, target_part_size), &request);
let discovery = discover_obj(&transfer, &request, false).await.unwrap();
assert!(discovery.remaining.is_none());
let initial_chunk = discovery
.initial_chunk
.expect("initial chunk")
.body
.collect()
.await
.expect("valid body");
assert_eq!(50, initial_chunk.remaining());
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn test_discover_obj_with_empty_object() {
let target_part_size = 500;
let get_range_rule = mock!(Client::get_object)
.match_requests(|r| r.range() == Some("bytes=0-499"))
.then_error(|| {
GetObjectError::generic(ErrorMetadata::builder().code("InvalidRange").build())
});
let get_first_part_rule = mock!(Client::get_object)
.match_requests(|r| r.part_number() == Some(1))
.then_output(|| GetObjectOutput::builder().content_length(0).build());
let client = mock_client!(aws_sdk_s3, &[&get_range_rule, &get_first_part_rule]);
let request = DownloadInput::builder()
.bucket("test-bucket")
.key("test-key")
.build()
.unwrap();
let transfer = test_transfer(test_handle(client, target_part_size), &request);
let discovery = discover_obj(&transfer, &request, false).await.unwrap();
assert!(discovery.remaining.is_none());
assert!(discovery.initial_chunk.is_none());
}
#[test]
fn whole_object_head_requires_valid_length_and_nonempty_object_etag() {
for response in [
HeadObjectOutput::builder().build(),
HeadObjectOutput::builder()
.content_length(-1)
.e_tag("etag")
.build(),
HeadObjectOutput::builder().content_length(10).build(),
HeadObjectOutput::builder()
.content_length(10)
.e_tag("")
.build(),
] {
let metadata = crate::sdk_v1::object_metadata_from_head_object(&response);
assert_eq!(
super::validate_whole_object_head(&metadata)
.unwrap_err()
.kind(),
&crate::error::ErrorKind::ObjectNotDiscoverable
);
}
let response = HeadObjectOutput::builder()
.content_length(10)
.e_tag("etag")
.content_range("bytes 0-9/10")
.build();
let metadata = crate::sdk_v1::object_metadata_from_head_object(&response);
let error = super::validate_whole_object_head(&metadata).unwrap_err();
assert_eq!(error.kind(), &crate::error::ErrorKind::ServiceError);
assert_eq!(error.operation_name(), Some("HeadObject"));
let response = HeadObjectOutput::builder().content_length(0).build();
let metadata = crate::sdk_v1::object_metadata_from_head_object(&response);
assert!(super::validate_whole_object_head(&metadata)
.unwrap()
.is_none());
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn validated_whole_objects_set_head_checksum_mode_and_keep_configured_ranges() {
for size in [7 * 1024 * 1024, 20 * 1024 * 1024] {
let head = mock!(Client::head_object)
.match_requests(|r| {
r.range().is_none()
&& r.checksum_mode() == Some(&aws_sdk_s3::types::ChecksumMode::Enabled)
})
.then_output(move || {
HeadObjectOutput::builder()
.content_length(size)
.e_tag("\"multipart-3\"")
.checksum_crc64_nvme("AAAAAAAAAAA=")
.checksum_type(aws_sdk_s3::types::ChecksumType::FullObject)
.build()
});
let client = mock_client!(aws_sdk_s3, &[&head]);
let input = DownloadInput::builder()
.bucket("bucket")
.key("key")
.build()
.unwrap();
let transfer = test_transfer(test_handle_auto(client), &input);
let discovery = discover_obj(&transfer, &input, true).await.unwrap();
assert_eq!(head.num_calls(), 1);
assert_eq!(discovery.remaining, Some(0..=size as u64 - 1));
assert_eq!(
discovery.effective_part_size,
crate::runtime::sizing::DEFAULT_PART_SIZE
);
assert!(discovery.initial_chunk.is_none());
assert!(discovery.chunk_meta.is_none());
assert!(matches!(
discovery.validation_plan,
super::ValidationPlan::WholeObject(_)
));
assert_eq!(discovery.object_meta.content_length, Some(size));
assert_eq!(transfer.ctx().metrics.request_metrics().requests, 1);
}
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn whole_object_head_preserves_caller_conditions_and_encryption_fields() {
let head = mock!(Client::head_object)
.match_requests(|r| {
r.range().is_none()
&& r.checksum_mode() == Some(&aws_sdk_s3::types::ChecksumMode::Enabled)
&& r.version_id() == Some("version")
&& r.if_match() == Some("caller-etag")
&& r.sse_customer_algorithm() == Some("AES256")
&& r.sse_customer_key() == Some("secret")
&& r.sse_customer_key_md5() == Some("key-md5")
&& r.request_payer() == Some(&aws_sdk_s3::types::RequestPayer::Requester)
&& r.expected_bucket_owner() == Some("owner")
})
.then_output(|| {
HeadObjectOutput::builder()
.content_length(500)
.e_tag("caller-etag")
.build()
});
let client = mock_client!(aws_sdk_s3, &[&head]);
let input = DownloadInput::builder()
.bucket("bucket")
.key("key")
.version_id("version")
.if_match("caller-etag")
.sse_customer_algorithm("AES256")
.sse_customer_key("secret")
.sse_customer_key_md5("key-md5")
.request_payer(crate::model::RequestPayer::Requester)
.expected_bucket_owner("owner")
.build()
.unwrap();
let transfer = test_transfer(test_handle_auto(client), &input);
let discovery = discover_obj(&transfer, &input, true).await.unwrap();
assert_eq!(head.num_calls(), 1);
assert_eq!(discovery.remaining, Some(0..=499));
assert!(matches!(
discovery.validation_plan,
super::ValidationPlan::PerResponse
));
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn empty_objects_discover_their_plan_without_a_get() {
for crc in [true, false] {
let head = mock!(Client::head_object).then_output(move || {
let builder = HeadObjectOutput::builder().content_length(0);
if crc {
builder.checksum_crc32("AAAAAA==")
} else {
builder.checksum_sha256("stored hash")
}
.build()
});
let client = mock_client!(aws_sdk_s3, &[&head]);
let input = DownloadInput::builder()
.bucket("bucket")
.key("key")
.build()
.unwrap();
let transfer = test_transfer(test_handle_auto(client), &input);
let discovery = discover_obj(&transfer, &input, true).await.unwrap();
assert!(discovery.remaining.is_none());
assert!(discovery.initial_chunk.is_none());
assert_eq!(
matches!(
discovery.validation_plan,
super::ValidationPlan::WholeObject(_)
),
crc
);
assert_eq!(head.num_calls(), 1);
}
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn explicit_part_size_keeps_composite_downloads_on_configured_ranges() {
let head = mock!(Client::head_object).then_output(|| {
HeadObjectOutput::builder()
.content_length(700)
.e_tag("\"abc-2\"")
.checksum_crc32("AAAAAA==-2")
.checksum_type(aws_sdk_s3::types::ChecksumType::Composite)
.build()
});
let client = mock_client!(aws_sdk_s3, &[&head]);
let input = DownloadInput::builder()
.bucket("bucket")
.key("key")
.build()
.unwrap();
let transfer = test_transfer(test_handle(client, 500), &input);
let discovery = discover_obj(&transfer, &input, true).await.unwrap();
assert!(discovery.initial_chunk.is_none());
assert_eq!(discovery.remaining, Some(0..=699));
assert_eq!(discovery.effective_part_size, 500);
assert!(matches!(
discovery.validation_plan,
super::ValidationPlan::PerResponse
));
assert_eq!(head.num_calls(), 1);
}
#[test]
fn composite_layout_must_match_head_size_and_etag() {
let head = HeadObjectOutput::builder()
.content_length(700)
.e_tag("etag")
.build();
let head = crate::sdk_v1::object_metadata_from_head_object(&head);
for (range, etag) in [
("bytes 0-499/701", "etag"),
("bytes 1-500/700", "etag"),
("bytes 0-499/700", "changed"),
] {
let part = GetObjectOutput::builder()
.content_length(500)
.content_range(range)
.e_tag(etag)
.build();
let part = crate::sdk_v1::object_metadata_from_get_object(&part);
assert_eq!(
super::validate_layout_response(&head, &part)
.unwrap_err()
.kind(),
&crate::error::ErrorKind::ObjectNotDiscoverable
);
}
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn changed_composite_object_fails_the_pinned_layout_request() {
let head = mock!(Client::head_object).then_output(|| {
HeadObjectOutput::builder()
.content_length(700)
.e_tag("\"abc-2\"")
.checksum_crc32("AAAAAA==-2")
.build()
});
let part = mock!(Client::get_object)
.match_requests(|r| r.if_match() == Some("\"abc-2\"") && r.part_number() == Some(1))
.then_error(|| {
GetObjectError::generic(
ErrorMetadata::builder()
.code("PreconditionFailed")
.message("object changed")
.build(),
)
});
let client = mock_client!(aws_sdk_s3, &[&head, &part]);
let input = DownloadInput::builder()
.bucket("bucket")
.key("key")
.build()
.unwrap();
let transfer = test_transfer(test_handle_auto(client), &input);
let error = discover_obj(&transfer, &input, true).await.unwrap_err();
assert_eq!(error.code(), Some("PreconditionFailed"));
assert_eq!(head.num_calls(), 1);
assert_eq!(part.num_calls(), 1);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn multipart_etag_without_a_composite_checksum_does_not_fetch_a_layout() {
let head = mock!(Client::head_object).then_output(|| {
HeadObjectOutput::builder()
.content_length(700)
.e_tag("\"abc-2\"")
.build()
});
let client = mock_client!(aws_sdk_s3, &[&head]);
let input = DownloadInput::builder()
.bucket("bucket")
.key("key")
.build()
.unwrap();
let transfer = test_transfer(test_handle_auto(client), &input);
let discovery = discover_obj(&transfer, &input, true).await.unwrap();
assert_eq!(head.num_calls(), 1);
assert!(discovery.initial_chunk.is_none());
assert!(matches!(
discovery.validation_plan,
super::ValidationPlan::PerResponse
));
}
}