mod crc;
pub(super) mod stored;
use crc::{digest_segments, CrcFold};
pub(super) use crc::{ComputedChecksum, CrcAlgorithm};
use stored::ExpectedValue;
use crate::error::{checksum_algorithm, Error, ErrorKind};
use crate::model::{ChecksumAlgorithm, ChecksumType, ObjectMetadata};
use crate::runtime::buffer_pool::SegmentedBytes;
use crate::types::{ChecksumValidation, IntegrityChecks, NotValidatedReason};
use aws_smithy_observability::unit_of_work::checksums::{
ChecksumOutcome, NotValidatedReason as ResponseNotValidatedReason,
};
#[derive(Debug)]
pub(super) enum ValidationPlan {
Disabled,
PerResponse,
WholeObject(WholeObjectPlan),
}
impl ValidationPlan {
pub(super) fn log(&self, meta: &ObjectMetadata, reason: &str) {
let (target, method, algorithm) = match self {
Self::Disabled => ("disabled", None, None),
Self::PerResponse => ("per_response", None, None),
Self::WholeObject(plan) => {
let ObjectChecksumMethod::CombineRanges { algorithm } = plan.method;
(
"whole_object",
Some("combine_ranges"),
Some(algorithm.model()),
)
}
};
tracing::debug!(
target: crate::telemetry::TARGET_TRANSFER_DOWNLOAD,
validation_target = target,
method,
?algorithm,
stored_algorithm = ?stored::StoredChecksum::algorithm(meta),
stored_type = ?meta.checksum_type,
reason,
"download validation plan",
);
}
}
#[derive(Debug)]
pub(super) struct WholeObjectPlan {
method: ObjectChecksumMethod,
expected: ExpectedValue,
object_len: u64,
}
#[derive(Debug, Clone, Copy)]
pub(super) enum ObjectChecksumMethod {
CombineRanges {
algorithm: CrcAlgorithm,
},
}
impl ObjectChecksumMethod {
fn range_digest_plan(self) -> RangeDigestPlan {
match self {
Self::CombineRanges { algorithm } => RangeDigestPlan::Calculate { algorithm },
}
}
}
#[derive(Debug, Clone, Copy)]
pub(super) enum RangeDigestPlan {
None,
Calculate {
algorithm: CrcAlgorithm,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub(super) struct CompletedRange {
pub(super) start: u64,
pub(super) end: u64,
}
#[derive(Debug)]
pub(super) struct RangeCompletion {
pub(super) range: CompletedRange,
pub(super) response_outcome: Option<ChecksumOutcome>,
pub(super) computed: Option<ComputedChecksum>,
}
impl RangeCompletion {
pub(super) fn capture(
offset: u64,
bytes: &SegmentedBytes,
response_outcome: Option<ChecksumOutcome>,
digest_plan: RangeDigestPlan,
) -> Result<Self, Error> {
let len = u64::try_from(bytes.len())
.map_err(|_| Error::new(ErrorKind::RuntimeError, "range length exceeds u64"))?;
let end = offset
.checked_add(len)
.ok_or_else(|| Error::new(ErrorKind::RuntimeError, "range checksum bounds overflow"))?;
let computed = match digest_plan {
RangeDigestPlan::None => None,
RangeDigestPlan::Calculate { algorithm } => Some(digest_segments(algorithm, bytes)),
};
Ok(Self {
range: CompletedRange { start: offset, end },
response_outcome,
computed,
})
}
}
#[derive(Debug)]
pub(super) struct DownloadIntegrity {
plan: ValidationPlan,
whole_object_meta: Option<ObjectMetadata>,
fold: Option<CrcFold>,
failure: Option<Error>,
delivered_bytes: u64,
validated_bytes: u64,
first_validated: Option<(u64, ChecksumAlgorithm)>,
part_level: bool,
unsupported_algorithm: bool,
}
impl Default for DownloadIntegrity {
fn default() -> Self {
Self::new(ValidationPlan::PerResponse, None)
}
}
impl DownloadIntegrity {
pub(super) fn new(plan: ValidationPlan, object_meta: Option<ObjectMetadata>) -> Self {
let fold = match &plan {
ValidationPlan::WholeObject(WholeObjectPlan {
method: ObjectChecksumMethod::CombineRanges { algorithm },
object_len,
..
}) => Some(CrcFold::new(*algorithm, *object_len)),
_ => None,
};
Self {
plan,
whole_object_meta: object_meta,
fold,
failure: None,
delivered_bytes: 0,
validated_bytes: 0,
first_validated: None,
part_level: false,
unsupported_algorithm: false,
}
}
pub(super) fn range_digest_plan(&self) -> RangeDigestPlan {
match &self.plan {
ValidationPlan::WholeObject(plan) => plan.method.range_digest_plan(),
_ => RangeDigestPlan::None,
}
}
pub(super) fn record(&mut self, completion: RangeCompletion) {
if self.failure.is_some() {
return;
}
let RangeCompletion {
range,
response_outcome,
computed,
} = completion;
let Some(len) = range.end.checked_sub(range.start) else {
self.failure.get_or_insert_with(|| {
Error::new(
ErrorKind::RuntimeError,
"range checksum has reversed bounds",
)
});
return;
};
if self.delivered_bytes.checked_add(len).is_none() {
self.failure.get_or_insert_with(|| {
Error::new(
ErrorKind::RuntimeError,
"accepted checksum byte count overflows",
)
});
return;
}
self.record_response_outcome(range.start, len, response_outcome);
if let Some(fold) = &mut self.fold {
match computed {
Some(computed) => fold.record(range, computed),
None => fold.fail("accepted range is missing its calculated CRC"),
}
}
}
pub(super) fn finish(mut self) -> Result<IntegrityChecks, Error> {
if let Some(error) = self.failure.take() {
return Err(error);
}
let mut validation = self.response_validation();
if let ValidationPlan::WholeObject(plan) = &self.plan {
let fold = self.fold.take().ok_or_else(|| {
Error::new(
ErrorKind::RuntimeError,
"whole-object plan has no CRC accumulator",
)
})?;
fold.finish(&plan.expected)?;
if !matches!(validation, ChecksumValidation::Validated { .. }) {
let ObjectChecksumMethod::CombineRanges { algorithm } = plan.method;
validation = ChecksumValidation::Validated {
algorithm: algorithm.model(),
};
}
}
Ok(self.checks(validation))
}
fn record_response_outcome(&mut self, offset: u64, len: u64, outcome: Option<ChecksumOutcome>) {
self.delivered_bytes += len;
match outcome {
Some(ChecksumOutcome::Validated { algorithm, .. }) => {
self.validated_bytes += len;
if self
.first_validated
.as_ref()
.is_none_or(|(first, _)| offset < *first)
{
self.first_validated = Some((offset, checksum_algorithm(algorithm)));
}
}
Some(ChecksumOutcome::NotValidated {
reason: ResponseNotValidatedReason::PartLevelChecksum,
}) => self.part_level = true,
Some(ChecksumOutcome::NotValidated {
reason: ResponseNotValidatedReason::UnsupportedAlgorithm,
}) => self.unsupported_algorithm = true,
_ => {}
}
}
fn response_validation(&self) -> ChecksumValidation {
let not_validated = |reason| ChecksumValidation::NotValidated { reason };
if matches!(self.plan, ValidationPlan::Disabled) {
return not_validated(NotValidatedReason::Disabled);
}
if self.delivered_bytes > 0 && self.validated_bytes == self.delivered_bytes {
if let Some((_, algorithm)) = &self.first_validated {
return ChecksumValidation::Validated {
algorithm: algorithm.clone(),
};
}
}
not_validated(if self.validated_bytes > 0 {
NotValidatedReason::PartialCoverage
} else if self.part_level {
NotValidatedReason::CompositeChecksum
} else if self.unsupported_algorithm {
NotValidatedReason::UnsupportedAlgorithm
} else {
NotValidatedReason::Unavailable
})
}
fn checks(&self, validation: ChecksumValidation) -> IntegrityChecks {
let meta = self.whole_object_meta.as_ref();
let values = meta.filter(|m| m.checksum_type != Some(ChecksumType::Composite));
IntegrityChecks::new(
values,
meta.and_then(|m| m.checksum_type.clone()),
validation,
)
}
}
#[cfg(test)]
mod test_util {
use super::*;
use aws_smithy_observability::unit_of_work::checksums::ResponseChecksumAlgorithm;
pub(super) fn validated(algorithm: ResponseChecksumAlgorithm) -> Option<ChecksumOutcome> {
Some(ChecksumOutcome::Validated {
algorithm,
computed: "AAAAAA==".to_owned(),
})
}
pub(super) fn not_validated(reason: ResponseNotValidatedReason) -> Option<ChecksumOutcome> {
Some(ChecksumOutcome::NotValidated { reason })
}
pub(super) fn reason(reason: NotValidatedReason) -> ChecksumValidation {
ChecksumValidation::NotValidated { reason }
}
pub(super) fn whole_owner(bytes: &[u8]) -> DownloadIntegrity {
let mut checksum = aws_smithy_checksums::ChecksumAlgorithm::Crc32.into_impl();
checksum.update(bytes);
let mut meta = ObjectMetadata::default();
meta.checksum_crc32 = Some(aws_smithy_types::base64::encode(checksum.finalize()));
meta.checksum_type = Some(ChecksumType::FullObject);
let plan = stored::StoredChecksum::from_metadata(&meta)
.unwrap()
.into_plan(bytes.len() as u64);
DownloadIntegrity::new(plan, Some(meta))
}
pub(super) fn record_response(
integrity: &mut DownloadIntegrity,
offset: u64,
len: u64,
outcome: Option<ChecksumOutcome>,
) {
integrity.record(RangeCompletion {
range: CompletedRange {
start: offset,
end: offset.checked_add(len).expect("test range bounds"),
},
response_outcome: outcome,
computed: None,
});
}
}
#[cfg(test)]
mod tests {
use super::test_util::*;
use super::*;
use crate::model::ChecksumType;
use aws_smithy_observability::unit_of_work::checksums::ResponseChecksumAlgorithm;
#[test]
fn disabled_when_validation_not_enabled() {
let mut tally = DownloadIntegrity::new(ValidationPlan::Disabled, None);
record_response(
&mut tally,
0,
8,
validated(ResponseChecksumAlgorithm::Crc32),
);
assert_eq!(
tally.response_validation(),
reason(NotValidatedReason::Disabled)
);
}
#[test]
fn validated_takes_the_lowest_offset_chunks_algorithm() {
let mut tally = DownloadIntegrity::default();
record_response(
&mut tally,
8,
8,
validated(ResponseChecksumAlgorithm::Sha256),
);
record_response(
&mut tally,
0,
8,
validated(ResponseChecksumAlgorithm::Crc32),
);
assert_eq!(
tally.response_validation(),
ChecksumValidation::Validated {
algorithm: ChecksumAlgorithm::Crc32
}
);
}
#[test]
fn partial_coverage_when_some_delivered_bytes_were_validated() {
let mut tally = DownloadIntegrity::default();
record_response(
&mut tally,
0,
8,
validated(ResponseChecksumAlgorithm::Crc32),
);
record_response(
&mut tally,
8,
8,
not_validated(ResponseNotValidatedReason::PartLevelChecksum),
);
assert_eq!(
tally.response_validation(),
reason(NotValidatedReason::PartialCoverage)
);
}
#[test]
fn composite_checksum_when_a_chunk_had_only_a_part_level_checksum() {
let mut tally = DownloadIntegrity::default();
record_response(
&mut tally,
0,
8,
not_validated(ResponseNotValidatedReason::PartLevelChecksum),
);
record_response(
&mut tally,
8,
8,
not_validated(ResponseNotValidatedReason::NoChecksum),
);
assert_eq!(
tally.response_validation(),
reason(NotValidatedReason::CompositeChecksum)
);
}
#[test]
fn unsupported_algorithm_when_a_chunk_had_only_an_unsupported_checksum() {
let mut tally = DownloadIntegrity::default();
record_response(
&mut tally,
0,
8,
not_validated(ResponseNotValidatedReason::UnsupportedAlgorithm),
);
record_response(
&mut tally,
8,
8,
not_validated(ResponseNotValidatedReason::NoChecksum),
);
assert_eq!(
tally.response_validation(),
reason(NotValidatedReason::UnsupportedAlgorithm)
);
}
#[test]
fn partial_coverage_outranks_an_unsupported_algorithm() {
let mut tally = DownloadIntegrity::default();
record_response(
&mut tally,
0,
8,
validated(ResponseChecksumAlgorithm::Crc32),
);
record_response(
&mut tally,
8,
8,
not_validated(ResponseNotValidatedReason::UnsupportedAlgorithm),
);
assert_eq!(
tally.response_validation(),
reason(NotValidatedReason::PartialCoverage)
);
}
#[test]
fn a_part_level_checksum_outranks_an_unsupported_algorithm() {
let mut tally = DownloadIntegrity::default();
record_response(
&mut tally,
0,
8,
not_validated(ResponseNotValidatedReason::UnsupportedAlgorithm),
);
record_response(
&mut tally,
8,
8,
not_validated(ResponseNotValidatedReason::PartLevelChecksum),
);
assert_eq!(
tally.response_validation(),
reason(NotValidatedReason::CompositeChecksum)
);
}
#[test]
fn unavailable_when_no_bytes_were_delivered() {
assert_eq!(
DownloadIntegrity::default().response_validation(),
reason(NotValidatedReason::Unavailable)
);
}
#[test]
fn other_outcomes_leave_the_bytes_unvalidated() {
for outcome in [
None,
not_validated(ResponseNotValidatedReason::NoChecksum),
not_validated(ResponseNotValidatedReason::Reused),
] {
let mut tally = DownloadIntegrity::default();
record_response(&mut tally, 0, 8, outcome.clone());
assert_eq!(
tally.response_validation(),
reason(NotValidatedReason::Unavailable),
"{outcome:?}"
);
}
}
#[test]
fn integrity_checks_surface_only_whole_object_metadata() {
let mut meta = ObjectMetadata::default();
meta.checksum_crc32 = Some("DUoRhQ==".to_owned());
meta.checksum_type = Some(ChecksumType::FullObject);
let whole = DownloadIntegrity::new(ValidationPlan::PerResponse, Some(meta));
let checks = whole.finish().unwrap();
assert_eq!(checks.checksum_crc32(), Some("DUoRhQ=="));
assert_eq!(checks.checksum_type(), Some(&ChecksumType::FullObject));
let part = DownloadIntegrity::new(ValidationPlan::PerResponse, None);
let checks = part.finish().unwrap();
assert_eq!(checks.checksum_crc32(), None);
assert_eq!(checks.checksum_type(), None);
}
#[test]
fn integrity_checks_omit_composite_values_and_keep_the_type() {
let mut meta = ObjectMetadata::default();
meta.checksum_crc32 = Some("DUoRhQ==-1".to_owned());
meta.checksum_crc32_c = Some("yZRlqg==-1".to_owned());
meta.checksum_sha1 = Some("Kq5sNclPz7QV2+lfQIuc6R7oRu0=-1".to_owned());
meta.checksum_sha256 = Some("uU0nuZNNPgilLlLX2n2r+sSE7+N6U4DukIj3rOLvzek=-1".to_owned());
meta.checksum_md5 = Some("1B2M2Y8AsgTpgAmY7PhCfg==-1".to_owned());
meta.checksum_type = Some(ChecksumType::Composite);
let checks = DownloadIntegrity::new(ValidationPlan::PerResponse, Some(meta))
.finish()
.unwrap();
assert_eq!(checks.checksum_crc32(), None);
assert_eq!(checks.checksum_crc32c(), None);
assert_eq!(checks.checksum_crc64_nvme(), None);
assert_eq!(checks.checksum_sha1(), None);
assert_eq!(checks.checksum_sha256(), None);
assert_eq!(checks.checksum_md5(), None);
assert_eq!(checks.checksum_type(), Some(&ChecksumType::Composite));
}
#[test]
fn integrity_checks_report_every_stored_full_object_value() {
let mut meta = ObjectMetadata::default();
meta.checksum_sha512 = Some("sha512".to_owned());
meta.checksum_md5 = Some("md5".to_owned());
meta.checksum_xxhash64 = Some("xxhash64".to_owned());
meta.checksum_xxhash3 = Some("xxhash3".to_owned());
meta.checksum_xxhash128 = Some("xxhash128".to_owned());
meta.checksum_type = Some(ChecksumType::FullObject);
let checks = DownloadIntegrity::new(ValidationPlan::PerResponse, Some(meta))
.finish()
.unwrap();
assert_eq!(checks.checksum_sha512(), Some("sha512"));
assert_eq!(checks.checksum_md5(), Some("md5"));
assert_eq!(checks.checksum_xxhash64(), Some("xxhash64"));
assert_eq!(checks.checksum_xxhash3(), Some("xxhash3"));
assert_eq!(checks.checksum_xxhash128(), Some("xxhash128"));
assert_eq!(checks.checksum_type(), Some(&ChecksumType::FullObject));
}
#[test]
fn whole_object_verification_validates_ranges_without_sdk_checksums() {
let mut integrity = whole_owner(b"abcdefgh");
let digest_plan = integrity.range_digest_plan();
for (offset, bytes) in [(4, b"efgh".as_slice()), (0, b"abcd".as_slice())] {
let bytes = SegmentedBytes::from(bytes::Bytes::copy_from_slice(bytes));
integrity.record(RangeCompletion::capture(offset, &bytes, None, digest_plan).unwrap());
}
assert_eq!(
integrity.finish().unwrap().checksum_validation(),
&ChecksumValidation::Validated {
algorithm: ChecksumAlgorithm::Crc32
}
);
}
#[test]
fn sdk_verdict_does_not_bypass_failed_whole_object_verification() {
let mut integrity = whole_owner(b"stored");
let bytes = SegmentedBytes::from(bytes::Bytes::from_static(b"broken"));
integrity.record(
RangeCompletion::capture(
0,
&bytes,
validated(ResponseChecksumAlgorithm::Sha256),
integrity.range_digest_plan(),
)
.unwrap(),
);
assert!(matches!(
integrity.finish().unwrap_err().kind(),
ErrorKind::IntegrityError(_)
));
}
#[test]
fn successful_sdk_verdict_retains_its_algorithm() {
let mut integrity = whole_owner(b"stored");
let bytes = SegmentedBytes::from(bytes::Bytes::from_static(b"stored"));
integrity.record(
RangeCompletion::capture(
0,
&bytes,
validated(ResponseChecksumAlgorithm::Sha256),
integrity.range_digest_plan(),
)
.unwrap(),
);
assert_eq!(
integrity.finish().unwrap().checksum_validation(),
&ChecksumValidation::Validated {
algorithm: ChecksumAlgorithm::Sha256
}
);
}
#[test]
fn missing_calculated_evidence_cannot_downgrade_verification() {
let mut integrity = whole_owner(b"stored");
record_response(
&mut integrity,
0,
6,
validated(ResponseChecksumAlgorithm::Crc32),
);
assert_eq!(
integrity.finish().unwrap_err().kind(),
&ErrorKind::RuntimeError
);
}
#[test]
fn empty_crc_object_is_verified_without_response_evidence() {
assert_eq!(
whole_owner(b"").finish().unwrap().checksum_validation(),
&ChecksumValidation::Validated {
algorithm: ChecksumAlgorithm::Crc32
}
);
}
#[test]
fn capture_rejects_offset_overflow() {
let bytes = SegmentedBytes::from(bytes::Bytes::from_static(b"a"));
assert_eq!(
RangeCompletion::capture(u64::MAX, &bytes, None, RangeDigestPlan::None)
.unwrap_err()
.kind(),
&ErrorKind::RuntimeError
);
}
}