aws-s3-transfer 1.0.0

Transfer Manager for Amazon S3 - a high-performance data plane client for uploading and downloading objects and directories.
Documentation
/*
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 * SPDX-License-Identifier: Apache-2.0
 */

//! Download a single S3 object using modeled request and response metadata.
//!
//! [`DownloadInput`](crate::operation::download::DownloadInput) and the metadata
//! types are re-exported from [`crate::types`].
//! Object metadata describes the discovery response (GET or HEAD), while each
//! [`ChunkOutput`](crate::operation::download::ChunkOutput) carries metadata from
//! its own GET response. Optional service
//! values remain absent when the corresponding response does not include them.
//! Reported checksum fields alone do not mean that the downloaded bytes were
//! validated; use
//! [`DownloadOutput::integrity_checks`](crate::operation::download::DownloadOutput::integrity_checks)
//! for that result.
//!
//! Whole-object downloads with checksum validation enabled start with checksum-mode
//! HEAD. A stored FULL_OBJECT CRC32, CRC32C or CRC64NVME checksum is verified by
//! hashing accepted ranges and combining their CRCs in object-offset order.
//! SDK response checks remain active. COMPOSITE objects use response checks;
//! automatic part sizing resolves the first stored part before issuing later ranges.
//! Caller ranges use response checks without whole-object CRC verification.
//!
//! Streamed bytes can arrive before the final whole-object check. Consume the body
//! and await [`DownloadHandle::join`](crate::operation::download::DownloadHandle::join)
//! before treating the download as successful.
//! An integrity failure prevents a path destination from replacing the existing file;
//! callers must discard bytes already received through a stream or caller-owned file.

pub use crate::model::builders::DownloadInputBuilder;
pub use crate::model::{ChunkMetadata, DownloadInput, ObjectMetadata};

/// Operation builders
pub mod builders;

/// Abstractions for responses and consuming data streams.
mod body;
pub use body::{Body, ChunkOutput};

/// In-order delivery buffer (out-of-order arrival → in-order stream).
pub(crate) mod recv_buffer;

/// Positioned-write targets for disk downloads.
pub(crate) mod sink;

#[cfg(test)]
pub(crate) mod test_util;

/// Read-ahead window — the occupancy bound on speculative issuance.
pub(crate) mod read_ahead;

/// The object-integrity result a completed download reports.
mod checksum;
mod context;
mod observability;

pub(crate) mod discovery;

mod handle;
pub use handle::DownloadHandle;
pub(crate) use handle::DownloadHandleInner;
pub use handle::DownloadIoCtl;
pub use handle::ManagedDownloadHandle;

mod output;
pub use output::DownloadOutput;

pub(crate) mod transfer;
pub(crate) use transfer::DownloadTransfer;

mod object_meta;

use crate::config::user_agent::OperationFeatures;
use crate::error;
use crate::operation::download::body::new_recv_body;
use crate::types::BucketType;
use aws_runtime::sdk_feature::AwsSdkFeature;
use std::sync::Arc;

/// Most names tried for the temporary file of one path download.
///
/// Bounded so that creation fails, rather than retrying indefinitely, when the
/// suffix source keeps naming entries that already exist.
pub(crate) const TEMP_FILE_ATTEMPTS: usize = 3;

/// Candidate paths for the temporary file a path download writes before
/// renaming it to `dest`.
///
/// Each is `{dest file name}.s3tmp.{suffix}` in `dest`'s directory, with the
/// suffix written as 8 lower-case hex digits, for the first
/// [`TEMP_FILE_ATTEMPTS`] values of `suffixes`. The suffixes are drawn here, on
/// the calling thread, so a source backed by thread-local state, such as
/// `fastrand`'s global generator, draws from the caller's state even when the
/// file is then created on the blocking pool.
pub(crate) fn temp_file_candidates(
    dest: &std::path::Path,
    suffixes: impl IntoIterator<Item = u32>,
) -> Vec<std::path::PathBuf> {
    let file_name = dest.file_name().unwrap_or_default().to_string_lossy();
    suffixes
        .into_iter()
        .take(TEMP_FILE_ATTEMPTS)
        .map(|suffix| dest.with_file_name(format!("{file_name}.s3tmp.{suffix:08x}")))
        .collect()
}

/// Operation struct for single object download
#[derive(Clone, Default, Debug)]
pub(crate) struct Download;

/// An object as a `download_objects` listing reported it.
///
/// Discovery of a child download uses it to skip HeadObject for an object that
/// fits in one range: a single checksum-mode GET pinned to the listed ETag covers
/// the whole object, and its response carries the object's checksum.
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct ListedObject {
    /// The listed size in bytes.
    pub(crate) size: u64,
    /// The listed ETag, sent as `If-Match`.
    pub(crate) e_tag: String,
}

impl ListedObject {
    /// The listing's size and ETag for `object`, or `None` when either is missing or
    /// the size is negative.
    pub(crate) fn from_listing(object: &crate::model::Object) -> Option<Self> {
        Some(Self {
            size: u64::try_from(object.size()?).ok()?,
            e_tag: object.e_tag().filter(|e_tag| !e_tag.is_empty())?.to_owned(),
        })
    }
}

impl Download {
    /// Execute a single `Download` transfer operation
    pub(crate) fn orchestrate(
        handle: Arc<crate::client::Handle>,
        input: DownloadInput,
        _use_current_span_as_parent_for_tasks: bool,
    ) -> Result<DownloadHandle, error::Error> {
        use crate::transfer::TransferContext;

        if input.part_number().is_some() {
            return Err(error::invalid_input(
                "downloading a single part by part number is not supported",
            ));
        }

        let bucket_type =
            BucketType::from_bucket_name(input.bucket().expect("bucket is available"));

        let (writer, consumer) = new_recv_body();

        let (ctx, completion_rx) = TransferContext::new(handle.clone());

        let transfer = DownloadTransfer::new(ctx.clone(), bucket_type, input, writer);
        handle
            .scheduler
            .enqueue_transfer(Box::new(transfer.clone()));

        Ok(DownloadHandle::new(transfer, consumer, completion_rx))
    }

    /// Orchestrate a download that writes to a file path (temp file + rename).
    ///
    /// The destination sink is created by `sinks` over the temp file, which the
    /// transfer manager owns and may therefore preallocate. A failure to create
    /// the temp file or the sink is returned as
    /// [`ErrorKind::IOError`](error::ErrorKind::IOError); a
    /// temp file whose sink could not be created is removed.
    ///
    /// When `parent_id` is `Some`, the transfer is linked as a child of the
    /// given composite transfer via [`TransferContext::new_child`](crate::transfer::TransferContext::new_child).
    #[cfg(any(unix, windows, target_os = "wasi"))]
    pub(crate) async fn orchestrate_to_path(
        handle: Arc<crate::client::Handle>,
        input: DownloadInput,
        dest_path: std::path::PathBuf,
        parent_id: Option<u64>,
        sinks: &dyn sink::SinkFactory,
    ) -> Result<ManagedDownloadHandle, error::Error> {
        // Generate temp file in the same directory as destination
        let candidates =
            temp_file_candidates(&dest_path, std::iter::repeat_with(|| fastrand::u32(..)));
        let (file, temp_path) = crate::io::fs::create_first_new_async(candidates)
            .await
            .map_err(|e| error::io_at("creating temporary file for", &dest_path, e))?;

        let sink = match sinks.create(file, true) {
            Ok(sink) => sink,
            Err(e) => {
                // No handle owns the temp file yet, so remove it here.
                let _ = crate::io::fs::remove_file_async(&temp_path).await;
                return Err(error::io_at("preparing temporary file for", &dest_path, e));
            }
        };
        let inner = Self::orchestrate_with_sink(handle, input, sink, parent_id, None)?;
        Ok(ManagedDownloadHandle::new(inner, temp_path, dest_path))
    }

    /// Orchestrate a download that writes to a caller-provided file.
    ///
    /// The destination sink is created by `sinks` over `file`. The caller owns
    /// `file`, so it is not preallocated.
    ///
    /// Returns [`ErrorKind::InputInvalid`](error::ErrorKind::InputInvalid)
    /// when `sinks` rejects `file` with [`std::io::ErrorKind::InvalidInput`],
    /// and [`ErrorKind::IOError`](error::ErrorKind::IOError) for any other
    /// failure to create the sink. Either happens before the transfer is
    /// scheduled, so no request is sent and `file` is not written.
    #[cfg(any(unix, windows, target_os = "wasi"))]
    pub(crate) fn orchestrate_to_file(
        handle: Arc<crate::client::Handle>,
        input: DownloadInput,
        file: std::fs::File,
        sinks: &dyn sink::SinkFactory,
    ) -> Result<ManagedDownloadHandle, error::Error> {
        let sink = sinks.create(file, false).map_err(|e| {
            if e.kind() == std::io::ErrorKind::InvalidInput {
                error::invalid_input(e)
            } else {
                error::from_kind(error::ErrorKind::IOError)(e)
            }
        })?;
        let inner = Self::orchestrate_with_sink(handle, input, sink, None, None)?;
        // No temp/dest paths — caller manages the file lifecycle
        Ok(ManagedDownloadHandle::new_unmanaged(inner))
    }

    /// Shared orchestration for disk downloads: the transfer writes the object
    /// through `sink`.
    ///
    /// When `parent_id` is `Some`, the transfer is linked as a child of the
    /// given composite transfer via [`TransferContext::new_child`](crate::transfer::TransferContext::new_child).
    pub(crate) fn orchestrate_with_sink(
        handle: Arc<crate::client::Handle>,
        input: DownloadInput,
        sink: Box<dyn sink::SinkWrite>,
        parent_id: Option<u64>,
        listed: Option<ListedObject>,
    ) -> Result<DownloadHandleInner, error::Error> {
        use crate::transfer::TransferContext;

        if input.part_number().is_some() {
            return Err(error::invalid_input(
                "downloading a single part by part number is not supported",
            ));
        }

        let bucket_type =
            BucketType::from_bucket_name(input.bucket().expect("bucket is available"));

        let (writer, _consumer) = body::new_recv_body_with_disk_mode(sink);

        let (ctx, completion_rx) = match parent_id {
            Some(pid) => TransferContext::new_child(handle.clone(), pid),
            None => TransferContext::new(handle.clone()),
        };

        let features = OperationFeatures::new(
            parent_id
                .map(|_| AwsSdkFeature::S3TransferDownloadDirectory)
                .into_iter()
                .chain([AwsSdkFeature::S3FilePath]),
        );
        let ctx = ctx.with_features(features);
        let transfer =
            DownloadTransfer::new_listed(ctx.clone(), bucket_type, input, writer, listed);
        handle
            .scheduler
            .enqueue_transfer(Box::new(transfer.clone()));

        Ok(DownloadHandleInner {
            transfer,
            completion_rx: Some(completion_rx),
        })
    }
}

#[cfg(test)]
mod tests {
    use super::sink::{FileSinkFactory, SinkFactory};
    use super::{temp_file_candidates, Download, DownloadInput, TEMP_FILE_ATTEMPTS};
    use crate::error::ErrorKind;
    use std::path::{Path, PathBuf};

    fn part_number_input() -> DownloadInput {
        DownloadInput::builder()
            .bucket("test-bucket")
            .key("test-key")
            .part_number(1)
            .build()
            .unwrap()
    }

    fn mock_handle() -> std::sync::Arc<crate::client::Handle> {
        crate::client::Handle::test_handle_tokio(
            crate::Config::builder()
                .sdk_client(aws_smithy_mocks::mock_client!(
                    aws_sdk_s3,
                    aws_smithy_mocks::RuleMode::MatchAny,
                    &[]
                ))
                .build()
                .expect("valid config"),
        )
    }

    // Single-part downloads are not a supported transfer. Both entry points
    // must reject a part number as invalid input rather than panic.
    // FIXME: crossbeam-epoch is incompatible with miri (https://github.com/crossbeam-rs/crossbeam/issues/1181)
    #[cfg_attr(miri, ignore)]
    #[tokio::test]
    async fn part_number_download_is_invalid_input() {
        let err = Download::orchestrate(mock_handle(), part_number_input(), false)
            .expect_err("a part-number download is rejected");
        assert_eq!(err.kind(), &ErrorKind::InputInvalid);
    }

    // FIXME: crossbeam-epoch is incompatible with miri (https://github.com/crossbeam-rs/crossbeam/issues/1181)
    #[cfg_attr(miri, ignore)]
    #[tokio::test]
    async fn part_number_download_to_sink_is_invalid_input() {
        let file = tempfile::tempfile().unwrap();
        let sink = FileSinkFactory.create(file, true).unwrap();
        let err =
            Download::orchestrate_with_sink(mock_handle(), part_number_input(), sink, None, None)
                .expect_err("a part-number download is rejected");
        assert_eq!(err.kind(), &ErrorKind::InputInvalid);
    }

    #[test]
    fn temp_file_candidates_name_files_beside_the_destination() {
        let candidates = temp_file_candidates(Path::new("dir/out.dat"), [0xaa, 0xbb]);

        assert_eq!(
            candidates,
            [
                PathBuf::from("dir/out.dat.s3tmp.000000aa"),
                PathBuf::from("dir/out.dat.s3tmp.000000bb"),
            ]
        );
    }

    #[test]
    fn temp_file_candidates_stop_at_the_attempt_bound() {
        let candidates = temp_file_candidates(Path::new("out.dat"), 0..100);

        assert_eq!(candidates.len(), TEMP_FILE_ATTEMPTS);
    }
}