pub use crate::model::builders::DownloadInputBuilder;
pub use crate::model::{ChunkMetadata, DownloadInput, ObjectMetadata};
pub mod builders;
mod body;
pub use body::{Body, ChunkOutput};
pub(crate) mod recv_buffer;
pub(crate) mod sink;
#[cfg(test)]
pub(crate) mod test_util;
pub(crate) mod read_ahead;
mod checksum;
mod context;
mod observability;
pub(crate) mod discovery;
mod handle;
pub use handle::DownloadHandle;
pub(crate) use handle::DownloadHandleInner;
pub use handle::DownloadIoCtl;
pub use handle::ManagedDownloadHandle;
mod output;
pub use output::DownloadOutput;
pub(crate) mod transfer;
pub(crate) use transfer::DownloadTransfer;
mod object_meta;
use crate::config::user_agent::OperationFeatures;
use crate::error;
use crate::operation::download::body::new_recv_body;
use crate::types::BucketType;
use aws_runtime::sdk_feature::AwsSdkFeature;
use std::sync::Arc;
pub(crate) const TEMP_FILE_ATTEMPTS: usize = 3;
pub(crate) fn temp_file_candidates(
dest: &std::path::Path,
suffixes: impl IntoIterator<Item = u32>,
) -> Vec<std::path::PathBuf> {
let file_name = dest.file_name().unwrap_or_default().to_string_lossy();
suffixes
.into_iter()
.take(TEMP_FILE_ATTEMPTS)
.map(|suffix| dest.with_file_name(format!("{file_name}.s3tmp.{suffix:08x}")))
.collect()
}
#[derive(Clone, Default, Debug)]
pub(crate) struct Download;
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct ListedObject {
pub(crate) size: u64,
pub(crate) e_tag: String,
}
impl ListedObject {
pub(crate) fn from_listing(object: &crate::model::Object) -> Option<Self> {
Some(Self {
size: u64::try_from(object.size()?).ok()?,
e_tag: object.e_tag().filter(|e_tag| !e_tag.is_empty())?.to_owned(),
})
}
}
impl Download {
pub(crate) fn orchestrate(
handle: Arc<crate::client::Handle>,
input: DownloadInput,
_use_current_span_as_parent_for_tasks: bool,
) -> Result<DownloadHandle, error::Error> {
use crate::transfer::TransferContext;
if input.part_number().is_some() {
return Err(error::invalid_input(
"downloading a single part by part number is not supported",
));
}
let bucket_type =
BucketType::from_bucket_name(input.bucket().expect("bucket is available"));
let (writer, consumer) = new_recv_body();
let (ctx, completion_rx) = TransferContext::new(handle.clone());
let transfer = DownloadTransfer::new(ctx.clone(), bucket_type, input, writer);
handle
.scheduler
.enqueue_transfer(Box::new(transfer.clone()));
Ok(DownloadHandle::new(transfer, consumer, completion_rx))
}
#[cfg(any(unix, windows, target_os = "wasi"))]
pub(crate) async fn orchestrate_to_path(
handle: Arc<crate::client::Handle>,
input: DownloadInput,
dest_path: std::path::PathBuf,
parent_id: Option<u64>,
sinks: &dyn sink::SinkFactory,
) -> Result<ManagedDownloadHandle, error::Error> {
let candidates =
temp_file_candidates(&dest_path, std::iter::repeat_with(|| fastrand::u32(..)));
let (file, temp_path) = crate::io::fs::create_first_new_async(candidates)
.await
.map_err(|e| error::io_at("creating temporary file for", &dest_path, e))?;
let sink = match sinks.create(file, true) {
Ok(sink) => sink,
Err(e) => {
let _ = crate::io::fs::remove_file_async(&temp_path).await;
return Err(error::io_at("preparing temporary file for", &dest_path, e));
}
};
let inner = Self::orchestrate_with_sink(handle, input, sink, parent_id, None)?;
Ok(ManagedDownloadHandle::new(inner, temp_path, dest_path))
}
#[cfg(any(unix, windows, target_os = "wasi"))]
pub(crate) fn orchestrate_to_file(
handle: Arc<crate::client::Handle>,
input: DownloadInput,
file: std::fs::File,
sinks: &dyn sink::SinkFactory,
) -> Result<ManagedDownloadHandle, error::Error> {
let sink = sinks.create(file, false).map_err(|e| {
if e.kind() == std::io::ErrorKind::InvalidInput {
error::invalid_input(e)
} else {
error::from_kind(error::ErrorKind::IOError)(e)
}
})?;
let inner = Self::orchestrate_with_sink(handle, input, sink, None, None)?;
Ok(ManagedDownloadHandle::new_unmanaged(inner))
}
pub(crate) fn orchestrate_with_sink(
handle: Arc<crate::client::Handle>,
input: DownloadInput,
sink: Box<dyn sink::SinkWrite>,
parent_id: Option<u64>,
listed: Option<ListedObject>,
) -> Result<DownloadHandleInner, error::Error> {
use crate::transfer::TransferContext;
if input.part_number().is_some() {
return Err(error::invalid_input(
"downloading a single part by part number is not supported",
));
}
let bucket_type =
BucketType::from_bucket_name(input.bucket().expect("bucket is available"));
let (writer, _consumer) = body::new_recv_body_with_disk_mode(sink);
let (ctx, completion_rx) = match parent_id {
Some(pid) => TransferContext::new_child(handle.clone(), pid),
None => TransferContext::new(handle.clone()),
};
let features = OperationFeatures::new(
parent_id
.map(|_| AwsSdkFeature::S3TransferDownloadDirectory)
.into_iter()
.chain([AwsSdkFeature::S3FilePath]),
);
let ctx = ctx.with_features(features);
let transfer =
DownloadTransfer::new_listed(ctx.clone(), bucket_type, input, writer, listed);
handle
.scheduler
.enqueue_transfer(Box::new(transfer.clone()));
Ok(DownloadHandleInner {
transfer,
completion_rx: Some(completion_rx),
})
}
}
#[cfg(test)]
mod tests {
use super::sink::{FileSinkFactory, SinkFactory};
use super::{temp_file_candidates, Download, DownloadInput, TEMP_FILE_ATTEMPTS};
use crate::error::ErrorKind;
use std::path::{Path, PathBuf};
fn part_number_input() -> DownloadInput {
DownloadInput::builder()
.bucket("test-bucket")
.key("test-key")
.part_number(1)
.build()
.unwrap()
}
fn mock_handle() -> std::sync::Arc<crate::client::Handle> {
crate::client::Handle::test_handle_tokio(
crate::Config::builder()
.sdk_client(aws_smithy_mocks::mock_client!(
aws_sdk_s3,
aws_smithy_mocks::RuleMode::MatchAny,
&[]
))
.build()
.expect("valid config"),
)
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn part_number_download_is_invalid_input() {
let err = Download::orchestrate(mock_handle(), part_number_input(), false)
.expect_err("a part-number download is rejected");
assert_eq!(err.kind(), &ErrorKind::InputInvalid);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn part_number_download_to_sink_is_invalid_input() {
let file = tempfile::tempfile().unwrap();
let sink = FileSinkFactory.create(file, true).unwrap();
let err =
Download::orchestrate_with_sink(mock_handle(), part_number_input(), sink, None, None)
.expect_err("a part-number download is rejected");
assert_eq!(err.kind(), &ErrorKind::InputInvalid);
}
#[test]
fn temp_file_candidates_name_files_beside_the_destination() {
let candidates = temp_file_candidates(Path::new("dir/out.dat"), [0xaa, 0xbb]);
assert_eq!(
candidates,
[
PathBuf::from("dir/out.dat.s3tmp.000000aa"),
PathBuf::from("dir/out.dat.s3tmp.000000bb"),
]
);
}
#[test]
fn temp_file_candidates_stop_at_the_attempt_bound() {
let candidates = temp_file_candidates(Path::new("out.dat"), 0..100);
assert_eq!(candidates.len(), TEMP_FILE_ATTEMPTS);
}
}