use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::time::Duration;
use bytes::Bytes;
use http::HeaderMap;
use super::{CompletedPart, MultipartError, UploadId};
use crate::operation::{ChecksumAlgorithm, ChecksumType, ObjectKey, RequestIds};
const MIN_PART_SIZE: u64 = 5 * 1024 * 1024;
const MAX_PART_SIZE: u64 = 5 * 1024 * 1024 * 1024;
const MAX_CONCURRENCY: usize = 64;
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct MultipartOptions {
part_size: u64,
concurrency: usize,
transfer_timeout: Duration,
cleanup_timeout: Duration,
}
impl MultipartOptions {
pub fn new(part_size: u64, concurrency: usize) -> Result<Self, crate::error::S3Error> {
if !(MIN_PART_SIZE..=MAX_PART_SIZE).contains(&part_size) {
return Err(crate::error::S3Error::configuration(
"multipart part size must be between 5 MiB and 5 GiB",
));
}
if !(1..=MAX_CONCURRENCY).contains(&concurrency) {
return Err(crate::error::S3Error::configuration(
"multipart concurrency must be between 1 and 64",
));
}
part_size
.checked_mul(u64::try_from(concurrency).map_err(|_| {
crate::error::S3Error::configuration("multipart concurrency does not fit in u64")
})?)
.ok_or_else(|| {
crate::error::S3Error::configuration("multipart buffered byte bound overflow")
})?;
Ok(Self {
part_size,
concurrency,
transfer_timeout: Duration::from_secs(5 * 60),
cleanup_timeout: Duration::from_secs(30),
})
}
pub fn with_transfer_timeout(
mut self,
timeout: Duration,
) -> Result<Self, crate::error::S3Error> {
if timeout.is_zero() {
return Err(crate::error::S3Error::configuration(
"multipart transfer timeout must be greater than zero",
));
}
validate_deadline(timeout, "multipart transfer timeout")?;
self.transfer_timeout = timeout;
Ok(self)
}
pub fn with_cleanup_timeout(
mut self,
timeout: Duration,
) -> Result<Self, crate::error::S3Error> {
if timeout.is_zero() {
return Err(crate::error::S3Error::configuration(
"multipart cleanup timeout must be greater than zero",
));
}
validate_deadline(timeout, "multipart cleanup timeout")?;
self.cleanup_timeout = timeout;
Ok(self)
}
pub const fn part_size(self) -> u64 {
self.part_size
}
pub const fn concurrency(self) -> usize {
self.concurrency
}
pub fn maximum_buffered_bytes(self) -> u64 {
self.part_size * u64::try_from(self.concurrency).expect("validated concurrency fits in u64")
}
pub const fn transfer_timeout(self) -> Duration {
self.transfer_timeout
}
pub const fn cleanup_timeout(self) -> Duration {
self.cleanup_timeout
}
}
fn validate_deadline(timeout: Duration, name: &str) -> Result<(), crate::error::S3Error> {
std::time::Instant::now()
.checked_add(timeout)
.map(|_| ())
.ok_or_else(|| {
crate::error::S3Error::configuration(format!(
"{name} is too large to represent as a deadline"
))
})
}
impl Default for MultipartOptions {
fn default() -> Self {
Self::new(8 * 1024 * 1024, 4).expect("default multipart options are valid")
}
}
#[derive(Clone, derive_more::Debug, Eq, PartialEq)]
pub struct CreateMultipartUploadRequest {
pub key: ObjectKey,
pub content_type: Option<String>,
#[debug("{:?}", self.user_metadata.keys().collect::<Vec<_>>())]
pub user_metadata: BTreeMap<String, String>,
pub checksum_algorithm: Option<ChecksumAlgorithm>,
pub checksum_type: Option<ChecksumType>,
#[debug("{:?}", "<redacted>")]
pub headers: HeaderMap,
}
impl CreateMultipartUploadRequest {
pub fn new(key: ObjectKey) -> Self {
Self {
key,
content_type: None,
user_metadata: BTreeMap::new(),
checksum_algorithm: None,
checksum_type: None,
headers: HeaderMap::new(),
}
}
pub fn with_headers(mut self, headers: HeaderMap) -> Self {
self.headers = headers;
self
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct CreateMultipartUploadOutput {
pub bucket: Option<String>,
pub key: ObjectKey,
upload_id: UploadId,
pub checksum_algorithm: Option<String>,
pub checksum_type: Option<ChecksumType>,
pub request_ids: RequestIds,
}
impl CreateMultipartUploadOutput {
pub(crate) fn new(
bucket: Option<String>,
key: ObjectKey,
upload_id: UploadId,
checksum_algorithm: Option<String>,
checksum_type: Option<ChecksumType>,
) -> Self {
Self {
bucket,
key,
upload_id,
checksum_algorithm,
checksum_type,
request_ids: RequestIds::default(),
}
}
pub const fn upload_id(&self) -> &UploadId {
&self.upload_id
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct MultipartUpload {
key: ObjectKey,
upload_id: UploadId,
completed_parts: Vec<CompletedPart>,
}
pub(crate) enum MultipartUploadSource {
Bytes(Bytes),
File(PathBuf),
}
#[derive(Clone, Default, derive_more::Debug, Eq, PartialEq)]
pub struct ManagedMultipartHeaders {
#[debug("{:?}", "<redacted>")]
pub create: HeaderMap,
#[debug("{:?}", "<redacted>")]
pub upload_part: HeaderMap,
#[debug("{:?}", "<redacted>")]
pub complete: HeaderMap,
#[debug("{:?}", "<redacted>")]
pub abort: HeaderMap,
}
#[derive(derive_more::Debug)]
pub struct ManagedMultipartUploadRequest {
pub(crate) key: ObjectKey,
#[debug(
"{:?}",
match self.source {
MultipartUploadSource::Bytes(_) => "bytes",
MultipartUploadSource::File(_) => "file",
}
)]
pub(crate) source: MultipartUploadSource,
pub(crate) content_type: Option<String>,
#[debug("{:?}", self.user_metadata.keys().collect::<Vec<_>>())]
pub(crate) user_metadata: BTreeMap<String, String>,
pub(crate) options: MultipartOptions,
pub(crate) checksum_algorithm: Option<ChecksumAlgorithm>,
pub headers: ManagedMultipartHeaders,
}
impl ManagedMultipartUploadRequest {
pub fn from_bytes(key: ObjectKey, bytes: impl Into<Bytes>) -> Self {
Self {
key,
source: MultipartUploadSource::Bytes(bytes.into()),
content_type: None,
user_metadata: BTreeMap::new(),
options: MultipartOptions::default(),
checksum_algorithm: None,
headers: ManagedMultipartHeaders::default(),
}
}
pub fn from_path(key: ObjectKey, path: impl AsRef<Path>) -> Self {
Self {
key,
source: MultipartUploadSource::File(path.as_ref().to_owned()),
content_type: None,
user_metadata: BTreeMap::new(),
options: MultipartOptions::default(),
checksum_algorithm: None,
headers: ManagedMultipartHeaders::default(),
}
}
pub fn with_headers(mut self, headers: ManagedMultipartHeaders) -> Self {
self.headers = headers;
self
}
pub fn with_content_type(mut self, content_type: impl Into<String>) -> Self {
self.content_type = Some(content_type.into());
self
}
pub fn with_metadata(mut self, name: impl Into<String>, value: impl Into<String>) -> Self {
self.user_metadata.insert(name.into(), value.into());
self
}
pub fn with_options(mut self, options: MultipartOptions) -> Self {
self.options = options;
self
}
pub fn with_checksum_algorithm(
mut self,
algorithm: ChecksumAlgorithm,
) -> Result<Self, crate::operation::ChecksumCalculationError> {
if algorithm == ChecksumAlgorithm::Sha1 {
return Err(crate::operation::ChecksumCalculationError { algorithm });
}
self.checksum_algorithm = Some(algorithm);
Ok(self)
}
pub const fn options(&self) -> MultipartOptions {
self.options
}
pub const fn key(&self) -> &ObjectKey {
&self.key
}
}
impl MultipartUpload {
pub fn new(key: ObjectKey, upload_id: UploadId) -> Self {
Self {
key,
upload_id,
completed_parts: Vec::new(),
}
}
pub fn record_part(&mut self, part: CompletedPart) -> Result<(), MultipartError> {
match self
.completed_parts
.binary_search_by_key(&part.part_number(), CompletedPart::part_number)
{
Ok(_) => Err(MultipartError::DuplicatePart(part.part_number())),
Err(index) => {
self.completed_parts.insert(index, part);
Ok(())
}
}
}
pub fn completed_parts(&self) -> &[CompletedPart] {
&self.completed_parts
}
pub const fn key(&self) -> &ObjectKey {
&self.key
}
pub const fn upload_id(&self) -> &UploadId {
&self.upload_id
}
}