use std::collections::HashMap;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use crate::{Error, ObjectInfo, Result};
use tokio::sync::Notify;
pub const S3_SINGLE_COPY_MAX_SIZE: u64 = 5 * 1024 * 1024 * 1024;
pub const S3_MULTIPART_COPY_MIN_PART_SIZE: u64 = 5 * 1024 * 1024;
pub const S3_MULTIPART_COPY_MAX_PART_SIZE: u64 = 5 * 1024 * 1024 * 1024;
pub const S3_MULTIPART_COPY_MAX_PARTS: usize = 10_000;
pub const S3_MAX_OBJECT_SIZE: u64 = 5 * 1024 * 1024 * 1024 * 1024;
pub const DEFAULT_MULTIPART_COPY_PART_SIZE: u64 = 64 * 1024 * 1024;
#[derive(Debug, Clone, Default)]
pub struct MultipartCopyCancellation {
inner: Arc<MultipartCopyCancellationInner>,
}
#[derive(Debug, Default)]
struct MultipartCopyCancellationInner {
cancelled: AtomicBool,
notify: Notify,
}
impl MultipartCopyCancellation {
pub fn new() -> Self {
Self::default()
}
pub fn cancel(&self) {
if !self.inner.cancelled.swap(true, Ordering::AcqRel) {
self.inner.notify.notify_waiters();
}
}
pub fn is_cancelled(&self) -> bool {
self.inner.cancelled.load(Ordering::Acquire)
}
pub async fn cancelled(&self) {
let notified = self.inner.notify.notified();
tokio::pin!(notified);
notified.as_mut().enable();
if self.is_cancelled() {
return;
}
notified.await;
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MultipartCopyPart {
pub part_number: i32,
pub start: u64,
pub end_inclusive: u64,
pub size: u64,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MultipartCopyPlan {
pub object_size: u64,
pub part_size: u64,
pub parts: Vec<MultipartCopyPart>,
}
impl MultipartCopyPlan {
pub fn new(object_size: u64, preferred_part_size: Option<u64>) -> Result<Self> {
if object_size == 0 {
return Err(Error::InvalidPath(
"Multipart copy source size must be greater than zero".to_string(),
));
}
if object_size > S3_MAX_OBJECT_SIZE {
return Err(Error::InvalidPath(format!(
"Multipart copy source size exceeds the S3 object limit of {S3_MAX_OBJECT_SIZE} bytes"
)));
}
let preferred_part_size = preferred_part_size.unwrap_or(DEFAULT_MULTIPART_COPY_PART_SIZE);
if !(S3_MULTIPART_COPY_MIN_PART_SIZE..=S3_MULTIPART_COPY_MAX_PART_SIZE)
.contains(&preferred_part_size)
{
return Err(Error::InvalidPath(format!(
"Multipart copy part size must be between {S3_MULTIPART_COPY_MIN_PART_SIZE} and {S3_MULTIPART_COPY_MAX_PART_SIZE} bytes"
)));
}
let minimum_size_for_part_limit = object_size.div_ceil(S3_MULTIPART_COPY_MAX_PARTS as u64);
let part_size = preferred_part_size
.max(minimum_size_for_part_limit)
.max(S3_MULTIPART_COPY_MIN_PART_SIZE);
if part_size > S3_MULTIPART_COPY_MAX_PART_SIZE {
return Err(Error::InvalidPath(format!(
"Multipart copy requires a part larger than the S3 limit of {S3_MULTIPART_COPY_MAX_PART_SIZE} bytes"
)));
}
let part_count = object_size.div_ceil(part_size);
if part_count > S3_MULTIPART_COPY_MAX_PARTS as u64 {
return Err(Error::InvalidPath(format!(
"Multipart copy requires {part_count} parts, exceeding the S3 limit of {S3_MULTIPART_COPY_MAX_PARTS}"
)));
}
let mut parts = Vec::with_capacity(part_count as usize);
let mut start = 0;
for part_index in 0..part_count {
let end_exclusive = (start + part_size).min(object_size);
parts.push(MultipartCopyPart {
part_number: i32::try_from(part_index + 1)
.expect("S3 multipart copy plans contain at most 10,000 parts"),
start,
end_inclusive: end_exclusive - 1,
size: end_exclusive - start,
});
start = end_exclusive;
}
Ok(Self {
object_size,
part_size,
parts,
})
}
}
pub const fn requires_multipart_copy(object_size: u64) -> bool {
object_size > S3_SINGLE_COPY_MAX_SIZE
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MultipartCopyOptions {
pub source_size: u64,
pub source_etag: String,
pub source_version_id: Option<String>,
pub preferred_part_size: Option<u64>,
pub content_type: Option<String>,
pub metadata: HashMap<String, String>,
}
impl MultipartCopyOptions {
pub fn new(source_size: u64, source_etag: impl Into<String>) -> Result<Self> {
let source_etag = source_etag.into();
if source_etag.is_empty() {
return Err(Error::InvalidPath(
"Multipart copy source ETag cannot be empty".to_string(),
));
}
Ok(Self {
source_size,
source_etag,
source_version_id: None,
preferred_part_size: None,
content_type: None,
metadata: HashMap::new(),
})
}
pub fn plan(&self) -> Result<MultipartCopyPlan> {
if self.source_etag.is_empty() {
return Err(Error::InvalidPath(
"Multipart copy source ETag cannot be empty".to_string(),
));
}
if self.source_version_id.as_deref().is_some_and(str::is_empty) {
return Err(Error::InvalidPath(
"Source version ID cannot be empty".to_string(),
));
}
MultipartCopyPlan::new(self.source_size, self.preferred_part_size)
}
}
#[derive(Debug, Clone)]
pub struct MultipartCopyResult {
pub object: ObjectInfo,
pub upload_id: String,
pub part_count: usize,
pub bytes_copied: u64,
}
pub type MultipartCopyProgress<'a> = dyn Fn(u64) + Send + Sync + 'a;
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn copy_threshold_is_strictly_greater_than_five_gibibytes() {
assert!(!requires_multipart_copy(S3_SINGLE_COPY_MAX_SIZE));
assert!(requires_multipart_copy(S3_SINGLE_COPY_MAX_SIZE + 1));
}
#[tokio::test]
async fn cancellation_control_remembers_early_and_late_signals() {
let early = MultipartCopyCancellation::new();
early.cancel();
early.cancel();
early.cancelled().await;
assert!(early.is_cancelled());
let late = MultipartCopyCancellation::new();
let waiter = late.clone();
let task = tokio::spawn(async move {
waiter.cancelled().await;
});
late.cancel();
task.await.expect("cancellation waiter");
}
#[tokio::test]
async fn cancellation_control_handles_registration_races() {
for _ in 0..1_000 {
let cancellation = MultipartCopyCancellation::new();
let waiter = cancellation.clone();
let task = tokio::spawn(async move {
waiter.cancelled().await;
});
tokio::task::yield_now().await;
cancellation.cancel();
tokio::time::timeout(std::time::Duration::from_secs(1), task)
.await
.expect("cancellation waiter must not lose notification")
.expect("cancellation waiter task");
}
}
#[test]
fn plan_uses_inclusive_gap_free_ranges() {
let object_size = S3_MULTIPART_COPY_MIN_PART_SIZE * 2 + 1;
let plan = MultipartCopyPlan::new(object_size, Some(S3_MULTIPART_COPY_MIN_PART_SIZE))
.expect("build multipart copy plan");
assert_eq!(plan.parts.len(), 3);
assert_eq!(
plan.parts,
vec![
MultipartCopyPart {
part_number: 1,
start: 0,
end_inclusive: S3_MULTIPART_COPY_MIN_PART_SIZE - 1,
size: S3_MULTIPART_COPY_MIN_PART_SIZE,
},
MultipartCopyPart {
part_number: 2,
start: S3_MULTIPART_COPY_MIN_PART_SIZE,
end_inclusive: S3_MULTIPART_COPY_MIN_PART_SIZE * 2 - 1,
size: S3_MULTIPART_COPY_MIN_PART_SIZE,
},
MultipartCopyPart {
part_number: 3,
start: S3_MULTIPART_COPY_MIN_PART_SIZE * 2,
end_inclusive: object_size - 1,
size: 1,
},
]
);
assert_eq!(
plan.parts.iter().map(|part| part.size).sum::<u64>(),
object_size
);
}
#[test]
fn plan_increases_part_size_to_stay_within_ten_thousand_parts() {
let object_size =
S3_MULTIPART_COPY_MIN_PART_SIZE * (S3_MULTIPART_COPY_MAX_PARTS as u64 + 1);
let plan = MultipartCopyPlan::new(object_size, Some(S3_MULTIPART_COPY_MIN_PART_SIZE))
.expect("adjust undersized preferred part");
assert_eq!(plan.parts.len(), S3_MULTIPART_COPY_MAX_PARTS);
assert_eq!(plan.part_size, object_size.div_ceil(10_000));
assert_eq!(
plan.parts.last().map(|part| part.end_inclusive),
Some(object_size - 1)
);
}
#[test]
fn plan_accepts_exact_s3_boundaries() {
let exact_max_parts_size =
S3_MULTIPART_COPY_MIN_PART_SIZE * S3_MULTIPART_COPY_MAX_PARTS as u64;
let plan =
MultipartCopyPlan::new(exact_max_parts_size, Some(S3_MULTIPART_COPY_MIN_PART_SIZE))
.expect("accept exactly ten thousand parts");
assert_eq!(plan.parts.len(), S3_MULTIPART_COPY_MAX_PARTS);
let max_object = MultipartCopyPlan::new(S3_MAX_OBJECT_SIZE, None)
.expect("accept maximum S3 object size");
assert!(max_object.parts.len() <= S3_MULTIPART_COPY_MAX_PARTS);
assert!(max_object.part_size <= S3_MULTIPART_COPY_MAX_PART_SIZE);
}
#[test]
fn plan_rejects_zero_oversized_objects_and_invalid_part_sizes() {
for (object_size, part_size) in [
(0, None),
(S3_MAX_OBJECT_SIZE + 1, None),
(1, Some(S3_MULTIPART_COPY_MIN_PART_SIZE - 1)),
(1, Some(S3_MULTIPART_COPY_MAX_PART_SIZE + 1)),
] {
assert!(
MultipartCopyPlan::new(object_size, part_size).is_err(),
"object_size={object_size}, part_size={part_size:?}"
);
}
}
#[test]
fn options_reject_empty_source_identity() {
assert!(MultipartCopyOptions::new(1, "").is_err());
let mut options =
MultipartCopyOptions::new(1, "etag").expect("valid multipart copy options");
options.source_etag.clear();
assert!(options.plan().is_err());
options.source_etag = "etag".to_string();
options.source_version_id = Some(String::new());
assert!(options.plan().is_err());
}
}