brows3_lib/s3/
multipart.rs1use crate::error::{AppError, Result};
2use aws_sdk_s3::Client;
3
4const MIB: u64 = 1024 * 1024;
5const DEFAULT_PART_SIZE: u64 = 128 * MIB;
6const MIN_PART_SIZE: u64 = 5 * MIB;
7const MAX_PART_SIZE: u64 = 5 * 1024 * MIB;
8const MAX_PARTS: u64 = 10_000;
9const MAX_OBJECT_SIZE: u64 = 50_000_000_000_000;
10
11pub const MULTIPART_UPLOAD_THRESHOLD: u64 = 100 * MIB;
14
15#[derive(Clone, Copy, Debug, Eq, PartialEq)]
16pub struct MultipartPart {
17 pub number: i32,
18 pub offset: u64,
19 pub length: u64,
20}
21
22#[derive(Clone, Debug, Eq, PartialEq)]
23pub struct MultipartPlan {
24 pub part_size: u64,
25 pub parts: Vec<MultipartPart>,
26}
27
28pub fn plan_multipart_upload(object_size: u64) -> Result<MultipartPlan> {
29 if object_size == 0 {
30 return Err(AppError::ConfigError(
31 "Cannot create a multipart upload for an empty object".to_string(),
32 ));
33 }
34 if object_size > MAX_OBJECT_SIZE {
35 return Err(AppError::ConfigError(format!(
36 "Object size {object_size} bytes exceeds the S3 multipart object limit of {MAX_OBJECT_SIZE} bytes"
37 )));
38 }
39
40 let minimum_for_part_limit = object_size.div_ceil(MAX_PARTS);
41 let part_size = round_up_to_mib(DEFAULT_PART_SIZE.max(minimum_for_part_limit));
42 if !(MIN_PART_SIZE..=MAX_PART_SIZE).contains(&part_size) {
43 return Err(AppError::ConfigError(format!(
44 "No valid S3 multipart part size is available for an object of {object_size} bytes"
45 )));
46 }
47
48 let part_count = object_size.div_ceil(part_size);
49 if part_count > MAX_PARTS {
50 return Err(AppError::ConfigError(format!(
51 "Multipart upload would require {part_count} parts, exceeding the S3 limit of {MAX_PARTS}"
52 )));
53 }
54
55 let mut parts = Vec::with_capacity(part_count as usize);
56 for index in 0..part_count {
57 let offset = index * part_size;
58 parts.push(MultipartPart {
59 number: (index + 1) as i32,
60 offset,
61 length: (object_size - offset).min(part_size),
62 });
63 }
64
65 Ok(MultipartPlan { part_size, parts })
66}
67
68fn round_up_to_mib(value: u64) -> u64 {
69 value.div_ceil(MIB) * MIB
70}
71
72pub struct MultipartUploadGuard {
75 client: Client,
76 bucket: String,
77 key: String,
78 upload_id: Option<String>,
79}
80
81impl MultipartUploadGuard {
82 pub fn new(client: Client, bucket: &str, key: &str, upload_id: String) -> Self {
83 Self {
84 client,
85 bucket: bucket.to_string(),
86 key: key.to_string(),
87 upload_id: Some(upload_id),
88 }
89 }
90
91 pub fn upload_id(&self) -> &str {
92 self.upload_id
93 .as_deref()
94 .expect("multipart upload guard must be armed")
95 }
96
97 pub fn disarm(&mut self) {
98 self.upload_id = None;
99 }
100
101 pub async fn abort(&mut self) -> Result<()> {
102 let Some(upload_id) = self.upload_id.take() else {
103 return Ok(());
104 };
105
106 tokio::time::timeout(
107 std::time::Duration::from_secs(60),
108 self.client
109 .abort_multipart_upload()
110 .bucket(&self.bucket)
111 .key(&self.key)
112 .upload_id(&upload_id)
113 .send(),
114 )
115 .await
116 .map_err(|_| {
117 AppError::S3Error(format!(
118 "Aborting multipart upload {upload_id} timed out after 60 seconds"
119 ))
120 })?
121 .map_err(|error| {
122 AppError::S3Error(format!(
123 "Failed to abort multipart upload {upload_id} for s3://{}/{}: {}",
124 self.bucket,
125 self.key,
126 aws_sdk_s3::error::DisplayErrorContext(&error)
127 ))
128 })?;
129
130 Ok(())
131 }
132}
133
134impl Drop for MultipartUploadGuard {
135 fn drop(&mut self) {
136 let Some(upload_id) = self.upload_id.take() else {
137 return;
138 };
139 let Ok(runtime) = tokio::runtime::Handle::try_current() else {
140 log::error!(
141 "Could not schedule abort for multipart upload {} because no Tokio runtime is available",
142 upload_id
143 );
144 return;
145 };
146
147 let client = self.client.clone();
148 let bucket = self.bucket.clone();
149 let key = self.key.clone();
150 runtime.spawn(async move {
151 let mut guard = MultipartUploadGuard::new(client, &bucket, &key, upload_id.clone());
152 if let Err(error) = guard.abort().await {
153 log::error!(
154 "Best-effort abort failed for multipart upload {} at s3://{}/{}: {}",
155 upload_id,
156 bucket,
157 key,
158 error
159 );
160 }
161 });
162 }
163}
164
165#[cfg(test)]
166mod tests {
167 use super::{
168 plan_multipart_upload, DEFAULT_PART_SIZE, MAX_OBJECT_SIZE, MAX_PARTS, MAX_PART_SIZE,
169 MIN_PART_SIZE,
170 };
171
172 #[tokio::test]
173 async fn abort_returns_when_the_provider_never_responds() {
174 let (endpoint, received, server) = crate::commands::test_s3::stalled_endpoint("").await;
175 let config = aws_sdk_s3::config::Builder::new()
176 .behavior_version_latest()
177 .region(aws_sdk_s3::config::Region::new("us-east-1"))
178 .credentials_provider(aws_sdk_s3::config::Credentials::new(
179 "TEST",
180 "test-secret",
181 None,
182 None,
183 "test",
184 ))
185 .endpoint_url(endpoint)
186 .force_path_style(true)
187 .build();
188 let mut guard = super::MultipartUploadGuard::new(
189 aws_sdk_s3::Client::from_conf(config),
190 "bucket",
191 "file",
192 "upload".into(),
193 );
194 let task = tokio::spawn(async move { guard.abort().await });
195 received.await.unwrap();
196 tokio::time::pause();
197 let error = task.await.unwrap().unwrap_err();
198 tokio::time::resume();
199 server.abort();
200 assert!(error.to_string().contains("timed out"));
201 }
202
203 #[test]
204 fn plans_normal_large_uploads_with_contiguous_parts() {
205 let size = 14_700_000_000;
206 let plan = plan_multipart_upload(size).unwrap();
207
208 assert_eq!(plan.part_size, DEFAULT_PART_SIZE);
209 assert!(plan.parts.len() > 1);
210 assert!(plan.parts.len() <= MAX_PARTS as usize);
211 assert_eq!(plan.parts.first().unwrap().offset, 0);
212 assert_eq!(plan.parts.last().unwrap().number, plan.parts.len() as i32);
213 assert_eq!(plan.parts.iter().map(|part| part.length).sum::<u64>(), size);
214
215 for pair in plan.parts.windows(2) {
216 assert_eq!(pair[0].offset + pair[0].length, pair[1].offset);
217 assert!(pair[0].length >= MIN_PART_SIZE);
218 }
219 }
220
221 #[test]
222 fn increases_part_size_without_exceeding_s3_limits() {
223 let plan = plan_multipart_upload(MAX_OBJECT_SIZE).unwrap();
224
225 assert!(plan.part_size <= MAX_PART_SIZE);
226 assert!(plan.parts.len() <= MAX_PARTS as usize);
227 assert_eq!(
228 plan.parts.iter().map(|part| part.length).sum::<u64>(),
229 MAX_OBJECT_SIZE
230 );
231 }
232
233 #[test]
234 fn rejects_empty_and_oversized_objects() {
235 assert!(plan_multipart_upload(0).is_err());
236 assert!(plan_multipart_upload(MAX_OBJECT_SIZE + 1).is_err());
237 }
238}