Skip to main content

brows3_lib/s3/
multipart.rs

1use crate::error::{AppError, Result};
2use aws_sdk_s3::Client;
3
4const MIB: u64 = 1024 * 1024;
5const DEFAULT_PART_SIZE: u64 = 128 * MIB;
6const MIN_PART_SIZE: u64 = 5 * MIB;
7const MAX_PART_SIZE: u64 = 5 * 1024 * MIB;
8const MAX_PARTS: u64 = 10_000;
9const MAX_OBJECT_SIZE: u64 = 50_000_000_000_000;
10
11/// AWS recommends considering multipart upload at 100 MB. Starting here also
12/// gives large uploads retryable chunks instead of restarting the whole file.
13pub const MULTIPART_UPLOAD_THRESHOLD: u64 = 100 * MIB;
14
15#[derive(Clone, Copy, Debug, Eq, PartialEq)]
16pub struct MultipartPart {
17    pub number: i32,
18    pub offset: u64,
19    pub length: u64,
20}
21
22#[derive(Clone, Debug, Eq, PartialEq)]
23pub struct MultipartPlan {
24    pub part_size: u64,
25    pub parts: Vec<MultipartPart>,
26}
27
28pub fn plan_multipart_upload(object_size: u64) -> Result<MultipartPlan> {
29    if object_size == 0 {
30        return Err(AppError::ConfigError(
31            "Cannot create a multipart upload for an empty object".to_string(),
32        ));
33    }
34    if object_size > MAX_OBJECT_SIZE {
35        return Err(AppError::ConfigError(format!(
36            "Object size {object_size} bytes exceeds the S3 multipart object limit of {MAX_OBJECT_SIZE} bytes"
37        )));
38    }
39
40    let minimum_for_part_limit = object_size.div_ceil(MAX_PARTS);
41    let part_size = round_up_to_mib(DEFAULT_PART_SIZE.max(minimum_for_part_limit));
42    if !(MIN_PART_SIZE..=MAX_PART_SIZE).contains(&part_size) {
43        return Err(AppError::ConfigError(format!(
44            "No valid S3 multipart part size is available for an object of {object_size} bytes"
45        )));
46    }
47
48    let part_count = object_size.div_ceil(part_size);
49    if part_count > MAX_PARTS {
50        return Err(AppError::ConfigError(format!(
51            "Multipart upload would require {part_count} parts, exceeding the S3 limit of {MAX_PARTS}"
52        )));
53    }
54
55    let mut parts = Vec::with_capacity(part_count as usize);
56    for index in 0..part_count {
57        let offset = index * part_size;
58        parts.push(MultipartPart {
59            number: (index + 1) as i32,
60            offset,
61            length: (object_size - offset).min(part_size),
62        });
63    }
64
65    Ok(MultipartPlan { part_size, parts })
66}
67
68fn round_up_to_mib(value: u64) -> u64 {
69    value.div_ceil(MIB) * MIB
70}
71
72/// Owns an in-progress multipart upload. Explicit error paths await abort;
73/// cancellation and task panics still trigger a best-effort asynchronous abort.
74pub struct MultipartUploadGuard {
75    client: Client,
76    bucket: String,
77    key: String,
78    upload_id: Option<String>,
79}
80
81impl MultipartUploadGuard {
82    pub fn new(client: Client, bucket: &str, key: &str, upload_id: String) -> Self {
83        Self {
84            client,
85            bucket: bucket.to_string(),
86            key: key.to_string(),
87            upload_id: Some(upload_id),
88        }
89    }
90
91    pub fn upload_id(&self) -> &str {
92        self.upload_id
93            .as_deref()
94            .expect("multipart upload guard must be armed")
95    }
96
97    pub fn disarm(&mut self) {
98        self.upload_id = None;
99    }
100
101    pub async fn abort(&mut self) -> Result<()> {
102        let Some(upload_id) = self.upload_id.take() else {
103            return Ok(());
104        };
105
106        tokio::time::timeout(
107            std::time::Duration::from_secs(60),
108            self.client
109                .abort_multipart_upload()
110                .bucket(&self.bucket)
111                .key(&self.key)
112                .upload_id(&upload_id)
113                .send(),
114        )
115        .await
116        .map_err(|_| {
117            AppError::S3Error(format!(
118                "Aborting multipart upload {upload_id} timed out after 60 seconds"
119            ))
120        })?
121        .map_err(|error| {
122            AppError::S3Error(format!(
123                "Failed to abort multipart upload {upload_id} for s3://{}/{}: {}",
124                self.bucket,
125                self.key,
126                aws_sdk_s3::error::DisplayErrorContext(&error)
127            ))
128        })?;
129
130        Ok(())
131    }
132}
133
134impl Drop for MultipartUploadGuard {
135    fn drop(&mut self) {
136        let Some(upload_id) = self.upload_id.take() else {
137            return;
138        };
139        let Ok(runtime) = tokio::runtime::Handle::try_current() else {
140            log::error!(
141                "Could not schedule abort for multipart upload {} because no Tokio runtime is available",
142                upload_id
143            );
144            return;
145        };
146
147        let client = self.client.clone();
148        let bucket = self.bucket.clone();
149        let key = self.key.clone();
150        runtime.spawn(async move {
151            let mut guard = MultipartUploadGuard::new(client, &bucket, &key, upload_id.clone());
152            if let Err(error) = guard.abort().await {
153                log::error!(
154                    "Best-effort abort failed for multipart upload {} at s3://{}/{}: {}",
155                    upload_id,
156                    bucket,
157                    key,
158                    error
159                );
160            }
161        });
162    }
163}
164
165#[cfg(test)]
166mod tests {
167    use super::{
168        plan_multipart_upload, DEFAULT_PART_SIZE, MAX_OBJECT_SIZE, MAX_PARTS, MAX_PART_SIZE,
169        MIN_PART_SIZE,
170    };
171
172    #[tokio::test]
173    async fn abort_returns_when_the_provider_never_responds() {
174        let (endpoint, received, server) = crate::commands::test_s3::stalled_endpoint("").await;
175        let config = aws_sdk_s3::config::Builder::new()
176            .behavior_version_latest()
177            .region(aws_sdk_s3::config::Region::new("us-east-1"))
178            .credentials_provider(aws_sdk_s3::config::Credentials::new(
179                "TEST",
180                "test-secret",
181                None,
182                None,
183                "test",
184            ))
185            .endpoint_url(endpoint)
186            .force_path_style(true)
187            .build();
188        let mut guard = super::MultipartUploadGuard::new(
189            aws_sdk_s3::Client::from_conf(config),
190            "bucket",
191            "file",
192            "upload".into(),
193        );
194        let task = tokio::spawn(async move { guard.abort().await });
195        received.await.unwrap();
196        tokio::time::pause();
197        let error = task.await.unwrap().unwrap_err();
198        tokio::time::resume();
199        server.abort();
200        assert!(error.to_string().contains("timed out"));
201    }
202
203    #[test]
204    fn plans_normal_large_uploads_with_contiguous_parts() {
205        let size = 14_700_000_000;
206        let plan = plan_multipart_upload(size).unwrap();
207
208        assert_eq!(plan.part_size, DEFAULT_PART_SIZE);
209        assert!(plan.parts.len() > 1);
210        assert!(plan.parts.len() <= MAX_PARTS as usize);
211        assert_eq!(plan.parts.first().unwrap().offset, 0);
212        assert_eq!(plan.parts.last().unwrap().number, plan.parts.len() as i32);
213        assert_eq!(plan.parts.iter().map(|part| part.length).sum::<u64>(), size);
214
215        for pair in plan.parts.windows(2) {
216            assert_eq!(pair[0].offset + pair[0].length, pair[1].offset);
217            assert!(pair[0].length >= MIN_PART_SIZE);
218        }
219    }
220
221    #[test]
222    fn increases_part_size_without_exceeding_s3_limits() {
223        let plan = plan_multipart_upload(MAX_OBJECT_SIZE).unwrap();
224
225        assert!(plan.part_size <= MAX_PART_SIZE);
226        assert!(plan.parts.len() <= MAX_PARTS as usize);
227        assert_eq!(
228            plan.parts.iter().map(|part| part.length).sum::<u64>(),
229            MAX_OBJECT_SIZE
230        );
231    }
232
233    #[test]
234    fn rejects_empty_and_oversized_objects() {
235        assert!(plan_multipart_upload(0).is_err());
236        assert!(plan_multipart_upload(MAX_OBJECT_SIZE + 1).is_err());
237    }
238}