1use std::{path::PathBuf, sync::Arc};
2
3use futures_util::future::BoxFuture;
4use thiserror::Error;
5use tokio::fs::File;
6
7mod checksum;
8mod complete_checksum;
9mod fs_util;
10mod model;
11mod multipart;
12mod object;
13mod temp;
14
15use crate::error::S3Error;
16use crate::{
17 body::checksum::ChecksumRequest,
18 config::{AccessKeyId, IoTuning, UploadLimits},
19 s3::types::{BucketName, ObjectKey, PartNumber},
20};
21
22pub use crate::s3::types::UploadId;
23pub use checksum::{ChecksumAlgorithm, ChecksumType};
24pub use model::{
25 CommittedPart, CompletedPart, ObjectMetadata, PartMetadata, UploadMetadata, UploadProcessing,
26 UploadSession, UploadState,
27};
28pub use multipart::FileMultipartStore;
29pub use object::FileObjectStore;
30pub use temp::{StagedObject, TempObjectWriter, TempPartWriter};
31
32pub type SharedObjectStore = Arc<dyn ObjectStore>;
34pub type SharedMultipartStore = Arc<dyn MultipartStore>;
36
37pub trait ObjectStore: Send + Sync + 'static {
43 fn create_temp_object<'a>(
45 &'a self,
46 bucket: &'a BucketName,
47 key: &'a ObjectKey,
48 ) -> BoxFuture<'a, Result<TempObjectWriter, StoreError>>;
49
50 fn commit_staged_object<'a>(
52 &'a self,
53 staged: StagedObject,
54 metadata: ObjectMetadata,
55 ) -> BoxFuture<'a, Result<(), StoreError>>;
56
57 fn head_object<'a>(
59 &'a self,
60 bucket: &'a BucketName,
61 key: &'a ObjectKey,
62 ) -> BoxFuture<'a, Result<Option<ObjectMetadata>, StoreError>>;
63
64 fn open_object<'a>(
66 &'a self,
67 bucket: &'a BucketName,
68 key: &'a ObjectKey,
69 ) -> BoxFuture<'a, Result<Option<(ObjectMetadata, File)>, StoreError>>;
70
71 fn delete_object<'a>(
73 &'a self,
74 bucket: &'a BucketName,
75 key: &'a ObjectKey,
76 ) -> BoxFuture<'a, Result<(), StoreError>>;
77}
78
79impl ObjectStore for FileObjectStore {
80 fn create_temp_object<'a>(
81 &'a self,
82 bucket: &'a BucketName,
83 key: &'a ObjectKey,
84 ) -> BoxFuture<'a, Result<TempObjectWriter, StoreError>> {
85 Box::pin(FileObjectStore::create_temp_object(self, bucket, key))
86 }
87
88 fn commit_staged_object<'a>(
89 &'a self,
90 staged: StagedObject,
91 metadata: ObjectMetadata,
92 ) -> BoxFuture<'a, Result<(), StoreError>> {
93 Box::pin(FileObjectStore::commit_staged_object(
94 self, staged, metadata,
95 ))
96 }
97
98 fn head_object<'a>(
99 &'a self,
100 bucket: &'a BucketName,
101 key: &'a ObjectKey,
102 ) -> BoxFuture<'a, Result<Option<ObjectMetadata>, StoreError>> {
103 Box::pin(FileObjectStore::head_object(self, bucket, key))
104 }
105
106 fn open_object<'a>(
107 &'a self,
108 bucket: &'a BucketName,
109 key: &'a ObjectKey,
110 ) -> BoxFuture<'a, Result<Option<(ObjectMetadata, File)>, StoreError>> {
111 Box::pin(FileObjectStore::open_object(self, bucket, key))
112 }
113
114 fn delete_object<'a>(
115 &'a self,
116 bucket: &'a BucketName,
117 key: &'a ObjectKey,
118 ) -> BoxFuture<'a, Result<(), StoreError>> {
119 Box::pin(FileObjectStore::delete_object(self, bucket, key))
120 }
121}
122
123pub struct CompleteUploadRequest<'a> {
125 pub object_store: &'a dyn ObjectStore,
127 pub upload_id: &'a UploadId,
129 pub requested_parts: &'a [CompletedPart],
131 pub checksum_request: &'a ChecksumRequest,
133 pub upload_limits: &'a UploadLimits,
135 pub io_tuning: &'a IoTuning,
137 pub processing: UploadProcessing<'a>,
139}
140
141pub trait MultipartStore: Send + Sync + 'static {
143 fn create_upload<'a>(
145 &'a self,
146 bucket: BucketName,
147 key: ObjectKey,
148 owner_access_key_id: Option<AccessKeyId>,
149 metadata: UploadMetadata,
150 ) -> BoxFuture<'a, Result<UploadSession, StoreError>>;
151
152 fn get_upload(&self, upload_id: &UploadId) -> Option<UploadSession>;
154
155 fn create_temp_part<'a>(
157 &'a self,
158 upload_id: &'a UploadId,
159 part_number: PartNumber,
160 ) -> BoxFuture<'a, Result<TempPartWriter, StoreError>>;
161
162 fn commit_part<'a>(
164 &'a self,
165 upload_id: &'a UploadId,
166 part_number: PartNumber,
167 temp: TempPartWriter,
168 part: CommittedPart,
169 ) -> BoxFuture<'a, Result<PartMetadata, StoreError>>;
170
171 fn abort_upload<'a>(&'a self, upload_id: &'a UploadId)
173 -> BoxFuture<'a, Result<(), StoreError>>;
174
175 fn complete_upload<'a>(
177 &'a self,
178 request: CompleteUploadRequest<'a>,
179 ) -> BoxFuture<'a, Result<ObjectMetadata, StoreError>>;
180}
181
182impl MultipartStore for FileMultipartStore {
183 fn create_upload<'a>(
184 &'a self,
185 bucket: BucketName,
186 key: ObjectKey,
187 owner_access_key_id: Option<AccessKeyId>,
188 metadata: UploadMetadata,
189 ) -> BoxFuture<'a, Result<UploadSession, StoreError>> {
190 Box::pin(FileMultipartStore::create_upload(
191 self,
192 bucket,
193 key,
194 owner_access_key_id,
195 metadata,
196 ))
197 }
198
199 fn get_upload(&self, upload_id: &UploadId) -> Option<UploadSession> {
200 FileMultipartStore::get_upload(self, upload_id)
201 }
202
203 fn create_temp_part<'a>(
204 &'a self,
205 upload_id: &'a UploadId,
206 part_number: PartNumber,
207 ) -> BoxFuture<'a, Result<TempPartWriter, StoreError>> {
208 Box::pin(FileMultipartStore::create_temp_part(
209 self,
210 upload_id,
211 part_number,
212 ))
213 }
214
215 fn commit_part<'a>(
216 &'a self,
217 upload_id: &'a UploadId,
218 part_number: PartNumber,
219 temp: TempPartWriter,
220 part: CommittedPart,
221 ) -> BoxFuture<'a, Result<PartMetadata, StoreError>> {
222 Box::pin(FileMultipartStore::commit_part(
223 self,
224 upload_id,
225 part_number,
226 temp,
227 part,
228 ))
229 }
230
231 fn abort_upload<'a>(
232 &'a self,
233 upload_id: &'a UploadId,
234 ) -> BoxFuture<'a, Result<(), StoreError>> {
235 Box::pin(FileMultipartStore::abort_upload(self, upload_id))
236 }
237
238 fn complete_upload<'a>(
239 &'a self,
240 request: CompleteUploadRequest<'a>,
241 ) -> BoxFuture<'a, Result<ObjectMetadata, StoreError>> {
242 Box::pin(FileMultipartStore::complete_upload(self, request))
243 }
244}
245
246#[derive(Debug, Error)]
247pub enum StoreError {
249 #[error("storage I/O failed")]
251 Io(#[from] std::io::Error),
252
253 #[error("failed to encode or decode storage metadata")]
255 Json(#[from] serde_json::Error),
256
257 #[error("invalid storage path: {0}")]
259 InvalidPath(PathBuf),
260
261 #[error("invalid upload ID")]
263 InvalidUploadId,
264
265 #[error("multipart upload does not exist")]
267 NoSuchUpload,
268
269 #[error("multipart upload part is invalid")]
271 InvalidPart,
272
273 #[error("multipart upload part is smaller than the minimum allowed size")]
275 EntityTooSmall,
276
277 #[error("entity is larger than the maximum allowed size")]
279 EntityTooLarge,
280
281 #[error("checksum validation failed")]
283 Checksum(S3Error),
284
285 #[error("upload processor failed")]
287 Processor(S3Error),
288}
289
290#[cfg(test)]
291mod tests {
292 use super::*;
293 use crate::{
294 body::checksum::ChecksumRequest,
295 config::UploadLimits,
296 s3::types::{BucketName, ETag, ObjectKey, PartNumber, RequestId},
297 storage::complete_checksum::MultipartCompleteChecksumState,
298 };
299 use axum::http::{HeaderMap, HeaderValue};
300 use base64::{Engine, engine::general_purpose::STANDARD as BASE64};
301 use chrono::{TimeZone, Utc};
302 use std::{collections::BTreeMap, sync::Arc, time::Duration};
303
304 #[test]
305 fn upload_session_deserialization_defaults_missing_state_to_open() {
306 let upload_id = UploadId::new();
307 let upload_id_value = upload_id.as_str();
308 let json = format!(
309 r#"{{
310 "upload_id": "{upload_id_value}",
311 "bucket": "test-bucket",
312 "key": "key.txt",
313 "initiated": "2026-06-16T00:00:00Z",
314 "metadata": {{}},
315 "parts": {{}}
316 }}"#
317 );
318
319 let session = serde_json::from_str::<UploadSession>(&json).expect("session");
320
321 assert_eq!(session.state, UploadState::Open);
322 assert!(session.owner_access_key_id.is_none());
323 assert!(session.is_open());
324 }
325
326 #[test]
327 fn upload_metadata_serializes_checksum_enums_as_s3_values() {
328 let metadata = UploadMetadata {
329 checksum_algorithm: Some(ChecksumAlgorithm::Crc32c),
330 checksum_type: Some(ChecksumType::FullObject),
331 ..UploadMetadata::default()
332 };
333
334 let json = serde_json::to_value(&metadata).expect("serialize metadata");
335
336 assert_eq!(json["checksum_algorithm"], "CRC32C");
337 assert_eq!(json["checksum_type"], "FULL_OBJECT");
338 assert_eq!(
339 serde_json::from_value::<UploadMetadata>(json)
340 .expect("deserialize metadata")
341 .checksum_algorithm,
342 Some(ChecksumAlgorithm::Crc32c)
343 );
344 }
345
346 #[tokio::test]
347 async fn multipart_store_rejects_part_creation_for_non_open_upload() {
348 let temp_dir = tempfile::tempdir().expect("create temp dir");
349 let store = FileMultipartStore::new(temp_dir.path().to_path_buf())
350 .await
351 .expect("store");
352 let upload_id = UploadId::new();
353 let session = UploadSession {
354 upload_id: upload_id.clone(),
355 bucket: BucketName::parse("test-bucket").expect("bucket"),
356 key: ObjectKey::parse("key.txt").expect("key"),
357 initiated: Utc::now(),
358 state: UploadState::Completing,
359 owner_access_key_id: None,
360 metadata: UploadMetadata::default(),
361 parts: BTreeMap::new(),
362 };
363 store.sessions.insert(upload_id.clone(), session);
364
365 let result = store
366 .create_temp_part(&upload_id, PartNumber::parse(1).expect("part number"))
367 .await;
368
369 assert!(matches!(result, Err(StoreError::NoSuchUpload)));
370 }
371
372 #[tokio::test]
373 async fn multipart_completion_failure_restores_upload_to_open() {
374 let temp_dir = tempfile::tempdir().expect("create temp dir");
375 let object_store = FileObjectStore::new(temp_dir.path().to_path_buf())
376 .await
377 .expect("object store");
378 let multipart_store = FileMultipartStore::new(temp_dir.path().to_path_buf())
379 .await
380 .expect("multipart store");
381 let upload = multipart_store
382 .create_upload(
383 BucketName::parse("test-bucket").expect("bucket"),
384 ObjectKey::parse("key.txt").expect("key"),
385 None,
386 UploadMetadata::default(),
387 )
388 .await
389 .expect("create upload");
390 let checksum_request =
391 ChecksumRequest::from_headers(&HeaderMap::new()).expect("empty checksum request");
392
393 let result = multipart_store
394 .complete_upload(CompleteUploadRequest {
395 object_store: &object_store,
396 upload_id: &upload.upload_id,
397 requested_parts: &[CompletedPart {
398 part_number: PartNumber::parse(1).expect("part number"),
399 etag: ETag::parse("\"missing\"").expect("etag"),
400 }],
401 checksum_request: &checksum_request,
402 upload_limits: &UploadLimits::default(),
403 io_tuning: &crate::config::IoTuning::default(),
404 processing: UploadProcessing {
405 request_id: &RequestId::new(),
406 upload_processors: &[],
407 },
408 })
409 .await;
410
411 assert!(matches!(result, Err(StoreError::InvalidPart)));
412 let upload = multipart_store
413 .get_upload(&upload.upload_id)
414 .expect("upload remains");
415 assert_eq!(upload.state, UploadState::Open);
416 multipart_store
417 .create_temp_part(
418 &upload.upload_id,
419 PartNumber::parse(1).expect("part number"),
420 )
421 .await
422 .expect("part creation after failed completion");
423 }
424
425 #[tokio::test]
426 async fn multipart_store_startup_removes_terminal_upload_sessions() {
427 let temp_dir = tempfile::tempdir().expect("create temp dir");
428 let open_upload_id = UploadId::new();
429 let completed_upload_id = UploadId::new();
430 let aborted_upload_id = UploadId::new();
431
432 for (upload_id, state) in [
433 (open_upload_id.clone(), UploadState::Open),
434 (completed_upload_id.clone(), UploadState::Completed),
435 (aborted_upload_id.clone(), UploadState::Aborted),
436 ] {
437 let upload_dir = temp_dir.path().join("multipart").join(upload_id.as_str());
438 std::fs::create_dir_all(upload_dir.join("parts")).expect("create upload dir");
439 let session = UploadSession {
440 upload_id,
441 bucket: BucketName::parse("test-bucket").expect("bucket"),
442 key: ObjectKey::parse("key.txt").expect("key"),
443 initiated: Utc::now(),
444 state,
445 owner_access_key_id: None,
446 metadata: UploadMetadata::default(),
447 parts: BTreeMap::new(),
448 };
449 std::fs::write(
450 upload_dir.join("session.json"),
451 serde_json::to_vec_pretty(&session).expect("session json"),
452 )
453 .expect("write session");
454 }
455
456 let store = FileMultipartStore::new(temp_dir.path().to_path_buf())
457 .await
458 .expect("store");
459
460 assert!(store.get_upload(&open_upload_id).is_some());
461 assert!(store.get_upload(&completed_upload_id).is_none());
462 assert!(store.get_upload(&aborted_upload_id).is_none());
463 assert!(
464 temp_dir
465 .path()
466 .join("multipart")
467 .join(open_upload_id.as_str())
468 .exists()
469 );
470 assert!(
471 !temp_dir
472 .path()
473 .join("multipart")
474 .join(completed_upload_id.as_str())
475 .exists()
476 );
477 assert!(
478 !temp_dir
479 .path()
480 .join("multipart")
481 .join(aborted_upload_id.as_str())
482 .exists()
483 );
484 }
485
486 #[tokio::test]
487 async fn multipart_store_startup_removes_expired_open_upload_sessions() {
488 let temp_dir = tempfile::tempdir().expect("create temp dir");
489 let upload_id = UploadId::new();
490 let upload_dir = temp_dir.path().join("multipart").join(upload_id.as_str());
491 std::fs::create_dir_all(upload_dir.join("parts")).expect("create upload dir");
492 let now = Utc
493 .with_ymd_and_hms(2026, 6, 17, 12, 0, 0)
494 .single()
495 .expect("current time");
496 let session = UploadSession {
497 upload_id: upload_id.clone(),
498 bucket: BucketName::parse("test-bucket").expect("bucket"),
499 key: ObjectKey::parse("key.txt").expect("key"),
500 initiated: now - chrono::Duration::hours(2),
501 state: UploadState::Open,
502 owner_access_key_id: None,
503 metadata: UploadMetadata::default(),
504 parts: BTreeMap::new(),
505 };
506 std::fs::write(
507 upload_dir.join("session.json"),
508 serde_json::to_vec_pretty(&session).expect("session json"),
509 )
510 .expect("write session");
511
512 let store = FileMultipartStore::new_with_options(
513 temp_dir.path().to_path_buf(),
514 crate::config::MultipartLifecycle::builder()
515 .abort_incomplete_after(Duration::from_secs(60 * 60))
516 .build(),
517 Arc::new(move || now),
518 )
519 .await
520 .expect("store");
521
522 assert!(store.get_upload(&upload_id).is_none());
523 assert!(!upload_dir.exists());
524 }
525
526 #[test]
527 fn multipart_complete_checksum_state_is_empty_without_requested_checksums() {
528 let headers = HeaderMap::new();
529 let request = ChecksumRequest::from_headers(&headers).expect("checksum request");
530
531 let state = MultipartCompleteChecksumState::new(&request);
532 let enabled = state.enabled_digests();
533
534 assert!(!enabled.md5);
535 assert!(!enabled.sha1);
536 assert!(!enabled.sha256);
537 assert!(!enabled.sha512);
538 assert!(!enabled.crc32);
539 assert!(!enabled.crc32c);
540 }
541
542 #[test]
543 fn multipart_complete_checksum_state_enables_requested_digests() {
544 let mut headers = HeaderMap::new();
545 headers.insert(
546 "content-md5",
547 HeaderValue::from_static("AAAAAAAAAAAAAAAAAAAAAA=="),
548 );
549 headers.insert(
550 "x-amz-checksum-crc32c",
551 HeaderValue::from_static("AAAAAA=="),
552 );
553 headers.insert(
554 "x-amz-checksum-sha1",
555 HeaderValue::from_str(&BASE64.encode([0_u8; 20])).expect("header"),
556 );
557 let request = ChecksumRequest::from_headers(&headers).expect("checksum request");
558
559 let state = MultipartCompleteChecksumState::new(&request);
560 let enabled = state.enabled_digests();
561
562 assert!(enabled.md5);
563 assert!(enabled.sha1);
564 assert!(!enabled.sha256);
565 assert!(!enabled.sha512);
566 assert!(!enabled.crc32);
567 assert!(enabled.crc32c);
568 }
569}