mountpoint_s3_client/
object_client.rs

1use std::fmt::{self, Debug};
2use std::ops::Range;
3use std::time::SystemTime;
4
5use async_trait::async_trait;
6use auto_impl::auto_impl;
7use futures::Stream;
8use mountpoint_s3_crt::s3::client::BufferPoolUsageStats;
9use std::collections::HashMap;
10use thiserror::Error;
11use time::OffsetDateTime;
12
13use crate::checksums::{
14    self, crc32_to_base64, crc32c_to_base64, crc64nvme_to_base64, sha1_to_base64, sha256_to_base64,
15};
16use crate::error_metadata::{ClientErrorMetadata, ProvideErrorMetadata};
17
18mod etag;
19pub use etag::ETag;
20
21/// A generic interface to S3-like object storage services.
22///
23/// This trait defines the common methods that all object services implement.
24///
25/// This is an async trait defined with the [async-trait](https://crates.io/crates/async-trait)
26/// crate, and so implementations of this trait must use the `#[async_trait::async_trait]`
27/// attribute.
28#[cfg_attr(not(docsrs), async_trait)]
29#[auto_impl(Arc)]
30pub trait ObjectClient {
31    type GetObjectResponse: GetObjectResponse<ClientError = Self::ClientError>;
32    type PutObjectRequest: PutObjectRequest<ClientError = Self::ClientError>;
33    type ClientError: std::error::Error + ProvideErrorMetadata + Send + Sync + 'static;
34
35    /// Query the part size this client uses for GET operations to the object store. This
36    /// can be `None` if the client does not do multi-part operations.
37    fn read_part_size(&self) -> Option<usize>;
38
39    /// Query the part size this client uses for PUT operations to the object store. This
40    /// can be `None` if the client does not do multi-part operations.
41    fn write_part_size(&self) -> Option<usize>;
42
43    /// Query the initial read window size this client uses for backpressure GetObject requests.
44    /// This can be `None` if backpressure is disabled.
45    fn initial_read_window_size(&self) -> Option<usize>;
46
47    /// Query current memory usage stats for the client. This can be `None` if the client
48    /// does not record the stats.
49    fn mem_usage_stats(&self) -> Option<BufferPoolUsageStats>;
50
51    /// Delete a single object from the object store.
52    ///
53    /// DeleteObject will succeed even if the object within the bucket does not exist.
54    async fn delete_object(
55        &self,
56        bucket: &str,
57        key: &str,
58    ) -> ObjectClientResult<DeleteObjectResult, DeleteObjectError, Self::ClientError>;
59
60    /// Create a copy of an existing object. Currently, this functionality has the following limitations:
61    /// - Supported only for copying between matching bucket types:
62    ///     - Standard S3 to Standard S3 buckets.
63    ///     - S3 Express to S3 Express buckets.
64    /// - Host header must use virtual host addressing style (path style is not supported) and both source and dest buckets must have dns compliant name.
65    /// - Only {bucket}/{key} format is supported for source and passing arn as source will not work.
66    /// - Source bucket is assumed to be in the same region as destination bucket.
67    async fn copy_object(
68        &self,
69        source_bucket: &str,
70        source_key: &str,
71        destination_bucket: &str,
72        destination_key: &str,
73        params: &CopyObjectParams,
74    ) -> ObjectClientResult<CopyObjectResult, CopyObjectError, Self::ClientError>;
75
76    /// Get an object from the object store. Returns a stream of body parts of the object. Parts are
77    /// guaranteed to be returned by the stream in order and contiguously.
78    async fn get_object(
79        &self,
80        bucket: &str,
81        key: &str,
82        params: &GetObjectParams,
83    ) -> ObjectClientResult<Self::GetObjectResponse, GetObjectError, Self::ClientError>;
84
85    /// List the objects in a bucket under a given prefix
86    async fn list_objects(
87        &self,
88        bucket: &str,
89        continuation_token: Option<&str>,
90        delimiter: &str,
91        max_keys: usize,
92        prefix: &str,
93    ) -> ObjectClientResult<ListObjectsResult, ListObjectsError, Self::ClientError>;
94
95    /// Retrieve object metadata without retrieving the object contents
96    async fn head_object(
97        &self,
98        bucket: &str,
99        key: &str,
100        params: &HeadObjectParams,
101    ) -> ObjectClientResult<HeadObjectResult, HeadObjectError, Self::ClientError>;
102
103    /// Put an object into the object store. Returns a [PutObjectRequest] for callers
104    /// to provide the content of the object.
105    async fn put_object(
106        &self,
107        bucket: &str,
108        key: &str,
109        params: &PutObjectParams,
110    ) -> ObjectClientResult<Self::PutObjectRequest, PutObjectError, Self::ClientError>;
111
112    /// Put an object into the object store.
113    async fn put_object_single<'a>(
114        &self,
115        bucket: &str,
116        key: &str,
117        params: &PutObjectSingleParams,
118        contents: impl AsRef<[u8]> + Send + 'a,
119    ) -> ObjectClientResult<PutObjectResult, PutObjectError, Self::ClientError>;
120
121    /// Retrieves all the metadata from an object without returning the object contents.
122    async fn get_object_attributes(
123        &self,
124        bucket: &str,
125        key: &str,
126        max_parts: Option<usize>,
127        part_number_marker: Option<usize>,
128        object_attributes: &[ObjectAttribute],
129    ) -> ObjectClientResult<GetObjectAttributesResult, GetObjectAttributesError, Self::ClientError>;
130}
131
132/// The top-level error type returned by calls to an [`ObjectClient`].
133///
134/// Errors that are explicitly modeled on a per-request-type basis are [`ServiceError`]s. Other
135/// generic or unhandled errors are [`ClientError`]s.
136///
137/// The distinction between these two types of error can sometimes be blurry. As a rough heuristic,
138/// [`ServiceError`]s are those that *any reasonable implementation* of an object client would be
139/// capable of experiencing, and [`ClientError`]s are anything else. For example, any object client
140/// could experience a "no such key" error, but only object clients that implement a permissions
141/// system could experience "permission denied" errors. When in doubt, we err towards *not* adding
142/// new [`ServiceError`]s, as they are public API for *every* object client.
143///
144/// [`ServiceError`]: ObjectClientError::ServiceError
145/// [`ClientError`]: ObjectClientError::ClientError
146#[derive(Debug, Error, PartialEq)]
147pub enum ObjectClientError<S, C> {
148    /// An error returned by the service itself
149    #[error("Service error")]
150    ServiceError(#[source] S),
151
152    /// An error within the object client (for example, an unexpected response, or a failure to
153    /// construct the request).
154    #[error("Client error")]
155    ClientError(#[from] C),
156}
157
158impl<S, C> ProvideErrorMetadata for ObjectClientError<S, C>
159where
160    C: ProvideErrorMetadata,
161{
162    fn meta(&self) -> ClientErrorMetadata {
163        match self {
164            Self::ClientError(err) => err.meta(),
165            _ => Default::default(),
166        }
167    }
168}
169
170/// Shorthand type for the result of an object client request
171pub type ObjectClientResult<T, S, C> = Result<T, ObjectClientError<S, C>>;
172
173/// Errors returned by a [`get_object`](ObjectClient::get_object) request
174#[derive(Debug, Error, PartialEq, Eq)]
175#[non_exhaustive]
176pub enum GetObjectError {
177    #[error("The bucket does not exist")]
178    NoSuchBucket,
179
180    #[error("The key does not exist")]
181    NoSuchKey,
182
183    #[error("At least one of the preconditions specified did not hold")]
184    PreconditionFailed,
185}
186
187/// Parameters to a [`get_object`](ObjectClient::get_object) request
188#[derive(Debug, Default, Clone)]
189#[non_exhaustive]
190pub struct GetObjectParams {
191    pub range: Option<Range<u64>>,
192    pub if_match: Option<ETag>,
193    pub checksum_mode: Option<ChecksumMode>,
194}
195
196impl GetObjectParams {
197    /// Create a default [GetObjectParams].
198    pub fn new() -> Self {
199        Self::default()
200    }
201
202    /// Set the range retrieved by the GetObject request
203    pub fn range(mut self, value: Option<Range<u64>>) -> Self {
204        self.range = value;
205        self
206    }
207
208    /// Set the required etag on the object
209    pub fn if_match(mut self, value: Option<ETag>) -> Self {
210        self.if_match = value;
211        self
212    }
213
214    /// Set option to retrieve checksum as part of the GetObject request
215    pub fn checksum_mode(mut self, value: Option<ChecksumMode>) -> Self {
216        self.checksum_mode = value;
217        self
218    }
219}
220
221/// Result of a [`list_objects`](ObjectClient::list_objects) request
222#[derive(Debug)]
223#[non_exhaustive]
224pub struct ListObjectsResult {
225    /// The list of objects.
226    pub objects: Vec<ObjectInfo>,
227
228    /// The list of common prefixes. This rolls up all of the objects with a common prefix up to
229    /// the next instance of the delimiter.
230    pub common_prefixes: Vec<String>,
231
232    /// If present, the continuation token to use to query more results.
233    pub next_continuation_token: Option<String>,
234}
235
236/// Errors returned by a [`list_objects`](ObjectClient::list_objects) request
237#[derive(Debug, Error, PartialEq, Eq)]
238#[non_exhaustive]
239pub enum ListObjectsError {
240    #[error("The bucket does not exist")]
241    NoSuchBucket,
242}
243
244/// Parameters to a [`head_object`](ObjectClient::head_object) request
245#[derive(Debug, Default, Clone)]
246#[non_exhaustive]
247pub struct HeadObjectParams {
248    /// Enable to retrieve checksum as part of the HeadObject request
249    pub checksum_mode: Option<ChecksumMode>,
250}
251
252impl HeadObjectParams {
253    /// Create a default [HeadObjectParams].
254    pub fn new() -> Self {
255        Self::default()
256    }
257
258    /// Set option to retrieve checksum as part of the HeadObject request
259    pub fn checksum_mode(mut self, value: Option<ChecksumMode>) -> Self {
260        self.checksum_mode = value;
261        self
262    }
263}
264
265/// Enable [ChecksumMode] to retrieve object checksums
266#[non_exhaustive]
267#[derive(Clone, Debug, PartialEq)]
268pub enum ChecksumMode {
269    /// Retrieve checksums
270    Enabled,
271}
272
273/// Result of a [`head_object`](ObjectClient::head_object) request
274#[derive(Debug)]
275#[non_exhaustive]
276pub struct HeadObjectResult {
277    /// Size of the object in bytes.
278    ///
279    /// Refers to the `Content-Length` HTTP header for HeadObject.
280    pub size: u64,
281
282    /// The time this object was last modified.
283    pub last_modified: OffsetDateTime,
284
285    /// Entity tag of this object.
286    pub etag: ETag,
287
288    /// Storage class for this object.
289    ///
290    /// The value is optional because HeadObject does not return the storage class in its response
291    /// for objects in the S3 Standard storage class.
292    /// See examples in the
293    /// [Amazon S3 API Reference](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html#API_HeadObject_Examples).
294    pub storage_class: Option<String>,
295
296    /// Objects in flexible retrieval storage classes (such as GLACIER and DEEP_ARCHIVE) are only
297    /// accessible after restoration
298    pub restore_status: Option<RestoreStatus>,
299    /// Checksum of the object.
300    ///
301    /// HeadObject must explicitly request for this field to be included,
302    /// otherwise the values will be empty.
303    pub checksum: Checksum,
304
305    /// Server-side encryption type that was used to store the object.
306    pub sse_type: Option<String>,
307
308    /// Server-side encryption KMS key ID that was used to store the object.
309    pub sse_kms_key_id: Option<String>,
310}
311
312/// Errors returned by a [`head_object`](ObjectClient::head_object) request
313#[derive(Debug, Error, PartialEq, Eq)]
314#[non_exhaustive]
315pub enum HeadObjectError {
316    /// Note that HeadObject cannot distinguish between NoSuchBucket and NoSuchKey errors
317    #[error("The object was not found")]
318    NotFound,
319}
320
321/// Result of a [`delete_object`](ObjectClient::delete_object) request
322///
323/// Note: DeleteObject requests on a non-existent object within a bucket are considered a success.
324// TODO: Populate this struct with return fields from the S3 API, e.g., version id, delete marker.
325#[derive(Debug)]
326#[non_exhaustive]
327pub struct DeleteObjectResult {}
328
329/// Errors returned by a [`delete_object`](ObjectClient::delete_object) request
330#[derive(Debug, Error, PartialEq, Eq)]
331#[non_exhaustive]
332pub enum DeleteObjectError {
333    #[error("The bucket does not exist")]
334    NoSuchBucket,
335}
336
337/// Result of a [`copy_object`](ObjectClient::copy_object) request
338#[derive(Debug)]
339#[non_exhaustive]
340pub struct CopyObjectResult {
341    // TODO: Populate this struct with return fields from the S3 API, e.g., etag.
342}
343
344/// Errors returned by a [`copy_object`](ObjectClient::copy_object) request
345#[derive(Debug, Error, PartialEq, Eq)]
346#[non_exhaustive]
347pub enum CopyObjectError {
348    /// Note that CopyObject cannot distinguish between NoSuchBucket and NoSuchKey errors
349    #[error("The object was not found")]
350    NotFound,
351
352    #[error("The source object of the COPY action is not in the active tier and is only stored in Amazon S3 Glacier.")]
353    ObjectNotInActiveTierError,
354}
355
356/// Parameters to a [`copy_object`](ObjectClient::copy_object) request
357#[derive(Debug, Default, Clone)]
358#[non_exhaustive]
359pub struct CopyObjectParams {
360    // TODO: Populate this struct with fields as and when required to satisfy various use cases.
361}
362
363impl CopyObjectParams {
364    /// Create a default [CopyObjectParams].
365    pub fn new() -> Self {
366        Self::default()
367    }
368}
369
370/// Result of a [`get_object_attributes`](ObjectClient::get_object_attributes) request
371#[derive(Debug, Default)]
372pub struct GetObjectAttributesResult {
373    /// ETag of the object
374    pub etag: Option<String>,
375
376    /// Checksum of the object
377    pub checksum: Option<Checksum>,
378
379    /// Object parts metadata for multi part object
380    pub object_parts: Option<GetObjectAttributesParts>,
381
382    /// Storage class of the object
383    pub storage_class: Option<String>,
384
385    /// Object size
386    pub object_size: Option<u64>,
387}
388
389/// Errors returned by a [`get_object_attributes`](ObjectClient::get_object_attributes) request
390#[derive(Debug, Error, PartialEq, Eq)]
391#[non_exhaustive]
392pub enum GetObjectAttributesError {
393    #[error("The bucket does not exist")]
394    NoSuchBucket,
395
396    #[error("The key does not exist")]
397    NoSuchKey,
398}
399
400pub type ObjectMetadata = HashMap<String, String>;
401
402/// Parameters to a [`put_object`](ObjectClient::put_object) request
403#[derive(Debug, Default, Clone)]
404#[non_exhaustive]
405pub struct PutObjectParams {
406    /// Enable Crc32c trailing checksums.
407    pub trailing_checksums: PutObjectTrailingChecksums,
408    /// Storage class to be used when creating new S3 object
409    pub storage_class: Option<String>,
410    /// The server-side encryption algorithm to be used for this object in Amazon S3 (for example, AES256, aws:kms, aws:kms:dsse)
411    pub server_side_encryption: Option<String>,
412    /// If `server_side_encryption` has a valid value of aws:kms or aws:kms:dsse, this value may be used to specify AWS KMS key ID to be used
413    /// when creating new S3 object
414    pub ssekms_key_id: Option<String>,
415    /// Custom headers to add to the request
416    pub custom_headers: Vec<(String, String)>,
417    /// User-defined object metadata
418    pub object_metadata: ObjectMetadata,
419}
420
421impl PutObjectParams {
422    /// Create a default [PutObjectParams].
423    pub fn new() -> Self {
424        Self::default()
425    }
426
427    /// Set Crc32c trailing checksums.
428    pub fn trailing_checksums(mut self, value: PutObjectTrailingChecksums) -> Self {
429        self.trailing_checksums = value;
430        self
431    }
432
433    /// Set the storage class.
434    pub fn storage_class(mut self, value: String) -> Self {
435        self.storage_class = Some(value);
436        self
437    }
438
439    /// Set server-side encryption type.
440    pub fn server_side_encryption(mut self, value: Option<String>) -> Self {
441        self.server_side_encryption = value;
442        self
443    }
444
445    /// Set KMS key ID to be used for server-side encryption.
446    pub fn ssekms_key_id(mut self, value: Option<String>) -> Self {
447        self.ssekms_key_id = value;
448        self
449    }
450
451    /// Add a custom header to the request.
452    pub fn add_custom_header(mut self, name: String, value: String) -> Self {
453        self.custom_headers.push((name, value));
454        self
455    }
456
457    /// Set user defined object metadata.
458    pub fn object_metadata(mut self, value: ObjectMetadata) -> Self {
459        self.object_metadata = value;
460        self
461    }
462}
463
464/// How CRC32c checksums are used for parts of a multi-part PutObject request
465#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
466pub enum PutObjectTrailingChecksums {
467    /// Checksums are computed, passed to upload review, and also sent to S3
468    Enabled,
469    /// Checksums are computed, passed to upload review, but not sent to S3
470    ReviewOnly,
471    /// Checksums are not computed on the client side
472    #[default]
473    Disabled,
474}
475
476/// Info for the caller to review before an upload completes.
477pub type UploadReview = mountpoint_s3_crt::s3::client::UploadReview;
478
479/// Info about a single part, for the caller to review before the upload completes.
480pub type UploadReviewPart = mountpoint_s3_crt::s3::client::UploadReviewPart;
481
482/// A checksum algorithm used by the object client for integrity checks on uploads and downloads.
483pub type ChecksumAlgorithm = mountpoint_s3_crt::s3::client::ChecksumAlgorithm;
484
485/// Parameters to a [`put_object_single`](ObjectClient::put_object_single) request
486#[derive(Debug, Default, Clone)]
487#[non_exhaustive]
488pub struct PutObjectSingleParams {
489    /// User-provided checksum of the data to upload.
490    pub checksum: Option<UploadChecksum>,
491    /// Storage class to be used when creating new S3 object
492    pub storage_class: Option<String>,
493    /// The server-side encryption algorithm to be used for this object in Amazon S3 (for example, AES256, aws:kms, aws:kms:dsse)
494    pub server_side_encryption: Option<String>,
495    /// If `server_side_encryption` has a valid value of aws:kms or aws:kms:dsse, this value may be used to specify AWS KMS key ID to be used
496    /// when creating new S3 object
497    pub ssekms_key_id: Option<String>,
498    /// Requires pre-existing object to match the given etag in order to perform the request
499    pub if_match: Option<ETag>,
500    /// Offset on the pre-existing object where to append the data in the request
501    pub write_offset_bytes: Option<u64>,
502    /// Custom headers to add to the request
503    pub custom_headers: Vec<(String, String)>,
504    /// User-defined object metadata
505    pub object_metadata: ObjectMetadata,
506}
507
508impl PutObjectSingleParams {
509    /// Create a default [PutObjectSingleParams].
510    pub fn new() -> Self {
511        Self::default()
512    }
513
514    /// Create a [PutObjectSingleParams] for an append request at the given offset.
515    pub fn new_for_append(offset: u64) -> Self {
516        Self::default().write_offset_bytes(offset)
517    }
518
519    /// Set checksum.
520    pub fn checksum(mut self, value: Option<UploadChecksum>) -> Self {
521        self.checksum = value;
522        self
523    }
524
525    /// Set the storage class.
526    pub fn storage_class(mut self, value: String) -> Self {
527        self.storage_class = Some(value);
528        self
529    }
530
531    /// Set server-side encryption type.
532    pub fn server_side_encryption(mut self, value: Option<String>) -> Self {
533        self.server_side_encryption = value;
534        self
535    }
536
537    /// Set KMS key ID to be used for server-side encryption.
538    pub fn ssekms_key_id(mut self, value: Option<String>) -> Self {
539        self.ssekms_key_id = value;
540        self
541    }
542
543    /// Set the required etag on the pre-existing object.
544    pub fn if_match(mut self, value: Option<ETag>) -> Self {
545        self.if_match = value;
546        self
547    }
548
549    /// Set the offset on the pre-existing object where to append the data in the request.
550    pub fn write_offset_bytes(mut self, value: u64) -> Self {
551        self.write_offset_bytes = Some(value);
552        self
553    }
554
555    /// Add a custom header to the request.
556    pub fn add_custom_header(mut self, name: String, value: String) -> Self {
557        self.custom_headers.push((name, value));
558        self
559    }
560
561    /// Set user defined object metadata.
562    pub fn object_metadata(mut self, value: ObjectMetadata) -> Self {
563        self.object_metadata = value;
564        self
565    }
566}
567
568/// A checksum used by the object client for integrity checks on uploads.
569#[derive(Debug, Clone)]
570#[non_exhaustive]
571pub enum UploadChecksum {
572    Crc64nvme(checksums::Crc64nvme),
573    Crc32c(checksums::Crc32c),
574    Crc32(checksums::Crc32),
575    Sha1(checksums::Sha1),
576    Sha256(checksums::Sha256),
577}
578
579impl UploadChecksum {
580    /// The checksum algorithm used to compute this checksum.
581    pub fn checksum_algorithm(&self) -> ChecksumAlgorithm {
582        match self {
583            UploadChecksum::Crc64nvme(_) => ChecksumAlgorithm::Crc64nvme,
584            UploadChecksum::Crc32c(_) => ChecksumAlgorithm::Crc32c,
585            UploadChecksum::Crc32(_) => ChecksumAlgorithm::Crc32,
586            UploadChecksum::Sha1(_) => ChecksumAlgorithm::Sha1,
587            UploadChecksum::Sha256(_) => ChecksumAlgorithm::Sha256,
588        }
589    }
590}
591
592/// A handle for controlling backpressure enabled requests.
593///
594/// If the client was created with `enable_read_backpressure` set true,
595/// each meta request has a flow-control window that shrinks as response
596/// body data is downloaded (headers do not affect the size of the window).
597/// The client's `initial_read_window` determines the starting size of each meta request's window.
598/// If a meta request's flow-control window reaches 0, no further data will be downloaded.
599/// If the `initial_read_window` is 0, the request will not start until the window is incremented.
600/// Maintain a larger window to keep up a high download throughput,
601/// parts cannot download in parallel unless the window is large enough to hold multiple parts.
602/// Maintain a smaller window to limit the amount of data buffered in memory.
603pub trait ClientBackpressureHandle {
604    /// Increment the flow-control read window, so that response data continues downloading.
605    fn increment_read_window(&mut self, len: usize);
606
607    /// Move the upper bound of the read window to the given offset if it's not already there.
608    fn ensure_read_window(&mut self, desired_end_offset: u64);
609
610    /// Get the upper bound of the read window. When backpressure is enabled, [GetObjectRequest] can
611    /// return data up to this offset *exclusively*.
612    fn read_window_end_offset(&self) -> u64;
613}
614
615/// A streaming response to a GetObject request.
616///
617/// This struct implements [`futures::Stream`], which you can use to read the body of the object.
618/// Each item of the stream is a part of the object body together with the part's offset within the
619/// object.
620#[cfg_attr(not(docsrs), async_trait)]
621pub trait GetObjectResponse:
622    Stream<Item = ObjectClientResult<GetBodyPart, GetObjectError, Self::ClientError>> + Send + Sync
623{
624    type BackpressureHandle: ClientBackpressureHandle + Clone + Send + Sync;
625    type ClientError: std::error::Error + Send + Sync + 'static;
626
627    /// Take the backpressure handle from the response.
628    ///
629    /// If `enable_read_backpressure` is false this call will return `None`,
630    /// no backpressure is being applied and data is being downloaded as fast as possible.
631    fn backpressure_handle(&mut self) -> Option<&mut Self::BackpressureHandle>;
632
633    /// Get the object's user defined metadata.
634    fn get_object_metadata(&self) -> ObjectMetadata;
635
636    /// Get the object's checksum, if uploaded with one
637    fn get_object_checksum(&self) -> Result<Checksum, ObjectChecksumError>;
638}
639
640/// Failures to return object checksum
641#[derive(Debug, Error)]
642pub enum ObjectChecksumError {
643    #[error("requested object checksums, but did not specify it in the request")]
644    DidNotRequestChecksums,
645    #[error("object checksum could not be retrieved from headers")]
646    HeadersError(#[source] Box<dyn std::error::Error + Send + Sync + 'static>),
647}
648
649/// A single element of a [`get_object`](ObjectClient::get_object) response stream is a pair of
650/// offset within the object and the bytes starting at that offset.
651pub type GetBodyPart = (u64, Box<[u8]>);
652
653/// A streaming put request which allows callers to asynchronously write the body of the request.
654///
655/// You can call the [`write`](Self::write) method to write data to the object, and then call
656/// [`complete`](Self::complete) to complete the upload. Alternatively, you can call
657/// [`review_and_complete`](Self::review_and_complete) to review the upload before completing it,
658/// giving the chance to cancel the request if the upload is not as expected.
659///
660/// This is an async trait defined with the [async-trait](https://crates.io/crates/async-trait)
661/// crate, and so implementations of this trait must use the `#[async_trait::async_trait]`
662/// attribute.
663#[cfg_attr(not(docsrs), async_trait)]
664pub trait PutObjectRequest: Send {
665    type ClientError: std::error::Error + Send + Sync + 'static;
666
667    /// Write the given slice to the put request body.
668    async fn write(&mut self, slice: &[u8]) -> ObjectClientResult<(), PutObjectError, Self::ClientError>;
669
670    /// Complete the put request and return a [`PutObjectResult`].
671    async fn complete(self) -> ObjectClientResult<PutObjectResult, PutObjectError, Self::ClientError>;
672
673    /// Review and complete the put request and return a [`PutObjectResult`].
674    async fn review_and_complete(
675        self,
676        review_callback: impl FnOnce(UploadReview) -> bool + Send + 'static,
677    ) -> ObjectClientResult<PutObjectResult, PutObjectError, Self::ClientError>;
678}
679
680/// Result of a [ObjectClient::put_object] request
681#[derive(Debug)]
682#[non_exhaustive]
683pub struct PutObjectResult {
684    /// ETag of the uploaded object
685    pub etag: ETag,
686    /// Server-side encryption type that was used to store new object (reported by S3)
687    pub sse_type: Option<String>,
688    /// Server-side encryption KMS key ID that was used to store new object (reported by S3)
689    pub sse_kms_key_id: Option<String>,
690}
691
692/// Errors returned by a [`put_object`](ObjectClient::put_object) request
693#[derive(Debug, Error, PartialEq, Eq)]
694#[non_exhaustive]
695pub enum PutObjectError {
696    #[error("The bucket does not exist")]
697    NoSuchBucket,
698
699    #[error("The key does not exist")]
700    NoSuchKey,
701
702    #[error("Request body cannot be empty when write offset is specified")]
703    EmptyBody,
704
705    #[error("The offset does not match the current object size")]
706    InvalidWriteOffset,
707
708    #[error("The provided checksum does not match the data")]
709    BadChecksum,
710
711    #[error("The provided checksum is not valid or does not match the existing checksum algorithm")]
712    InvalidChecksumType,
713
714    #[error("At least one of the pre-conditions you specified did not hold")]
715    PreconditionFailed,
716
717    #[error("The server does not support the functionality required to fulfill the request")]
718    NotImplemented,
719}
720
721/// Restoration status for S3 objects in flexible retrieval storage classes.
722///
723/// See [Checking restore status and expiration
724/// date](https://docs.aws.amazon.com/AmazonS3/latest/userguide/restoring-objects.html#restore-archived-objects-status)
725/// in the *Amazon S3 User Guide* for more details.
726#[derive(Debug, Clone, Copy)]
727pub enum RestoreStatus {
728    /// S3 returns this status after it accepted a restoration request, but not have completed it yet.
729    /// Objects with this status are not readable.
730    InProgress,
731
732    /// This status means that restoration is fully completed. Note that restored objects are stored only
733    /// for the number of days that was specified in the request.
734    Restored { expiry: SystemTime },
735}
736
737/// Metadata about a single S3 object.
738///
739/// See [Object](https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html) in the *Amazon S3
740/// API Reference* for more details.
741#[derive(Debug, Clone)]
742#[non_exhaustive]
743pub struct ObjectInfo {
744    /// Key for this object.
745    pub key: String,
746
747    /// Size of this object in bytes.
748    pub size: u64,
749
750    /// The time this object was last modified.
751    pub last_modified: OffsetDateTime,
752
753    /// Storage class for this object. Optional because head_object does not return
754    /// the storage class in its response for Standard objects. See examples in the [*Amazon S3 API
755    /// Reference*](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html#API_HeadObject_Examples)
756    pub storage_class: Option<String>,
757
758    /// Objects in flexible retrieval storage classes (such as GLACIER and DEEP_ARCHIVE) are only
759    /// accessible after restoration
760    pub restore_status: Option<RestoreStatus>,
761
762    /// Entity tag of this object.
763    pub etag: String,
764
765    /// The algorithm that was used to create a checksum of the object.
766    ///
767    /// The [Amazon S3 API Reference] specifies this field as a list of strings,
768    /// so we return here a [Vec] of [ChecksumAlgorithm].
769    ///
770    /// [Amazon S3 API Reference]:
771    ///     https://docs.aws.amazon.com/AmazonS3/latest/API/API_Object.html#AmazonS3-Type-Object-ChecksumAlgorithm
772    pub checksum_algorithms: Vec<ChecksumAlgorithm>,
773}
774
775/// All possible object attributes that can be retrived from [ObjectClient::get_object_attributes].
776/// Fields that you do not specify are not returned.
777#[derive(Debug)]
778pub enum ObjectAttribute {
779    /// ETag of the object
780    ETag,
781
782    /// Checksum of the object
783    Checksum,
784
785    /// Object parts metadata for multi part object
786    ObjectParts,
787
788    /// Storage class of the object
789    StorageClass,
790
791    /// Object size
792    ObjectSize,
793}
794
795impl fmt::Display for ObjectAttribute {
796    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
797        let attr_name = match self {
798            ObjectAttribute::ETag => "ETag",
799            ObjectAttribute::Checksum => "Checksum",
800            ObjectAttribute::ObjectParts => "ObjectParts",
801            ObjectAttribute::StorageClass => "StorageClass",
802            ObjectAttribute::ObjectSize => "ObjectSize",
803        };
804        write!(f, "{}", attr_name)
805    }
806}
807
808/// Metadata about object checksum.
809///
810/// See [Checksum](https://docs.aws.amazon.com/AmazonS3/latest/API/API_Checksum.html) in the *Amazon
811/// S3 API Reference* for more details.
812#[derive(Clone, Debug, PartialEq, Eq)]
813pub struct Checksum {
814    /// Base64-encoded, 64-bit CRC64NVME checksum of the object
815    pub checksum_crc64nvme: Option<String>,
816
817    /// Base64-encoded, 32-bit CRC32 checksum of the object
818    pub checksum_crc32: Option<String>,
819
820    /// Base64-encoded, 32-bit CRC32C checksum of the object
821    pub checksum_crc32c: Option<String>,
822
823    /// Base64-encoded, 160-bit SHA-1 digest of the object
824    pub checksum_sha1: Option<String>,
825
826    /// Base64-encoded, 256-bit SHA-256 digest of the object
827    pub checksum_sha256: Option<String>,
828}
829
830impl Checksum {
831    /// Construct an empty [Checksum]
832    pub fn empty() -> Self {
833        Self {
834            checksum_crc64nvme: None,
835            checksum_crc32: None,
836            checksum_crc32c: None,
837            checksum_sha1: None,
838            checksum_sha256: None,
839        }
840    }
841
842    /// Provide [ChecksumAlgorithm]s for the [Checksum], if set and recognized.
843    pub fn algorithms(&self) -> Vec<ChecksumAlgorithm> {
844        // We assume that at most one checksum will be set.
845        let mut algorithms = Vec::with_capacity(1);
846
847        // Pattern match forces us to accomodate any new fields when added.
848        let Self {
849            checksum_crc64nvme,
850            checksum_crc32,
851            checksum_crc32c,
852            checksum_sha1,
853            checksum_sha256,
854        } = &self;
855
856        if checksum_crc64nvme.is_some() {
857            algorithms.push(ChecksumAlgorithm::Crc64nvme);
858        }
859        if checksum_crc32.is_some() {
860            algorithms.push(ChecksumAlgorithm::Crc32);
861        }
862        if checksum_crc32c.is_some() {
863            algorithms.push(ChecksumAlgorithm::Crc32c);
864        }
865        if checksum_sha1.is_some() {
866            algorithms.push(ChecksumAlgorithm::Sha1);
867        }
868        if checksum_sha256.is_some() {
869            algorithms.push(ChecksumAlgorithm::Sha256);
870        }
871
872        algorithms
873    }
874}
875
876impl From<Option<UploadChecksum>> for Checksum {
877    fn from(value: Option<UploadChecksum>) -> Self {
878        let mut checksum = Checksum::empty();
879        match value.as_ref() {
880            Some(UploadChecksum::Crc64nvme(crc64)) => checksum.checksum_crc64nvme = Some(crc64nvme_to_base64(crc64)),
881            Some(UploadChecksum::Crc32c(crc32c)) => checksum.checksum_crc32c = Some(crc32c_to_base64(crc32c)),
882            Some(UploadChecksum::Crc32(crc32)) => checksum.checksum_crc32 = Some(crc32_to_base64(crc32)),
883            Some(UploadChecksum::Sha1(sha1)) => checksum.checksum_sha1 = Some(sha1_to_base64(sha1)),
884            Some(UploadChecksum::Sha256(sha256)) => checksum.checksum_sha256 = Some(sha256_to_base64(sha256)),
885            None => {}
886        };
887        checksum
888    }
889}
890
891/// Metadata about object parts from GetObjectAttributes API.
892///
893/// See [GetObjectAttributesParts](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectAttributesParts.html)
894/// in the *Amazon S3 API Reference* for more details.
895#[derive(Debug)]
896pub struct GetObjectAttributesParts {
897    /// Indicates whether the returned list of parts is truncated
898    pub is_truncated: Option<bool>,
899
900    /// Maximum number of parts allowed in the response
901    pub max_parts: Option<usize>,
902
903    /// When a list is truncated, this element specifies the next marker
904    pub next_part_number_marker: Option<usize>,
905
906    /// The marker for the current part
907    pub part_number_marker: Option<usize>,
908
909    /// Array of metadata for particular parts
910    pub parts: Option<Vec<ObjectPart>>,
911
912    /// Total number of parts
913    pub total_parts_count: Option<usize>,
914}
915
916/// Metadata for an individual object part.
917///
918/// See [ObjectPart](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ObjectPart.html) in the
919/// *Amazon S3 API Reference* for more details.
920#[derive(Debug)]
921pub struct ObjectPart {
922    /// Checksum of the object
923    pub checksum: Option<Checksum>,
924
925    /// Number of the part, this value is a positive integer between 1 and 10,000
926    pub part_number: usize,
927
928    /// Size of the part in bytes
929    pub size: usize,
930}
931
932#[cfg(test)]
933mod tests {
934    use super::*;
935
936    #[test]
937    fn test_checksum_algorithm_one_set() {
938        let checksum = Checksum {
939            checksum_crc64nvme: None,
940            checksum_crc32: None,
941            checksum_crc32c: None,
942            checksum_sha1: Some("checksum_sha1".to_string()),
943            checksum_sha256: None,
944        };
945        assert_eq!(checksum.algorithms(), vec![ChecksumAlgorithm::Sha1]);
946    }
947
948    #[test]
949    fn test_checksum_algorithm_none_set() {
950        let checksum = Checksum {
951            checksum_crc64nvme: None,
952            checksum_crc32: None,
953            checksum_crc32c: None,
954            checksum_sha1: None,
955            checksum_sha256: None,
956        };
957        assert_eq!(checksum.algorithms(), vec![]);
958    }
959
960    #[test]
961    fn test_checksum_algorithm_many_set() {
962        // Amazon S3 doesn't support more than one algorithm today, but just in case... let's show we don't panic.
963        let checksum = Checksum {
964            checksum_crc64nvme: None,
965            checksum_crc32: None,
966            checksum_crc32c: Some("checksum_crc32c".to_string()),
967            checksum_sha1: Some("checksum_sha1".to_string()),
968            checksum_sha256: None,
969        };
970        assert_eq!(
971            checksum.algorithms(),
972            vec![ChecksumAlgorithm::Crc32c, ChecksumAlgorithm::Sha1],
973        );
974    }
975}
mountpoint_s3_client/object_client.rs

mountpoint_s3_client/
object_client.rs
