Struct s3_sync::S3

pub struct S3 { /* private fields */ }

Wrapper of Rusoto S3 client that adds some high level imperative and declarative operations on S3 buckets and objects.

Implementations

impl S3

pub fn new(
    region: impl Into<Option<Region>>,
    region_endpoint: impl Into<Option<String>>,
    settings: impl Into<Option<Settings>>
) -> S3

Creates high level S3 client.

• region - the AWS region to connect to; when None autodetects the region value (see Region for detail)
• region_endpoint - use dedicated AWS endpoint within the region
• settings - use specific client setting

Examples found in repository

src/lib.rs (line 545)

    fn default() -> S3 {
        S3::new(None, None, None)
    }
}

impl S3 {
    /// Creates high level S3 client.
    ///
    /// * `region` - the AWS region to connect to; when `None` autodetects the region value (see [Region] for detail)
    /// * `region_endpoint` - use dedicated AWS endpoint within the region
    /// * `settings` - use specific client setting
    pub fn new(
        region: impl Into<Option<Region>>,
        region_endpoint: impl Into<Option<String>>,
        settings: impl Into<Option<Settings>>
    ) -> S3 {
        let region = match (region.into(), region_endpoint.into()) {
            (Some(region), Some(endpoint)) => Region::Custom { name: region.name().to_owned(), endpoint },
            (None, Some(endpoint)) => Region::Custom { name: Region::default().name().to_owned(), endpoint },
            (Some(region), None) => region,
            _ => Region::default(),
        };
        let settings = settings.into().unwrap_or_default();

        S3 {
            client: S3Client::new(region),
            on_upload_progress: None,
            part_size: settings.part_size,
            timeout: settings.timeout,
            data_timeout: settings.data_timeout,
            max_multipart_upload_parts: settings.max_multipart_upload_parts,
            max_delete_objects: settings.max_delete_objects,
        }
    }

    /// Creates high level S3 client with given region and default settings.
    pub fn with_region(region: Region) -> S3 {
        S3::new(region, None, None)
    }

pub fn with_region(region: Region) -> S3

Creates high level S3 client with given region and default settings.

pub fn part_size(&self) -> usize

Gets maximum size of the multipart upload part.

Useful to set up other I/O buffers accordingly.

pub fn max_upload_size(&self) -> usize

Returns maximum size of data in bytes that can be uploaded to a single object with current settings.

pub fn on_upload_progress(
    &mut self,
    callback: impl FnMut(&TransferStatus) + 'static
) -> Option<Box<dyn FnMut(&TransferStatus)>>

Set callback on body upload progress.

Examples found in repository

src/lib.rs (line 613)

    pub fn with_on_upload_progress<O>(
        &mut self,
        callback: impl FnMut(&TransferStatus) + 'static,
        f: impl FnOnce(&mut Self) -> O
    ) -> O {
        let old = self.on_upload_progress(callback);
        let ret = f(self);
        old.map(|callback| self.on_upload_progress(callback));
        ret
    }

pub fn with_on_upload_progress<O>(
    &mut self,
    callback: impl FnMut(&TransferStatus) + 'static,
    f: impl FnOnce(&mut Self) -> O
) -> O

Calls f with S3 client that has S3::on_upload_progress() set to callback and restores callback to previous state on return.

pub fn check_bucket_exists(
    &self,
    bucket: Bucket
) -> Result<Either<Present<Bucket>, Absent<Bucket>>, S3SyncError>

Checks if given bucket exists.

pub fn check_object_exists<'s, 'b>(
    &'s self,
    bucket_key: BucketKey<'b>,
    implementation: CheckObjectImpl
) -> Result<Either<Object<'b>, Absent<BucketKey<'b>>>, S3SyncError>

Checks if given object exists.

• implementation - select implementation of this function

Note:

• Object::last_modified() value will be in different format depending on implementation.

Examples found in repository

src/lib.rs (line 1066)

    pub fn object_present<'b, 's: 'b, R, F>(
        &'s self,
        bucket_key: BucketKey<'b>,
        check_impl: CheckObjectImpl,
        body: F
    ) -> impl Ensure<Present<BucketKey<'b>>, EnsureAction = impl Meet<Met = Present<BucketKey<'b>>, Error = S3SyncError> + Captures1<'s> + Captures2<'b>> + Captures1<'s> + Captures2<'b>
    where R: Read + 's, F: FnOnce() -> Result<(R, ObjectBodyMeta), std::io::Error> + 's {
        move || {
            Ok(match self.check_object_exists(bucket_key, check_impl)? {
                Left(present) => Met(present.into()),
                Right(absent) => EnsureAction(move || {
                    let (body, meta) = body()?;
                    self.put_object(absent, body, meta)
                })
            })
        }
    }

    /// Returns [Ensure] value that can be used to ensure that object is absent in the S3 bucket.
    ///
    /// Warning: There can be a race condition between check if object exists and the delete operation.
    pub fn object_absent<'b, 's: 'b>(
        &'s self,
        bucket_key: BucketKey<'b>,
        check_impl: CheckObjectImpl
    ) -> impl Ensure<Absent<BucketKey<'b>>, EnsureAction = impl Meet<Met = Absent<BucketKey<'b>>, Error = S3SyncError> + Captures1<'s> + Captures2<'b>> + Captures1<'s> + Captures2<'b> {
        move || {
            Ok(match self.check_object_exists(bucket_key, check_impl)? {
                Right(absent) => Met(absent),
                Left(present) => EnsureAction(move || {
                    self.delete_object(present)
                })
            })
        }
    }

pub fn check_object_exists_head<'s, 'b>(
    &'s self,
    bucket_key: BucketKey<'b>
) -> Result<Either<Object<'b>, Absent<BucketKey<'b>>>, S3SyncError>

Checks if given object exists by issuing HeadObject API request.

Note:

• Requires GetObject AWS permission.
• The Object::last_modified() value will be in RFC 2822 format.

Examples found in repository

src/lib.rs (line 652)

    pub fn check_object_exists<'s, 'b>(&'s self, bucket_key: BucketKey<'b>, implementation: CheckObjectImpl)
        -> Result<Either<Object<'b>, Absent<BucketKey<'b>>>, S3SyncError> {
        match implementation {
            CheckObjectImpl::List => self.check_object_exists_list(bucket_key),
            CheckObjectImpl::Head => self.check_object_exists_head(bucket_key),
        }
    }

pub fn check_object_exists_list<'s, 'b>(
    &'s self,
    bucket_key: BucketKey<'b>
) -> Result<Either<Object<'b>, Absent<BucketKey<'b>>>, S3SyncError>

Checks if given object exists by listing objects with ListObjcetsV2 API request.

Note:

• Requires ListBucket AWS permission.
• The Object::last_modified() value will be in RFC 3339 format.

Examples found in repository

src/lib.rs (line 651)

    pub fn check_object_exists<'s, 'b>(&'s self, bucket_key: BucketKey<'b>, implementation: CheckObjectImpl)
        -> Result<Either<Object<'b>, Absent<BucketKey<'b>>>, S3SyncError> {
        match implementation {
            CheckObjectImpl::List => self.check_object_exists_list(bucket_key),
            CheckObjectImpl::Head => self.check_object_exists_head(bucket_key),
        }
    }

pub fn list_objects<'b, 's: 'b>(
    &'s self,
    bucket: &'b Present<Bucket>,
    prefix: String
) -> impl Iterator<Item = Result<Object<'b>, S3SyncError>> + Captures1<'s> + Captures2<'b>

Provides iterator of objects in existing bucket that have key of given prefix.

Note:

• Requires ListBucket AWS permission.
• The Object::last_modified() value will be in RFC 3339 format.

pub fn get_body(
    &self,
    bucket_key: &impl PresentBucketKeyName
) -> Result<impl Read + Send + '_, S3SyncError>

Gets object body.

pub fn get_body_box(
    &self,
    bucket_key: &impl PresentBucketKeyName
) -> Result<Box<dyn Read + Send + '_>, S3SyncError>

Gets object body boxed.

Note: This is provided as a workaround for “impl Trait return type capcuring all input lifetimes” if needed.

pub fn get_body_with_retry<'s, F>(
    &'s self,
    bucket_key: &impl PresentBucketKeyName,
    retries: u32,
    on_error: F
) -> Result<impl Read + Send + 's, S3SyncError>where
    F: Fn(u32, &S3SyncError) -> bool,

Gets object body retrying the operation in case of an error.

• retires - retry get_body call up to that many times
• on_error - called when get_body call fails and there are still retries left; if gets number of retries left and the error and if it returns false the retry loop is aborted

Note:

• The on_error closure may need to pause the execution of the thread to delay next retry attempt.
• Once this function returns, the subsequent read operation failures are not retried.

pub fn put_object<'s, 'b>(
    &'s self,
    bucket_key: impl Into<BucketKey<'b>>,
    body: impl Read,
    meta: ObjectBodyMeta
) -> Result<Present<BucketKey<'b>>, S3SyncError>

Creates the S3 object with given body using multipart upload API.

Warning: Existing object will be overwritten (subject to bucket versioning settings).

The size of the body is limited to value returned by S3::max_upload_size(). Increase Settings::part_size to be able to upload more data (max_upload_size = part_size * 10_000 on AWS; with default settings the limit is 100_000 MiB).

Examples found in repository

src/lib.rs (line 1070)

    pub fn object_present<'b, 's: 'b, R, F>(
        &'s self,
        bucket_key: BucketKey<'b>,
        check_impl: CheckObjectImpl,
        body: F
    ) -> impl Ensure<Present<BucketKey<'b>>, EnsureAction = impl Meet<Met = Present<BucketKey<'b>>, Error = S3SyncError> + Captures1<'s> + Captures2<'b>> + Captures1<'s> + Captures2<'b>
    where R: Read + 's, F: FnOnce() -> Result<(R, ObjectBodyMeta), std::io::Error> + 's {
        move || {
            Ok(match self.check_object_exists(bucket_key, check_impl)? {
                Left(present) => Met(present.into()),
                Right(absent) => EnsureAction(move || {
                    let (body, meta) = body()?;
                    self.put_object(absent, body, meta)
                })
            })
        }
    }

pub fn delete_object<'b, 's: 'b>(
    &'s self,
    bucket_key: impl Into<BucketKey<'b>>
) -> Result<Absent<BucketKey<'b>>, S3SyncError>

Deletes single object.

Note: Delete call does not fail if object does not exist.

To delete many objects use S3::delete_objects() witch uses bulk delete API.

Examples found in repository

src/lib.rs (line 1088)

    pub fn object_absent<'b, 's: 'b>(
        &'s self,
        bucket_key: BucketKey<'b>,
        check_impl: CheckObjectImpl
    ) -> impl Ensure<Absent<BucketKey<'b>>, EnsureAction = impl Meet<Met = Absent<BucketKey<'b>>, Error = S3SyncError> + Captures1<'s> + Captures2<'b>> + Captures1<'s> + Captures2<'b> {
        move || {
            Ok(match self.check_object_exists(bucket_key, check_impl)? {
                Right(absent) => Met(absent),
                Left(present) => EnsureAction(move || {
                    self.delete_object(present)
                })
            })
        }
    }

pub fn delete_objects<'b, 's: 'b>(
    &'s self,
    bucket_keys: impl IntoIterator<Item = impl Into<BucketKey<'b>>>
) -> impl Iterator<Item = Result<Vec<Result<Absent<BucketKey<'b>>, (BucketKey<'b>, S3SyncError)>>, S3SyncError>> + Captures1<'s> + Captures2<'b>

Deletes list of objects in streaming fashion using bulk delete API.

Warning: If returned iterator is not completely consumed not all items from the list may be deleted.

It is not an error to delete non-existing S3 object.

Objects can live in different buckets but for best performance it is recommended to order the list by bucket so that biggest batches can be crated.

Each returned item represent batch delete call to S3 API.

Successful batch call will return Result::Ok variant containing vector of results for each individual object delete operation as provided by S3.

pub fn object_present<'b, 's: 'b, R, F>(
    &'s self,
    bucket_key: BucketKey<'b>,
    check_impl: CheckObjectImpl,
    body: F
) -> impl Ensure<Present<BucketKey<'b>>, EnsureAction = impl Meet<Met = Present<BucketKey<'b>>, Error = S3SyncError> + Captures1<'s> + Captures2<'b>> + Captures1<'s> + Captures2<'b>where
    R: Read + 's,
    F: FnOnce() -> Result<(R, ObjectBodyMeta), Error> + 's,

Returns Ensure value that can be used to ensure that object is present in the S3 bucket.

If S3 object does not exist this method will call body function to obtain Read and metadata values, then data read will be uploaded to the new S3 object with given metadata set on it.

Warning: There can be a race condition between check if object exists and the upload creating it.

pub fn object_absent<'b, 's: 'b>(
    &'s self,
    bucket_key: BucketKey<'b>,
    check_impl: CheckObjectImpl
) -> impl Ensure<Absent<BucketKey<'b>>, EnsureAction = impl Meet<Met = Absent<BucketKey<'b>>, Error = S3SyncError> + Captures1<'s> + Captures2<'b>> + Captures1<'s> + Captures2<'b>

Returns Ensure value that can be used to ensure that object is absent in the S3 bucket.

Warning: There can be a race condition between check if object exists and the delete operation.

Trait Implementations

impl Debug for S3

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.

impl Default for S3

fn default() -> S3

Returns the “default value” for a type.