pub struct S3 { /* private fields */ }
Expand description
Wrapper of Rusoto S3 client that adds some high level imperative and declarative operations on S3 buckets and objects.
Implementations§
source§impl S3
impl S3
sourcepub fn new(
region: impl Into<Option<Region>>,
region_endpoint: impl Into<Option<String>>,
settings: impl Into<Option<Settings>>
) -> S3
pub fn new(
region: impl Into<Option<Region>>,
region_endpoint: impl Into<Option<String>>,
settings: impl Into<Option<Settings>>
) -> S3
Creates high level S3 client.
region
- the AWS region to connect to; whenNone
autodetects the region value (see Region for detail)region_endpoint
- use dedicated AWS endpoint within the regionsettings
- use specific client setting
Examples found in repository?
544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582
fn default() -> S3 {
S3::new(None, None, None)
}
}
impl S3 {
/// Creates high level S3 client.
///
/// * `region` - the AWS region to connect to; when `None` autodetects the region value (see [Region] for detail)
/// * `region_endpoint` - use dedicated AWS endpoint within the region
/// * `settings` - use specific client setting
pub fn new(
region: impl Into<Option<Region>>,
region_endpoint: impl Into<Option<String>>,
settings: impl Into<Option<Settings>>
) -> S3 {
let region = match (region.into(), region_endpoint.into()) {
(Some(region), Some(endpoint)) => Region::Custom { name: region.name().to_owned(), endpoint },
(None, Some(endpoint)) => Region::Custom { name: Region::default().name().to_owned(), endpoint },
(Some(region), None) => region,
_ => Region::default(),
};
let settings = settings.into().unwrap_or_default();
S3 {
client: S3Client::new(region),
on_upload_progress: None,
part_size: settings.part_size,
timeout: settings.timeout,
data_timeout: settings.data_timeout,
max_multipart_upload_parts: settings.max_multipart_upload_parts,
max_delete_objects: settings.max_delete_objects,
}
}
/// Creates high level S3 client with given region and default settings.
pub fn with_region(region: Region) -> S3 {
S3::new(region, None, None)
}
sourcepub fn with_region(region: Region) -> S3
pub fn with_region(region: Region) -> S3
Creates high level S3 client with given region and default settings.
sourcepub fn part_size(&self) -> usize
pub fn part_size(&self) -> usize
Gets maximum size of the multipart upload part.
Useful to set up other I/O buffers accordingly.
sourcepub fn max_upload_size(&self) -> usize
pub fn max_upload_size(&self) -> usize
Returns maximum size of data in bytes that can be uploaded to a single object with current settings.
sourcepub fn on_upload_progress(
&mut self,
callback: impl FnMut(&TransferStatus) + 'static
) -> Option<Box<dyn FnMut(&TransferStatus)>>
pub fn on_upload_progress(
&mut self,
callback: impl FnMut(&TransferStatus) + 'static
) -> Option<Box<dyn FnMut(&TransferStatus)>>
Set callback on body upload progress.
sourcepub fn with_on_upload_progress<O>(
&mut self,
callback: impl FnMut(&TransferStatus) + 'static,
f: impl FnOnce(&mut Self) -> O
) -> O
pub fn with_on_upload_progress<O>(
&mut self,
callback: impl FnMut(&TransferStatus) + 'static,
f: impl FnOnce(&mut Self) -> O
) -> O
Calls f
with S3 client that has S3::on_upload_progress() set to callback
and restores
callback to previous state on return.
sourcepub fn check_bucket_exists(
&self,
bucket: Bucket
) -> Result<Either<Present<Bucket>, Absent<Bucket>>, S3SyncError>
pub fn check_bucket_exists(
&self,
bucket: Bucket
) -> Result<Either<Present<Bucket>, Absent<Bucket>>, S3SyncError>
Checks if given bucket exists.
sourcepub fn check_object_exists<'s, 'b>(
&'s self,
bucket_key: BucketKey<'b>,
implementation: CheckObjectImpl
) -> Result<Either<Object<'b>, Absent<BucketKey<'b>>>, S3SyncError>
pub fn check_object_exists<'s, 'b>(
&'s self,
bucket_key: BucketKey<'b>,
implementation: CheckObjectImpl
) -> Result<Either<Object<'b>, Absent<BucketKey<'b>>>, S3SyncError>
Checks if given object exists.
implementation
- select implementation of this function
Note:
- Object::last_modified() value will be in different format depending on implementation.
Examples found in repository?
1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092
pub fn object_present<'b, 's: 'b, R, F>(
&'s self,
bucket_key: BucketKey<'b>,
check_impl: CheckObjectImpl,
body: F
) -> impl Ensure<Present<BucketKey<'b>>, EnsureAction = impl Meet<Met = Present<BucketKey<'b>>, Error = S3SyncError> + Captures1<'s> + Captures2<'b>> + Captures1<'s> + Captures2<'b>
where R: Read + 's, F: FnOnce() -> Result<(R, ObjectBodyMeta), std::io::Error> + 's {
move || {
Ok(match self.check_object_exists(bucket_key, check_impl)? {
Left(present) => Met(present.into()),
Right(absent) => EnsureAction(move || {
let (body, meta) = body()?;
self.put_object(absent, body, meta)
})
})
}
}
/// Returns [Ensure] value that can be used to ensure that object is absent in the S3 bucket.
///
/// Warning: There can be a race condition between check if object exists and the delete operation.
pub fn object_absent<'b, 's: 'b>(
&'s self,
bucket_key: BucketKey<'b>,
check_impl: CheckObjectImpl
) -> impl Ensure<Absent<BucketKey<'b>>, EnsureAction = impl Meet<Met = Absent<BucketKey<'b>>, Error = S3SyncError> + Captures1<'s> + Captures2<'b>> + Captures1<'s> + Captures2<'b> {
move || {
Ok(match self.check_object_exists(bucket_key, check_impl)? {
Right(absent) => Met(absent),
Left(present) => EnsureAction(move || {
self.delete_object(present)
})
})
}
}
sourcepub fn check_object_exists_head<'s, 'b>(
&'s self,
bucket_key: BucketKey<'b>
) -> Result<Either<Object<'b>, Absent<BucketKey<'b>>>, S3SyncError>
pub fn check_object_exists_head<'s, 'b>(
&'s self,
bucket_key: BucketKey<'b>
) -> Result<Either<Object<'b>, Absent<BucketKey<'b>>>, S3SyncError>
Checks if given object exists by issuing HeadObject
API request.
Note:
- Requires
GetObject
AWS permission. - The Object::last_modified() value will be in RFC 2822 format.
sourcepub fn check_object_exists_list<'s, 'b>(
&'s self,
bucket_key: BucketKey<'b>
) -> Result<Either<Object<'b>, Absent<BucketKey<'b>>>, S3SyncError>
pub fn check_object_exists_list<'s, 'b>(
&'s self,
bucket_key: BucketKey<'b>
) -> Result<Either<Object<'b>, Absent<BucketKey<'b>>>, S3SyncError>
Checks if given object exists by listing objects with ListObjcetsV2
API request.
Note:
- Requires
ListBucket
AWS permission. - The Object::last_modified() value will be in RFC 3339 format.
sourcepub fn list_objects<'b, 's: 'b>(
&'s self,
bucket: &'b Present<Bucket>,
prefix: String
) -> impl Iterator<Item = Result<Object<'b>, S3SyncError>> + Captures1<'s> + Captures2<'b>
pub fn list_objects<'b, 's: 'b>(
&'s self,
bucket: &'b Present<Bucket>,
prefix: String
) -> impl Iterator<Item = Result<Object<'b>, S3SyncError>> + Captures1<'s> + Captures2<'b>
Provides iterator of objects in existing bucket that have key of given prefix.
Note:
- Requires
ListBucket
AWS permission. - The Object::last_modified() value will be in RFC 3339 format.
sourcepub fn get_body(
&self,
bucket_key: &impl PresentBucketKeyName
) -> Result<impl Read + Send + '_, S3SyncError>
pub fn get_body(
&self,
bucket_key: &impl PresentBucketKeyName
) -> Result<impl Read + Send + '_, S3SyncError>
Gets object body.
sourcepub fn get_body_box(
&self,
bucket_key: &impl PresentBucketKeyName
) -> Result<Box<dyn Read + Send + '_>, S3SyncError>
pub fn get_body_box(
&self,
bucket_key: &impl PresentBucketKeyName
) -> Result<Box<dyn Read + Send + '_>, S3SyncError>
Gets object body boxed.
Note: This is provided as a workaround for “impl Trait return type capcuring all input lifetimes” if needed.
sourcepub fn get_body_with_retry<'s, F>(
&'s self,
bucket_key: &impl PresentBucketKeyName,
retries: u32,
on_error: F
) -> Result<impl Read + Send + 's, S3SyncError>where
F: Fn(u32, &S3SyncError) -> bool,
pub fn get_body_with_retry<'s, F>(
&'s self,
bucket_key: &impl PresentBucketKeyName,
retries: u32,
on_error: F
) -> Result<impl Read + Send + 's, S3SyncError>where
F: Fn(u32, &S3SyncError) -> bool,
Gets object body retrying the operation in case of an error.
retires
- retry get_body call up to that many timeson_error
- called when get_body call fails and there are still retries left; if gets number of retries left and the error and if it returns false the retry loop is aborted
Note:
- The
on_error
closure may need to pause the execution of the thread to delay next retry attempt. - Once this function returns, the subsequent read operation failures are not retried.
sourcepub fn put_object<'s, 'b>(
&'s self,
bucket_key: impl Into<BucketKey<'b>>,
body: impl Read,
meta: ObjectBodyMeta
) -> Result<Present<BucketKey<'b>>, S3SyncError>
pub fn put_object<'s, 'b>(
&'s self,
bucket_key: impl Into<BucketKey<'b>>,
body: impl Read,
meta: ObjectBodyMeta
) -> Result<Present<BucketKey<'b>>, S3SyncError>
Creates the S3 object with given body using multipart upload API.
Warning: Existing object will be overwritten (subject to bucket versioning settings).
The size of the body is limited to value returned by S3::max_upload_size().
Increase Settings::part_size to be able to upload more data (max_upload_size = part_size * 10_000
on AWS; with default settings the limit is 100_000 MiB).
Examples found in repository?
1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074
pub fn object_present<'b, 's: 'b, R, F>(
&'s self,
bucket_key: BucketKey<'b>,
check_impl: CheckObjectImpl,
body: F
) -> impl Ensure<Present<BucketKey<'b>>, EnsureAction = impl Meet<Met = Present<BucketKey<'b>>, Error = S3SyncError> + Captures1<'s> + Captures2<'b>> + Captures1<'s> + Captures2<'b>
where R: Read + 's, F: FnOnce() -> Result<(R, ObjectBodyMeta), std::io::Error> + 's {
move || {
Ok(match self.check_object_exists(bucket_key, check_impl)? {
Left(present) => Met(present.into()),
Right(absent) => EnsureAction(move || {
let (body, meta) = body()?;
self.put_object(absent, body, meta)
})
})
}
}
sourcepub fn delete_object<'b, 's: 'b>(
&'s self,
bucket_key: impl Into<BucketKey<'b>>
) -> Result<Absent<BucketKey<'b>>, S3SyncError>
pub fn delete_object<'b, 's: 'b>(
&'s self,
bucket_key: impl Into<BucketKey<'b>>
) -> Result<Absent<BucketKey<'b>>, S3SyncError>
Deletes single object.
Note: Delete call does not fail if object does not exist.
To delete many objects use S3::delete_objects() witch uses bulk delete API.
Examples found in repository?
1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092
pub fn object_absent<'b, 's: 'b>(
&'s self,
bucket_key: BucketKey<'b>,
check_impl: CheckObjectImpl
) -> impl Ensure<Absent<BucketKey<'b>>, EnsureAction = impl Meet<Met = Absent<BucketKey<'b>>, Error = S3SyncError> + Captures1<'s> + Captures2<'b>> + Captures1<'s> + Captures2<'b> {
move || {
Ok(match self.check_object_exists(bucket_key, check_impl)? {
Right(absent) => Met(absent),
Left(present) => EnsureAction(move || {
self.delete_object(present)
})
})
}
}
sourcepub fn delete_objects<'b, 's: 'b>(
&'s self,
bucket_keys: impl IntoIterator<Item = impl Into<BucketKey<'b>>>
) -> impl Iterator<Item = Result<Vec<Result<Absent<BucketKey<'b>>, (BucketKey<'b>, S3SyncError)>>, S3SyncError>> + Captures1<'s> + Captures2<'b>
pub fn delete_objects<'b, 's: 'b>(
&'s self,
bucket_keys: impl IntoIterator<Item = impl Into<BucketKey<'b>>>
) -> impl Iterator<Item = Result<Vec<Result<Absent<BucketKey<'b>>, (BucketKey<'b>, S3SyncError)>>, S3SyncError>> + Captures1<'s> + Captures2<'b>
Deletes list of objects in streaming fashion using bulk delete API.
Warning: If returned iterator is not completely consumed not all items from the list may be deleted.
It is not an error to delete non-existing S3 object.
Objects can live in different buckets but for best performance it is recommended to order the list by bucket so that biggest batches can be crated.
Each returned item represent batch delete call to S3 API.
Successful batch call will return Result::Ok variant containing vector of results for each individual object delete operation as provided by S3.
sourcepub fn object_present<'b, 's: 'b, R, F>(
&'s self,
bucket_key: BucketKey<'b>,
check_impl: CheckObjectImpl,
body: F
) -> impl Ensure<Present<BucketKey<'b>>, EnsureAction = impl Meet<Met = Present<BucketKey<'b>>, Error = S3SyncError> + Captures1<'s> + Captures2<'b>> + Captures1<'s> + Captures2<'b>where
R: Read + 's,
F: FnOnce() -> Result<(R, ObjectBodyMeta), Error> + 's,
pub fn object_present<'b, 's: 'b, R, F>(
&'s self,
bucket_key: BucketKey<'b>,
check_impl: CheckObjectImpl,
body: F
) -> impl Ensure<Present<BucketKey<'b>>, EnsureAction = impl Meet<Met = Present<BucketKey<'b>>, Error = S3SyncError> + Captures1<'s> + Captures2<'b>> + Captures1<'s> + Captures2<'b>where
R: Read + 's,
F: FnOnce() -> Result<(R, ObjectBodyMeta), Error> + 's,
Returns Ensure value that can be used to ensure that object is present in the S3 bucket.
If S3 object does not exist this method will call body
function to obtain Read and metadata values,
then data read will be uploaded to the new S3 object with given metadata set on it.
Warning: There can be a race condition between check if object exists and the upload creating it.
sourcepub fn object_absent<'b, 's: 'b>(
&'s self,
bucket_key: BucketKey<'b>,
check_impl: CheckObjectImpl
) -> impl Ensure<Absent<BucketKey<'b>>, EnsureAction = impl Meet<Met = Absent<BucketKey<'b>>, Error = S3SyncError> + Captures1<'s> + Captures2<'b>> + Captures1<'s> + Captures2<'b>
pub fn object_absent<'b, 's: 'b>(
&'s self,
bucket_key: BucketKey<'b>,
check_impl: CheckObjectImpl
) -> impl Ensure<Absent<BucketKey<'b>>, EnsureAction = impl Meet<Met = Absent<BucketKey<'b>>, Error = S3SyncError> + Captures1<'s> + Captures2<'b>> + Captures1<'s> + Captures2<'b>
Returns Ensure value that can be used to ensure that object is absent in the S3 bucket.
Warning: There can be a race condition between check if object exists and the delete operation.