1use std::collections::HashMap;
2use std::fmt;
3use std::fmt::{Debug, Formatter};
4use std::path::PathBuf;
5use std::sync::{Arc, Mutex};
6
7use aws_sdk_s3::operation::get_object::GetObjectOutput;
8use aws_sdk_s3::operation::head_object::HeadObjectOutput;
9use aws_sdk_s3::primitives::DateTime;
10use aws_sdk_s3::types::{
11 ChecksumAlgorithm, ChecksumType, DeleteMarkerEntry, Object, ObjectPart, ObjectVersion, Tag,
12};
13use sha1::{Digest, Sha1};
14use zeroize_derive::{Zeroize, ZeroizeOnDrop};
15
16pub mod async_callback;
17pub mod error;
18pub mod event_callback;
19
20pub mod filter_callback;
21pub mod filter_message;
22pub mod preprocess_callback;
23pub mod token;
24
25pub const S3SYNC_ORIGIN_VERSION_ID_METADATA_KEY: &str = "s3sync_origin_version_id";
26pub const S3SYNC_ORIGIN_LAST_MODIFIED_METADATA_KEY: &str = "s3sync_origin_last_modified";
27pub const SYNC_REPORT_SUMMERY_NAME: &str = "REPORT_SUMMARY";
28pub const SYNC_REPORT_RECORD_NAME: &str = "SYNC_STATUS";
29pub const SYNC_REPORT_EXISTENCE_TYPE: &str = "EXISTENCE";
30pub const SYNC_REPORT_ETAG_TYPE: &str = "ETAG";
31pub const SYNC_REPORT_CHECKSUM_TYPE: &str = "CHECKSUM";
32pub const SYNC_REPORT_METADATA_TYPE: &str = "METADATA";
33pub const SYNC_REPORT_TAGGING_TYPE: &str = "TAGGING";
34pub const SYNC_REPORT_CONTENT_DISPOSITION_METADATA_KEY: &str = "Content-Disposition";
35pub const SYNC_REPORT_CONTENT_ENCODING_METADATA_KEY: &str = "Content-Encoding";
36pub const SYNC_REPORT_CONTENT_LANGUAGE_METADATA_KEY: &str = "Content-Language";
37pub const SYNC_REPORT_CONTENT_TYPE_METADATA_KEY: &str = "Content-Type";
38pub const SYNC_REPORT_CACHE_CONTROL_METADATA_KEY: &str = "Cache-Control";
39pub const SYNC_REPORT_EXPIRES_METADATA_KEY: &str = "Expires";
40pub const SYNC_REPORT_WEBSITE_REDIRECT_METADATA_KEY: &str = "x-amz-website-redirect-location";
41pub const SYNC_REPORT_USER_DEFINED_METADATA_KEY: &str = "x-amz-meta-";
42
43pub const METADATA_SYNC_REPORT_LOG_NAME: &str = "METADATA_SYNC_STATUS";
44pub const TAGGING_SYNC_REPORT_LOG_NAME: &str = "TAGGING_SYNC_STATUS";
45pub const SYNC_STATUS_MATCHES: &str = "MATCHES";
46pub const SYNC_STATUS_MISMATCH: &str = "MISMATCH";
47pub const SYNC_STATUS_NOT_FOUND: &str = "NOT_FOUND";
48pub const SYNC_STATUS_UNKNOWN: &str = "UNKNOWN";
49pub(crate) const MINIMUM_CHUNKSIZE: usize = 5 * 1024 * 1024;
50
51pub type Sha1Digest = [u8; 20];
52
53#[derive(Debug, Clone, PartialEq, Eq, Hash)]
54pub enum ObjectKey {
55 KeyString(String),
56 KeySHA1Digest(Sha1Digest),
57}
58
59#[derive(Debug, Clone, PartialEq)]
60pub struct ObjectEntry {
61 pub last_modified: DateTime,
62 pub content_length: i64,
63 pub e_tag: Option<String>,
64}
65
66pub type ObjectKeyMap = Arc<Mutex<HashMap<ObjectKey, ObjectEntry>>>;
67
68pub type ObjectVersions = Vec<S3syncObject>;
69
70#[derive(Debug, Clone, PartialEq)]
71pub struct PackedObjectVersions {
72 pub key: String,
73 pub packed_object_versions: ObjectVersions,
74}
75
76#[derive(Debug, Clone, Default)]
77pub struct SyncStatsReport {
78 pub number_of_objects: usize,
79 pub not_found: usize,
80 pub etag_matches: usize,
81 pub etag_mismatch: usize,
82 pub etag_unknown: usize,
83 pub checksum_matches: usize,
84 pub checksum_mismatch: usize,
85 pub checksum_unknown: usize,
86 pub metadata_matches: usize,
87 pub metadata_mismatch: usize,
88 pub tagging_matches: usize,
89 pub tagging_mismatch: usize,
90}
91
92impl SyncStatsReport {
93 pub fn increment_number_of_objects(&mut self) {
94 self.number_of_objects += 1;
95 }
96 pub fn increment_not_found(&mut self) {
97 self.not_found += 1;
98 }
99 pub fn increment_etag_matches(&mut self) {
100 self.etag_matches += 1;
101 }
102 pub fn increment_etag_mismatch(&mut self) {
103 self.etag_mismatch += 1;
104 }
105 pub fn increment_etag_unknown(&mut self) {
106 self.etag_unknown += 1;
107 }
108 pub fn increment_checksum_matches(&mut self) {
109 self.checksum_matches += 1;
110 }
111 pub fn increment_checksum_mismatch(&mut self) {
112 self.checksum_mismatch += 1;
113 }
114 pub fn increment_checksum_unknown(&mut self) {
115 self.checksum_unknown += 1;
116 }
117 pub fn increment_metadata_matches(&mut self) {
118 self.metadata_matches += 1;
119 }
120 pub fn increment_metadata_mismatch(&mut self) {
121 self.metadata_mismatch += 1;
122 }
123 pub fn increment_tagging_matches(&mut self) {
124 self.tagging_matches += 1;
125 }
126 pub fn increment_tagging_mismatch(&mut self) {
127 self.tagging_mismatch += 1;
128 }
129}
130
131#[derive(Debug, Clone, PartialEq)]
132pub enum S3syncObject {
133 NotVersioning(Object),
134 Versioning(ObjectVersion),
135 DeleteMarker(DeleteMarkerEntry),
136 PackedVersions(PackedObjectVersions),
137}
138
139#[derive(Clone, Default)]
140pub struct ObjectChecksum {
141 pub key: String,
142 pub version_id: Option<String>,
143 pub checksum_algorithm: Option<ChecksumAlgorithm>,
144 pub checksum_type: Option<ChecksumType>,
145 pub object_parts: Option<Vec<ObjectPart>>,
146 pub final_checksum: Option<String>,
147}
148
149pub fn pack_object_versions(key: &str, object_versions: &ObjectVersions) -> S3syncObject {
150 S3syncObject::PackedVersions(PackedObjectVersions {
151 key: key.to_string(),
152 packed_object_versions: object_versions.clone(),
153 })
154}
155
156pub fn unpack_object_versions(object: &S3syncObject) -> ObjectVersions {
157 match &object {
158 S3syncObject::PackedVersions(packed) => packed.packed_object_versions.clone(),
159 _ => {
160 panic!("not PackedVersions")
161 }
162 }
163}
164
165pub fn format_metadata(metadata: &HashMap<String, String>) -> String {
166 let mut sorted_keys: Vec<&String> = metadata.keys().collect();
167 sorted_keys.sort();
168
169 sorted_keys
170 .iter()
171 .map(|key| {
172 let value = urlencoding::encode(&metadata[*key]).to_string();
173 format!("{key}={value}")
174 })
175 .collect::<Vec<String>>()
176 .join(",")
177}
178
179pub fn format_tags(tags: &[Tag]) -> String {
180 let mut tags = tags
181 .iter()
182 .map(|tag| (tag.key(), tag.value()))
183 .collect::<Vec<_>>();
184
185 tags.sort_by(|a, b| a.0.cmp(b.0));
186
187 tags.iter()
188 .map(|(key, value)| {
189 let escaped_key = urlencoding::encode(key).to_string();
190 let encoded_value = urlencoding::encode(value).to_string();
191 format!("{escaped_key}={encoded_value}")
192 })
193 .collect::<Vec<String>>()
194 .join("&")
195}
196
197impl S3syncObject {
198 pub fn key(&self) -> &str {
199 match &self {
200 Self::Versioning(object) => object.key().unwrap(),
201 Self::NotVersioning(object) => object.key().unwrap(),
202 Self::DeleteMarker(maker) => maker.key().unwrap(),
203 Self::PackedVersions(packed_object_versions) => &packed_object_versions.key,
204 }
205 }
206
207 pub fn last_modified(&self) -> &DateTime {
208 match &self {
209 Self::Versioning(object) => object.last_modified().unwrap(),
210 Self::NotVersioning(object) => object.last_modified().unwrap(),
211 Self::DeleteMarker(maker) => maker.last_modified().unwrap(),
212 Self::PackedVersions(_) => {
213 panic!("PackedVersions doesn't have last_modified.")
214 }
215 }
216 }
217
218 pub fn size(&self) -> i64 {
219 match &self {
220 Self::Versioning(object) => object.size().unwrap(),
221 Self::NotVersioning(object) => object.size().unwrap(),
222 _ => panic!("doesn't have size."),
223 }
224 }
225
226 pub fn version_id(&self) -> Option<&str> {
227 match &self {
228 Self::Versioning(object) => object.version_id(),
229 Self::NotVersioning(_) => None,
230 Self::DeleteMarker(object) => object.version_id(),
231 _ => panic!("unsupported."),
232 }
233 }
234
235 pub fn e_tag(&self) -> Option<&str> {
236 match &self {
237 Self::Versioning(object) => object.e_tag(),
238 Self::NotVersioning(object) => object.e_tag(),
239 _ => panic!("doesn't have ETag."),
240 }
241 }
242
243 pub fn checksum_algorithm(&self) -> Option<&[ChecksumAlgorithm]> {
244 match &self {
245 Self::Versioning(object) => {
246 if object.checksum_algorithm().is_empty() {
247 None
248 } else {
249 Some(object.checksum_algorithm())
250 }
251 }
252 Self::NotVersioning(object) => {
253 if object.checksum_algorithm().is_empty() {
254 None
255 } else {
256 Some(object.checksum_algorithm())
257 }
258 }
259 _ => panic!("doesn't have checksum_algorithm."),
260 }
261 }
262
263 pub fn checksum_type(&self) -> Option<&ChecksumType> {
264 match &self {
265 Self::Versioning(object) => object.checksum_type(),
266 Self::NotVersioning(object) => object.checksum_type(),
267 Self::DeleteMarker(_) => panic!("DeleteMarker doesn't have checksum_type."),
268 Self::PackedVersions(_) => {
269 panic!("PackedVersions doesn't have checksum_type.")
270 }
271 }
272 }
273
274 pub fn is_latest(&self) -> bool {
275 match &self {
276 Self::Versioning(object) => object.is_latest().unwrap(),
277 Self::NotVersioning(_) => false,
278 Self::DeleteMarker(maker) => maker.is_latest().unwrap(),
279 Self::PackedVersions(_) => false,
280 }
281 }
282
283 pub fn is_delete_marker(&self) -> bool {
284 match &self {
285 Self::Versioning(_) => false,
286 Self::NotVersioning(_) => false,
287 Self::DeleteMarker(_) => true,
288 Self::PackedVersions(_) => false,
289 }
290 }
291
292 pub fn clone_non_versioning_object_with_key(object: &Object, key: &str) -> Self {
293 S3syncObject::NotVersioning(clone_object_with_key(object, key))
294 }
295
296 pub fn clone_versioning_object_with_key(object: &ObjectVersion, key: &str) -> Self {
297 S3syncObject::Versioning(clone_object_version_with_key(object, key))
298 }
299
300 pub fn clone_delete_marker_with_key(delete_marker: &DeleteMarkerEntry, key: &str) -> Self {
301 S3syncObject::DeleteMarker(
302 DeleteMarkerEntry::builder()
303 .key(key)
304 .version_id(delete_marker.version_id().unwrap().to_string())
305 .is_latest(delete_marker.is_latest().unwrap())
306 .set_last_modified(delete_marker.last_modified().cloned())
307 .set_owner(delete_marker.owner().cloned())
308 .build(),
309 )
310 }
311}
312
313pub fn sha1_digest_from_key(key: &str) -> Sha1Digest {
314 let digest = Sha1::digest(key);
315 TryInto::<Sha1Digest>::try_into(digest.as_slice()).unwrap()
316}
317
318pub fn clone_object_with_key(object: &Object, key: &str) -> Object {
319 let checksum_algorithm = if object.checksum_algorithm().is_empty() {
320 None
321 } else {
322 Some(
323 object
324 .checksum_algorithm()
325 .iter()
326 .map(|checksum_algorithm| checksum_algorithm.to_owned())
327 .collect(),
328 )
329 };
330
331 Object::builder()
332 .key(key)
333 .size(object.size().unwrap())
334 .set_last_modified(object.last_modified().cloned())
335 .set_e_tag(object.e_tag().map(|e_tag| e_tag.to_string()))
336 .set_owner(object.owner().cloned())
337 .set_storage_class(object.storage_class().cloned())
338 .set_checksum_algorithm(checksum_algorithm)
339 .set_checksum_type(object.checksum_type().cloned())
340 .build()
341}
342
343pub fn clone_object_version_with_key(object: &ObjectVersion, key: &str) -> ObjectVersion {
344 let checksum_algorithm = if object.checksum_algorithm().is_empty() {
345 None
346 } else {
347 Some(
348 object
349 .checksum_algorithm()
350 .iter()
351 .map(|checksum_algorithm| checksum_algorithm.to_owned())
352 .collect(),
353 )
354 };
355
356 ObjectVersion::builder()
357 .key(key)
358 .version_id(object.version_id().unwrap().to_string())
359 .is_latest(object.is_latest().unwrap())
360 .size(object.size().unwrap())
361 .set_last_modified(object.last_modified().cloned())
362 .set_e_tag(object.e_tag().map(|e_tag| e_tag.to_string()))
363 .set_owner(object.owner().cloned())
364 .set_storage_class(object.storage_class().cloned())
365 .set_checksum_algorithm(checksum_algorithm)
366 .set_checksum_type(object.checksum_type().cloned())
367 .build()
368}
369
370pub fn get_additional_checksum(
371 get_object_output: &GetObjectOutput,
372 checksum_algorithm: Option<ChecksumAlgorithm>,
373) -> Option<String> {
374 checksum_algorithm.as_ref()?;
375
376 match checksum_algorithm.unwrap() {
377 ChecksumAlgorithm::Sha256 => get_object_output
378 .checksum_sha256()
379 .map(|checksum| checksum.to_string()),
380 ChecksumAlgorithm::Sha1 => get_object_output
381 .checksum_sha1()
382 .map(|checksum| checksum.to_string()),
383 ChecksumAlgorithm::Crc32 => get_object_output
384 .checksum_crc32()
385 .map(|checksum| checksum.to_string()),
386 ChecksumAlgorithm::Crc32C => get_object_output
387 .checksum_crc32_c()
388 .map(|checksum| checksum.to_string()),
389 ChecksumAlgorithm::Crc64Nvme => get_object_output
390 .checksum_crc64_nvme()
391 .map(|checksum| checksum.to_string()),
392 _ => {
393 panic!("unknown algorithm")
394 }
395 }
396}
397
398pub fn get_additional_checksum_with_head_object(
399 head_object_output: &HeadObjectOutput,
400 checksum_algorithm: Option<ChecksumAlgorithm>,
401) -> Option<String> {
402 checksum_algorithm.as_ref()?;
403
404 match checksum_algorithm.unwrap() {
405 ChecksumAlgorithm::Sha256 => head_object_output
406 .checksum_sha256()
407 .map(|checksum| checksum.to_string()),
408 ChecksumAlgorithm::Sha1 => head_object_output
409 .checksum_sha1()
410 .map(|checksum| checksum.to_string()),
411 ChecksumAlgorithm::Crc32 => head_object_output
412 .checksum_crc32()
413 .map(|checksum| checksum.to_string()),
414 ChecksumAlgorithm::Crc32C => head_object_output
415 .checksum_crc32_c()
416 .map(|checksum| checksum.to_string()),
417 ChecksumAlgorithm::Crc64Nvme => head_object_output
418 .checksum_crc64_nvme()
419 .map(|checksum| checksum.to_string()),
420 _ => {
421 panic!("unknown algorithm")
422 }
423 }
424}
425
426pub fn is_full_object_checksum(checksum: &Option<String>) -> bool {
427 if checksum.is_none() {
428 return false;
429 }
430
431 let find_result = checksum.as_ref().unwrap().find('-');
434 find_result.is_none()
435}
436
437#[derive(Debug, PartialEq)]
438pub enum SyncStatistics {
439 SyncBytes(u64),
440 SyncComplete { key: String },
441 SyncSkip { key: String },
442 SyncDelete { key: String },
443 SyncError { key: String },
444 SyncWarning { key: String },
445 ETagVerified { key: String },
446 ChecksumVerified { key: String },
447}
448
449#[derive(Debug, Clone)]
450pub enum StoragePath {
451 S3 { bucket: String, prefix: String },
452 Local(PathBuf),
453}
454
455#[derive(Debug, Clone)]
456pub struct ClientConfigLocation {
457 pub aws_config_file: Option<PathBuf>,
458 pub aws_shared_credentials_file: Option<PathBuf>,
459}
460
461#[derive(Debug, Clone)]
462pub enum S3Credentials {
463 Profile(String),
464 Credentials { access_keys: AccessKeys },
465 FromEnvironment,
466}
467
468#[derive(Clone, Zeroize, ZeroizeOnDrop)]
469pub struct AccessKeys {
470 pub access_key: String,
471 pub secret_access_key: String,
472 pub session_token: Option<String>,
473}
474
475impl Debug for AccessKeys {
476 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
477 let mut keys = f.debug_struct("AccessKeys");
478 let session_token = self
479 .session_token
480 .as_ref()
481 .map_or("None", |_| "** redacted **");
482 keys.field("access_key", &self.access_key)
483 .field("secret_access_key", &"** redacted **")
484 .field("session_token", &session_token);
485 keys.finish()
486 }
487}
488
489#[derive(Clone, Zeroize, ZeroizeOnDrop)]
490pub struct SseKmsKeyId {
491 pub id: Option<String>,
492}
493
494impl Debug for SseKmsKeyId {
495 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
496 let mut keys = f.debug_struct("SseKmsKeyId");
497 let sse_kms_key_id = self.id.as_ref().map_or("None", |_| "** redacted **");
498 keys.field("sse_kms_key_id", &sse_kms_key_id);
499 keys.finish()
500 }
501}
502
503#[derive(Clone, Zeroize, ZeroizeOnDrop)]
504pub struct SseCustomerKey {
505 pub key: Option<String>,
506}
507
508impl Debug for SseCustomerKey {
509 fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
510 let mut keys = f.debug_struct("SseCustomerKey");
511 let sse_c_key = self.key.as_ref().map_or("None", |_| "** redacted **");
512 keys.field("key", &sse_c_key);
513 keys.finish()
514 }
515}
516
517#[cfg(test)]
518mod tests {
519 use aws_sdk_s3::types::{
520 ChecksumAlgorithm, ObjectStorageClass, ObjectVersionStorageClass, Owner,
521 };
522
523 use super::*;
524
525 #[test]
526 fn clone_non_versioning_object_with_key_test() {
527 init_dummy_tracing_subscriber();
528
529 let source_object = Object::builder()
530 .key("source")
531 .size(1)
532 .e_tag("my-etag")
533 .storage_class(ObjectStorageClass::Glacier)
534 .checksum_algorithm(ChecksumAlgorithm::Sha256)
535 .checksum_type(ChecksumType::FullObject)
536 .owner(
537 Owner::builder()
538 .id("test_id")
539 .display_name("test_name")
540 .build(),
541 )
542 .last_modified(DateTime::from_secs(777))
543 .build();
544
545 let expected_object = S3syncObject::NotVersioning(
546 Object::builder()
547 .key("cloned")
548 .size(1)
549 .e_tag("my-etag")
550 .storage_class(ObjectStorageClass::Glacier)
551 .checksum_algorithm(ChecksumAlgorithm::Sha256)
552 .checksum_type(ChecksumType::FullObject)
553 .owner(
554 Owner::builder()
555 .id("test_id")
556 .display_name("test_name")
557 .build(),
558 )
559 .last_modified(DateTime::from_secs(777))
560 .build(),
561 );
562
563 let cloned_object =
564 S3syncObject::clone_non_versioning_object_with_key(&source_object, "cloned");
565
566 assert_eq!(cloned_object, expected_object);
567 }
568 #[test]
569 fn clone_non_versioning_object_with_key_test_no_checksum() {
570 init_dummy_tracing_subscriber();
571
572 let source_object = Object::builder()
573 .key("source")
574 .size(1)
575 .e_tag("my-etag")
576 .storage_class(ObjectStorageClass::Glacier)
577 .owner(
578 Owner::builder()
579 .id("test_id")
580 .display_name("test_name")
581 .build(),
582 )
583 .last_modified(DateTime::from_secs(777))
584 .build();
585
586 let expected_object = S3syncObject::NotVersioning(
587 Object::builder()
588 .key("cloned")
589 .size(1)
590 .e_tag("my-etag")
591 .storage_class(ObjectStorageClass::Glacier)
592 .owner(
593 Owner::builder()
594 .id("test_id")
595 .display_name("test_name")
596 .build(),
597 )
598 .last_modified(DateTime::from_secs(777))
599 .build(),
600 );
601
602 let cloned_object =
603 S3syncObject::clone_non_versioning_object_with_key(&source_object, "cloned");
604
605 assert_eq!(cloned_object, expected_object);
606 }
607
608 #[test]
609 fn clone_versioning_object_with_key_test() {
610 init_dummy_tracing_subscriber();
611
612 let source_object = ObjectVersion::builder()
613 .key("source")
614 .version_id("version1".to_string())
615 .is_latest(false)
616 .size(1)
617 .e_tag("my-etag")
618 .storage_class(ObjectVersionStorageClass::Standard)
619 .checksum_algorithm(ChecksumAlgorithm::Sha256)
620 .checksum_type(ChecksumType::FullObject)
621 .owner(
622 Owner::builder()
623 .id("test_id")
624 .display_name("test_name")
625 .build(),
626 )
627 .last_modified(DateTime::from_secs(777))
628 .build();
629
630 let expected_object = S3syncObject::Versioning(
631 ObjectVersion::builder()
632 .key("cloned")
633 .version_id("version1".to_string())
634 .is_latest(false)
635 .size(1)
636 .e_tag("my-etag")
637 .storage_class(ObjectVersionStorageClass::Standard)
638 .checksum_algorithm(ChecksumAlgorithm::Sha256)
639 .checksum_type(ChecksumType::FullObject)
640 .owner(
641 Owner::builder()
642 .id("test_id")
643 .display_name("test_name")
644 .build(),
645 )
646 .last_modified(DateTime::from_secs(777))
647 .build(),
648 );
649
650 let cloned_object =
651 S3syncObject::clone_versioning_object_with_key(&source_object, "cloned");
652
653 assert_eq!(cloned_object, expected_object);
654 }
655
656 #[test]
657 fn clone_versioning_object_with_key_test_no_checksum() {
658 init_dummy_tracing_subscriber();
659
660 let source_object = ObjectVersion::builder()
661 .key("source")
662 .version_id("version1".to_string())
663 .is_latest(false)
664 .size(1)
665 .e_tag("my-etag")
666 .storage_class(ObjectVersionStorageClass::Standard)
667 .owner(
668 Owner::builder()
669 .id("test_id")
670 .display_name("test_name")
671 .build(),
672 )
673 .last_modified(DateTime::from_secs(777))
674 .build();
675
676 let expected_object = S3syncObject::Versioning(
677 ObjectVersion::builder()
678 .key("cloned")
679 .version_id("version1".to_string())
680 .is_latest(false)
681 .size(1)
682 .e_tag("my-etag")
683 .storage_class(ObjectVersionStorageClass::Standard)
684 .owner(
685 Owner::builder()
686 .id("test_id")
687 .display_name("test_name")
688 .build(),
689 )
690 .last_modified(DateTime::from_secs(777))
691 .build(),
692 );
693
694 let cloned_object =
695 S3syncObject::clone_versioning_object_with_key(&source_object, "cloned");
696
697 assert_eq!(cloned_object, expected_object);
698 }
699
700 #[test]
701 fn versioning_object_getter_test() {
702 init_dummy_tracing_subscriber();
703
704 let versioning_object = ObjectVersion::builder()
705 .key("source")
706 .version_id("version1".to_string())
707 .is_latest(false)
708 .size(1)
709 .e_tag("my-etag")
710 .storage_class(ObjectVersionStorageClass::Standard)
711 .checksum_algorithm(ChecksumAlgorithm::Sha256)
712 .checksum_type(ChecksumType::FullObject)
713 .owner(
714 Owner::builder()
715 .id("test_id")
716 .display_name("test_name")
717 .build(),
718 )
719 .last_modified(DateTime::from_secs(777))
720 .build();
721
722 let s3sync_object =
723 S3syncObject::clone_versioning_object_with_key(&versioning_object, "cloned");
724
725 assert_eq!(s3sync_object.key(), "cloned");
726 assert_eq!(s3sync_object.version_id().unwrap(), "version1");
727 assert!(!s3sync_object.is_latest());
728 assert_eq!(s3sync_object.size(), 1);
729 assert_eq!(s3sync_object.e_tag().unwrap(), "my-etag");
730 assert_eq!(
731 s3sync_object.checksum_type().unwrap(),
732 &ChecksumType::FullObject
733 );
734 assert_eq!(
735 s3sync_object.checksum_algorithm().unwrap(),
736 &[ChecksumAlgorithm::Sha256]
737 );
738 }
739
740 #[test]
741 fn non_versioning_object_getter_test() {
742 init_dummy_tracing_subscriber();
743
744 let versioning_object = Object::builder()
745 .key("source")
746 .size(1)
747 .e_tag("my-etag")
748 .storage_class(ObjectStorageClass::Standard)
749 .checksum_algorithm(ChecksumAlgorithm::Sha256)
750 .checksum_type(ChecksumType::FullObject)
751 .owner(
752 Owner::builder()
753 .id("test_id")
754 .display_name("test_name")
755 .build(),
756 )
757 .last_modified(DateTime::from_secs(777))
758 .build();
759
760 let s3sync_object =
761 S3syncObject::clone_non_versioning_object_with_key(&versioning_object, "cloned");
762
763 assert_eq!(s3sync_object.key(), "cloned");
764 assert_eq!(s3sync_object.size(), 1);
765 assert_eq!(s3sync_object.e_tag().unwrap(), "my-etag");
766 assert_eq!(
767 s3sync_object.checksum_type().unwrap(),
768 &ChecksumType::FullObject
769 );
770 assert_eq!(
771 s3sync_object.checksum_algorithm().unwrap(),
772 &[ChecksumAlgorithm::Sha256]
773 );
774 }
775
776 #[test]
777 fn versioning_object_getter_test_no_checksum() {
778 init_dummy_tracing_subscriber();
779
780 let versioning_object = ObjectVersion::builder()
781 .key("source")
782 .version_id("version1".to_string())
783 .is_latest(false)
784 .size(1)
785 .e_tag("my-etag")
786 .storage_class(ObjectVersionStorageClass::Standard)
787 .owner(
788 Owner::builder()
789 .id("test_id")
790 .display_name("test_name")
791 .build(),
792 )
793 .last_modified(DateTime::from_secs(777))
794 .build();
795
796 let s3sync_object =
797 S3syncObject::clone_versioning_object_with_key(&versioning_object, "cloned");
798
799 assert_eq!(s3sync_object.key(), "cloned");
800 assert_eq!(s3sync_object.version_id().unwrap(), "version1");
801 assert!(!s3sync_object.is_latest());
802 assert_eq!(s3sync_object.size(), 1);
803 assert_eq!(s3sync_object.e_tag().unwrap(), "my-etag");
804 assert!(s3sync_object.checksum_algorithm().is_none());
805 }
806
807 #[test]
808 fn clone_delete_marker_with_key_test() {
809 init_dummy_tracing_subscriber();
810
811 let source_object = DeleteMarkerEntry::builder()
812 .key("source")
813 .version_id("version1".to_string())
814 .is_latest(true)
815 .owner(
816 Owner::builder()
817 .id("test_id")
818 .display_name("test_name")
819 .build(),
820 )
821 .last_modified(DateTime::from_secs(777))
822 .build();
823
824 let expected_object = S3syncObject::DeleteMarker(
825 DeleteMarkerEntry::builder()
826 .key("cloned")
827 .version_id("version1".to_string())
828 .is_latest(true)
829 .owner(
830 Owner::builder()
831 .id("test_id")
832 .display_name("test_name")
833 .build(),
834 )
835 .last_modified(DateTime::from_secs(777))
836 .build(),
837 );
838
839 let cloned_object = S3syncObject::clone_delete_marker_with_key(&source_object, "cloned");
840
841 assert_eq!(cloned_object, expected_object);
842 }
843
844 #[test]
845 fn debug_print_access_keys() {
846 init_dummy_tracing_subscriber();
847
848 let access_keys = AccessKeys {
849 access_key: "access_key".to_string(),
850 secret_access_key: "secret_access_key".to_string(),
851 session_token: Some("session_token".to_string()),
852 };
853 let debug_string = format!("{access_keys:?}");
854
855 assert!(debug_string.contains("secret_access_key: \"** redacted **\""));
856 assert!(debug_string.contains("session_token: \"** redacted **\""));
857 }
858
859 #[test]
860 #[should_panic]
861 fn last_modified_test_should_panic() {
862 init_dummy_tracing_subscriber();
863
864 let object_versions = vec![S3syncObject::Versioning(ObjectVersion::builder().build())];
865
866 let packed = S3syncObject::PackedVersions(PackedObjectVersions {
867 key: "test".to_string(),
868 packed_object_versions: object_versions.clone(),
869 });
870
871 let _ = packed.last_modified();
872 }
873
874 #[test]
875 #[should_panic]
876 fn size_test_should_panic() {
877 init_dummy_tracing_subscriber();
878
879 let object_versions = vec![S3syncObject::Versioning(ObjectVersion::builder().build())];
880
881 let packed = S3syncObject::PackedVersions(PackedObjectVersions {
882 key: "test".to_string(),
883 packed_object_versions: object_versions.clone(),
884 });
885
886 let _ = packed.size();
887 }
888
889 #[test]
890 #[should_panic]
891 fn version_id_test_should_panic() {
892 init_dummy_tracing_subscriber();
893
894 let object_versions = vec![S3syncObject::Versioning(ObjectVersion::builder().build())];
895
896 let packed = S3syncObject::PackedVersions(PackedObjectVersions {
897 key: "test".to_string(),
898 packed_object_versions: object_versions.clone(),
899 });
900
901 let _ = packed.version_id();
902 }
903
904 #[test]
905 #[should_panic]
906 fn checksum_algorithm_test_should_panic() {
907 init_dummy_tracing_subscriber();
908
909 let object_versions = vec![S3syncObject::Versioning(ObjectVersion::builder().build())];
910
911 let packed = S3syncObject::PackedVersions(PackedObjectVersions {
912 key: "test".to_string(),
913 packed_object_versions: object_versions.clone(),
914 });
915
916 let _ = packed.checksum_algorithm();
917 }
918
919 #[test]
920 #[should_panic]
921 fn checksum_type_packed_test_should_panic() {
922 init_dummy_tracing_subscriber();
923
924 let object_versions = vec![S3syncObject::Versioning(ObjectVersion::builder().build())];
925
926 let packed = S3syncObject::PackedVersions(PackedObjectVersions {
927 key: "test".to_string(),
928 packed_object_versions: object_versions.clone(),
929 });
930
931 let _ = packed.checksum_type();
932 }
933
934 #[test]
935 #[should_panic]
936 fn checksum_type_delete_marker_test_should_panic() {
937 init_dummy_tracing_subscriber();
938
939 let delete_marker = S3syncObject::DeleteMarker(DeleteMarkerEntry::builder().build());
940 let _ = delete_marker.checksum_type();
941 }
942
943 #[test]
944 fn is_latest_test() {
945 init_dummy_tracing_subscriber();
946
947 let object_versions = vec![S3syncObject::Versioning(ObjectVersion::builder().build())];
948
949 let packed = S3syncObject::PackedVersions(PackedObjectVersions {
950 key: "test".to_string(),
951 packed_object_versions: object_versions.clone(),
952 });
953
954 assert!(!packed.is_latest())
955 }
956
957 #[test]
958 #[should_panic]
959 fn e_tag_test_should_panic() {
960 init_dummy_tracing_subscriber();
961
962 let object_versions = vec![S3syncObject::Versioning(ObjectVersion::builder().build())];
963
964 let packed = S3syncObject::PackedVersions(PackedObjectVersions {
965 key: "test".to_string(),
966 packed_object_versions: object_versions.clone(),
967 });
968
969 let _ = packed.e_tag();
970 }
971
972 #[test]
973 fn unpack_object_versions_test() {
974 init_dummy_tracing_subscriber();
975
976 let object_versions = vec![
977 S3syncObject::Versioning(ObjectVersion::builder().build()),
978 S3syncObject::Versioning(ObjectVersion::builder().build()),
979 S3syncObject::Versioning(ObjectVersion::builder().build()),
980 S3syncObject::Versioning(ObjectVersion::builder().build()),
981 S3syncObject::Versioning(ObjectVersion::builder().build()),
982 ];
983
984 let packed = S3syncObject::PackedVersions(PackedObjectVersions {
985 key: "test".to_string(),
986 packed_object_versions: object_versions.clone(),
987 });
988
989 let unpacked = unpack_object_versions(&packed);
990
991 assert_eq!(object_versions, unpacked);
992 }
993
994 #[test]
995 #[should_panic]
996 fn unpack_object_versions_test_panic() {
997 init_dummy_tracing_subscriber();
998
999 unpack_object_versions(&S3syncObject::Versioning(ObjectVersion::builder().build()));
1000 }
1001
1002 #[test]
1003 fn is_delete_marker_test() {
1004 init_dummy_tracing_subscriber();
1005
1006 assert!(
1007 S3syncObject::DeleteMarker(DeleteMarkerEntry::builder().build()).is_delete_marker()
1008 );
1009
1010 assert!(!S3syncObject::NotVersioning(Object::builder().build()).is_delete_marker());
1011 assert!(!S3syncObject::Versioning(ObjectVersion::builder().build()).is_delete_marker());
1012 assert!(
1013 !S3syncObject::PackedVersions(PackedObjectVersions {
1014 key: "test".to_string(),
1015 packed_object_versions: vec![]
1016 })
1017 .is_delete_marker()
1018 );
1019 }
1020
1021 #[test]
1022 fn test_sse_kms_keyid_debug_string() {
1023 let secret = SseKmsKeyId {
1024 id: Some("secret".to_string()),
1025 };
1026
1027 let debug_string = format!("{:?}", secret);
1028 assert!(debug_string.contains("redacted"))
1029 }
1030
1031 #[test]
1032 fn test_sse_customer_key_debug_string() {
1033 let secret = SseCustomerKey {
1034 key: Some("secret".to_string()),
1035 };
1036
1037 let debug_string = format!("{:?}", secret);
1038 assert!(debug_string.contains("redacted"))
1039 }
1040
1041 #[test]
1042 fn test_format_metadata() {
1043 let metadata = HashMap::from([
1044 ("key3".to_string(), "value3".to_string()),
1045 ("key1".to_string(), "value1".to_string()),
1046 ("abc".to_string(), "☃".to_string()),
1047 ("xyz_abc".to_string(), "value_xyz".to_string()),
1048 ("key_comma".to_string(), "value,comma".to_string()),
1049 ("key2".to_string(), "value2".to_string()),
1050 ]);
1051
1052 let formatted = format_metadata(&metadata);
1053 assert_eq!(
1054 formatted,
1055 "abc=%E2%98%83,key1=value1,key2=value2,key3=value3,key_comma=value%2Ccomma,xyz_abc=value_xyz"
1056 );
1057 }
1058
1059 #[test]
1060 fn test_format_tags() {
1061 let tags = vec![
1062 Tag::builder().key("key3").value("value3").build().unwrap(),
1063 Tag::builder().key("key1").value("value1").build().unwrap(),
1064 Tag::builder().key("abc").value("☃").build().unwrap(),
1065 Tag::builder().key("☃").value("value").build().unwrap(),
1066 Tag::builder()
1067 .key("xyz_abc")
1068 .value("value_xyz")
1069 .build()
1070 .unwrap(),
1071 Tag::builder()
1072 .key("key_comma")
1073 .value("value,comma")
1074 .build()
1075 .unwrap(),
1076 Tag::builder()
1077 .key("key_and")
1078 .value("value&and")
1079 .build()
1080 .unwrap(),
1081 Tag::builder().key("key2").value("value2").build().unwrap(),
1082 ];
1083
1084 let formatted = format_tags(tags.as_slice());
1085 assert_eq!(
1086 formatted,
1087 "abc=%E2%98%83&key1=value1&key2=value2&key3=value3&key_and=value%26and&key_comma=value%2Ccomma&xyz_abc=value_xyz&%E2%98%83=value"
1088 );
1089 }
1090
1091 fn init_dummy_tracing_subscriber() {
1092 let _ = tracing_subscriber::fmt()
1093 .with_env_filter("dummy=trace")
1094 .try_init();
1095 }
1096}