s3util_rs/types/
mod.rs

1use std::collections::HashMap;
2use std::fmt;
3use std::fmt::{Debug, Formatter};
4use std::path::PathBuf;
5use std::sync::{Arc, Mutex};
6
7use aws_sdk_s3::operation::get_object::GetObjectOutput;
8use aws_sdk_s3::operation::head_object::HeadObjectOutput;
9use aws_sdk_s3::primitives::DateTime;
10use aws_sdk_s3::types::{ChecksumAlgorithm, ChecksumType, ObjectPart, Tag};
11use sha1::{Digest, Sha1};
12use zeroize_derive::{Zeroize, ZeroizeOnDrop};
13
14pub mod async_callback;
15pub mod error;
16
17pub mod filter_message;
18pub mod token;
19
20pub const S3SYNC_ORIGIN_LAST_MODIFIED_METADATA_KEY: &str = "s3sync_origin_last_modified";
21pub const SYNC_REPORT_SUMMERY_NAME: &str = "REPORT_SUMMARY";
22pub const SYNC_REPORT_RECORD_NAME: &str = "SYNC_STATUS";
23pub const SYNC_REPORT_EXISTENCE_TYPE: &str = "EXISTENCE";
24pub const SYNC_REPORT_ETAG_TYPE: &str = "ETAG";
25pub const SYNC_REPORT_CHECKSUM_TYPE: &str = "CHECKSUM";
26pub const SYNC_REPORT_METADATA_TYPE: &str = "METADATA";
27pub const SYNC_REPORT_TAGGING_TYPE: &str = "TAGGING";
28pub const SYNC_REPORT_CONTENT_DISPOSITION_METADATA_KEY: &str = "Content-Disposition";
29pub const SYNC_REPORT_CONTENT_ENCODING_METADATA_KEY: &str = "Content-Encoding";
30pub const SYNC_REPORT_CONTENT_LANGUAGE_METADATA_KEY: &str = "Content-Language";
31pub const SYNC_REPORT_CONTENT_TYPE_METADATA_KEY: &str = "Content-Type";
32pub const SYNC_REPORT_CACHE_CONTROL_METADATA_KEY: &str = "Cache-Control";
33pub const SYNC_REPORT_EXPIRES_METADATA_KEY: &str = "Expires";
34pub const SYNC_REPORT_WEBSITE_REDIRECT_METADATA_KEY: &str = "x-amz-website-redirect-location";
35pub const SYNC_REPORT_USER_DEFINED_METADATA_KEY: &str = "x-amz-meta-";
36
37pub const METADATA_SYNC_REPORT_LOG_NAME: &str = "METADATA_SYNC_STATUS";
38pub const TAGGING_SYNC_REPORT_LOG_NAME: &str = "TAGGING_SYNC_STATUS";
39pub const SYNC_STATUS_MATCHES: &str = "MATCHES";
40pub const SYNC_STATUS_MISMATCH: &str = "MISMATCH";
41pub const SYNC_STATUS_NOT_FOUND: &str = "NOT_FOUND";
42pub const SYNC_STATUS_UNKNOWN: &str = "UNKNOWN";
43
44pub type Sha1Digest = [u8; 20];
45
46#[derive(Debug, Clone, PartialEq, Eq, Hash)]
47pub enum ObjectKey {
48    KeyString(String),
49    KeySHA1Digest(Sha1Digest),
50}
51
52#[derive(Debug, Clone, PartialEq)]
53pub struct ObjectEntry {
54    pub last_modified: DateTime,
55    pub content_length: i64,
56    pub e_tag: Option<String>,
57}
58
59pub type ObjectKeyMap = Arc<Mutex<HashMap<ObjectKey, ObjectEntry>>>;
60
61#[derive(Debug, Clone, Default)]
62pub struct SyncStatsReport {
63    pub number_of_objects: usize,
64    pub not_found: usize,
65    pub etag_matches: usize,
66    pub etag_mismatch: usize,
67    pub etag_unknown: usize,
68    pub checksum_matches: usize,
69    pub checksum_mismatch: usize,
70    pub checksum_unknown: usize,
71    pub metadata_matches: usize,
72    pub metadata_mismatch: usize,
73    pub tagging_matches: usize,
74    pub tagging_mismatch: usize,
75}
76
77impl SyncStatsReport {
78    pub fn increment_number_of_objects(&mut self) {
79        self.number_of_objects += 1;
80    }
81    pub fn increment_not_found(&mut self) {
82        self.not_found += 1;
83    }
84    pub fn increment_etag_matches(&mut self) {
85        self.etag_matches += 1;
86    }
87    pub fn increment_etag_mismatch(&mut self) {
88        self.etag_mismatch += 1;
89    }
90    pub fn increment_etag_unknown(&mut self) {
91        self.etag_unknown += 1;
92    }
93    pub fn increment_checksum_matches(&mut self) {
94        self.checksum_matches += 1;
95    }
96    pub fn increment_checksum_mismatch(&mut self) {
97        self.checksum_mismatch += 1;
98    }
99    pub fn increment_checksum_unknown(&mut self) {
100        self.checksum_unknown += 1;
101    }
102    pub fn increment_metadata_matches(&mut self) {
103        self.metadata_matches += 1;
104    }
105    pub fn increment_metadata_mismatch(&mut self) {
106        self.metadata_mismatch += 1;
107    }
108    pub fn increment_tagging_matches(&mut self) {
109        self.tagging_matches += 1;
110    }
111    pub fn increment_tagging_mismatch(&mut self) {
112        self.tagging_mismatch += 1;
113    }
114}
115
116#[derive(Clone, Default)]
117pub struct ObjectChecksum {
118    pub key: String,
119    pub version_id: Option<String>,
120    pub checksum_algorithm: Option<ChecksumAlgorithm>,
121    pub checksum_type: Option<ChecksumType>,
122    pub object_parts: Option<Vec<ObjectPart>>,
123    pub final_checksum: Option<String>,
124}
125
126pub fn format_metadata(metadata: &HashMap<String, String>) -> String {
127    let mut sorted_keys: Vec<&String> = metadata.keys().collect();
128    sorted_keys.sort();
129
130    sorted_keys
131        .iter()
132        .map(|key| {
133            let value = urlencoding::encode(&metadata[*key]).to_string();
134            format!("{key}={value}")
135        })
136        .collect::<Vec<String>>()
137        .join(",")
138}
139
140pub fn format_tags(tags: &[Tag]) -> String {
141    let mut tags = tags
142        .iter()
143        .map(|tag| (tag.key(), tag.value()))
144        .collect::<Vec<_>>();
145
146    tags.sort_by(|a, b| a.0.cmp(b.0));
147
148    tags.iter()
149        .map(|(key, value)| {
150            let escaped_key = urlencoding::encode(key).to_string();
151            let encoded_value = urlencoding::encode(value).to_string();
152            format!("{escaped_key}={encoded_value}")
153        })
154        .collect::<Vec<String>>()
155        .join("&")
156}
157
158// sha1 uses generic-array v0.x internally, which is deprecated.
159// Suppress warnings until the underlying library is updated.
160#[allow(deprecated)]
161pub fn sha1_digest_from_key(key: &str) -> Sha1Digest {
162    let digest = Sha1::digest(key);
163    TryInto::<Sha1Digest>::try_into(digest.as_slice()).unwrap()
164}
165
166/// Detect which checksum algorithm the source object uses by inspecting the GetObjectOutput fields.
167/// Returns the algorithm and its value if found.
168///
169/// Priority favors explicitly user-chosen algorithms (SHA256/SHA1/CRC32/CRC32C) over CRC64NVME,
170/// which S3 often auto-adds to objects uploaded with a different explicit algorithm. This ensures
171/// a multipart object uploaded with `--additional-checksum-algorithm SHA256` is verified with
172/// SHA256, not with the auto-added full-object CRC64NVME.
173pub fn detect_additional_checksum(
174    get_object_output: &GetObjectOutput,
175) -> Option<(ChecksumAlgorithm, String)> {
176    if let Some(v) = get_object_output.checksum_sha256() {
177        return Some((ChecksumAlgorithm::Sha256, v.to_string()));
178    }
179    if let Some(v) = get_object_output.checksum_sha1() {
180        return Some((ChecksumAlgorithm::Sha1, v.to_string()));
181    }
182    if let Some(v) = get_object_output.checksum_crc32_c() {
183        return Some((ChecksumAlgorithm::Crc32C, v.to_string()));
184    }
185    if let Some(v) = get_object_output.checksum_crc32() {
186        return Some((ChecksumAlgorithm::Crc32, v.to_string()));
187    }
188    if let Some(v) = get_object_output.checksum_crc64_nvme() {
189        return Some((ChecksumAlgorithm::Crc64Nvme, v.to_string()));
190    }
191    None
192}
193
194/// Detect which checksum algorithm the source object uses by inspecting the HeadObjectOutput fields.
195/// Returns the algorithm and its value if found.
196///
197/// Priority favors explicitly user-chosen algorithms (SHA256/SHA1/CRC32/CRC32C) over CRC64NVME,
198/// which S3 often auto-adds to objects uploaded with a different explicit algorithm. This ensures
199/// a multipart object uploaded with `--additional-checksum-algorithm SHA256` is verified with
200/// SHA256, not with the auto-added full-object CRC64NVME.
201pub fn detect_additional_checksum_with_head_object(
202    head_object_output: &HeadObjectOutput,
203) -> Option<(ChecksumAlgorithm, String)> {
204    if let Some(v) = head_object_output.checksum_sha256() {
205        return Some((ChecksumAlgorithm::Sha256, v.to_string()));
206    }
207    if let Some(v) = head_object_output.checksum_sha1() {
208        return Some((ChecksumAlgorithm::Sha1, v.to_string()));
209    }
210    if let Some(v) = head_object_output.checksum_crc32_c() {
211        return Some((ChecksumAlgorithm::Crc32C, v.to_string()));
212    }
213    if let Some(v) = head_object_output.checksum_crc32() {
214        return Some((ChecksumAlgorithm::Crc32, v.to_string()));
215    }
216    if let Some(v) = head_object_output.checksum_crc64_nvme() {
217        return Some((ChecksumAlgorithm::Crc64Nvme, v.to_string()));
218    }
219    None
220}
221
222pub fn get_additional_checksum(
223    get_object_output: &GetObjectOutput,
224    checksum_algorithm: Option<ChecksumAlgorithm>,
225) -> Option<String> {
226    checksum_algorithm.as_ref()?;
227
228    match checksum_algorithm.unwrap() {
229        ChecksumAlgorithm::Sha256 => get_object_output
230            .checksum_sha256()
231            .map(|checksum| checksum.to_string()),
232        ChecksumAlgorithm::Sha1 => get_object_output
233            .checksum_sha1()
234            .map(|checksum| checksum.to_string()),
235        ChecksumAlgorithm::Crc32 => get_object_output
236            .checksum_crc32()
237            .map(|checksum| checksum.to_string()),
238        ChecksumAlgorithm::Crc32C => get_object_output
239            .checksum_crc32_c()
240            .map(|checksum| checksum.to_string()),
241        ChecksumAlgorithm::Crc64Nvme => get_object_output
242            .checksum_crc64_nvme()
243            .map(|checksum| checksum.to_string()),
244        _ => {
245            panic!("unknown algorithm")
246        }
247    }
248}
249
250pub fn get_additional_checksum_with_head_object(
251    head_object_output: &HeadObjectOutput,
252    checksum_algorithm: Option<ChecksumAlgorithm>,
253) -> Option<String> {
254    checksum_algorithm.as_ref()?;
255
256    match checksum_algorithm.unwrap() {
257        ChecksumAlgorithm::Sha256 => head_object_output
258            .checksum_sha256()
259            .map(|checksum| checksum.to_string()),
260        ChecksumAlgorithm::Sha1 => head_object_output
261            .checksum_sha1()
262            .map(|checksum| checksum.to_string()),
263        ChecksumAlgorithm::Crc32 => head_object_output
264            .checksum_crc32()
265            .map(|checksum| checksum.to_string()),
266        ChecksumAlgorithm::Crc32C => head_object_output
267            .checksum_crc32_c()
268            .map(|checksum| checksum.to_string()),
269        ChecksumAlgorithm::Crc64Nvme => head_object_output
270            .checksum_crc64_nvme()
271            .map(|checksum| checksum.to_string()),
272        _ => {
273            panic!("unknown algorithm")
274        }
275    }
276}
277
278pub fn is_full_object_checksum(checksum: &Option<String>) -> bool {
279    if checksum.is_none() {
280        return false;
281    }
282
283    // As of February 2, 2025, Amazon S3 GetObject does not return ChecksumType::Composite.
284    // So, we can't get the checksum type from GetObjectOutput and decide where checksum has '-' or not.
285    let find_result = checksum.as_ref().unwrap().find('-');
286    find_result.is_none()
287}
288
289#[derive(Debug, PartialEq)]
290pub enum SyncStatistics {
291    SyncBytes(u64),
292    SyncComplete { key: String },
293    SyncError { key: String },
294    SyncWarning { key: String },
295    ETagVerified { key: String },
296    ETagMismatch { key: String },
297    ChecksumVerified { key: String },
298    ChecksumMismatch { key: String },
299}
300
301#[derive(Debug, Clone)]
302pub enum StoragePath {
303    S3 { bucket: String, prefix: String },
304    Local(PathBuf),
305    Stdio,
306}
307
308#[derive(Debug, Clone)]
309pub struct ClientConfigLocation {
310    pub aws_config_file: Option<PathBuf>,
311    pub aws_shared_credentials_file: Option<PathBuf>,
312}
313
314#[derive(Debug, Clone)]
315#[non_exhaustive]
316pub enum S3Credentials {
317    Profile(String),
318    Credentials { access_keys: AccessKeys },
319    FromEnvironment,
320    NoSignRequest,
321}
322
323#[derive(Clone, Zeroize, ZeroizeOnDrop)]
324pub struct AccessKeys {
325    pub access_key: String,
326    pub secret_access_key: String,
327    pub session_token: Option<String>,
328}
329
330impl Debug for AccessKeys {
331    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
332        let mut keys = f.debug_struct("AccessKeys");
333        let session_token = self
334            .session_token
335            .as_ref()
336            .map_or("None", |_| "** redacted **");
337        keys.field("access_key", &self.access_key)
338            .field("secret_access_key", &"** redacted **")
339            .field("session_token", &session_token);
340        keys.finish()
341    }
342}
343
344#[derive(Clone, Zeroize, ZeroizeOnDrop)]
345pub struct SseKmsKeyId {
346    pub id: Option<String>,
347}
348
349impl Debug for SseKmsKeyId {
350    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
351        let mut keys = f.debug_struct("SseKmsKeyId");
352        let sse_kms_key_id = self.id.as_ref().map_or("None", |_| "** redacted **");
353        keys.field("sse_kms_key_id", &sse_kms_key_id);
354        keys.finish()
355    }
356}
357
358#[derive(Clone, Zeroize, ZeroizeOnDrop)]
359pub struct SseCustomerKey {
360    pub key: Option<String>,
361}
362
363impl Debug for SseCustomerKey {
364    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
365        let mut keys = f.debug_struct("SseCustomerKey");
366        let sse_c_key = self.key.as_ref().map_or("None", |_| "** redacted **");
367        keys.field("key", &sse_c_key);
368        keys.finish()
369    }
370}
371
372#[cfg(test)]
373mod tests {
374    use super::*;
375
376    #[test]
377    fn debug_print_access_keys() {
378        init_dummy_tracing_subscriber();
379
380        let access_keys = AccessKeys {
381            access_key: "access_key".to_string(),
382            secret_access_key: "secret_access_key".to_string(),
383            session_token: Some("session_token".to_string()),
384        };
385        let debug_string = format!("{access_keys:?}");
386
387        assert!(debug_string.contains("secret_access_key: \"** redacted **\""));
388        assert!(debug_string.contains("session_token: \"** redacted **\""));
389    }
390
391    #[test]
392    fn test_sse_kms_keyid_debug_string() {
393        let secret = SseKmsKeyId {
394            id: Some("secret".to_string()),
395        };
396
397        let debug_string = format!("{:?}", secret);
398        assert!(debug_string.contains("redacted"))
399    }
400
401    #[test]
402    fn test_sse_customer_key_debug_string() {
403        let secret = SseCustomerKey {
404            key: Some("secret".to_string()),
405        };
406
407        let debug_string = format!("{:?}", secret);
408        assert!(debug_string.contains("redacted"))
409    }
410
411    #[test]
412    fn test_format_metadata() {
413        let metadata = HashMap::from([
414            ("key3".to_string(), "value3".to_string()),
415            ("key1".to_string(), "value1".to_string()),
416            ("abc".to_string(), "\u{2603}".to_string()),
417            ("xyz_abc".to_string(), "value_xyz".to_string()),
418            ("key_comma".to_string(), "value,comma".to_string()),
419            ("key2".to_string(), "value2".to_string()),
420        ]);
421
422        let formatted = format_metadata(&metadata);
423        assert_eq!(
424            formatted,
425            "abc=%E2%98%83,key1=value1,key2=value2,key3=value3,key_comma=value%2Ccomma,xyz_abc=value_xyz"
426        );
427    }
428
429    #[test]
430    fn test_format_tags() {
431        let tags = vec![
432            Tag::builder().key("key3").value("value3").build().unwrap(),
433            Tag::builder().key("key1").value("value1").build().unwrap(),
434            Tag::builder().key("abc").value("\u{2603}").build().unwrap(),
435            Tag::builder()
436                .key("\u{2603}")
437                .value("value")
438                .build()
439                .unwrap(),
440            Tag::builder()
441                .key("xyz_abc")
442                .value("value_xyz")
443                .build()
444                .unwrap(),
445            Tag::builder()
446                .key("key_comma")
447                .value("value,comma")
448                .build()
449                .unwrap(),
450            Tag::builder()
451                .key("key_and")
452                .value("value&and")
453                .build()
454                .unwrap(),
455            Tag::builder().key("key2").value("value2").build().unwrap(),
456        ];
457
458        let formatted = format_tags(tags.as_slice());
459        assert_eq!(
460            formatted,
461            "abc=%E2%98%83&key1=value1&key2=value2&key3=value3&key_and=value%26and&key_comma=value%2Ccomma&xyz_abc=value_xyz&%E2%98%83=value"
462        );
463    }
464
465    fn init_dummy_tracing_subscriber() {
466        let _ = tracing_subscriber::fmt()
467            .with_env_filter("dummy=trace")
468            .try_init();
469    }
470
471    #[test]
472    fn is_full_object_checksum_none() {
473        assert!(!is_full_object_checksum(&None));
474    }
475
476    #[test]
477    fn is_full_object_checksum_no_dash_means_full_object() {
478        assert!(is_full_object_checksum(&Some("abc123==".to_string())));
479    }
480
481    #[test]
482    fn is_full_object_checksum_dash_means_composite() {
483        // Multipart composite checksums have the form "<base64>-<partcount>".
484        assert!(!is_full_object_checksum(&Some("abc123==-4".to_string())));
485    }
486
487    #[test]
488    fn detect_additional_checksum_returns_none_when_no_checksum_present() {
489        let get = GetObjectOutput::builder().build();
490        assert!(detect_additional_checksum(&get).is_none());
491    }
492
493    #[test]
494    fn detect_additional_checksum_returns_sha256_when_present() {
495        let get = GetObjectOutput::builder()
496            .checksum_sha256("sha256-value")
497            .build();
498        let (algo, value) = detect_additional_checksum(&get).unwrap();
499        assert!(matches!(algo, ChecksumAlgorithm::Sha256));
500        assert_eq!(value, "sha256-value");
501    }
502
503    #[test]
504    fn detect_additional_checksum_prefers_explicit_over_auto_added_crc64nvme() {
505        // S3 may auto-add CRC64NVME alongside an explicitly chosen algorithm.
506        // Per the function's documented contract, the explicit choice wins.
507        let get = GetObjectOutput::builder()
508            .checksum_sha256("sha256-value")
509            .checksum_crc64_nvme("crc64-value")
510            .build();
511        let (algo, value) = detect_additional_checksum(&get).unwrap();
512        assert!(matches!(algo, ChecksumAlgorithm::Sha256));
513        assert_eq!(value, "sha256-value");
514    }
515
516    #[test]
517    fn detect_additional_checksum_returns_crc64nvme_when_only_one_present() {
518        let get = GetObjectOutput::builder()
519            .checksum_crc64_nvme("crc64-value")
520            .build();
521        let (algo, value) = detect_additional_checksum(&get).unwrap();
522        assert!(matches!(algo, ChecksumAlgorithm::Crc64Nvme));
523        assert_eq!(value, "crc64-value");
524    }
525
526    #[test]
527    fn detect_additional_checksum_with_head_object_returns_none_when_no_checksum_present() {
528        let head = HeadObjectOutput::builder().build();
529        assert!(detect_additional_checksum_with_head_object(&head).is_none());
530    }
531
532    #[test]
533    fn detect_additional_checksum_with_head_object_returns_sha256_when_present() {
534        let head = HeadObjectOutput::builder()
535            .checksum_sha256("sha256-value")
536            .build();
537        let (algo, value) = detect_additional_checksum_with_head_object(&head).unwrap();
538        assert!(matches!(algo, ChecksumAlgorithm::Sha256));
539        assert_eq!(value, "sha256-value");
540    }
541
542    #[test]
543    fn detect_additional_checksum_with_head_object_prefers_explicit_over_auto_added_crc64nvme() {
544        // S3 may auto-add CRC64NVME alongside an explicitly chosen algorithm.
545        // Per the function's documented contract, the explicit choice wins.
546        let head = HeadObjectOutput::builder()
547            .checksum_sha256("sha256-value")
548            .checksum_crc64_nvme("crc64-value")
549            .build();
550        let (algo, value) = detect_additional_checksum_with_head_object(&head).unwrap();
551        assert!(matches!(algo, ChecksumAlgorithm::Sha256));
552        assert_eq!(value, "sha256-value");
553    }
554
555    #[test]
556    fn detect_additional_checksum_with_head_object_returns_crc64nvme_when_only_one_present() {
557        let head = HeadObjectOutput::builder()
558            .checksum_crc64_nvme("crc64-value")
559            .build();
560        let (algo, value) = detect_additional_checksum_with_head_object(&head).unwrap();
561        assert!(matches!(algo, ChecksumAlgorithm::Crc64Nvme));
562        assert_eq!(value, "crc64-value");
563    }
564
565    #[test]
566    fn get_additional_checksum_returns_none_when_algorithm_none() {
567        let get = GetObjectOutput::builder()
568            .checksum_sha256("ignored")
569            .build();
570        assert!(get_additional_checksum(&get, None).is_none());
571    }
572
573    #[test]
574    fn get_additional_checksum_extracts_requested_algorithm() {
575        let get = GetObjectOutput::builder()
576            .checksum_sha256("sha256-value")
577            .checksum_sha1("sha1-value")
578            .checksum_crc32("crc32-value")
579            .checksum_crc32_c("crc32c-value")
580            .checksum_crc64_nvme("crc64-value")
581            .build();
582        assert_eq!(
583            get_additional_checksum(&get, Some(ChecksumAlgorithm::Sha256)).unwrap(),
584            "sha256-value"
585        );
586        assert_eq!(
587            get_additional_checksum(&get, Some(ChecksumAlgorithm::Sha1)).unwrap(),
588            "sha1-value"
589        );
590        assert_eq!(
591            get_additional_checksum(&get, Some(ChecksumAlgorithm::Crc32)).unwrap(),
592            "crc32-value"
593        );
594        assert_eq!(
595            get_additional_checksum(&get, Some(ChecksumAlgorithm::Crc32C)).unwrap(),
596            "crc32c-value"
597        );
598        assert_eq!(
599            get_additional_checksum(&get, Some(ChecksumAlgorithm::Crc64Nvme)).unwrap(),
600            "crc64-value"
601        );
602        // Requested but absent on an empty output → None.
603        let empty = GetObjectOutput::builder().build();
604        assert!(get_additional_checksum(&empty, Some(ChecksumAlgorithm::Sha1)).is_none());
605    }
606
607    #[test]
608    fn get_additional_checksum_with_head_object_extracts_correct_field() {
609        use aws_sdk_s3::operation::head_object::HeadObjectOutput;
610        let head = HeadObjectOutput::builder()
611            .checksum_sha256("head-sha256")
612            .checksum_sha1("head-sha1")
613            .checksum_crc32("head-crc32")
614            .checksum_crc32_c("head-crc32c")
615            .checksum_crc64_nvme("head-crc64")
616            .build();
617        assert!(get_additional_checksum_with_head_object(&head, None).is_none());
618        assert_eq!(
619            get_additional_checksum_with_head_object(&head, Some(ChecksumAlgorithm::Sha256))
620                .unwrap(),
621            "head-sha256"
622        );
623        assert_eq!(
624            get_additional_checksum_with_head_object(&head, Some(ChecksumAlgorithm::Sha1)).unwrap(),
625            "head-sha1"
626        );
627        assert_eq!(
628            get_additional_checksum_with_head_object(&head, Some(ChecksumAlgorithm::Crc32))
629                .unwrap(),
630            "head-crc32"
631        );
632        assert_eq!(
633            get_additional_checksum_with_head_object(&head, Some(ChecksumAlgorithm::Crc32C))
634                .unwrap(),
635            "head-crc32c"
636        );
637        assert_eq!(
638            get_additional_checksum_with_head_object(&head, Some(ChecksumAlgorithm::Crc64Nvme))
639                .unwrap(),
640            "head-crc64"
641        );
642        let empty = HeadObjectOutput::builder().build();
643        assert!(
644            get_additional_checksum_with_head_object(&empty, Some(ChecksumAlgorithm::Sha256))
645                .is_none()
646        );
647    }
648
649    #[test]
650    fn sha1_digest_from_key_is_deterministic_and_correct() {
651        let key = "some-object-key.dat";
652        let a = sha1_digest_from_key(key);
653        let b = sha1_digest_from_key(key);
654        assert_eq!(a, b);
655        // Length is enforced by the type (Sha1Digest = [u8; 20]); also verify
656        // a different key yields a different digest.
657        assert_ne!(sha1_digest_from_key("different-key"), a);
658    }
659
660    #[test]
661    fn sync_stats_report_increments_advance_each_field() {
662        let mut r = SyncStatsReport::default();
663        r.increment_number_of_objects();
664        r.increment_not_found();
665        r.increment_etag_matches();
666        r.increment_etag_mismatch();
667        r.increment_etag_unknown();
668        r.increment_checksum_matches();
669        r.increment_checksum_mismatch();
670        r.increment_checksum_unknown();
671        r.increment_metadata_matches();
672        r.increment_metadata_mismatch();
673        r.increment_tagging_matches();
674        r.increment_tagging_mismatch();
675
676        assert_eq!(r.number_of_objects, 1);
677        assert_eq!(r.not_found, 1);
678        assert_eq!(r.etag_matches, 1);
679        assert_eq!(r.etag_mismatch, 1);
680        assert_eq!(r.etag_unknown, 1);
681        assert_eq!(r.checksum_matches, 1);
682        assert_eq!(r.checksum_mismatch, 1);
683        assert_eq!(r.checksum_unknown, 1);
684        assert_eq!(r.metadata_matches, 1);
685        assert_eq!(r.metadata_mismatch, 1);
686        assert_eq!(r.tagging_matches, 1);
687        assert_eq!(r.tagging_mismatch, 1);
688    }
689
690    #[test]
691    fn sync_stats_report_increments_accumulate() {
692        let mut r = SyncStatsReport::default();
693        for _ in 0..5 {
694            r.increment_etag_matches();
695        }
696        assert_eq!(r.etag_matches, 5);
697    }
698
699    #[test]
700    #[should_panic(expected = "unknown algorithm")]
701    fn get_additional_checksum_panics_on_unsupported_algorithm() {
702        // ChecksumAlgorithm has variants (Md5, Sha512, Xxhash*, Unknown) that the
703        // function does not support. Selecting any of them must panic — guarding
704        // against accidental future additions slipping through silently.
705        let get = GetObjectOutput::builder().build();
706        let _ = get_additional_checksum(&get, Some(ChecksumAlgorithm::Md5));
707    }
708
709    #[test]
710    #[should_panic(expected = "unknown algorithm")]
711    fn get_additional_checksum_with_head_object_panics_on_unsupported_algorithm() {
712        let head = HeadObjectOutput::builder().build();
713        let _ = get_additional_checksum_with_head_object(&head, Some(ChecksumAlgorithm::Md5));
714    }
715
716    #[test]
717    fn detect_additional_checksum_returns_sha1_when_only_sha1_present() {
718        let get = GetObjectOutput::builder()
719            .checksum_sha1("sha1-value")
720            .build();
721        let (algo, value) = detect_additional_checksum(&get).unwrap();
722        assert!(matches!(algo, ChecksumAlgorithm::Sha1));
723        assert_eq!(value, "sha1-value");
724    }
725
726    #[test]
727    fn detect_additional_checksum_returns_crc32c_when_only_crc32c_present() {
728        let get = GetObjectOutput::builder()
729            .checksum_crc32_c("crc32c-value")
730            .build();
731        let (algo, value) = detect_additional_checksum(&get).unwrap();
732        assert!(matches!(algo, ChecksumAlgorithm::Crc32C));
733        assert_eq!(value, "crc32c-value");
734    }
735
736    #[test]
737    fn detect_additional_checksum_returns_crc32_when_only_crc32_present() {
738        let get = GetObjectOutput::builder()
739            .checksum_crc32("crc32-value")
740            .build();
741        let (algo, value) = detect_additional_checksum(&get).unwrap();
742        assert!(matches!(algo, ChecksumAlgorithm::Crc32));
743        assert_eq!(value, "crc32-value");
744    }
745
746    #[test]
747    fn detect_additional_checksum_with_head_object_returns_sha1_when_only_sha1_present() {
748        let head = HeadObjectOutput::builder().checksum_sha1("v").build();
749        let (algo, value) = detect_additional_checksum_with_head_object(&head).unwrap();
750        assert!(matches!(algo, ChecksumAlgorithm::Sha1));
751        assert_eq!(value, "v");
752    }
753
754    #[test]
755    fn detect_additional_checksum_with_head_object_returns_crc32c_when_only_crc32c_present() {
756        let head = HeadObjectOutput::builder().checksum_crc32_c("v").build();
757        let (algo, value) = detect_additional_checksum_with_head_object(&head).unwrap();
758        assert!(matches!(algo, ChecksumAlgorithm::Crc32C));
759        assert_eq!(value, "v");
760    }
761
762    #[test]
763    fn detect_additional_checksum_with_head_object_returns_crc32_when_only_crc32_present() {
764        let head = HeadObjectOutput::builder().checksum_crc32("v").build();
765        let (algo, value) = detect_additional_checksum_with_head_object(&head).unwrap();
766        assert!(matches!(algo, ChecksumAlgorithm::Crc32));
767        assert_eq!(value, "v");
768    }
769
770    #[test]
771    fn is_full_object_checksum_dash_at_start_treated_as_composite() {
772        // Any '-' anywhere in the value is treated as composite. A pathological
773        // leading dash — which S3 should never return — is still classified as
774        // composite, documenting the current contract.
775        assert!(!is_full_object_checksum(&Some("-foo".to_string())));
776    }
777
778    #[test]
779    fn object_key_equality_and_hash_consistency() {
780        // ObjectKey is used as a HashMap key — equality and hashing must agree.
781        use std::collections::HashSet;
782        let a = ObjectKey::KeyString("foo/bar".to_string());
783        let a2 = ObjectKey::KeyString("foo/bar".to_string());
784        let b = ObjectKey::KeyString("foo/baz".to_string());
785        let digest = sha1_digest_from_key("foo/bar");
786        let c = ObjectKey::KeySHA1Digest(digest);
787        let c2 = ObjectKey::KeySHA1Digest(digest);
788
789        assert_eq!(a, a2);
790        assert_ne!(a, b);
791        assert_eq!(c, c2);
792        // Different variants are never equal even with semantically related contents.
793        assert_ne!(a, c);
794
795        let mut set = HashSet::new();
796        set.insert(a.clone());
797        set.insert(c.clone());
798        assert!(set.contains(&a2));
799        assert!(set.contains(&c2));
800        assert!(!set.contains(&b));
801    }
802
803    #[test]
804    fn object_checksum_default_is_empty() {
805        let oc = ObjectChecksum::default();
806        assert_eq!(oc.key, "");
807        assert!(oc.version_id.is_none());
808        assert!(oc.checksum_algorithm.is_none());
809        assert!(oc.checksum_type.is_none());
810        assert!(oc.object_parts.is_none());
811        assert!(oc.final_checksum.is_none());
812    }
813
814    #[test]
815    fn format_metadata_empty_returns_empty_string() {
816        let formatted = format_metadata(&HashMap::new());
817        assert_eq!(formatted, "");
818    }
819
820    #[test]
821    fn format_tags_empty_returns_empty_string() {
822        let formatted = format_tags(&[]);
823        assert_eq!(formatted, "");
824    }
825
826    #[test]
827    fn s3_credentials_variants_are_constructible() {
828        // Smoke test that the non-exhaustive enum can be matched on each known variant.
829        let c1 = S3Credentials::Profile("p".to_string());
830        let c2 = S3Credentials::FromEnvironment;
831        let c3 = S3Credentials::NoSignRequest;
832        let c4 = S3Credentials::Credentials {
833            access_keys: AccessKeys {
834                access_key: "a".to_string(),
835                secret_access_key: "s".to_string(),
836                session_token: None,
837            },
838        };
839        for c in [c1, c2, c3, c4] {
840            // Ensure Debug doesn't panic for any variant.
841            let s = format!("{c:?}");
842            assert!(!s.is_empty());
843        }
844    }
845
846    #[test]
847    fn access_keys_debug_with_no_session_token_redacts_correctly() {
848        let access_keys = AccessKeys {
849            access_key: "AKIA".to_string(),
850            secret_access_key: "secret".to_string(),
851            session_token: None,
852        };
853        let debug_string = format!("{access_keys:?}");
854        assert!(debug_string.contains("session_token: \"None\""));
855        assert!(!debug_string.contains("redacted") || debug_string.contains("secret_access_key"));
856    }
857
858    #[test]
859    fn sse_kms_keyid_debug_with_none_id_does_not_redact() {
860        let secret = SseKmsKeyId { id: None };
861        let debug_string = format!("{secret:?}");
862        assert!(debug_string.contains("None"));
863        assert!(!debug_string.contains("redacted"));
864    }
865
866    #[test]
867    fn sse_customer_key_debug_with_none_key_does_not_redact() {
868        let secret = SseCustomerKey { key: None };
869        let debug_string = format!("{secret:?}");
870        assert!(debug_string.contains("None"));
871        assert!(!debug_string.contains("redacted"));
872    }
873}
s3util_rs/types/mod.rs

s3util_rs/types/
mod.rs
