s3/
utils.rs

1use std::str::FromStr;
2
3use crate::error::S3Error;
4use crate::request::ResponseData;
5use crate::{bucket::CHUNK_SIZE, serde_types::HeadObjectResult};
6
7use std::fs::File;
8
9use std::io::Read;
10use std::path::Path;
11
12#[cfg(feature = "with-tokio")]
13use tokio::io::{AsyncRead, AsyncReadExt};
14
15#[cfg(feature = "with-async-std")]
16use futures::io::{AsyncRead, AsyncReadExt};
17
18/// # Example
19/// ```rust,no_run
20/// use s3::utils::etag_for_path;
21///
22/// let path = "test_etag";
23/// let etag = etag_for_path(path).unwrap();
24/// println!("{}", etag);
25/// ```
26pub fn etag_for_path(path: impl AsRef<Path>) -> Result<String, S3Error> {
27    let mut file = File::open(path)?;
28    let mut digests = Vec::new();
29    let mut chunks = 0;
30    loop {
31        let chunk = read_chunk(&mut file)?;
32        let digest: [u8; 16] = md5::compute(&chunk).into();
33        digests.extend_from_slice(&digest);
34        chunks += 1;
35        if chunk.len() < CHUNK_SIZE {
36            break;
37        }
38    }
39    let digest = format!("{:x}", md5::compute(digests));
40    let etag = if chunks <= 1 {
41        digest
42    } else {
43        format!("{}-{}", digest, chunks)
44    };
45    Ok(etag)
46}
47
48pub fn read_chunk<R: Read>(reader: &mut R) -> Result<Vec<u8>, S3Error> {
49    let mut chunk = Vec::with_capacity(CHUNK_SIZE);
50    let mut take = reader.take(CHUNK_SIZE as u64);
51    take.read_to_end(&mut chunk)?;
52
53    Ok(chunk)
54}
55
56#[cfg(any(feature = "with-tokio", feature = "with-async-std"))]
57pub async fn read_chunk_async<R: AsyncRead + Unpin>(reader: &mut R) -> Result<Vec<u8>, S3Error> {
58    let mut chunk = Vec::with_capacity(CHUNK_SIZE);
59    let mut take = reader.take(CHUNK_SIZE as u64);
60    take.read_to_end(&mut chunk).await?;
61
62    Ok(chunk)
63}
64
65pub trait GetAndConvertHeaders {
66    fn get_and_convert<T: FromStr>(&self, header: &str) -> Option<T>;
67    fn get_string(&self, header: &str) -> Option<String>;
68}
69
70impl GetAndConvertHeaders for http::header::HeaderMap {
71    fn get_and_convert<T: FromStr>(&self, header: &str) -> Option<T> {
72        self.get(header)?.to_str().ok()?.parse::<T>().ok()
73    }
74    fn get_string(&self, header: &str) -> Option<String> {
75        Some(self.get(header)?.to_str().ok()?.to_owned())
76    }
77}
78
79impl From<&http::HeaderMap> for HeadObjectResult {
80    fn from(headers: &http::HeaderMap) -> Self {
81        let mut result = HeadObjectResult {
82            accept_ranges: headers.get_string("accept-ranges"),
83            cache_control: headers.get_string("Cache-Control"),
84            content_disposition: headers.get_string("Content-Disposition"),
85            content_encoding: headers.get_string("Content-Encoding"),
86            content_language: headers.get_string("Content-Language"),
87            content_length: headers.get_and_convert("Content-Length"),
88            content_type: headers.get_string("Content-Type"),
89            delete_marker: headers.get_and_convert("x-amz-delete-marker"),
90            e_tag: headers.get_string("ETag"),
91            expiration: headers.get_string("x-amz-expiration"),
92            expires: headers.get_string("Expires"),
93            last_modified: headers.get_string("Last-Modified"),
94            ..Default::default()
95        };
96        let mut values = ::std::collections::HashMap::new();
97        for (key, value) in headers.iter() {
98            if key.as_str().starts_with("x-amz-meta-") {
99                if let Ok(value) = value.to_str() {
100                    values.insert(
101                        key.as_str()["x-amz-meta-".len()..].to_owned(),
102                        value.to_owned(),
103                    );
104                }
105            }
106        }
107        result.metadata = Some(values);
108        result.missing_meta = headers.get_and_convert("x-amz-missing-meta");
109        result.object_lock_legal_hold_status = headers.get_string("x-amz-object-lock-legal-hold");
110        result.object_lock_mode = headers.get_string("x-amz-object-lock-mode");
111        result.object_lock_retain_until_date =
112            headers.get_string("x-amz-object-lock-retain-until-date");
113        result.parts_count = headers.get_and_convert("x-amz-mp-parts-count");
114        result.replication_status = headers.get_string("x-amz-replication-status");
115        result.request_charged = headers.get_string("x-amz-request-charged");
116        result.restore = headers.get_string("x-amz-restore");
117        result.sse_customer_algorithm =
118            headers.get_string("x-amz-server-side-encryption-customer-algorithm");
119        result.sse_customer_key_md5 =
120            headers.get_string("x-amz-server-side-encryption-customer-key-MD5");
121        result.ssekms_key_id = headers.get_string("x-amz-server-side-encryption-aws-kms-key-id");
122        result.server_side_encryption = headers.get_string("x-amz-server-side-encryption");
123        result.storage_class = headers.get_string("x-amz-storage-class");
124        result.version_id = headers.get_string("x-amz-version-id");
125        result.website_redirect_location = headers.get_string("x-amz-website-redirect-location");
126        result
127    }
128}
129
130pub(crate) fn error_from_response_data(response_data: ResponseData) -> Result<S3Error, S3Error> {
131    let utf8_content = String::from_utf8(response_data.as_slice().to_vec())?;
132    Err(S3Error::Http(response_data.status_code(), utf8_content))
133}
134
135#[cfg(test)]
136mod test {
137    use crate::utils::etag_for_path;
138    use std::fs::File;
139    use std::io::prelude::*;
140    use std::io::Cursor;
141
142    fn object(size: u32) -> Vec<u8> {
143        (0..size).map(|_| 33).collect()
144    }
145
146    #[test]
147    fn test_etag() {
148        let path = "test_etag";
149        std::fs::remove_file(path).unwrap_or_else(|_| {});
150        let test: Vec<u8> = object(10_000_000);
151
152        let mut file = File::create(path).unwrap();
153        file.write_all(&test).unwrap();
154
155        let etag = etag_for_path(path).unwrap();
156
157        std::fs::remove_file(path).unwrap_or_else(|_| {});
158
159        assert_eq!(etag, "e438487f09f09c042b2de097765e5ac2-2");
160    }
161
162    #[test]
163    fn test_read_chunk_all_zero() {
164        let blob = vec![0u8; 10_000_000];
165        let mut blob = Cursor::new(blob);
166
167        let result = super::read_chunk(&mut blob).unwrap();
168
169        assert_eq!(result.len(), crate::bucket::CHUNK_SIZE);
170    }
171
172    #[test]
173    fn test_read_chunk_multi_chunk() {
174        let blob = vec![1u8; 10_000_000];
175        let mut blob = Cursor::new(blob);
176
177        let result = super::read_chunk(&mut blob).unwrap();
178        assert_eq!(result.len(), crate::bucket::CHUNK_SIZE);
179
180        let result = super::read_chunk(&mut blob).unwrap();
181        assert_eq!(result.len(), 1_611_392);
182    }
183}
s3/utils.rs

s3/
utils.rs
