s3/
utils.rs

1use std::str::FromStr;
2
3use crate::error::S3Error;
4use crate::request::ResponseData;
5use crate::{bucket::CHUNK_SIZE, serde_types::HeadObjectResult};
6
7use crate::request::{AsyncRead, AsyncReadExt};
8use std::fs::File;
9use std::io::Read;
10use std::path::Path;
11
12pub struct PutStreamResponse {
13    status_code: u16,
14    uploaded_bytes: usize,
15}
16
17impl PutStreamResponse {
18    pub fn new(status_code: u16, uploaded_bytes: usize) -> Self {
19        Self {
20            status_code,
21            uploaded_bytes,
22        }
23    }
24    pub fn status_code(&self) -> u16 {
25        self.status_code
26    }
27
28    pub fn uploaded_bytes(&self) -> usize {
29        self.uploaded_bytes
30    }
31}
32
33/// # Example
34/// ```rust,no_run
35/// use s3::utils::etag_for_path;
36///
37/// let path = "test_etag";
38/// let etag = etag_for_path(path).unwrap();
39/// println!("{}", etag);
40/// ```
41pub fn etag_for_path(path: impl AsRef<Path>) -> Result<String, S3Error> {
42    let mut file = File::open(path)?;
43    let mut last_digest: [u8; 16];
44    let mut digests = Vec::new();
45    let mut chunks = 0;
46    loop {
47        let chunk = read_chunk(&mut file)?;
48        last_digest = md5::compute(&chunk).into();
49        digests.extend_from_slice(&last_digest);
50        chunks += 1;
51        if chunk.len() < CHUNK_SIZE {
52            break;
53        }
54    }
55    let etag = if chunks <= 1 {
56        format!("{:x}", md5::Digest(last_digest))
57    } else {
58        let digest = format!("{:x}", md5::compute(digests));
59        format!("{}-{}", digest, chunks)
60    };
61    Ok(etag)
62}
63
64pub fn read_chunk<R: Read>(reader: &mut R) -> Result<Vec<u8>, S3Error> {
65    let mut chunk = Vec::with_capacity(CHUNK_SIZE);
66    let mut take = reader.take(CHUNK_SIZE as u64);
67    take.read_to_end(&mut chunk)?;
68
69    Ok(chunk)
70}
71
72pub async fn read_chunk_async<R: AsyncRead + Unpin>(reader: &mut R) -> Result<Vec<u8>, S3Error> {
73    let mut chunk = Vec::with_capacity(CHUNK_SIZE);
74    let mut take = reader.take(CHUNK_SIZE as u64);
75    take.read_to_end(&mut chunk).await?;
76
77    Ok(chunk)
78}
79
80pub trait GetAndConvertHeaders {
81    fn get_and_convert<T: FromStr>(&self, header: &str) -> Option<T>;
82    fn get_string(&self, header: &str) -> Option<String>;
83}
84
85impl GetAndConvertHeaders for http::header::HeaderMap {
86    fn get_and_convert<T: FromStr>(&self, header: &str) -> Option<T> {
87        self.get(header)?.to_str().ok()?.parse::<T>().ok()
88    }
89    fn get_string(&self, header: &str) -> Option<String> {
90        Some(self.get(header)?.to_str().ok()?.to_owned())
91    }
92}
93
94impl From<&http::HeaderMap> for HeadObjectResult {
95    fn from(headers: &http::HeaderMap) -> Self {
96        let mut result = HeadObjectResult {
97            accept_ranges: headers.get_string("accept-ranges"),
98            cache_control: headers.get_string("Cache-Control"),
99            content_disposition: headers.get_string("Content-Disposition"),
100            content_encoding: headers.get_string("Content-Encoding"),
101            content_language: headers.get_string("Content-Language"),
102            content_length: headers.get_and_convert("Content-Length"),
103            content_type: headers.get_string("Content-Type"),
104            delete_marker: headers.get_and_convert("x-amz-delete-marker"),
105            e_tag: headers.get_string("ETag"),
106            expiration: headers.get_string("x-amz-expiration"),
107            expires: headers.get_and_convert("Expires"),
108            last_modified: headers.get_string("Last-Modified"),
109            ..Default::default()
110        };
111        let mut values = ::std::collections::HashMap::new();
112        for (key, value) in headers.iter() {
113            if key.as_str().starts_with("x-amz-meta-") {
114                if let Ok(value) = value.to_str() {
115                    values.insert(
116                        key.as_str()["x-amz-meta-".len()..].to_owned(),
117                        value.to_owned(),
118                    );
119                }
120            }
121        }
122        result.metadata = Some(values);
123        result.missing_meta = headers.get_and_convert("x-amz-missing-meta");
124        result.object_lock_legal_hold_status = headers.get_string("x-amz-object-lock-legal-hold");
125        result.object_lock_mode = headers.get_string("x-amz-object-lock-mode");
126        result.object_lock_retain_until_date =
127            headers.get_string("x-amz-object-lock-retain-until-date");
128        result.parts_count = headers.get_and_convert("x-amz-mp-parts-count");
129        result.replication_status = headers.get_string("x-amz-replication-status");
130        result.request_charged = headers.get_string("x-amz-request-charged");
131        result.restore = headers.get_string("x-amz-restore");
132        result.sse_customer_algorithm =
133            headers.get_string("x-amz-server-side-encryption-customer-algorithm");
134        result.sse_customer_key_md5 =
135            headers.get_string("x-amz-server-side-encryption-customer-key-MD5");
136        result.ssekms_key_id = headers.get_string("x-amz-server-side-encryption-aws-kms-key-id");
137        result.server_side_encryption = headers.get_string("x-amz-server-side-encryption");
138        result.storage_class = headers.get_string("x-amz-storage-class");
139        result.version_id = headers.get_string("x-amz-version-id");
140        result.website_redirect_location = headers.get_string("x-amz-website-redirect-location");
141        result
142    }
143}
144
145pub(crate) fn error_from_response_data(response_data: ResponseData) -> Result<S3Error, S3Error> {
146    let utf8_content = String::from_utf8(response_data.as_slice().to_vec())?;
147    Err(S3Error::HttpFailWithBody(
148        response_data.status_code(),
149        utf8_content,
150    ))
151}
152
153#[cfg(test)]
154mod test {
155    use crate::utils::etag_for_path;
156    use std::fs::File;
157    use std::io::prelude::*;
158    use std::io::Cursor;
159    use std::path::Path;
160
161    struct TestFile<'a> {
162        path: &'a Path,
163    }
164
165    impl<'a> TestFile<'a> {
166        pub fn new(path: &'a Path) -> Self {
167            Self { path }
168        }
169    }
170
171    impl AsRef<Path> for TestFile<'_> {
172        fn as_ref(&self) -> &Path {
173            self.path
174        }
175    }
176
177    impl Drop for TestFile<'_> {
178        fn drop(&mut self) {
179            std::fs::remove_file(self.path).unwrap_or(());
180        }
181    }
182
183    fn object(size: u32) -> Vec<u8> {
184        (0..size).map(|_| 33).collect()
185    }
186
187    #[test]
188    fn test_etag_large_file() {
189        let path = &TestFile::new(Path::new("test_etag"));
190        let test: Vec<u8> = object(10_000_000);
191
192        let mut file = File::create(path).unwrap();
193        file.write_all(&test).unwrap();
194
195        let etag = etag_for_path(path).unwrap();
196
197        assert_eq!(etag, "e438487f09f09c042b2de097765e5ac2-2");
198    }
199
200    #[test]
201    fn test_etag_small_file() {
202        let path = &TestFile::new(Path::new("test_etag"));
203        let test: Vec<u8> = object(1000);
204
205        let mut file = File::create(path).unwrap();
206        file.write_all(&test).unwrap();
207
208        let etag = etag_for_path(path).unwrap();
209
210        assert_eq!(etag, "8122ef1c2b2331f7986349560248cf56");
211    }
212
213    #[test]
214    fn test_read_chunk_all_zero() {
215        let blob = vec![0u8; 10_000_000];
216        let mut blob = Cursor::new(blob);
217
218        let result = super::read_chunk(&mut blob).unwrap();
219
220        assert_eq!(result.len(), crate::bucket::CHUNK_SIZE);
221    }
222
223    #[test]
224    fn test_read_chunk_multi_chunk() {
225        let blob = vec![1u8; 10_000_000];
226        let mut blob = Cursor::new(blob);
227
228        let result = super::read_chunk(&mut blob).unwrap();
229        assert_eq!(result.len(), crate::bucket::CHUNK_SIZE);
230
231        let result = super::read_chunk(&mut blob).unwrap();
232        assert_eq!(result.len(), 1_611_392);
233    }
234}
s3/utils.rs

s3/
utils.rs
