gcs_rsync/gcp/storage/resources/
object.rs

1use std::{convert::TryInto, fmt::Display, str::FromStr};
2
3use base64::Engine;
4
5use crate::storage::{Error, StorageResult};
6
7#[derive(Debug, PartialEq, Eq, serde::Serialize, Clone)]
8#[serde(rename_all = "camelCase")]
9pub enum Projection {
10    Full,
11    NoAcl,
12}
13
14/// See [GCS list API reference](https://cloud.google.com/storage/docs/json_api/v1/objects/list)
15#[derive(Debug, PartialEq, Eq, serde::Serialize, Default, Clone)]
16#[serde(rename_all = "camelCase")]
17pub struct ObjectsListRequest {
18    /// [Partial Response](https://cloud.google.com/storage/docs/json_api#partial-response)
19    pub fields: Option<String>,
20    pub delimiter: Option<String>,
21    pub end_offset: Option<String>,
22    pub include_trailing_delimiter: Option<bool>,
23    pub max_results: Option<usize>,
24    pub page_token: Option<String>,
25    pub prefix: Option<String>,
26    pub projection: Option<Projection>,
27    pub start_offset: Option<String>,
28    pub versions: Option<bool>,
29}
30
31#[derive(Debug, PartialEq, Eq, serde::Serialize, Default, Clone)]
32#[serde(rename_all = "camelCase")]
33pub struct ObjectMetadata {
34    pub metadata: Metadata,
35}
36#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize, Default, Clone)]
37#[serde(rename_all = "camelCase")]
38pub struct Metadata {
39    #[serde(
40        rename = "goog-reserved-file-mtime",
41        deserialize_with = "from_string_option"
42    )] //compat with gsutil rsync
43    pub modification_time: Option<i64>,
44}
45
46/// ObjectList response
47#[derive(Debug, serde::Deserialize, Default)]
48#[serde(rename_all = "camelCase")]
49pub struct Objects {
50    pub kind: Option<String>,
51
52    #[serde(default = "Vec::new")]
53    pub items: Vec<PartialObject>,
54
55    #[serde(default = "Vec::new")]
56    pub prefixes: Vec<String>,
57
58    pub next_page_token: Option<String>,
59}
60
61#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
62#[serde(rename_all = "camelCase")]
63pub struct Object {
64    pub bucket: String,
65    pub name: String,
66}
67
68impl Display for Object {
69    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
70        write!(f, "{}", self.gs_url())
71    }
72}
73
74impl FromStr for Object {
75    type Err = Error;
76
77    fn from_str(s: &str) -> Result<Self, Self::Err> {
78        s.strip_prefix("gs://")
79            .and_then(|part| part.split_once('/'))
80            .ok_or(Error::GcsInvalidUrl {
81                url: s.to_owned(),
82                message: "gs url should be gs://bucket/object/path/name".to_owned(),
83            })
84            .and_then(|(bucket, name)| Object::new(bucket, name))
85    }
86}
87
88type GsUrl = String;
89
90const BASE_URL: &str = "https://storage.googleapis.com/storage/v1";
91const UPLOAD_BASE_URL: &str = "https://storage.googleapis.com/upload/storage/v1";
92
93fn percent_encode(input: &str) -> String {
94    percent_encoding::utf8_percent_encode(input, percent_encoding::NON_ALPHANUMERIC).to_string()
95}
96
97impl Object {
98    pub fn gs_url(&self) -> GsUrl {
99        format!("gs://{}/{}", &self.bucket, &self.name)
100    }
101
102    /// References: `<https://cloud.google.com/storage/docs/naming-objects>`
103    pub fn new(bucket: &str, name: &str) -> StorageResult<Self> {
104        if bucket.is_empty() {
105            return Err(Error::GcsInvalidObjectName);
106        }
107
108        if name.is_empty() || name.starts_with('.') {
109            return Err(Error::GcsInvalidObjectName);
110        }
111
112        Ok(Self {
113            bucket: bucket.to_owned(),
114            name: name.to_owned(),
115        })
116    }
117
118    pub fn url(&self) -> String {
119        format!(
120            "{}/b/{}/o/{}",
121            BASE_URL,
122            percent_encode(&self.bucket),
123            percent_encode(&self.name)
124        )
125    }
126
127    pub fn upload_url(&self, upload_type: &str) -> String {
128        format!(
129            "{}/b/{}/o?uploadType={}&name={}",
130            UPLOAD_BASE_URL,
131            percent_encode(&self.bucket),
132            upload_type,
133            percent_encode(&self.name)
134        )
135    }
136}
137
138#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
139#[serde(rename_all = "camelCase")]
140pub struct Bucket {
141    name: String,
142}
143
144impl Bucket {
145    pub fn new(name: &str) -> Self {
146        Self {
147            name: name.to_owned(),
148        }
149    }
150
151    pub fn url(&self) -> String {
152        format!("{}/b/{}/o", BASE_URL, percent_encode(&self.name))
153    }
154}
155
156impl TryInto<Object> for PartialObject {
157    type Error = Error;
158
159    fn try_into(self) -> Result<Object, Self::Error> {
160        fn err(e: &str) -> Error {
161            Error::GcsPartialResponseError(e.to_owned())
162        }
163
164        match (self.bucket, self.name) {
165            (Some(bucket), Some(name)) => Ok(Object { bucket, name }),
166            (None, Some(_)) => Err(err("bucket field is missing")),
167            (Some(_), None) => Err(err("name field is missing")),
168            (None, None) => Err(err("bucket and name fields are missing")),
169        }
170    }
171}
172
173#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize, Default)]
174#[serde(rename_all = "camelCase")]
175pub struct PartialObject {
176    pub bucket: Option<String>,
177    pub id: Option<String>,
178    pub self_link: Option<String>,
179    pub name: Option<String>,
180    pub content_type: Option<String>,
181    pub time_created: Option<chrono::DateTime<chrono::Utc>>,
182    pub updated: Option<chrono::DateTime<chrono::Utc>>,
183    pub time_deleted: Option<chrono::DateTime<chrono::Utc>>,
184    pub retention_expiration_time: Option<chrono::DateTime<chrono::Utc>>,
185    pub storage_class: Option<String>,
186    #[serde(default, deserialize_with = "from_string_option")]
187    pub size: Option<u64>,
188    pub media_link: Option<String>,
189    pub content_encoding: Option<String>,
190    pub content_disposition: Option<String>,
191    pub content_language: Option<String>,
192    pub cache_control: Option<String>,
193    pub metadata: Option<Metadata>,
194    #[serde(default, deserialize_with = "from_string_option")]
195    pub crc32c: Option<CRC32C>,
196    pub etag: Option<String>,
197}
198
199#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize, Default)]
200#[serde(rename_all = "camelCase")]
201pub struct CRC32C {
202    value: u32,
203}
204
205impl CRC32C {
206    pub fn new(value: u32) -> Self {
207        Self { value }
208    }
209    pub fn to_u32(&self) -> u32 {
210        self.value
211    }
212}
213#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
214pub enum Base64EncodedCRC32CError {
215    Base64DecodeError(String),
216    Base64ToU32BigEndianError(Vec<u8>),
217}
218
219impl Display for Base64EncodedCRC32CError {
220    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
221        write!(f, "{:?}", self)
222    }
223}
224
225impl FromStr for CRC32C {
226    type Err = Base64EncodedCRC32CError;
227
228    fn from_str(base64crc32c: &str) -> Result<Self, Self::Err> {
229        let decoded = base64::engine::general_purpose::STANDARD
230            .decode(base64crc32c)
231            .map_err(|err| Base64EncodedCRC32CError::Base64DecodeError(format!("{:?}", err)))?;
232        let crc32c = decoded
233            .try_into()
234            .map(u32::from_be_bytes)
235            .map_err(Base64EncodedCRC32CError::Base64ToU32BigEndianError)?;
236        Ok(CRC32C::new(crc32c))
237    }
238}
239
240fn from_string_option<'de, T, D>(deserializer: D) -> std::result::Result<Option<T>, D::Error>
241where
242    T: std::str::FromStr,
243    T::Err: std::fmt::Display,
244    D: serde::Deserializer<'de>,
245{
246    use serde::{de::Error, Deserialize};
247    use serde_json::Value;
248    match Deserialize::deserialize(deserializer) {
249        Ok(Value::String(s)) => T::from_str(&s).map(Option::from).map_err(Error::custom),
250        Ok(Value::Number(num)) => T::from_str(&num.to_string())
251            .map(Option::from)
252            .map_err(Error::custom),
253        Ok(value) => Err(Error::custom(format!(
254            "Wrong type, expected type {} but got value {:?}",
255            std::any::type_name::<T>(),
256            value,
257        ))),
258        Err(_) => Ok(None),
259    }
260}
261
262#[cfg(test)]
263mod tests {
264    use std::{convert::TryInto, str::FromStr};
265
266    use crate::storage::{Bucket, Error, Object};
267
268    use super::PartialObject;
269
270    #[test]
271    fn fn_gs_url_parsing_to_object() {
272        assert_eq!(
273            Object::new("hello", "world/object").unwrap(),
274            Object::from_str("gs://hello/world/object").unwrap(),
275        )
276    }
277
278    #[test]
279    fn test_invalid_object() {
280        fn assert_object_error(bucket: &str, name: &str) {
281            assert!(matches!(
282                Object::new(bucket, name).unwrap_err(),
283                Error::GcsInvalidObjectName
284            ))
285        }
286        assert_object_error("", "name");
287        assert_object_error("bucket", "");
288        assert_object_error("bucket", ".");
289        assert_object_error("bucket", "..");
290    }
291    #[test]
292    fn test_object_display() {
293        let o = Object::new("hello", "world").unwrap();
294        assert_eq!("gs://hello/world", o.gs_url());
295        assert_eq!("gs://hello/world", format!("{}", o));
296    }
297
298    #[test]
299    fn test_object_url() {
300        let o = Object::new("hello/hello", "world/world").unwrap();
301        assert_eq!(
302            "https://storage.googleapis.com/storage/v1/b/hello%2Fhello/o/world%2Fworld",
303            o.url()
304        );
305    }
306
307    #[test]
308    fn test_object_upload_url() {
309        let o = Object::new("hello/hello", "world/world").unwrap();
310        assert_eq!(
311            "https://storage.googleapis.com/upload/storage/v1/b/hello%2Fhello/o?uploadType=media&name=world%2Fworld",
312            o.upload_url("media")
313        );
314    }
315
316    #[test]
317    fn test_bucket_url() {
318        let b = Bucket::new("hello/hello");
319        assert_eq!(
320            "https://storage.googleapis.com/storage/v1/b/hello%2Fhello/o",
321            b.url()
322        );
323    }
324
325    #[test]
326    fn test_partial_object_into_object() {
327        let p = PartialObject {
328            bucket: Some("hello".to_owned()),
329            name: Some("world".to_owned()),
330            ..Default::default()
331        };
332
333        assert_eq!(
334            Object::new("hello", "world").unwrap(),
335            p.try_into().unwrap()
336        );
337    }
338}
gcs_rsync/gcp/storage/resources/object.rs

gcs_rsync/gcp/storage/resources/
object.rs
