gcs_rsync/gcp/storage/resources/
object.rs

1use std::{convert::TryInto, fmt::Display, str::FromStr};
2
3use base64::Engine;
4
5use crate::storage::{Error, StorageResult};
6
7#[derive(Debug, PartialEq, Eq, serde::Serialize, Clone)]
8#[serde(rename_all = "camelCase")]
9pub enum Projection {
10    Full,
11    NoAcl,
12}
13
14/// See [GCS list API reference](https://cloud.google.com/storage/docs/json_api/v1/objects/list)
15#[derive(Debug, PartialEq, Eq, serde::Serialize, Default, Clone)]
16#[serde(rename_all = "camelCase")]
17pub struct ObjectsListRequest {
18    /// [Partial Response](https://cloud.google.com/storage/docs/json_api#partial-response)
19    pub fields: Option<String>,
20    pub delimiter: Option<String>,
21    pub end_offset: Option<String>,
22    pub include_trailing_delimiter: Option<bool>,
23    pub max_results: Option<usize>,
24    pub page_token: Option<String>,
25    pub prefix: Option<String>,
26    pub projection: Option<Projection>,
27    pub start_offset: Option<String>,
28    pub versions: Option<bool>,
29}
30
31#[derive(Debug, PartialEq, Eq, serde::Serialize, Default, Clone)]
32#[serde(rename_all = "camelCase")]
33pub struct ObjectMetadata {
34    pub metadata: Metadata,
35}
36#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize, Default, Clone)]
37#[serde(rename_all = "camelCase")]
38pub struct Metadata {
39    #[serde(
40        rename = "goog-reserved-file-mtime",
41        deserialize_with = "from_string_option"
42    )] //compat with gsutil rsync
43    pub modification_time: Option<i64>,
44}
45
46/// ObjectList response
47#[derive(Debug, serde::Deserialize, Default)]
48#[serde(rename_all = "camelCase")]
49pub struct Objects {
50    pub kind: Option<String>,
51
52    #[serde(default = "Vec::new")]
53    pub items: Vec<PartialObject>,
54
55    #[serde(default = "Vec::new")]
56    pub prefixes: Vec<String>,
57
58    pub next_page_token: Option<String>,
59}
60
61#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
62#[serde(rename_all = "camelCase")]
63pub struct Object {
64    pub bucket: String,
65    pub name: String,
66}
67
68impl Display for Object {
69    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
70        write!(f, "{}", self.gs_url())
71    }
72}
73
74impl FromStr for Object {
75    type Err = Error;
76
77    fn from_str(s: &str) -> Result<Self, Self::Err> {
78        s.strip_prefix("gs://")
79            .and_then(|part| part.split_once('/'))
80            .ok_or(Error::GcsInvalidUrl {
81                url: s.to_owned(),
82                message: "gs url should be gs://bucket/object/path/name".to_owned(),
83            })
84            .and_then(|(bucket, name)| Object::new(bucket, name))
85    }
86}
87
88type GsUrl = String;
89
90// https://storage.googleapis.com/
91const BASE_URL: &str = "storage/v1";
92const UPLOAD_BASE_URL: &str = "upload/storage/v1";
93
94fn percent_encode(input: &str) -> String {
95    percent_encoding::utf8_percent_encode(input, percent_encoding::NON_ALPHANUMERIC).to_string()
96}
97
98impl Object {
99    pub fn gs_url(&self) -> GsUrl {
100        format!("gs://{}/{}", &self.bucket, &self.name)
101    }
102
103    /// References: `<https://cloud.google.com/storage/docs/naming-objects>`
104    pub fn new(bucket: &str, name: &str) -> StorageResult<Self> {
105        if bucket.is_empty() {
106            return Err(Error::GcsInvalidObjectName);
107        }
108
109        if name.is_empty() || name.starts_with('.') {
110            return Err(Error::GcsInvalidObjectName);
111        }
112
113        Ok(Self {
114            bucket: bucket.to_owned(),
115            name: name.to_owned(),
116        })
117    }
118
119    pub fn url(&self) -> String {
120        format!(
121            "{}/b/{}/o/{}",
122            BASE_URL,
123            percent_encode(&self.bucket),
124            percent_encode(&self.name)
125        )
126    }
127
128    pub fn upload_url(&self, upload_type: &str) -> String {
129        format!(
130            "{}/b/{}/o?uploadType={}&name={}",
131            UPLOAD_BASE_URL,
132            percent_encode(&self.bucket),
133            upload_type,
134            percent_encode(&self.name)
135        )
136    }
137}
138
139#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
140#[serde(rename_all = "camelCase")]
141pub struct Bucket {
142    name: String,
143}
144
145impl Bucket {
146    pub fn new(name: &str) -> Self {
147        Self {
148            name: name.to_owned(),
149        }
150    }
151
152    pub fn url(&self) -> String {
153        format!("{}/b/{}", BASE_URL, percent_encode(&self.name))
154    }
155}
156
157impl TryInto<Object> for PartialObject {
158    type Error = Error;
159
160    fn try_into(self) -> Result<Object, Self::Error> {
161        fn err(e: &str) -> Error {
162            Error::GcsPartialResponseError(e.to_owned())
163        }
164
165        match (self.bucket, self.name) {
166            (Some(bucket), Some(name)) => Ok(Object { bucket, name }),
167            (None, Some(_)) => Err(err("bucket field is missing")),
168            (Some(_), None) => Err(err("name field is missing")),
169            (None, None) => Err(err("bucket and name fields are missing")),
170        }
171    }
172}
173
174#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize, Default)]
175#[serde(rename_all = "camelCase")]
176pub struct PartialObject {
177    pub bucket: Option<String>,
178    pub id: Option<String>,
179    pub self_link: Option<String>,
180    pub name: Option<String>,
181    pub content_type: Option<String>,
182    pub time_created: Option<chrono::DateTime<chrono::Utc>>,
183    pub updated: Option<chrono::DateTime<chrono::Utc>>,
184    pub time_deleted: Option<chrono::DateTime<chrono::Utc>>,
185    pub retention_expiration_time: Option<chrono::DateTime<chrono::Utc>>,
186    pub storage_class: Option<String>,
187    #[serde(default, deserialize_with = "from_string_option")]
188    pub size: Option<u64>,
189    pub media_link: Option<String>,
190    pub content_encoding: Option<String>,
191    pub content_disposition: Option<String>,
192    pub content_language: Option<String>,
193    pub cache_control: Option<String>,
194    pub metadata: Option<Metadata>,
195    #[serde(default, deserialize_with = "from_string_option")]
196    pub crc32c: Option<CRC32C>,
197    pub etag: Option<String>,
198}
199
200#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize, Default)]
201#[serde(rename_all = "camelCase")]
202pub struct CRC32C {
203    value: u32,
204}
205
206impl CRC32C {
207    pub fn new(value: u32) -> Self {
208        Self { value }
209    }
210    pub fn to_u32(&self) -> u32 {
211        self.value
212    }
213}
214#[derive(Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
215pub enum Base64EncodedCRC32CError {
216    Base64DecodeError(String),
217    Base64ToU32BigEndianError(Vec<u8>),
218}
219
220impl Display for Base64EncodedCRC32CError {
221    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
222        write!(f, "{:?}", self)
223    }
224}
225
226impl FromStr for CRC32C {
227    type Err = Base64EncodedCRC32CError;
228
229    fn from_str(base64crc32c: &str) -> Result<Self, Self::Err> {
230        let decoded = base64::engine::general_purpose::STANDARD
231            .decode(base64crc32c)
232            .map_err(|err| Base64EncodedCRC32CError::Base64DecodeError(format!("{:?}", err)))?;
233        let crc32c = decoded
234            .try_into()
235            .map(u32::from_be_bytes)
236            .map_err(Base64EncodedCRC32CError::Base64ToU32BigEndianError)?;
237        Ok(CRC32C::new(crc32c))
238    }
239}
240
241fn from_string_option<'de, T, D>(deserializer: D) -> std::result::Result<Option<T>, D::Error>
242where
243    T: std::str::FromStr,
244    T::Err: std::fmt::Display,
245    D: serde::Deserializer<'de>,
246{
247    use serde::{de::Error, Deserialize};
248    use serde_json::Value;
249    match Deserialize::deserialize(deserializer) {
250        Ok(Value::String(s)) => T::from_str(&s).map(Option::from).map_err(Error::custom),
251        Ok(Value::Number(num)) => T::from_str(&num.to_string())
252            .map(Option::from)
253            .map_err(Error::custom),
254        Ok(value) => Err(Error::custom(format!(
255            "Wrong type, expected type {} but got value {:?}",
256            std::any::type_name::<T>(),
257            value,
258        ))),
259        Err(_) => Ok(None),
260    }
261}
262
263#[cfg(test)]
264mod tests {
265    use std::{convert::TryInto, str::FromStr};
266
267    use crate::storage::{Bucket, Error, Object};
268
269    use super::PartialObject;
270
271    #[test]
272    fn fn_gs_url_parsing_to_object() {
273        assert_eq!(
274            Object::new("hello", "world/object").unwrap(),
275            Object::from_str("gs://hello/world/object").unwrap(),
276        )
277    }
278
279    #[test]
280    fn test_invalid_object() {
281        fn assert_object_error(bucket: &str, name: &str) {
282            assert!(matches!(
283                Object::new(bucket, name).unwrap_err(),
284                Error::GcsInvalidObjectName
285            ))
286        }
287        assert_object_error("", "name");
288        assert_object_error("bucket", "");
289        assert_object_error("bucket", ".");
290        assert_object_error("bucket", "..");
291    }
292    #[test]
293    fn test_object_display() {
294        let o = Object::new("hello", "world").unwrap();
295        assert_eq!("gs://hello/world", o.gs_url());
296        assert_eq!("gs://hello/world", format!("{}", o));
297    }
298
299    #[test]
300    fn test_object_url() {
301        let o = Object::new("hello/hello", "world/world").unwrap();
302        assert_eq!("storage/v1/b/hello%2Fhello/o/world%2Fworld", o.url());
303    }
304
305    #[test]
306    fn test_object_upload_url() {
307        let o = Object::new("hello/hello", "world/world").unwrap();
308        assert_eq!(
309            "upload/storage/v1/b/hello%2Fhello/o?uploadType=media&name=world%2Fworld",
310            o.upload_url("media")
311        );
312    }
313
314    #[test]
315    fn test_bucket_url() {
316        let b = Bucket::new("hello/hello");
317        assert_eq!("storage/v1/b/hello%2Fhello", b.url());
318    }
319
320    #[test]
321    fn test_partial_object_into_object() {
322        let p = PartialObject {
323            bucket: Some("hello".to_owned()),
324            name: Some("world".to_owned()),
325            ..Default::default()
326        };
327
328        assert_eq!(
329            Object::new("hello", "world").unwrap(),
330            p.try_into().unwrap()
331        );
332    }
333}
gcs_rsync/gcp/storage/resources/object.rs

gcs_rsync/gcp/storage/resources/
object.rs
