objectstore_types/
lib.rs

1//! This is a collection of types shared among various objectstore crates.
2//!
3//! It primarily includes metadata-related structures being used by both the client and server/service
4//! components.
5
6#![warn(missing_docs)]
7#![warn(missing_debug_implementations)]
8
9use std::borrow::Cow;
10use std::collections::BTreeMap;
11use std::fmt;
12use std::str::FromStr;
13use std::time::{Duration, SystemTime};
14
15use http::header::{self, HeaderMap, HeaderName};
16use humantime::{format_duration, format_rfc3339_seconds, parse_duration};
17use serde::{Deserialize, Serialize};
18
19/// The custom HTTP header that contains the serialized [`ExpirationPolicy`].
20pub const HEADER_EXPIRATION: &str = "x-sn-expiration";
21/// The prefix for custom HTTP headers containing custom per-object metadata.
22pub const HEADER_META_PREFIX: &str = "x-snme-";
23
24/// HTTP request query parameter that contains the request scope.
25pub const PARAM_SCOPE: &str = "scope";
26/// HTTP request query parameter that contains the request usecase.
27pub const PARAM_USECASE: &str = "usecase";
28
29/// The default content type for objects without a known content type.
30pub const DEFAULT_CONTENT_TYPE: &str = "application/octet-stream";
31
32/// The per-object expiration policy
33///
34/// We support automatic time-to-live and time-to-idle policies.
35/// Setting this to `Manual` means that the object has no automatic policy, and will not be
36/// garbage-collected automatically. It essentially lives forever until manually deleted.
37#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
38pub enum ExpirationPolicy {
39    /// Manual expiration, meaning no automatic cleanup.
40    // IMPORTANT: Do not change the default, we rely on this for persisted objects.
41    #[default]
42    Manual,
43    /// Time to live, with expiration after the specified duration.
44    TimeToLive(Duration),
45    /// Time to idle, with expiration once the object has not been accessed within the specified duration.
46    TimeToIdle(Duration),
47}
48impl ExpirationPolicy {
49    /// Returns the duration after which the object expires.
50    pub fn expires_in(&self) -> Option<Duration> {
51        match self {
52            ExpirationPolicy::Manual => None,
53            ExpirationPolicy::TimeToLive(duration) => Some(*duration),
54            ExpirationPolicy::TimeToIdle(duration) => Some(*duration),
55        }
56    }
57
58    /// Returns `true` if this policy indicates time-based expiry.
59    pub fn is_timeout(&self) -> bool {
60        match self {
61            ExpirationPolicy::TimeToLive(_) => true,
62            ExpirationPolicy::TimeToIdle(_) => true,
63            ExpirationPolicy::Manual => false,
64        }
65    }
66
67    /// Returns `true` if this policy is `Manual`.
68    pub fn is_manual(&self) -> bool {
69        *self == ExpirationPolicy::Manual
70    }
71}
72impl fmt::Display for ExpirationPolicy {
73    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
74        match self {
75            ExpirationPolicy::TimeToLive(duration) => {
76                write!(f, "ttl:{}", format_duration(*duration))
77            }
78            ExpirationPolicy::TimeToIdle(duration) => {
79                write!(f, "tti:{}", format_duration(*duration))
80            }
81            ExpirationPolicy::Manual => f.write_str("manual"),
82        }
83    }
84}
85impl FromStr for ExpirationPolicy {
86    type Err = anyhow::Error;
87
88    fn from_str(s: &str) -> Result<Self, Self::Err> {
89        if s == "manual" {
90            return Ok(ExpirationPolicy::Manual);
91        }
92        if let Some(duration) = s.strip_prefix("ttl:") {
93            return Ok(ExpirationPolicy::TimeToLive(parse_duration(duration)?));
94        }
95        if let Some(duration) = s.strip_prefix("tti:") {
96            return Ok(ExpirationPolicy::TimeToIdle(parse_duration(duration)?));
97        }
98        anyhow::bail!("invalid expiration policy")
99    }
100}
101
102/// The compression algorithm of an object to upload.
103#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
104pub enum Compression {
105    /// Compressed using `zstd`.
106    Zstd,
107    // /// Compressed using `gzip`.
108    // Gzip,
109    // /// Compressed using `lz4`.
110    // Lz4,
111}
112
113impl Compression {
114    /// Returns a string representation of the compression algorithm.
115    pub fn as_str(&self) -> &str {
116        match self {
117            Compression::Zstd => "zstd",
118            // Compression::Gzip => "gzip",
119            // Compression::Lz4 => "lz4",
120        }
121    }
122}
123
124impl fmt::Display for Compression {
125    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
126        f.write_str(self.as_str())
127    }
128}
129
130impl FromStr for Compression {
131    type Err = anyhow::Error;
132
133    fn from_str(s: &str) -> Result<Self, Self::Err> {
134        Ok(match s {
135            "zstd" => Compression::Zstd,
136            // "gzip" => Compression::Gzip,
137            // "lz4" => Compression::Lz4,
138            _ => anyhow::bail!("unknown compression algorithm"),
139        })
140    }
141}
142
143/// Per-object Metadata.
144///
145/// This includes special metadata like the expiration policy and compression used,
146/// as well as arbitrary user-provided metadata.
147#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
148#[serde(default)]
149pub struct Metadata {
150    /// The expiration policy of the object.
151    #[serde(skip_serializing_if = "ExpirationPolicy::is_manual")]
152    pub expiration_policy: ExpirationPolicy,
153
154    /// The content type of the object, if known.
155    pub content_type: Cow<'static, str>,
156
157    /// The compression algorithm used for this object, if any.
158    #[serde(skip_serializing_if = "Option::is_none")]
159    pub compression: Option<Compression>,
160
161    /// Size of the data in bytes, if known.
162    #[serde(skip_serializing_if = "Option::is_none")]
163    pub size: Option<usize>,
164
165    /// Some arbitrary user-provided metadata.
166    #[serde(skip_serializing_if = "BTreeMap::is_empty")]
167    pub custom: BTreeMap<String, String>,
168}
169
170impl Metadata {
171    /// Extracts metadata from the given [`HeaderMap`].
172    ///
173    /// A prefix can be also be provided which is being stripped from custom non-standard headers.
174    pub fn from_headers(headers: &HeaderMap, prefix: &str) -> anyhow::Result<Self> {
175        let mut metadata = Metadata::default();
176
177        for (name, value) in headers {
178            if name == header::CONTENT_TYPE {
179                let content_type = value.to_str()?;
180                metadata.content_type = content_type.to_owned().into();
181            } else if name == header::CONTENT_ENCODING {
182                let compression = value.to_str()?;
183                metadata.compression = Some(Compression::from_str(compression)?);
184            } else if let Some(name) = name.as_str().strip_prefix(prefix) {
185                if name == HEADER_EXPIRATION {
186                    let expiration_policy = value.to_str()?;
187                    metadata.expiration_policy = ExpirationPolicy::from_str(expiration_policy)?;
188                } else if let Some(name) = name.strip_prefix(HEADER_META_PREFIX) {
189                    let value = value.to_str()?;
190                    metadata.custom.insert(name.into(), value.into());
191                }
192            }
193        }
194
195        Ok(metadata)
196    }
197
198    /// Turns the metadata into a [`HeaderMap`].
199    ///
200    /// It will prefix any non-standard headers with the given `prefix`.
201    /// If the `with_expiration` parameter is set, it will additionally resolve the expiration policy
202    /// into a specific RFC3339 datetime, and set that as the `Custom-Time` header.
203    pub fn to_headers(&self, prefix: &str, with_expiration: bool) -> anyhow::Result<HeaderMap> {
204        let Self {
205            content_type,
206            compression,
207            expiration_policy,
208            size: _,
209            custom,
210        } = self;
211
212        let mut headers = HeaderMap::new();
213        headers.append(header::CONTENT_TYPE, content_type.parse()?);
214
215        if let Some(compression) = compression {
216            headers.append(header::CONTENT_ENCODING, compression.as_str().parse()?);
217        }
218
219        if *expiration_policy != ExpirationPolicy::Manual {
220            let name = HeaderName::try_from(format!("{prefix}{HEADER_EXPIRATION}"))?;
221            headers.append(name, expiration_policy.to_string().parse()?);
222            if with_expiration {
223                let expires_in = expiration_policy.expires_in().unwrap_or_default();
224                let expires_at = format_rfc3339_seconds(SystemTime::now() + expires_in);
225                headers.append("x-goog-custom-time", expires_at.to_string().parse()?);
226            }
227        }
228
229        for (key, value) in custom {
230            let name = HeaderName::try_from(format!("{prefix}{HEADER_META_PREFIX}{key}"))?;
231            headers.append(name, value.parse()?);
232        }
233
234        Ok(headers)
235    }
236}
237
238impl Default for Metadata {
239    fn default() -> Self {
240        Self {
241            expiration_policy: ExpirationPolicy::Manual,
242            content_type: DEFAULT_CONTENT_TYPE.into(),
243            compression: None,
244            size: None,
245            custom: BTreeMap::new(),
246        }
247    }
248}
objectstore_types/lib.rs

objectstore_types/
lib.rs
