Skip to main content

ch_database/s3/
mod.rs

1use std::fmt;
2use std::fs;
3use std::path::PathBuf;
4use std::sync::Arc;
5
6use bytes::Bytes;
7use object_store::aws::AmazonS3Builder;
8use object_store::local::LocalFileSystem;
9use object_store::path::Path as ObjectPath;
10use object_store::{ObjectStore, ObjectStoreExt, PutPayload};
11use url::Url;
12use zeroize::Zeroizing;
13
14/// Default maximum size collected by the convenience read methods.
15pub const DEFAULT_MAX_BUFFERED_OBJECT_SIZE: u64 = 64 * 1024 * 1024;
16
17#[derive(Debug, thiserror::Error)]
18#[non_exhaustive]
19/// Errors returned while configuring or accessing object storage.
20pub enum ObjectStoreError {
21    /// The supplied object-store settings are incomplete or inconsistent.
22    #[error("{message}")]
23    InvalidConfiguration {
24        /// Description of the invalid setting.
25        message: &'static str,
26    },
27    /// A local filesystem operation failed.
28    #[error("{operation}: {source}")]
29    Io {
30        /// Operation that was attempted.
31        operation: &'static str,
32        #[source]
33        /// Underlying filesystem error.
34        source: std::io::Error,
35    },
36    /// The object-store backend returned an error.
37    #[error("{operation}: {source}")]
38    Store {
39        /// Operation that was attempted.
40        operation: &'static str,
41        #[source]
42        /// Underlying object-store error.
43        source: object_store::Error,
44    },
45    /// The requested object does not exist.
46    #[error("object storage read failed: object not found")]
47    NotFound,
48    /// An object key contains an unsupported path segment.
49    #[error("{message}")]
50    InvalidKey {
51        /// Description of the invalid key.
52        message: &'static str,
53    },
54    /// An object key could not be converted to a backend path.
55    #[error("object storage key is invalid: {source}")]
56    InvalidPath {
57        #[source]
58        /// Underlying path parser error.
59        source: object_store::path::Error,
60    },
61    /// The object exceeds the caller's in-memory read limit.
62    #[error("object storage read exceeds the {limit}-byte buffer limit ({actual} bytes)")]
63    ObjectTooLarge {
64        /// Configured maximum number of bytes.
65        limit: u64,
66        /// Object size reported by the backend.
67        actual: u64,
68    },
69}
70
71/// Result type used by object-store operations.
72pub type ObjectStoreResult<T> = Result<T, ObjectStoreError>;
73
74#[derive(Debug, PartialEq, Eq)]
75/// Selects a local filesystem or S3-compatible object store.
76pub enum ObjectStoreConfig {
77    /// Stores objects below a local directory.
78    Local(LocalObjectStoreConfig),
79    /// Stores objects in an S3-compatible bucket.
80    S3(S3ObjectStoreConfig),
81}
82
83#[derive(Debug, PartialEq, Eq)]
84/// Configuration for a local filesystem object store.
85pub struct LocalObjectStoreConfig {
86    /// Root directory below which all objects are stored.
87    pub root: PathBuf,
88}
89
90#[derive(PartialEq, Eq)]
91/// Configuration for an S3-compatible object store.
92pub struct S3ObjectStoreConfig {
93    /// Bucket that receives objects.
94    pub bucket: String,
95    /// Optional custom endpoint for S3-compatible services.
96    pub endpoint: Option<String>,
97    /// Optional AWS region.
98    pub region: Option<String>,
99    /// Optional access key; must be paired with `secret_access_key`.
100    pub access_key_id: Option<SecretString>,
101    /// Optional secret key; must be paired with `access_key_id`.
102    pub secret_access_key: Option<SecretString>,
103    /// Allows plain HTTP endpoints, typically for local development.
104    pub allow_http: bool,
105    /// Uses virtual-hosted-style bucket addressing when enabled.
106    pub virtual_hosted_style_request: bool,
107}
108
109impl fmt::Debug for S3ObjectStoreConfig {
110    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
111        formatter
112            .debug_struct("S3ObjectStoreConfig")
113            .field("bucket", &self.bucket)
114            .field("endpoint", &self.endpoint)
115            .field("region", &self.region)
116            .field(
117                "access_key_id",
118                &self.access_key_id.as_ref().map(|_| "[REDACTED]"),
119            )
120            .field(
121                "secret_access_key",
122                &self.secret_access_key.as_ref().map(|_| "[REDACTED]"),
123            )
124            .field("allow_http", &self.allow_http)
125            .field(
126                "virtual_hosted_style_request",
127                &self.virtual_hosted_style_request,
128            )
129            .finish()
130    }
131}
132
133/// Heap-allocated secret that is redacted in diagnostics and cleared on drop.
134#[derive(PartialEq, Eq)]
135pub struct SecretString(Zeroizing<String>);
136
137impl SecretString {
138    /// Wraps a secret value.
139    pub fn new(value: impl Into<String>) -> Self {
140        Self(Zeroizing::new(value.into()))
141    }
142
143    /// Borrows the secret for an outbound provider request.
144    pub fn expose_secret(&self) -> &str {
145        self.0.as_str()
146    }
147}
148
149impl fmt::Debug for SecretString {
150    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
151        formatter.write_str("[REDACTED]")
152    }
153}
154
155impl S3ObjectStoreConfig {
156    /// Creates secure path-style settings for a MinIO-compatible endpoint.
157    ///
158    /// Plain HTTP remains disabled. Callers using an isolated local endpoint
159    /// must opt in by setting [`Self::allow_http`] explicitly.
160    pub fn minio(
161        bucket: impl Into<String>,
162        endpoint: impl Into<String>,
163        access_key_id: impl Into<String>,
164        secret_access_key: impl Into<String>,
165    ) -> Self {
166        let endpoint = endpoint.into();
167        Self {
168            bucket: bucket.into(),
169            allow_http: false,
170            endpoint: Some(endpoint),
171            region: Some("us-east-1".to_string()),
172            access_key_id: Some(SecretString::new(access_key_id)),
173            secret_access_key: Some(SecretString::new(secret_access_key)),
174            virtual_hosted_style_request: false,
175        }
176    }
177}
178
179#[derive(Clone, Debug)]
180/// Cloneable client for local and S3-compatible object storage.
181pub struct ObjectStoreClient {
182    inner: Arc<dyn ObjectStore>,
183    max_buffered_object_size: u64,
184}
185
186impl ObjectStoreClient {
187    /// Creates a client from either supported configuration variant.
188    pub fn from_config(config: ObjectStoreConfig) -> ObjectStoreResult<Self> {
189        match config {
190            ObjectStoreConfig::Local(config) => Self::local(config.root),
191            ObjectStoreConfig::S3(config) => Self::s3(config),
192        }
193    }
194
195    /// Creates a local filesystem store and ensures its root exists.
196    pub fn local(root: impl Into<PathBuf>) -> ObjectStoreResult<Self> {
197        let root = root.into();
198        fs::create_dir_all(root.as_path()).map_err(|source| ObjectStoreError::Io {
199            operation: "object storage root creation failed",
200            source,
201        })?;
202        let store = LocalFileSystem::new_with_prefix(root.as_path()).map_err(|source| {
203            ObjectStoreError::Store {
204                operation: "local object store initialization failed",
205                source,
206            }
207        })?;
208
209        Ok(Self {
210            inner: Arc::new(store),
211            max_buffered_object_size: DEFAULT_MAX_BUFFERED_OBJECT_SIZE,
212        })
213    }
214
215    /// Creates an S3-compatible store from validated settings.
216    pub fn s3(config: S3ObjectStoreConfig) -> ObjectStoreResult<Self> {
217        let bucket =
218            non_empty(config.bucket.as_str()).ok_or(ObjectStoreError::InvalidConfiguration {
219                message: "s3 object store bucket must not be empty",
220            })?;
221
222        let endpoint = non_empty_option(config.endpoint.as_deref());
223        if let Some(endpoint) = endpoint.as_deref() {
224            validate_endpoint(endpoint, config.allow_http)?;
225        } else if config.allow_http {
226            return Err(ObjectStoreError::InvalidConfiguration {
227                message: "plain HTTP requires an explicit S3 endpoint",
228            });
229        }
230
231        let mut builder = AmazonS3Builder::new()
232            .with_bucket_name(bucket)
233            .with_virtual_hosted_style_request(config.virtual_hosted_style_request)
234            .with_allow_http(config.allow_http);
235
236        if let Some(region) = non_empty_option(config.region.as_deref()) {
237            builder = builder.with_region(region);
238        }
239
240        if let Some(endpoint) = endpoint {
241            builder = builder.with_endpoint(endpoint);
242        }
243
244        match (
245            non_empty_option(
246                config
247                    .access_key_id
248                    .as_ref()
249                    .map(SecretString::expose_secret),
250            ),
251            non_empty_option(
252                config
253                    .secret_access_key
254                    .as_ref()
255                    .map(SecretString::expose_secret),
256            ),
257        ) {
258            (Some(access_key_id), Some(secret_access_key)) => {
259                builder = builder
260                    .with_access_key_id(access_key_id)
261                    .with_secret_access_key(secret_access_key);
262            }
263            (None, None) => {}
264            _ => {
265                return Err(ObjectStoreError::InvalidConfiguration {
266                    message: "s3 object store credentials require both access key id and secret access key",
267                });
268            }
269        }
270
271        let store = builder.build().map_err(|source| ObjectStoreError::Store {
272            operation: "s3 object store build failed",
273            source,
274        })?;
275
276        Ok(Self {
277            inner: Arc::new(store),
278            max_buffered_object_size: DEFAULT_MAX_BUFFERED_OBJECT_SIZE,
279        })
280    }
281
282    /// Overrides the maximum size collected by [`Self::read_object`] and
283    /// [`Self::read_object_optional`].
284    #[must_use]
285    pub fn with_max_buffered_object_size(mut self, limit: u64) -> Self {
286        self.max_buffered_object_size = limit;
287        self
288    }
289
290    /// Validates and converts an object key to a backend path.
291    pub fn object_path(&self, key: &str) -> ObjectStoreResult<ObjectPath> {
292        object_path(key)
293    }
294
295    /// Writes or replaces an object.
296    pub async fn write_object(&self, key: &str, payload: &[u8]) -> ObjectStoreResult<()> {
297        self.write_bytes(key, Bytes::copy_from_slice(payload)).await
298    }
299
300    /// Writes or replaces an object without copying an existing byte buffer.
301    pub async fn write_bytes(&self, key: &str, payload: Bytes) -> ObjectStoreResult<()> {
302        let path = self.object_path(key)?;
303        self.inner
304            .put(&path, PutPayload::from(payload))
305            .await
306            .map_err(|source| ObjectStoreError::Store {
307                operation: "object storage write failed",
308                source,
309            })?;
310        Ok(())
311    }
312
313    /// Reads an object or returns [`ObjectStoreError::NotFound`].
314    pub async fn read_object(&self, key: &str) -> ObjectStoreResult<Vec<u8>> {
315        self.read_object_optional(key)
316            .await?
317            .ok_or(ObjectStoreError::NotFound)
318    }
319
320    /// Reads an object, returning `None` when it does not exist.
321    pub async fn read_object_optional(&self, key: &str) -> ObjectStoreResult<Option<Vec<u8>>> {
322        self.read_object_optional_with_limit(key, self.max_buffered_object_size)
323            .await
324    }
325
326    /// Reads an object into memory with a caller-selected size limit.
327    pub async fn read_object_with_limit(
328        &self,
329        key: &str,
330        limit: u64,
331    ) -> ObjectStoreResult<Vec<u8>> {
332        self.read_object_optional_with_limit(key, limit)
333            .await?
334            .ok_or(ObjectStoreError::NotFound)
335    }
336
337    /// Reads an optional object into memory with a caller-selected size limit.
338    pub async fn read_object_optional_with_limit(
339        &self,
340        key: &str,
341        limit: u64,
342    ) -> ObjectStoreResult<Option<Vec<u8>>> {
343        let Some(get_result) = self.get_object_optional(key, limit).await? else {
344            return Ok(None);
345        };
346        let bytes = get_result
347            .bytes()
348            .await
349            .map_err(|source| ObjectStoreError::Store {
350                operation: "object storage byte read failed",
351                source,
352            })?;
353        Ok(Some(bytes.to_vec()))
354    }
355
356    /// Returns the backend read result without collecting its payload.
357    ///
358    /// Consumers can stream `payload` directly; the backend-reported object
359    /// size is checked before the result is returned.
360    pub async fn get_object(
361        &self,
362        key: &str,
363        limit: u64,
364    ) -> ObjectStoreResult<object_store::GetResult> {
365        self.get_object_optional(key, limit)
366            .await?
367            .ok_or(ObjectStoreError::NotFound)
368    }
369
370    /// Returns an optional backend read result without collecting its payload.
371    pub async fn get_object_optional(
372        &self,
373        key: &str,
374        limit: u64,
375    ) -> ObjectStoreResult<Option<object_store::GetResult>> {
376        let path = self.object_path(key)?;
377        let get_result = match self.inner.get(&path).await {
378            Ok(result) => result,
379            Err(object_store::Error::NotFound { .. }) => return Ok(None),
380            Err(source) => {
381                return Err(ObjectStoreError::Store {
382                    operation: "object storage read failed",
383                    source,
384                });
385            }
386        };
387        if get_result.meta.size > limit {
388            return Err(ObjectStoreError::ObjectTooLarge {
389                limit,
390                actual: get_result.meta.size,
391            });
392        }
393        Ok(Some(get_result))
394    }
395
396    /// Deletes an object; missing objects are treated as already deleted.
397    pub async fn delete_object(&self, key: &str) -> ObjectStoreResult<()> {
398        let path = self.object_path(key)?;
399        match self.inner.delete(&path).await {
400            Ok(()) => Ok(()),
401            Err(object_store::Error::NotFound { .. }) => Ok(()),
402            Err(source) => Err(ObjectStoreError::Store {
403                operation: "object storage delete failed",
404                source,
405            }),
406        }
407    }
408}
409
410fn object_path(key: &str) -> ObjectStoreResult<ObjectPath> {
411    let mut segments = Vec::new();
412    for segment in key.split('/') {
413        let trimmed = segment.trim();
414        if trimmed != segment
415            || segment.is_empty()
416            || segment == "."
417            || segment == ".."
418            || segment.contains('\\')
419            || segment.contains(':')
420        {
421            return Err(ObjectStoreError::InvalidKey {
422                message: "object storage key contains an invalid segment",
423            });
424        }
425        segments.push(segment);
426    }
427    if segments.is_empty() {
428        return Err(ObjectStoreError::InvalidKey {
429            message: "object storage key must not be empty",
430        });
431    }
432
433    ObjectPath::parse(segments.join("/")).map_err(|source| ObjectStoreError::InvalidPath { source })
434}
435
436fn non_empty(value: &str) -> Option<String> {
437    let trimmed = value.trim();
438    (!trimmed.is_empty()).then(|| trimmed.to_string())
439}
440
441fn non_empty_option(value: Option<&str>) -> Option<String> {
442    value.and_then(non_empty)
443}
444
445fn validate_endpoint(endpoint: &str, allow_http: bool) -> ObjectStoreResult<()> {
446    let endpoint = Url::parse(endpoint).map_err(|_| ObjectStoreError::InvalidConfiguration {
447        message: "s3 object store endpoint must be an absolute URL",
448    })?;
449    if !endpoint.username().is_empty() || endpoint.password().is_some() {
450        return Err(ObjectStoreError::InvalidConfiguration {
451            message: "s3 object store endpoint must not contain credentials",
452        });
453    }
454    match endpoint.scheme() {
455        "https" => Ok(()),
456        "http" if allow_http => Ok(()),
457        "http" => Err(ObjectStoreError::InvalidConfiguration {
458            message: "plain HTTP S3 endpoints require an explicit allow_http opt-in",
459        }),
460        _ => Err(ObjectStoreError::InvalidConfiguration {
461            message: "s3 object store endpoint must use HTTP or HTTPS",
462        }),
463    }
464}
465
466#[cfg(test)]
467mod tests {
468    use std::time::{SystemTime, UNIX_EPOCH};
469
470    use super::*;
471
472    fn temp_store() -> ObjectStoreClient {
473        let unique = SystemTime::now()
474            .duration_since(UNIX_EPOCH)
475            .expect("clock")
476            .as_nanos();
477        ObjectStoreClient::local(std::env::temp_dir().join(format!("s3-local-store-{unique}")))
478            .expect("local object store")
479    }
480
481    #[test]
482    fn object_store_rejects_unsafe_keys() {
483        let store = temp_store();
484
485        assert!(store.object_path("").is_err());
486        assert!(store.object_path("../asset").is_err());
487        assert!(store.object_path("media/../asset").is_err());
488        assert!(store.object_path("media\\asset").is_err());
489        assert!(store.object_path("media/c:asset").is_err());
490        assert!(store.object_path("media/ asset").is_err());
491    }
492
493    #[tokio::test]
494    async fn local_object_store_writes_reads_and_deletes_objects() {
495        let store = temp_store();
496        let key = "media/account/asset/original";
497
498        store
499            .write_object(key, b"payload")
500            .await
501            .expect("write object");
502
503        assert_eq!(
504            store.read_object(key).await.expect("read object"),
505            b"payload"
506        );
507        assert_eq!(
508            store
509                .read_object_optional(key)
510                .await
511                .expect("optional read object"),
512            Some(b"payload".to_vec())
513        );
514
515        store.delete_object(key).await.expect("delete object");
516
517        assert!(store.read_object(key).await.is_err());
518        assert_eq!(
519            store
520                .read_object_optional(key)
521                .await
522                .expect("missing object should be optional"),
523            None
524        );
525    }
526
527    #[test]
528    fn s3_config_requires_credentials_as_a_pair() {
529        let config = S3ObjectStoreConfig {
530            bucket: "library".to_string(),
531            endpoint: Some("http://127.0.0.1:9000".to_string()),
532            region: Some("us-east-1".to_string()),
533            access_key_id: Some(SecretString::new("minio")),
534            secret_access_key: None,
535            allow_http: true,
536            virtual_hosted_style_request: false,
537        };
538
539        let error = ObjectStoreClient::s3(config).expect_err("missing secret must fail");
540
541        assert!(
542            error
543                .to_string()
544                .contains("both access key id and secret access key")
545        );
546    }
547
548    #[test]
549    fn minio_config_uses_path_style_http_endpoint() {
550        let config =
551            S3ObjectStoreConfig::minio("library", "http://127.0.0.1:9000", "minio", "minio-secret");
552
553        assert_eq!(config.bucket, "library");
554        assert_eq!(config.endpoint.as_deref(), Some("http://127.0.0.1:9000"));
555        assert_eq!(config.region.as_deref(), Some("us-east-1"));
556        assert!(!config.allow_http);
557        assert!(!config.virtual_hosted_style_request);
558    }
559
560    #[test]
561    fn s3_debug_output_redacts_credentials() {
562        let config = S3ObjectStoreConfig::minio(
563            "library",
564            "http://127.0.0.1:9000",
565            "visible-access-key",
566            "visible-secret-key",
567        );
568
569        let debug = format!("{config:?}");
570
571        assert!(!debug.contains("visible-access-key"));
572        assert!(!debug.contains("visible-secret-key"));
573        assert_eq!(debug.matches("[REDACTED]").count(), 2);
574    }
575
576    #[tokio::test]
577    async fn buffered_reads_enforce_the_configured_limit() {
578        let store = temp_store();
579        store.write_object("media/large", b"12345").await.unwrap();
580        assert!(matches!(
581            store.read_object_with_limit("media/large", 4).await,
582            Err(ObjectStoreError::ObjectTooLarge {
583                limit: 4,
584                actual: 5
585            })
586        ));
587    }
588}