1use std::fmt;
2use std::fs;
3use std::path::PathBuf;
4use std::sync::Arc;
5
6use bytes::Bytes;
7use object_store::aws::AmazonS3Builder;
8use object_store::local::LocalFileSystem;
9use object_store::path::Path as ObjectPath;
10use object_store::{ObjectStore, ObjectStoreExt, PutPayload};
11use url::Url;
12use zeroize::Zeroizing;
13
14pub const DEFAULT_MAX_BUFFERED_OBJECT_SIZE: u64 = 64 * 1024 * 1024;
16
17#[derive(Debug, thiserror::Error)]
18#[non_exhaustive]
19pub enum ObjectStoreError {
21 #[error("{message}")]
23 InvalidConfiguration {
24 message: &'static str,
26 },
27 #[error("{operation}: {source}")]
29 Io {
30 operation: &'static str,
32 #[source]
33 source: std::io::Error,
35 },
36 #[error("{operation}: {source}")]
38 Store {
39 operation: &'static str,
41 #[source]
42 source: object_store::Error,
44 },
45 #[error("object storage read failed: object not found")]
47 NotFound,
48 #[error("{message}")]
50 InvalidKey {
51 message: &'static str,
53 },
54 #[error("object storage key is invalid: {source}")]
56 InvalidPath {
57 #[source]
58 source: object_store::path::Error,
60 },
61 #[error("object storage read exceeds the {limit}-byte buffer limit ({actual} bytes)")]
63 ObjectTooLarge {
64 limit: u64,
66 actual: u64,
68 },
69}
70
71pub type ObjectStoreResult<T> = Result<T, ObjectStoreError>;
73
74#[derive(Debug, PartialEq, Eq)]
75pub enum ObjectStoreConfig {
77 Local(LocalObjectStoreConfig),
79 S3(S3ObjectStoreConfig),
81}
82
83#[derive(Debug, PartialEq, Eq)]
84pub struct LocalObjectStoreConfig {
86 pub root: PathBuf,
88}
89
90#[derive(PartialEq, Eq)]
91pub struct S3ObjectStoreConfig {
93 pub bucket: String,
95 pub endpoint: Option<String>,
97 pub region: Option<String>,
99 pub access_key_id: Option<SecretString>,
101 pub secret_access_key: Option<SecretString>,
103 pub allow_http: bool,
105 pub virtual_hosted_style_request: bool,
107}
108
109impl fmt::Debug for S3ObjectStoreConfig {
110 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
111 formatter
112 .debug_struct("S3ObjectStoreConfig")
113 .field("bucket", &self.bucket)
114 .field("endpoint", &self.endpoint)
115 .field("region", &self.region)
116 .field(
117 "access_key_id",
118 &self.access_key_id.as_ref().map(|_| "[REDACTED]"),
119 )
120 .field(
121 "secret_access_key",
122 &self.secret_access_key.as_ref().map(|_| "[REDACTED]"),
123 )
124 .field("allow_http", &self.allow_http)
125 .field(
126 "virtual_hosted_style_request",
127 &self.virtual_hosted_style_request,
128 )
129 .finish()
130 }
131}
132
133#[derive(PartialEq, Eq)]
135pub struct SecretString(Zeroizing<String>);
136
137impl SecretString {
138 pub fn new(value: impl Into<String>) -> Self {
140 Self(Zeroizing::new(value.into()))
141 }
142
143 pub fn expose_secret(&self) -> &str {
145 self.0.as_str()
146 }
147}
148
149impl fmt::Debug for SecretString {
150 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
151 formatter.write_str("[REDACTED]")
152 }
153}
154
155impl S3ObjectStoreConfig {
156 pub fn minio(
161 bucket: impl Into<String>,
162 endpoint: impl Into<String>,
163 access_key_id: impl Into<String>,
164 secret_access_key: impl Into<String>,
165 ) -> Self {
166 let endpoint = endpoint.into();
167 Self {
168 bucket: bucket.into(),
169 allow_http: false,
170 endpoint: Some(endpoint),
171 region: Some("us-east-1".to_string()),
172 access_key_id: Some(SecretString::new(access_key_id)),
173 secret_access_key: Some(SecretString::new(secret_access_key)),
174 virtual_hosted_style_request: false,
175 }
176 }
177}
178
179#[derive(Clone, Debug)]
180pub struct ObjectStoreClient {
182 inner: Arc<dyn ObjectStore>,
183 max_buffered_object_size: u64,
184}
185
186impl ObjectStoreClient {
187 pub fn from_config(config: ObjectStoreConfig) -> ObjectStoreResult<Self> {
189 match config {
190 ObjectStoreConfig::Local(config) => Self::local(config.root),
191 ObjectStoreConfig::S3(config) => Self::s3(config),
192 }
193 }
194
195 pub fn local(root: impl Into<PathBuf>) -> ObjectStoreResult<Self> {
197 let root = root.into();
198 fs::create_dir_all(root.as_path()).map_err(|source| ObjectStoreError::Io {
199 operation: "object storage root creation failed",
200 source,
201 })?;
202 let store = LocalFileSystem::new_with_prefix(root.as_path()).map_err(|source| {
203 ObjectStoreError::Store {
204 operation: "local object store initialization failed",
205 source,
206 }
207 })?;
208
209 Ok(Self {
210 inner: Arc::new(store),
211 max_buffered_object_size: DEFAULT_MAX_BUFFERED_OBJECT_SIZE,
212 })
213 }
214
215 pub fn s3(config: S3ObjectStoreConfig) -> ObjectStoreResult<Self> {
217 let bucket =
218 non_empty(config.bucket.as_str()).ok_or(ObjectStoreError::InvalidConfiguration {
219 message: "s3 object store bucket must not be empty",
220 })?;
221
222 let endpoint = non_empty_option(config.endpoint.as_deref());
223 if let Some(endpoint) = endpoint.as_deref() {
224 validate_endpoint(endpoint, config.allow_http)?;
225 } else if config.allow_http {
226 return Err(ObjectStoreError::InvalidConfiguration {
227 message: "plain HTTP requires an explicit S3 endpoint",
228 });
229 }
230
231 let mut builder = AmazonS3Builder::new()
232 .with_bucket_name(bucket)
233 .with_virtual_hosted_style_request(config.virtual_hosted_style_request)
234 .with_allow_http(config.allow_http);
235
236 if let Some(region) = non_empty_option(config.region.as_deref()) {
237 builder = builder.with_region(region);
238 }
239
240 if let Some(endpoint) = endpoint {
241 builder = builder.with_endpoint(endpoint);
242 }
243
244 match (
245 non_empty_option(
246 config
247 .access_key_id
248 .as_ref()
249 .map(SecretString::expose_secret),
250 ),
251 non_empty_option(
252 config
253 .secret_access_key
254 .as_ref()
255 .map(SecretString::expose_secret),
256 ),
257 ) {
258 (Some(access_key_id), Some(secret_access_key)) => {
259 builder = builder
260 .with_access_key_id(access_key_id)
261 .with_secret_access_key(secret_access_key);
262 }
263 (None, None) => {}
264 _ => {
265 return Err(ObjectStoreError::InvalidConfiguration {
266 message: "s3 object store credentials require both access key id and secret access key",
267 });
268 }
269 }
270
271 let store = builder.build().map_err(|source| ObjectStoreError::Store {
272 operation: "s3 object store build failed",
273 source,
274 })?;
275
276 Ok(Self {
277 inner: Arc::new(store),
278 max_buffered_object_size: DEFAULT_MAX_BUFFERED_OBJECT_SIZE,
279 })
280 }
281
282 #[must_use]
285 pub fn with_max_buffered_object_size(mut self, limit: u64) -> Self {
286 self.max_buffered_object_size = limit;
287 self
288 }
289
290 pub fn object_path(&self, key: &str) -> ObjectStoreResult<ObjectPath> {
292 object_path(key)
293 }
294
295 pub async fn write_object(&self, key: &str, payload: &[u8]) -> ObjectStoreResult<()> {
297 self.write_bytes(key, Bytes::copy_from_slice(payload)).await
298 }
299
300 pub async fn write_bytes(&self, key: &str, payload: Bytes) -> ObjectStoreResult<()> {
302 let path = self.object_path(key)?;
303 self.inner
304 .put(&path, PutPayload::from(payload))
305 .await
306 .map_err(|source| ObjectStoreError::Store {
307 operation: "object storage write failed",
308 source,
309 })?;
310 Ok(())
311 }
312
313 pub async fn read_object(&self, key: &str) -> ObjectStoreResult<Vec<u8>> {
315 self.read_object_optional(key)
316 .await?
317 .ok_or(ObjectStoreError::NotFound)
318 }
319
320 pub async fn read_object_optional(&self, key: &str) -> ObjectStoreResult<Option<Vec<u8>>> {
322 self.read_object_optional_with_limit(key, self.max_buffered_object_size)
323 .await
324 }
325
326 pub async fn read_object_with_limit(
328 &self,
329 key: &str,
330 limit: u64,
331 ) -> ObjectStoreResult<Vec<u8>> {
332 self.read_object_optional_with_limit(key, limit)
333 .await?
334 .ok_or(ObjectStoreError::NotFound)
335 }
336
337 pub async fn read_object_optional_with_limit(
339 &self,
340 key: &str,
341 limit: u64,
342 ) -> ObjectStoreResult<Option<Vec<u8>>> {
343 let Some(get_result) = self.get_object_optional(key, limit).await? else {
344 return Ok(None);
345 };
346 let bytes = get_result
347 .bytes()
348 .await
349 .map_err(|source| ObjectStoreError::Store {
350 operation: "object storage byte read failed",
351 source,
352 })?;
353 Ok(Some(bytes.to_vec()))
354 }
355
356 pub async fn get_object(
361 &self,
362 key: &str,
363 limit: u64,
364 ) -> ObjectStoreResult<object_store::GetResult> {
365 self.get_object_optional(key, limit)
366 .await?
367 .ok_or(ObjectStoreError::NotFound)
368 }
369
370 pub async fn get_object_optional(
372 &self,
373 key: &str,
374 limit: u64,
375 ) -> ObjectStoreResult<Option<object_store::GetResult>> {
376 let path = self.object_path(key)?;
377 let get_result = match self.inner.get(&path).await {
378 Ok(result) => result,
379 Err(object_store::Error::NotFound { .. }) => return Ok(None),
380 Err(source) => {
381 return Err(ObjectStoreError::Store {
382 operation: "object storage read failed",
383 source,
384 });
385 }
386 };
387 if get_result.meta.size > limit {
388 return Err(ObjectStoreError::ObjectTooLarge {
389 limit,
390 actual: get_result.meta.size,
391 });
392 }
393 Ok(Some(get_result))
394 }
395
396 pub async fn delete_object(&self, key: &str) -> ObjectStoreResult<()> {
398 let path = self.object_path(key)?;
399 match self.inner.delete(&path).await {
400 Ok(()) => Ok(()),
401 Err(object_store::Error::NotFound { .. }) => Ok(()),
402 Err(source) => Err(ObjectStoreError::Store {
403 operation: "object storage delete failed",
404 source,
405 }),
406 }
407 }
408}
409
410fn object_path(key: &str) -> ObjectStoreResult<ObjectPath> {
411 let mut segments = Vec::new();
412 for segment in key.split('/') {
413 let trimmed = segment.trim();
414 if trimmed != segment
415 || segment.is_empty()
416 || segment == "."
417 || segment == ".."
418 || segment.contains('\\')
419 || segment.contains(':')
420 {
421 return Err(ObjectStoreError::InvalidKey {
422 message: "object storage key contains an invalid segment",
423 });
424 }
425 segments.push(segment);
426 }
427 if segments.is_empty() {
428 return Err(ObjectStoreError::InvalidKey {
429 message: "object storage key must not be empty",
430 });
431 }
432
433 ObjectPath::parse(segments.join("/")).map_err(|source| ObjectStoreError::InvalidPath { source })
434}
435
436fn non_empty(value: &str) -> Option<String> {
437 let trimmed = value.trim();
438 (!trimmed.is_empty()).then(|| trimmed.to_string())
439}
440
441fn non_empty_option(value: Option<&str>) -> Option<String> {
442 value.and_then(non_empty)
443}
444
445fn validate_endpoint(endpoint: &str, allow_http: bool) -> ObjectStoreResult<()> {
446 let endpoint = Url::parse(endpoint).map_err(|_| ObjectStoreError::InvalidConfiguration {
447 message: "s3 object store endpoint must be an absolute URL",
448 })?;
449 if !endpoint.username().is_empty() || endpoint.password().is_some() {
450 return Err(ObjectStoreError::InvalidConfiguration {
451 message: "s3 object store endpoint must not contain credentials",
452 });
453 }
454 match endpoint.scheme() {
455 "https" => Ok(()),
456 "http" if allow_http => Ok(()),
457 "http" => Err(ObjectStoreError::InvalidConfiguration {
458 message: "plain HTTP S3 endpoints require an explicit allow_http opt-in",
459 }),
460 _ => Err(ObjectStoreError::InvalidConfiguration {
461 message: "s3 object store endpoint must use HTTP or HTTPS",
462 }),
463 }
464}
465
466#[cfg(test)]
467mod tests {
468 use std::time::{SystemTime, UNIX_EPOCH};
469
470 use super::*;
471
472 fn temp_store() -> ObjectStoreClient {
473 let unique = SystemTime::now()
474 .duration_since(UNIX_EPOCH)
475 .expect("clock")
476 .as_nanos();
477 ObjectStoreClient::local(std::env::temp_dir().join(format!("s3-local-store-{unique}")))
478 .expect("local object store")
479 }
480
481 #[test]
482 fn object_store_rejects_unsafe_keys() {
483 let store = temp_store();
484
485 assert!(store.object_path("").is_err());
486 assert!(store.object_path("../asset").is_err());
487 assert!(store.object_path("media/../asset").is_err());
488 assert!(store.object_path("media\\asset").is_err());
489 assert!(store.object_path("media/c:asset").is_err());
490 assert!(store.object_path("media/ asset").is_err());
491 }
492
493 #[tokio::test]
494 async fn local_object_store_writes_reads_and_deletes_objects() {
495 let store = temp_store();
496 let key = "media/account/asset/original";
497
498 store
499 .write_object(key, b"payload")
500 .await
501 .expect("write object");
502
503 assert_eq!(
504 store.read_object(key).await.expect("read object"),
505 b"payload"
506 );
507 assert_eq!(
508 store
509 .read_object_optional(key)
510 .await
511 .expect("optional read object"),
512 Some(b"payload".to_vec())
513 );
514
515 store.delete_object(key).await.expect("delete object");
516
517 assert!(store.read_object(key).await.is_err());
518 assert_eq!(
519 store
520 .read_object_optional(key)
521 .await
522 .expect("missing object should be optional"),
523 None
524 );
525 }
526
527 #[test]
528 fn s3_config_requires_credentials_as_a_pair() {
529 let config = S3ObjectStoreConfig {
530 bucket: "library".to_string(),
531 endpoint: Some("http://127.0.0.1:9000".to_string()),
532 region: Some("us-east-1".to_string()),
533 access_key_id: Some(SecretString::new("minio")),
534 secret_access_key: None,
535 allow_http: true,
536 virtual_hosted_style_request: false,
537 };
538
539 let error = ObjectStoreClient::s3(config).expect_err("missing secret must fail");
540
541 assert!(
542 error
543 .to_string()
544 .contains("both access key id and secret access key")
545 );
546 }
547
548 #[test]
549 fn minio_config_uses_path_style_http_endpoint() {
550 let config =
551 S3ObjectStoreConfig::minio("library", "http://127.0.0.1:9000", "minio", "minio-secret");
552
553 assert_eq!(config.bucket, "library");
554 assert_eq!(config.endpoint.as_deref(), Some("http://127.0.0.1:9000"));
555 assert_eq!(config.region.as_deref(), Some("us-east-1"));
556 assert!(!config.allow_http);
557 assert!(!config.virtual_hosted_style_request);
558 }
559
560 #[test]
561 fn s3_debug_output_redacts_credentials() {
562 let config = S3ObjectStoreConfig::minio(
563 "library",
564 "http://127.0.0.1:9000",
565 "visible-access-key",
566 "visible-secret-key",
567 );
568
569 let debug = format!("{config:?}");
570
571 assert!(!debug.contains("visible-access-key"));
572 assert!(!debug.contains("visible-secret-key"));
573 assert_eq!(debug.matches("[REDACTED]").count(), 2);
574 }
575
576 #[tokio::test]
577 async fn buffered_reads_enforce_the_configured_limit() {
578 let store = temp_store();
579 store.write_object("media/large", b"12345").await.unwrap();
580 assert!(matches!(
581 store.read_object_with_limit("media/large", 4).await,
582 Err(ObjectStoreError::ObjectTooLarge {
583 limit: 4,
584 actual: 5
585 })
586 ));
587 }
588}