1use std::fmt;
2use std::fs;
3use std::path::PathBuf;
4use std::sync::Arc;
5
6use bytes::Bytes;
7use object_store::aws::{AmazonS3Builder, S3CopyIfNotExists};
8use object_store::local::LocalFileSystem;
9use object_store::path::Path as ObjectPath;
10use object_store::{ObjectStore, ObjectStoreExt, PutMode, PutPayload};
11use url::Url;
12use zeroize::Zeroizing;
13
14pub const DEFAULT_MAX_BUFFERED_OBJECT_SIZE: u64 = 64 * 1024 * 1024;
16
17#[derive(Debug, thiserror::Error)]
18#[non_exhaustive]
19pub enum ObjectStoreError {
21 #[error("{message}")]
23 InvalidConfiguration {
24 message: &'static str,
26 },
27 #[error("{operation}: {source}")]
29 Io {
30 operation: &'static str,
32 #[source]
33 source: std::io::Error,
35 },
36 #[error("{operation}: {source}")]
38 Store {
39 operation: &'static str,
41 #[source]
42 source: object_store::Error,
44 },
45 #[error("object storage read failed: object not found")]
47 NotFound,
48 #[error("{message}")]
50 InvalidKey {
51 message: &'static str,
53 },
54 #[error("object storage key is invalid: {source}")]
56 InvalidPath {
57 #[source]
58 source: object_store::path::Error,
60 },
61 #[error("object storage read exceeds the {limit}-byte buffer limit ({actual} bytes)")]
63 ObjectTooLarge {
64 limit: u64,
66 actual: u64,
68 },
69}
70
71pub type ObjectStoreResult<T> = Result<T, ObjectStoreError>;
73
74#[derive(Clone, Copy, Debug, Eq, PartialEq)]
75pub enum ObjectCreateOutcome {
77 Created,
79 AlreadyPresent,
81}
82
83#[derive(Clone, Copy, Debug, Eq, PartialEq)]
84pub enum S3CopyIfAbsentMode {
86 AwsMultipart,
88 CloudflareR2,
90}
91
92#[derive(Debug, PartialEq, Eq)]
93pub enum ObjectStoreConfig {
95 Local(LocalObjectStoreConfig),
97 S3(S3ObjectStoreConfig),
99}
100
101#[derive(Debug, PartialEq, Eq)]
102pub struct LocalObjectStoreConfig {
104 pub root: PathBuf,
106}
107
108#[derive(PartialEq, Eq)]
109pub struct S3ObjectStoreConfig {
111 pub bucket: String,
113 pub endpoint: Option<String>,
115 pub region: Option<String>,
117 pub access_key_id: Option<SecretString>,
119 pub secret_access_key: Option<SecretString>,
121 pub allow_http: bool,
123 pub virtual_hosted_style_request: bool,
125}
126
127impl fmt::Debug for S3ObjectStoreConfig {
128 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
129 formatter
130 .debug_struct("S3ObjectStoreConfig")
131 .field("bucket", &self.bucket)
132 .field("endpoint", &self.endpoint)
133 .field("region", &self.region)
134 .field(
135 "access_key_id",
136 &self.access_key_id.as_ref().map(|_| "[REDACTED]"),
137 )
138 .field(
139 "secret_access_key",
140 &self.secret_access_key.as_ref().map(|_| "[REDACTED]"),
141 )
142 .field("allow_http", &self.allow_http)
143 .field(
144 "virtual_hosted_style_request",
145 &self.virtual_hosted_style_request,
146 )
147 .finish()
148 }
149}
150
151#[derive(PartialEq, Eq)]
153pub struct SecretString(Zeroizing<String>);
154
155impl SecretString {
156 pub fn new(value: impl Into<String>) -> Self {
158 Self(Zeroizing::new(value.into()))
159 }
160
161 pub fn expose_secret(&self) -> &str {
163 self.0.as_str()
164 }
165}
166
167impl fmt::Debug for SecretString {
168 fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
169 formatter.write_str("[REDACTED]")
170 }
171}
172
173impl S3ObjectStoreConfig {
174 pub fn minio(
179 bucket: impl Into<String>,
180 endpoint: impl Into<String>,
181 access_key_id: impl Into<String>,
182 secret_access_key: impl Into<String>,
183 ) -> Self {
184 let endpoint = endpoint.into();
185 Self {
186 bucket: bucket.into(),
187 allow_http: false,
188 endpoint: Some(endpoint),
189 region: Some("us-east-1".to_string()),
190 access_key_id: Some(SecretString::new(access_key_id)),
191 secret_access_key: Some(SecretString::new(secret_access_key)),
192 virtual_hosted_style_request: false,
193 }
194 }
195}
196
197#[derive(Clone, Debug)]
198pub struct ObjectStoreClient {
200 inner: Arc<dyn ObjectStore>,
201 max_buffered_object_size: u64,
202}
203
204impl ObjectStoreClient {
205 pub fn from_config(config: ObjectStoreConfig) -> ObjectStoreResult<Self> {
207 match config {
208 ObjectStoreConfig::Local(config) => Self::local(config.root),
209 ObjectStoreConfig::S3(config) => Self::s3(config),
210 }
211 }
212
213 pub fn local(root: impl Into<PathBuf>) -> ObjectStoreResult<Self> {
215 let root = root.into();
216 fs::create_dir_all(root.as_path()).map_err(|source| ObjectStoreError::Io {
217 operation: "object storage root creation failed",
218 source,
219 })?;
220 let store = LocalFileSystem::new_with_prefix(root.as_path()).map_err(|source| {
221 ObjectStoreError::Store {
222 operation: "local object store initialization failed",
223 source,
224 }
225 })?;
226
227 Ok(Self {
228 inner: Arc::new(store),
229 max_buffered_object_size: DEFAULT_MAX_BUFFERED_OBJECT_SIZE,
230 })
231 }
232
233 pub fn s3(config: S3ObjectStoreConfig) -> ObjectStoreResult<Self> {
235 Self::s3_inner(config, None)
236 }
237
238 pub fn s3_with_copy_if_absent(
243 config: S3ObjectStoreConfig,
244 mode: S3CopyIfAbsentMode,
245 ) -> ObjectStoreResult<Self> {
246 Self::s3_inner(config, Some(mode))
247 }
248
249 fn s3_inner(
250 config: S3ObjectStoreConfig,
251 copy_if_absent: Option<S3CopyIfAbsentMode>,
252 ) -> ObjectStoreResult<Self> {
253 let bucket =
254 non_empty(config.bucket.as_str()).ok_or(ObjectStoreError::InvalidConfiguration {
255 message: "s3 object store bucket must not be empty",
256 })?;
257
258 let endpoint = config.endpoint.as_deref().and_then(non_empty);
259 if let Some(endpoint) = endpoint {
260 validate_endpoint(endpoint, config.allow_http)?;
261 } else if config.allow_http {
262 return Err(ObjectStoreError::InvalidConfiguration {
263 message: "plain HTTP requires an explicit S3 endpoint",
264 });
265 }
266
267 let mut builder = AmazonS3Builder::new()
268 .with_bucket_name(bucket)
269 .with_virtual_hosted_style_request(config.virtual_hosted_style_request)
270 .with_allow_http(config.allow_http);
271
272 if let Some(region) = config.region.as_deref().and_then(non_empty) {
273 builder = builder.with_region(region);
274 }
275
276 if let Some(endpoint) = endpoint {
277 builder = builder.with_endpoint(endpoint);
278 }
279
280 if let Some(mode) = copy_if_absent {
281 builder = builder.with_copy_if_not_exists(match mode {
282 S3CopyIfAbsentMode::AwsMultipart => S3CopyIfNotExists::Multipart,
283 S3CopyIfAbsentMode::CloudflareR2 => S3CopyIfNotExists::Header(
284 "cf-copy-destination-if-none-match".to_owned(),
285 "*".to_owned(),
286 ),
287 });
288 }
289
290 match (
291 config
292 .access_key_id
293 .as_ref()
294 .map(SecretString::expose_secret)
295 .and_then(non_empty),
296 config
297 .secret_access_key
298 .as_ref()
299 .map(SecretString::expose_secret)
300 .and_then(non_empty),
301 ) {
302 (Some(access_key_id), Some(secret_access_key)) => {
303 builder = builder
304 .with_access_key_id(access_key_id)
305 .with_secret_access_key(secret_access_key);
306 }
307 (None, None) => {}
308 _ => {
309 return Err(ObjectStoreError::InvalidConfiguration {
310 message: "s3 object store credentials require both access key id and secret access key",
311 });
312 }
313 }
314
315 let store = builder.build().map_err(|source| ObjectStoreError::Store {
316 operation: "s3 object store build failed",
317 source,
318 })?;
319
320 Ok(Self {
321 inner: Arc::new(store),
322 max_buffered_object_size: DEFAULT_MAX_BUFFERED_OBJECT_SIZE,
323 })
324 }
325
326 #[must_use]
329 pub fn with_max_buffered_object_size(mut self, limit: u64) -> Self {
330 self.max_buffered_object_size = limit;
331 self
332 }
333
334 pub fn object_path(&self, key: &str) -> ObjectStoreResult<ObjectPath> {
336 object_path(key)
337 }
338
339 pub async fn write_object(&self, key: &str, payload: &[u8]) -> ObjectStoreResult<()> {
341 self.write_bytes(key, Bytes::copy_from_slice(payload)).await
342 }
343
344 pub async fn write_bytes(&self, key: &str, payload: Bytes) -> ObjectStoreResult<()> {
346 let path = self.object_path(key)?;
347 self.inner
348 .put(&path, PutPayload::from(payload))
349 .await
350 .map_err(|source| ObjectStoreError::Store {
351 operation: "object storage write failed",
352 source,
353 })?;
354 Ok(())
355 }
356
357 pub async fn write_object_if_absent(
359 &self,
360 key: &str,
361 payload: &[u8],
362 ) -> ObjectStoreResult<ObjectCreateOutcome> {
363 self.write_bytes_if_absent(key, Bytes::copy_from_slice(payload))
364 .await
365 }
366
367 pub async fn write_bytes_if_absent(
370 &self,
371 key: &str,
372 payload: Bytes,
373 ) -> ObjectStoreResult<ObjectCreateOutcome> {
374 let path = self.object_path(key)?;
375 match self
376 .inner
377 .put_opts(&path, PutPayload::from(payload), PutMode::Create.into())
378 .await
379 {
380 Ok(_) => Ok(ObjectCreateOutcome::Created),
381 Err(object_store::Error::AlreadyExists { .. }) => {
382 Ok(ObjectCreateOutcome::AlreadyPresent)
383 }
384 Err(source) => Err(ObjectStoreError::Store {
385 operation: "object storage conditional write failed",
386 source,
387 }),
388 }
389 }
390
391 pub async fn copy_object_if_absent(
396 &self,
397 source_key: &str,
398 destination_key: &str,
399 ) -> ObjectStoreResult<ObjectCreateOutcome> {
400 let source = self.object_path(source_key)?;
401 let destination = self.object_path(destination_key)?;
402 match self.inner.copy_if_not_exists(&source, &destination).await {
403 Ok(()) => Ok(ObjectCreateOutcome::Created),
404 Err(object_store::Error::AlreadyExists { .. }) => {
405 Ok(ObjectCreateOutcome::AlreadyPresent)
406 }
407 Err(object_store::Error::NotFound { .. }) => Err(ObjectStoreError::NotFound),
408 Err(source) => Err(ObjectStoreError::Store {
409 operation: "object storage conditional copy failed",
410 source,
411 }),
412 }
413 }
414
415 pub async fn head_object(&self, key: &str) -> ObjectStoreResult<object_store::ObjectMeta> {
417 self.head_object_optional(key)
418 .await?
419 .ok_or(ObjectStoreError::NotFound)
420 }
421
422 pub async fn head_object_optional(
425 &self,
426 key: &str,
427 ) -> ObjectStoreResult<Option<object_store::ObjectMeta>> {
428 let path = self.object_path(key)?;
429 match self.inner.head(&path).await {
430 Ok(metadata) => Ok(Some(metadata)),
431 Err(object_store::Error::NotFound { .. }) => Ok(None),
432 Err(source) => Err(ObjectStoreError::Store {
433 operation: "object storage metadata read failed",
434 source,
435 }),
436 }
437 }
438
439 pub async fn read_object(&self, key: &str) -> ObjectStoreResult<Vec<u8>> {
441 self.read_object_optional(key)
442 .await?
443 .ok_or(ObjectStoreError::NotFound)
444 }
445
446 pub async fn read_object_optional(&self, key: &str) -> ObjectStoreResult<Option<Vec<u8>>> {
448 self.read_object_optional_with_limit(key, self.max_buffered_object_size)
449 .await
450 }
451
452 pub async fn read_object_with_limit(
454 &self,
455 key: &str,
456 limit: u64,
457 ) -> ObjectStoreResult<Vec<u8>> {
458 self.read_object_optional_with_limit(key, limit)
459 .await?
460 .ok_or(ObjectStoreError::NotFound)
461 }
462
463 pub async fn read_object_optional_with_limit(
465 &self,
466 key: &str,
467 limit: u64,
468 ) -> ObjectStoreResult<Option<Vec<u8>>> {
469 let Some(get_result) = self.get_object_optional(key, limit).await? else {
470 return Ok(None);
471 };
472 let bytes = get_result
473 .bytes()
474 .await
475 .map_err(|source| ObjectStoreError::Store {
476 operation: "object storage byte read failed",
477 source,
478 })?;
479 Ok(Some(bytes.to_vec()))
480 }
481
482 pub async fn get_object(
487 &self,
488 key: &str,
489 limit: u64,
490 ) -> ObjectStoreResult<object_store::GetResult> {
491 self.get_object_optional(key, limit)
492 .await?
493 .ok_or(ObjectStoreError::NotFound)
494 }
495
496 pub async fn get_object_optional(
498 &self,
499 key: &str,
500 limit: u64,
501 ) -> ObjectStoreResult<Option<object_store::GetResult>> {
502 let path = self.object_path(key)?;
503 let get_result = match self.inner.get(&path).await {
504 Ok(result) => result,
505 Err(object_store::Error::NotFound { .. }) => return Ok(None),
506 Err(source) => {
507 return Err(ObjectStoreError::Store {
508 operation: "object storage read failed",
509 source,
510 });
511 }
512 };
513 if get_result.meta.size > limit {
514 return Err(ObjectStoreError::ObjectTooLarge {
515 limit,
516 actual: get_result.meta.size,
517 });
518 }
519 Ok(Some(get_result))
520 }
521
522 pub async fn delete_object(&self, key: &str) -> ObjectStoreResult<()> {
524 let path = self.object_path(key)?;
525 match self.inner.delete(&path).await {
526 Ok(()) => Ok(()),
527 Err(object_store::Error::NotFound { .. }) => Ok(()),
528 Err(source) => Err(ObjectStoreError::Store {
529 operation: "object storage delete failed",
530 source,
531 }),
532 }
533 }
534}
535
536fn object_path(key: &str) -> ObjectStoreResult<ObjectPath> {
537 for segment in key.split('/') {
538 let trimmed = segment.trim();
539 if trimmed != segment
540 || segment.is_empty()
541 || segment == "."
542 || segment == ".."
543 || segment.contains('\\')
544 || segment.contains(':')
545 {
546 return Err(ObjectStoreError::InvalidKey {
547 message: "object storage key contains an invalid segment",
548 });
549 }
550 }
551
552 ObjectPath::parse(key).map_err(|source| ObjectStoreError::InvalidPath { source })
553}
554
555fn non_empty(value: &str) -> Option<&str> {
556 let trimmed = value.trim();
557 (!trimmed.is_empty()).then_some(trimmed)
558}
559
560fn validate_endpoint(endpoint: &str, allow_http: bool) -> ObjectStoreResult<()> {
561 let endpoint = Url::parse(endpoint).map_err(|_| ObjectStoreError::InvalidConfiguration {
562 message: "s3 object store endpoint must be an absolute URL",
563 })?;
564 if !endpoint.username().is_empty() || endpoint.password().is_some() {
565 return Err(ObjectStoreError::InvalidConfiguration {
566 message: "s3 object store endpoint must not contain credentials",
567 });
568 }
569 match endpoint.scheme() {
570 "https" => Ok(()),
571 "http" if allow_http => Ok(()),
572 "http" => Err(ObjectStoreError::InvalidConfiguration {
573 message: "plain HTTP S3 endpoints require an explicit allow_http opt-in",
574 }),
575 _ => Err(ObjectStoreError::InvalidConfiguration {
576 message: "s3 object store endpoint must use HTTP or HTTPS",
577 }),
578 }
579}
580
581#[cfg(test)]
582mod tests {
583 use std::time::{SystemTime, UNIX_EPOCH};
584
585 use super::*;
586
587 fn temp_store() -> ObjectStoreClient {
588 let unique = SystemTime::now()
589 .duration_since(UNIX_EPOCH)
590 .expect("clock")
591 .as_nanos();
592 ObjectStoreClient::local(std::env::temp_dir().join(format!("s3-local-store-{unique}")))
593 .expect("local object store")
594 }
595
596 #[test]
597 fn object_store_rejects_unsafe_keys() {
598 let store = temp_store();
599
600 assert!(store.object_path("").is_err());
601 assert!(store.object_path("../asset").is_err());
602 assert!(store.object_path("media/../asset").is_err());
603 assert!(store.object_path("media\\asset").is_err());
604 assert!(store.object_path("media/c:asset").is_err());
605 assert!(store.object_path("media/ asset").is_err());
606 }
607
608 #[tokio::test]
609 async fn local_object_store_writes_reads_and_deletes_objects() {
610 let store = temp_store();
611 let key = "media/account/asset/original";
612
613 store
614 .write_object(key, b"payload")
615 .await
616 .expect("write object");
617
618 assert_eq!(
619 store.read_object(key).await.expect("read object"),
620 b"payload"
621 );
622 assert_eq!(
623 store
624 .read_object_optional(key)
625 .await
626 .expect("optional read object"),
627 Some(b"payload".to_vec())
628 );
629
630 store.delete_object(key).await.expect("delete object");
631
632 assert!(store.read_object(key).await.is_err());
633 assert_eq!(
634 store
635 .read_object_optional(key)
636 .await
637 .expect("missing object should be optional"),
638 None
639 );
640 }
641
642 #[tokio::test]
643 async fn local_object_store_creates_and_copies_without_overwriting() {
644 let store = temp_store();
645
646 assert_eq!(
647 store
648 .write_object_if_absent("quarantine/release", b"signed-release")
649 .await
650 .unwrap(),
651 ObjectCreateOutcome::Created
652 );
653 assert_eq!(
654 store
655 .write_object_if_absent("quarantine/release", b"tampered")
656 .await
657 .unwrap(),
658 ObjectCreateOutcome::AlreadyPresent
659 );
660 assert_eq!(
661 store.read_object("quarantine/release").await.unwrap(),
662 b"signed-release"
663 );
664
665 assert_eq!(
666 store
667 .copy_object_if_absent("quarantine/release", "releases/digest/release-id")
668 .await
669 .unwrap(),
670 ObjectCreateOutcome::Created
671 );
672 store
673 .write_object("quarantine/release", b"different-source")
674 .await
675 .unwrap();
676 assert_eq!(
677 store
678 .copy_object_if_absent("quarantine/release", "releases/digest/release-id")
679 .await
680 .unwrap(),
681 ObjectCreateOutcome::AlreadyPresent
682 );
683 assert_eq!(
684 store
685 .read_object("releases/digest/release-id")
686 .await
687 .unwrap(),
688 b"signed-release"
689 );
690 assert_eq!(
691 store
692 .head_object("releases/digest/release-id")
693 .await
694 .unwrap()
695 .size,
696 b"signed-release".len() as u64
697 );
698 assert!(
699 store
700 .head_object_optional("releases/missing")
701 .await
702 .unwrap()
703 .is_none()
704 );
705 assert!(matches!(
706 store
707 .copy_object_if_absent("quarantine/missing", "releases/missing")
708 .await,
709 Err(ObjectStoreError::NotFound)
710 ));
711 assert!(
712 store
713 .read_object_optional("releases/missing")
714 .await
715 .unwrap()
716 .is_none()
717 );
718 }
719
720 #[test]
721 fn s3_config_requires_credentials_as_a_pair() {
722 let config = S3ObjectStoreConfig {
723 bucket: "library".to_string(),
724 endpoint: Some("http://127.0.0.1:9000".to_string()),
725 region: Some("us-east-1".to_string()),
726 access_key_id: Some(SecretString::new("minio")),
727 secret_access_key: None,
728 allow_http: true,
729 virtual_hosted_style_request: false,
730 };
731
732 let error = ObjectStoreClient::s3(config).expect_err("missing secret must fail");
733
734 assert!(
735 error
736 .to_string()
737 .contains("both access key id and secret access key")
738 );
739 }
740
741 #[test]
742 fn minio_config_uses_path_style_http_endpoint() {
743 let config =
744 S3ObjectStoreConfig::minio("library", "http://127.0.0.1:9000", "minio", "minio-secret");
745
746 assert_eq!(config.bucket, "library");
747 assert_eq!(config.endpoint.as_deref(), Some("http://127.0.0.1:9000"));
748 assert_eq!(config.region.as_deref(), Some("us-east-1"));
749 assert!(!config.allow_http);
750 assert!(!config.virtual_hosted_style_request);
751 }
752
753 #[test]
754 fn s3_debug_output_redacts_credentials() {
755 let config = S3ObjectStoreConfig::minio(
756 "library",
757 "http://127.0.0.1:9000",
758 "visible-access-key",
759 "visible-secret-key",
760 );
761
762 let debug = format!("{config:?}");
763
764 assert!(!debug.contains("visible-access-key"));
765 assert!(!debug.contains("visible-secret-key"));
766 assert_eq!(debug.matches("[REDACTED]").count(), 2);
767 }
768
769 #[tokio::test]
770 async fn buffered_reads_enforce_the_configured_limit() {
771 let store = temp_store();
772 store.write_object("media/large", b"12345").await.unwrap();
773 assert!(matches!(
774 store.read_object_with_limit("media/large", 4).await,
775 Err(ObjectStoreError::ObjectTooLarge {
776 limit: 4,
777 actual: 5
778 })
779 ));
780 }
781}