use std::fmt;
use std::fs;
use std::path::PathBuf;
use std::sync::Arc;
use bytes::Bytes;
use object_store::aws::AmazonS3Builder;
use object_store::local::LocalFileSystem;
use object_store::path::Path as ObjectPath;
use object_store::{ObjectStore, ObjectStoreExt, PutPayload};
use url::Url;
use zeroize::Zeroizing;
pub const DEFAULT_MAX_BUFFERED_OBJECT_SIZE: u64 = 64 * 1024 * 1024;
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
pub enum ObjectStoreError {
#[error("{message}")]
InvalidConfiguration {
message: &'static str,
},
#[error("{operation}: {source}")]
Io {
operation: &'static str,
#[source]
source: std::io::Error,
},
#[error("{operation}: {source}")]
Store {
operation: &'static str,
#[source]
source: object_store::Error,
},
#[error("object storage read failed: object not found")]
NotFound,
#[error("{message}")]
InvalidKey {
message: &'static str,
},
#[error("object storage key is invalid: {source}")]
InvalidPath {
#[source]
source: object_store::path::Error,
},
#[error("object storage read exceeds the {limit}-byte buffer limit ({actual} bytes)")]
ObjectTooLarge {
limit: u64,
actual: u64,
},
}
pub type ObjectStoreResult<T> = Result<T, ObjectStoreError>;
#[derive(Debug, PartialEq, Eq)]
pub enum ObjectStoreConfig {
Local(LocalObjectStoreConfig),
S3(S3ObjectStoreConfig),
}
#[derive(Debug, PartialEq, Eq)]
pub struct LocalObjectStoreConfig {
pub root: PathBuf,
}
#[derive(PartialEq, Eq)]
pub struct S3ObjectStoreConfig {
pub bucket: String,
pub endpoint: Option<String>,
pub region: Option<String>,
pub access_key_id: Option<SecretString>,
pub secret_access_key: Option<SecretString>,
pub allow_http: bool,
pub virtual_hosted_style_request: bool,
}
impl fmt::Debug for S3ObjectStoreConfig {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("S3ObjectStoreConfig")
.field("bucket", &self.bucket)
.field("endpoint", &self.endpoint)
.field("region", &self.region)
.field(
"access_key_id",
&self.access_key_id.as_ref().map(|_| "[REDACTED]"),
)
.field(
"secret_access_key",
&self.secret_access_key.as_ref().map(|_| "[REDACTED]"),
)
.field("allow_http", &self.allow_http)
.field(
"virtual_hosted_style_request",
&self.virtual_hosted_style_request,
)
.finish()
}
}
#[derive(PartialEq, Eq)]
pub struct SecretString(Zeroizing<String>);
impl SecretString {
pub fn new(value: impl Into<String>) -> Self {
Self(Zeroizing::new(value.into()))
}
pub fn expose_secret(&self) -> &str {
self.0.as_str()
}
}
impl fmt::Debug for SecretString {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("[REDACTED]")
}
}
impl S3ObjectStoreConfig {
pub fn minio(
bucket: impl Into<String>,
endpoint: impl Into<String>,
access_key_id: impl Into<String>,
secret_access_key: impl Into<String>,
) -> Self {
let endpoint = endpoint.into();
Self {
bucket: bucket.into(),
allow_http: false,
endpoint: Some(endpoint),
region: Some("us-east-1".to_string()),
access_key_id: Some(SecretString::new(access_key_id)),
secret_access_key: Some(SecretString::new(secret_access_key)),
virtual_hosted_style_request: false,
}
}
}
#[derive(Clone, Debug)]
pub struct ObjectStoreClient {
inner: Arc<dyn ObjectStore>,
max_buffered_object_size: u64,
}
impl ObjectStoreClient {
pub fn from_config(config: ObjectStoreConfig) -> ObjectStoreResult<Self> {
match config {
ObjectStoreConfig::Local(config) => Self::local(config.root),
ObjectStoreConfig::S3(config) => Self::s3(config),
}
}
pub fn local(root: impl Into<PathBuf>) -> ObjectStoreResult<Self> {
let root = root.into();
fs::create_dir_all(root.as_path()).map_err(|source| ObjectStoreError::Io {
operation: "object storage root creation failed",
source,
})?;
let store = LocalFileSystem::new_with_prefix(root.as_path()).map_err(|source| {
ObjectStoreError::Store {
operation: "local object store initialization failed",
source,
}
})?;
Ok(Self {
inner: Arc::new(store),
max_buffered_object_size: DEFAULT_MAX_BUFFERED_OBJECT_SIZE,
})
}
pub fn s3(config: S3ObjectStoreConfig) -> ObjectStoreResult<Self> {
let bucket =
non_empty(config.bucket.as_str()).ok_or(ObjectStoreError::InvalidConfiguration {
message: "s3 object store bucket must not be empty",
})?;
let endpoint = non_empty_option(config.endpoint.as_deref());
if let Some(endpoint) = endpoint.as_deref() {
validate_endpoint(endpoint, config.allow_http)?;
} else if config.allow_http {
return Err(ObjectStoreError::InvalidConfiguration {
message: "plain HTTP requires an explicit S3 endpoint",
});
}
let mut builder = AmazonS3Builder::new()
.with_bucket_name(bucket)
.with_virtual_hosted_style_request(config.virtual_hosted_style_request)
.with_allow_http(config.allow_http);
if let Some(region) = non_empty_option(config.region.as_deref()) {
builder = builder.with_region(region);
}
if let Some(endpoint) = endpoint {
builder = builder.with_endpoint(endpoint);
}
match (
non_empty_option(
config
.access_key_id
.as_ref()
.map(SecretString::expose_secret),
),
non_empty_option(
config
.secret_access_key
.as_ref()
.map(SecretString::expose_secret),
),
) {
(Some(access_key_id), Some(secret_access_key)) => {
builder = builder
.with_access_key_id(access_key_id)
.with_secret_access_key(secret_access_key);
}
(None, None) => {}
_ => {
return Err(ObjectStoreError::InvalidConfiguration {
message: "s3 object store credentials require both access key id and secret access key",
});
}
}
let store = builder.build().map_err(|source| ObjectStoreError::Store {
operation: "s3 object store build failed",
source,
})?;
Ok(Self {
inner: Arc::new(store),
max_buffered_object_size: DEFAULT_MAX_BUFFERED_OBJECT_SIZE,
})
}
#[must_use]
pub fn with_max_buffered_object_size(mut self, limit: u64) -> Self {
self.max_buffered_object_size = limit;
self
}
pub fn object_path(&self, key: &str) -> ObjectStoreResult<ObjectPath> {
object_path(key)
}
pub async fn write_object(&self, key: &str, payload: &[u8]) -> ObjectStoreResult<()> {
self.write_bytes(key, Bytes::copy_from_slice(payload)).await
}
pub async fn write_bytes(&self, key: &str, payload: Bytes) -> ObjectStoreResult<()> {
let path = self.object_path(key)?;
self.inner
.put(&path, PutPayload::from(payload))
.await
.map_err(|source| ObjectStoreError::Store {
operation: "object storage write failed",
source,
})?;
Ok(())
}
pub async fn read_object(&self, key: &str) -> ObjectStoreResult<Vec<u8>> {
self.read_object_optional(key)
.await?
.ok_or(ObjectStoreError::NotFound)
}
pub async fn read_object_optional(&self, key: &str) -> ObjectStoreResult<Option<Vec<u8>>> {
self.read_object_optional_with_limit(key, self.max_buffered_object_size)
.await
}
pub async fn read_object_with_limit(
&self,
key: &str,
limit: u64,
) -> ObjectStoreResult<Vec<u8>> {
self.read_object_optional_with_limit(key, limit)
.await?
.ok_or(ObjectStoreError::NotFound)
}
pub async fn read_object_optional_with_limit(
&self,
key: &str,
limit: u64,
) -> ObjectStoreResult<Option<Vec<u8>>> {
let Some(get_result) = self.get_object_optional(key, limit).await? else {
return Ok(None);
};
let bytes = get_result
.bytes()
.await
.map_err(|source| ObjectStoreError::Store {
operation: "object storage byte read failed",
source,
})?;
Ok(Some(bytes.to_vec()))
}
pub async fn get_object(
&self,
key: &str,
limit: u64,
) -> ObjectStoreResult<object_store::GetResult> {
self.get_object_optional(key, limit)
.await?
.ok_or(ObjectStoreError::NotFound)
}
pub async fn get_object_optional(
&self,
key: &str,
limit: u64,
) -> ObjectStoreResult<Option<object_store::GetResult>> {
let path = self.object_path(key)?;
let get_result = match self.inner.get(&path).await {
Ok(result) => result,
Err(object_store::Error::NotFound { .. }) => return Ok(None),
Err(source) => {
return Err(ObjectStoreError::Store {
operation: "object storage read failed",
source,
});
}
};
if get_result.meta.size > limit {
return Err(ObjectStoreError::ObjectTooLarge {
limit,
actual: get_result.meta.size,
});
}
Ok(Some(get_result))
}
pub async fn delete_object(&self, key: &str) -> ObjectStoreResult<()> {
let path = self.object_path(key)?;
match self.inner.delete(&path).await {
Ok(()) => Ok(()),
Err(object_store::Error::NotFound { .. }) => Ok(()),
Err(source) => Err(ObjectStoreError::Store {
operation: "object storage delete failed",
source,
}),
}
}
}
fn object_path(key: &str) -> ObjectStoreResult<ObjectPath> {
let mut segments = Vec::new();
for segment in key.split('/') {
let trimmed = segment.trim();
if trimmed != segment
|| segment.is_empty()
|| segment == "."
|| segment == ".."
|| segment.contains('\\')
|| segment.contains(':')
{
return Err(ObjectStoreError::InvalidKey {
message: "object storage key contains an invalid segment",
});
}
segments.push(segment);
}
if segments.is_empty() {
return Err(ObjectStoreError::InvalidKey {
message: "object storage key must not be empty",
});
}
ObjectPath::parse(segments.join("/")).map_err(|source| ObjectStoreError::InvalidPath { source })
}
fn non_empty(value: &str) -> Option<String> {
let trimmed = value.trim();
(!trimmed.is_empty()).then(|| trimmed.to_string())
}
fn non_empty_option(value: Option<&str>) -> Option<String> {
value.and_then(non_empty)
}
fn validate_endpoint(endpoint: &str, allow_http: bool) -> ObjectStoreResult<()> {
let endpoint = Url::parse(endpoint).map_err(|_| ObjectStoreError::InvalidConfiguration {
message: "s3 object store endpoint must be an absolute URL",
})?;
if !endpoint.username().is_empty() || endpoint.password().is_some() {
return Err(ObjectStoreError::InvalidConfiguration {
message: "s3 object store endpoint must not contain credentials",
});
}
match endpoint.scheme() {
"https" => Ok(()),
"http" if allow_http => Ok(()),
"http" => Err(ObjectStoreError::InvalidConfiguration {
message: "plain HTTP S3 endpoints require an explicit allow_http opt-in",
}),
_ => Err(ObjectStoreError::InvalidConfiguration {
message: "s3 object store endpoint must use HTTP or HTTPS",
}),
}
}
#[cfg(test)]
mod tests {
use std::time::{SystemTime, UNIX_EPOCH};
use super::*;
fn temp_store() -> ObjectStoreClient {
let unique = SystemTime::now()
.duration_since(UNIX_EPOCH)
.expect("clock")
.as_nanos();
ObjectStoreClient::local(std::env::temp_dir().join(format!("s3-local-store-{unique}")))
.expect("local object store")
}
#[test]
fn object_store_rejects_unsafe_keys() {
let store = temp_store();
assert!(store.object_path("").is_err());
assert!(store.object_path("../asset").is_err());
assert!(store.object_path("media/../asset").is_err());
assert!(store.object_path("media\\asset").is_err());
assert!(store.object_path("media/c:asset").is_err());
assert!(store.object_path("media/ asset").is_err());
}
#[tokio::test]
async fn local_object_store_writes_reads_and_deletes_objects() {
let store = temp_store();
let key = "media/account/asset/original";
store
.write_object(key, b"payload")
.await
.expect("write object");
assert_eq!(
store.read_object(key).await.expect("read object"),
b"payload"
);
assert_eq!(
store
.read_object_optional(key)
.await
.expect("optional read object"),
Some(b"payload".to_vec())
);
store.delete_object(key).await.expect("delete object");
assert!(store.read_object(key).await.is_err());
assert_eq!(
store
.read_object_optional(key)
.await
.expect("missing object should be optional"),
None
);
}
#[test]
fn s3_config_requires_credentials_as_a_pair() {
let config = S3ObjectStoreConfig {
bucket: "library".to_string(),
endpoint: Some("http://127.0.0.1:9000".to_string()),
region: Some("us-east-1".to_string()),
access_key_id: Some(SecretString::new("minio")),
secret_access_key: None,
allow_http: true,
virtual_hosted_style_request: false,
};
let error = ObjectStoreClient::s3(config).expect_err("missing secret must fail");
assert!(
error
.to_string()
.contains("both access key id and secret access key")
);
}
#[test]
fn minio_config_uses_path_style_http_endpoint() {
let config =
S3ObjectStoreConfig::minio("library", "http://127.0.0.1:9000", "minio", "minio-secret");
assert_eq!(config.bucket, "library");
assert_eq!(config.endpoint.as_deref(), Some("http://127.0.0.1:9000"));
assert_eq!(config.region.as_deref(), Some("us-east-1"));
assert!(!config.allow_http);
assert!(!config.virtual_hosted_style_request);
}
#[test]
fn s3_debug_output_redacts_credentials() {
let config = S3ObjectStoreConfig::minio(
"library",
"http://127.0.0.1:9000",
"visible-access-key",
"visible-secret-key",
);
let debug = format!("{config:?}");
assert!(!debug.contains("visible-access-key"));
assert!(!debug.contains("visible-secret-key"));
assert_eq!(debug.matches("[REDACTED]").count(), 2);
}
#[tokio::test]
async fn buffered_reads_enforce_the_configured_limit() {
let store = temp_store();
store.write_object("media/large", b"12345").await.unwrap();
assert!(matches!(
store.read_object_with_limit("media/large", 4).await,
Err(ObjectStoreError::ObjectTooLarge {
limit: 4,
actual: 5
})
));
}
}