use async_trait::async_trait;
use bytes::Bytes;
use futures::stream::{BoxStream, TryStreamExt};
use loonfs_api::StorageChecksum;
use serde::{Deserialize, Serialize};
use std::fmt::Debug;
use std::sync::Arc;
use thiserror::Error;
pub type SharedObjectStore = Arc<dyn ObjectStore>;
pub type ByteStream = BoxStream<'static, Result<Bytes>>;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ObjectMetadata {
pub etag: Option<String>,
pub version: Option<String>,
pub size_bytes: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub last_modified_ms: Option<u64>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct StoredObjectChecksum {
pub size_bytes: u64,
pub storage_checksum: StorageChecksum,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct MultipartPart {
pub part_number: u32,
pub etag: String,
pub checksum: StorageChecksum,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum MultipartCompletion {
Assembled,
UnknownUpload,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ObjectBody {
pub metadata: ObjectMetadata,
pub bytes: Vec<u8>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub enum PutMode {
Overwrite,
CreateIfAbsent,
CompareAndSwap {
expected_etag: String,
},
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct ByteRange {
pub start_inclusive: u64,
pub end_exclusive: u64,
}
#[derive(Debug, Error)]
#[non_exhaustive]
pub enum ObjectStoreError {
#[error("object not found `{object_key}`")]
NotFound {
object_key: String,
},
#[error("invalid object key `{object_key}`: {message}")]
InvalidKey {
object_key: String,
message: String,
},
#[error("invalid content ref: {0}")]
InvalidContentRef(String),
#[error("invalid byte range for `{object_key}`")]
InvalidRange {
object_key: String,
},
#[error("precondition failed for `{object_key}`")]
PreconditionFailed {
object_key: String,
},
#[error("permission denied for `{object_key}`: {message}")]
PermissionDenied {
object_key: String,
message: String,
},
#[error("unsupported capability: {0}")]
Unsupported(&'static str),
#[error("invalid object store configuration: {0}")]
Configuration(String),
#[error("transport error for `{object_key}`: {message}")]
Transport {
object_key: String,
message: String,
},
}
impl ObjectStoreError {
pub fn transport(object_key: impl Into<String>, message: impl Into<String>) -> Self {
Self::Transport {
object_key: object_key.into(),
message: message.into(),
}
}
pub fn object_key(&self) -> Option<&str> {
match self {
Self::NotFound { object_key }
| Self::InvalidKey { object_key, .. }
| Self::InvalidRange { object_key }
| Self::PreconditionFailed { object_key }
| Self::PermissionDenied { object_key, .. }
| Self::Transport { object_key, .. } => Some(object_key),
Self::InvalidContentRef(_) | Self::Unsupported(_) | Self::Configuration(_) => None,
}
}
pub fn message(&self) -> String {
match self {
Self::NotFound { .. } => "object not found".to_owned(),
Self::InvalidKey { message, .. } => format!("invalid object key: {message}"),
Self::InvalidContentRef(message) => format!("invalid content ref: {message}"),
Self::InvalidRange { .. } => "invalid byte range".to_owned(),
Self::PreconditionFailed { .. } => "precondition failed".to_owned(),
Self::PermissionDenied { message, .. } => {
format!("permission denied: {message}")
}
Self::Unsupported(capability) => format!("unsupported capability: {capability}"),
Self::Configuration(message) => {
format!("invalid object store configuration: {message}")
}
Self::Transport { message, .. } => message.clone(),
}
}
}
pub type Result<T> = std::result::Result<T, ObjectStoreError>;
pub(crate) async fn collect_stream(mut body: ByteStream) -> Result<Bytes> {
use futures::StreamExt as _;
let mut buffered = bytes::BytesMut::new();
while let Some(chunk) = body.next().await {
buffered.extend_from_slice(&chunk?);
}
Ok(buffered.freeze())
}
#[async_trait]
pub trait ObjectStore: Send + Sync + Debug {
async fn head(&self, key: &str) -> Result<Option<ObjectMetadata>>;
async fn head_stored_checksum(&self, key: &str) -> Result<Option<StoredObjectChecksum>> {
let _ = key;
Err(ObjectStoreError::Unsupported(
"stored full-object checksum readback",
))
}
async fn create_multipart_upload(&self, key: &str) -> Result<String> {
let _ = key;
Err(ObjectStoreError::Unsupported(
"client-driven multipart upload",
))
}
async fn complete_multipart_upload(
&self,
key: &str,
provider_upload_id: &str,
parts: &[MultipartPart],
full_object_checksum: &StorageChecksum,
) -> Result<MultipartCompletion> {
let (_, _, _, _) = (key, provider_upload_id, parts, full_object_checksum);
Err(ObjectStoreError::Unsupported(
"client-driven multipart upload",
))
}
async fn abort_multipart_upload(&self, key: &str, provider_upload_id: &str) -> Result<()> {
let _ = (key, provider_upload_id);
Err(ObjectStoreError::Unsupported(
"client-driven multipart upload",
))
}
async fn get_with_metadata(&self, key: &str) -> Result<Option<ObjectBody>>;
async fn get(&self, key: &str, range: Option<ByteRange>) -> Result<Option<Bytes>>;
async fn put(&self, key: &str, bytes: Bytes, mode: PutMode) -> Result<ObjectMetadata>;
async fn put_streamed(&self, key: &str, body: ByteStream, mode: PutMode) -> Result<u64> {
let bytes = collect_stream(body).await?;
let size_bytes = bytes.len() as u64;
self.put(key, bytes, mode).await?;
Ok(size_bytes)
}
async fn delete(&self, key: &str) -> Result<()>;
fn list_prefix_stream(&self, prefix: &str) -> BoxStream<'static, Result<String>>;
async fn list_prefix(&self, prefix: &str) -> Result<Vec<String>> {
let mut keys: Vec<String> = self.list_prefix_stream(prefix).try_collect().await?;
keys.sort();
Ok(keys)
}
async fn put_overwrite(&self, key: &str, bytes: Bytes) -> Result<ObjectMetadata> {
self.put(key, bytes, PutMode::Overwrite).await
}
async fn put_if_absent(&self, key: &str, bytes: Bytes) -> Result<ObjectMetadata> {
self.put(key, bytes, PutMode::CreateIfAbsent).await
}
async fn put_immutable_verified(
&self,
key: &str,
bytes: Bytes,
) -> std::result::Result<(), crate::ImmutableWriteError> {
crate::immutable_write::put(self, key, bytes).await
}
async fn compare_and_swap(
&self,
key: &str,
expected_etag: &str,
bytes: Bytes,
) -> Result<ObjectMetadata> {
self.put(
key,
bytes,
PutMode::CompareAndSwap {
expected_etag: expected_etag.to_owned(),
},
)
.await
}
}
#[async_trait]
impl<T: ObjectStore + ?Sized> ObjectStore for Arc<T> {
async fn head(&self, key: &str) -> Result<Option<ObjectMetadata>> {
self.as_ref().head(key).await
}
async fn head_stored_checksum(&self, key: &str) -> Result<Option<StoredObjectChecksum>> {
self.as_ref().head_stored_checksum(key).await
}
async fn create_multipart_upload(&self, key: &str) -> Result<String> {
self.as_ref().create_multipart_upload(key).await
}
async fn complete_multipart_upload(
&self,
key: &str,
provider_upload_id: &str,
parts: &[MultipartPart],
full_object_checksum: &StorageChecksum,
) -> Result<MultipartCompletion> {
self.as_ref()
.complete_multipart_upload(key, provider_upload_id, parts, full_object_checksum)
.await
}
async fn abort_multipart_upload(&self, key: &str, provider_upload_id: &str) -> Result<()> {
self.as_ref()
.abort_multipart_upload(key, provider_upload_id)
.await
}
async fn get_with_metadata(&self, key: &str) -> Result<Option<ObjectBody>> {
self.as_ref().get_with_metadata(key).await
}
async fn get(&self, key: &str, range: Option<ByteRange>) -> Result<Option<Bytes>> {
self.as_ref().get(key, range).await
}
async fn put(&self, key: &str, bytes: Bytes, mode: PutMode) -> Result<ObjectMetadata> {
self.as_ref().put(key, bytes, mode).await
}
async fn put_streamed(&self, key: &str, body: ByteStream, mode: PutMode) -> Result<u64> {
self.as_ref().put_streamed(key, body, mode).await
}
async fn delete(&self, key: &str) -> Result<()> {
self.as_ref().delete(key).await
}
fn list_prefix_stream(&self, prefix: &str) -> BoxStream<'static, Result<String>> {
self.as_ref().list_prefix_stream(prefix)
}
async fn list_prefix(&self, prefix: &str) -> Result<Vec<String>> {
self.as_ref().list_prefix(prefix).await
}
async fn put_overwrite(&self, key: &str, bytes: Bytes) -> Result<ObjectMetadata> {
self.as_ref().put_overwrite(key, bytes).await
}
async fn put_if_absent(&self, key: &str, bytes: Bytes) -> Result<ObjectMetadata> {
self.as_ref().put_if_absent(key, bytes).await
}
async fn put_immutable_verified(
&self,
key: &str,
bytes: Bytes,
) -> std::result::Result<(), crate::ImmutableWriteError> {
self.as_ref().put_immutable_verified(key, bytes).await
}
async fn compare_and_swap(
&self,
key: &str,
expected_etag: &str,
bytes: Bytes,
) -> Result<ObjectMetadata> {
self.as_ref()
.compare_and_swap(key, expected_etag, bytes)
.await
}
}
#[async_trait]
impl<T: ObjectStore + ?Sized> ObjectStore for &T {
async fn head(&self, key: &str) -> Result<Option<ObjectMetadata>> {
(*self).head(key).await
}
async fn head_stored_checksum(&self, key: &str) -> Result<Option<StoredObjectChecksum>> {
(*self).head_stored_checksum(key).await
}
async fn create_multipart_upload(&self, key: &str) -> Result<String> {
(*self).create_multipart_upload(key).await
}
async fn complete_multipart_upload(
&self,
key: &str,
provider_upload_id: &str,
parts: &[MultipartPart],
full_object_checksum: &StorageChecksum,
) -> Result<MultipartCompletion> {
(*self)
.complete_multipart_upload(key, provider_upload_id, parts, full_object_checksum)
.await
}
async fn abort_multipart_upload(&self, key: &str, provider_upload_id: &str) -> Result<()> {
(*self)
.abort_multipart_upload(key, provider_upload_id)
.await
}
async fn get_with_metadata(&self, key: &str) -> Result<Option<ObjectBody>> {
(*self).get_with_metadata(key).await
}
async fn get(&self, key: &str, range: Option<ByteRange>) -> Result<Option<Bytes>> {
(*self).get(key, range).await
}
async fn put(&self, key: &str, bytes: Bytes, mode: PutMode) -> Result<ObjectMetadata> {
(*self).put(key, bytes, mode).await
}
async fn put_streamed(&self, key: &str, body: ByteStream, mode: PutMode) -> Result<u64> {
(*self).put_streamed(key, body, mode).await
}
async fn delete(&self, key: &str) -> Result<()> {
(*self).delete(key).await
}
fn list_prefix_stream(&self, prefix: &str) -> BoxStream<'static, Result<String>> {
(*self).list_prefix_stream(prefix)
}
async fn list_prefix(&self, prefix: &str) -> Result<Vec<String>> {
(*self).list_prefix(prefix).await
}
async fn put_overwrite(&self, key: &str, bytes: Bytes) -> Result<ObjectMetadata> {
(*self).put_overwrite(key, bytes).await
}
async fn put_if_absent(&self, key: &str, bytes: Bytes) -> Result<ObjectMetadata> {
(*self).put_if_absent(key, bytes).await
}
async fn put_immutable_verified(
&self,
key: &str,
bytes: Bytes,
) -> std::result::Result<(), crate::ImmutableWriteError> {
(*self).put_immutable_verified(key, bytes).await
}
async fn compare_and_swap(
&self,
key: &str,
expected_etag: &str,
bytes: Bytes,
) -> Result<ObjectMetadata> {
(*self).compare_and_swap(key, expected_etag, bytes).await
}
}
#[cfg(test)]
mod tests {
use super::*;
use futures::stream;
use std::sync::atomic::{AtomicBool, Ordering};
#[derive(Debug)]
struct ListOverrideStore {
override_reached: Arc<AtomicBool>,
}
#[async_trait]
impl ObjectStore for ListOverrideStore {
async fn head(&self, _key: &str) -> Result<Option<ObjectMetadata>> {
Ok(None)
}
async fn get_with_metadata(&self, _key: &str) -> Result<Option<ObjectBody>> {
Ok(None)
}
async fn get(&self, _key: &str, _range: Option<ByteRange>) -> Result<Option<Bytes>> {
Ok(None)
}
async fn put(&self, key: &str, _bytes: Bytes, _mode: PutMode) -> Result<ObjectMetadata> {
Err(ObjectStoreError::PreconditionFailed {
object_key: key.to_owned(),
})
}
async fn delete(&self, _key: &str) -> Result<()> {
Ok(())
}
fn list_prefix_stream(&self, _prefix: &str) -> BoxStream<'static, Result<String>> {
Box::pin(stream::empty())
}
async fn list_prefix(&self, _prefix: &str) -> Result<Vec<String>> {
self.override_reached.store(true, Ordering::SeqCst);
Ok(vec!["overridden".to_owned()])
}
}
#[tokio::test]
async fn arc_dyn_store_forwards_overridden_list_prefix() {
let override_reached = Arc::new(AtomicBool::new(false));
let store: Arc<dyn ObjectStore> = Arc::new(ListOverrideStore {
override_reached: Arc::clone(&override_reached),
});
let keys = <Arc<dyn ObjectStore> as ObjectStore>::list_prefix(&store, "prefix/")
.await
.expect("overridden list should succeed");
assert_eq!(keys, vec!["overridden"]);
assert!(override_reached.load(Ordering::SeqCst));
}
}