use std::collections::HashMap;
use std::sync::Arc;
use bytes::Bytes;
use crate::storage::config::StorageConfig;
use crate::storage::error::{StorageConnectError, StorageError};
use crate::storage::path::StoragePath;
#[derive(Clone)]
pub struct Storage {
inner: Arc<StorageInner>,
}
struct StorageInner {
disks: HashMap<String, Disk>,
default_name: String,
}
#[derive(Clone)]
pub struct Disk {
pub(crate) operator: opendal::Operator,
}
impl Storage {
#[must_use]
pub fn builder() -> StorageBuilder {
StorageBuilder {
disks: Vec::new(),
default_name: None,
}
}
pub async fn connect(config: StorageConfig) -> Result<Storage, StorageConnectError> {
config.validate()?;
let operator = build_operator(&config).await?;
let mut disks = HashMap::new();
disks.insert("default".to_string(), Disk { operator });
Ok(Storage {
inner: Arc::new(StorageInner {
disks,
default_name: "default".to_string(),
}),
})
}
#[must_use]
pub fn disk(&self, name: &str) -> Disk {
self.try_disk(name)
.unwrap_or_else(|| panic!("storage disk `{name}` is not registered"))
}
#[must_use]
pub fn try_disk(&self, name: &str) -> Option<Disk> {
self.inner.disks.get(name).cloned()
}
#[must_use]
pub fn default_disk(&self) -> Disk {
self.disk(&self.inner.default_name)
}
#[must_use]
pub fn disk_names(&self) -> Vec<&str> {
self.inner.disks.keys().map(String::as_str).collect()
}
}
impl Disk {
#[must_use]
pub fn from_operator(operator: opendal::Operator) -> Self {
Self { operator }
}
#[must_use]
pub fn operator(&self) -> &opendal::Operator {
&self.operator
}
pub async fn put(&self, path: &StoragePath, data: &[u8]) -> Result<(), StorageError> {
self.operator
.write(path.as_str(), opendal::Buffer::from(data.to_vec()))
.await?;
Ok(())
}
pub async fn get(&self, path: &StoragePath) -> Result<Bytes, StorageError> {
let buffer = self.operator.read(path.as_str()).await?;
Ok(buffer.to_bytes())
}
pub async fn delete(&self, path: &StoragePath) -> Result<(), StorageError> {
self.operator.delete(path.as_str()).await?;
Ok(())
}
pub async fn exists(&self, path: &StoragePath) -> Result<bool, StorageError> {
let exists = self.operator.exists(path.as_str()).await?;
Ok(exists)
}
pub async fn stat(&self, path: &StoragePath) -> Result<opendal::Metadata, StorageError> {
let meta = self.operator.stat(path.as_str()).await?;
Ok(meta)
}
pub async fn list(&self, path: &StoragePath) -> Result<Vec<opendal::Entry>, StorageError> {
use futures::TryStreamExt;
let lister = self.operator.lister(path.as_str()).await?;
let mut entries = Vec::new();
let mut lister = lister;
while let Some(entry) = lister.try_next().await? {
entries.push(entry);
}
Ok(entries)
}
pub async fn reader(&self, path: &StoragePath) -> Result<opendal::Reader, StorageError> {
let reader = self.operator.reader(path.as_str()).await?;
Ok(reader)
}
pub async fn writer(&self, path: &StoragePath) -> Result<opendal::Writer, StorageError> {
let writer = self.operator.writer(path.as_str()).await?;
Ok(writer)
}
pub async fn copy(&self, from: &StoragePath, to: &StoragePath) -> Result<(), StorageError> {
self.operator.copy(from.as_str(), to.as_str()).await?;
Ok(())
}
pub async fn rename(&self, from: &StoragePath, to: &StoragePath) -> Result<(), StorageError> {
self.operator.rename(from.as_str(), to.as_str()).await?;
Ok(())
}
}
#[cfg(feature = "uploads")]
pub const STAGING_PREFIX: &str = "_staging";
#[cfg(feature = "uploads")]
static STAGING_COUNTER: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
#[cfg(feature = "uploads")]
fn staging_path() -> StoragePath {
let counter = STAGING_COUNTER.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
let nanos = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_or(0, |elapsed| elapsed.as_nanos());
let pid = std::process::id();
StoragePath::new(&format!(
"{STAGING_PREFIX}/{pid:08x}-{nanos:032x}-{counter:016x}.part"
))
.expect("a hex-only key under a fixed ASCII prefix is always a valid object key")
}
#[cfg(feature = "uploads")]
impl Disk {
pub async fn begin_upload(&self) -> Result<UploadWriter, StorageError> {
self.open_upload(None).await
}
pub async fn begin_upload_under(&self, prefix: &str) -> Result<UploadWriter, StorageError> {
let prefix = prefix.trim_end_matches('/');
StoragePath::new(prefix)?;
self.open_upload(Some(prefix.to_string())).await
}
async fn open_upload(&self, prefix: Option<String>) -> Result<UploadWriter, StorageError> {
let staging = staging_path();
let writer = self.writer(&staging).await?;
Ok(UploadWriter {
disk: self.clone(),
staging,
prefix,
writer,
hasher: crate::storage::content::ContentHasher::new(),
head: Vec::new(),
})
}
}
#[cfg(feature = "uploads")]
pub struct UploadWriter {
disk: Disk,
staging: StoragePath,
prefix: Option<String>,
writer: opendal::Writer,
hasher: crate::storage::content::ContentHasher,
head: Vec<u8>,
}
#[cfg(feature = "uploads")]
impl std::fmt::Debug for UploadWriter {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("UploadWriter")
.field("staging", &self.staging)
.field("prefix", &self.prefix)
.field("byte_len", &self.hasher.byte_len())
.finish_non_exhaustive()
}
}
#[cfg(feature = "uploads")]
impl UploadWriter {
#[must_use]
pub fn staging_path(&self) -> &StoragePath {
&self.staging
}
#[must_use]
pub fn byte_len(&self) -> u64 {
self.hasher.byte_len()
}
pub async fn write(&mut self, chunk: impl Into<bytes::Bytes>) -> Result<(), StorageError> {
let chunk = chunk.into();
self.hasher.update(&chunk);
if self.head.len() < crate::storage::sniff::SNIFF_BYTES {
let room = crate::storage::sniff::SNIFF_BYTES - self.head.len();
self.head.extend_from_slice(&chunk[..chunk.len().min(room)]);
}
self.writer.write(chunk).await?;
Ok(())
}
pub async fn finish(
mut self,
extension: crate::storage::filename::Extension,
) -> Result<crate::storage::content::ContentAddress, StorageError> {
self.writer.close().await?;
let address = self.hasher.finish(extension);
let destination = match &self.prefix {
Some(prefix) => address.path_under(prefix)?,
None => address.path(),
};
if self.disk.exists(&destination).await? {
self.disk.delete(&self.staging).await?;
} else {
self.disk.rename(&self.staging, &destination).await?;
}
Ok(address)
}
pub async fn abort(mut self) -> Result<(), StorageError> {
let _ = self.writer.abort().await;
self.disk.delete(&self.staging).await?;
Ok(())
}
#[must_use]
pub fn head(&self) -> &[u8] {
&self.head
}
#[must_use]
pub fn sniffed(&self) -> Option<crate::storage::sniff::SniffedType> {
crate::storage::sniff::sniff(&self.head)
}
pub fn verify(
&self,
extension: &crate::storage::filename::Extension,
) -> Result<Option<crate::storage::sniff::SniffedType>, crate::storage::error::SniffError> {
crate::storage::sniff::verify(&self.head, extension)
}
pub async fn finish_verified(
self,
extension: crate::storage::filename::Extension,
) -> Result<crate::storage::content::ContentAddress, crate::storage::error::UploadError> {
if let Err(rejection) = self.verify(&extension) {
self.abort().await?;
return Err(rejection.into());
}
Ok(self.finish(extension).await?)
}
}
async fn build_operator(config: &StorageConfig) -> Result<opendal::Operator, opendal::Error> {
match config {
#[cfg(feature = "storage-fs")]
StorageConfig::Fs(fs) => {
let builder = fs.clone().into_builder();
Operator::new(builder)
}
#[cfg(feature = "storage-s3")]
StorageConfig::S3(s3) => {
opendal::install_default();
let builder = s3.clone().into_builder();
Operator::new(builder)
}
#[allow(unreachable_patterns)]
_ => unreachable!("no storage backend feature is enabled"),
}
}
#[cfg(any(feature = "storage-fs", feature = "storage-s3"))]
use opendal::Operator;
pub struct StorageBuilder {
disks: Vec<(String, StorageConfig)>,
default_name: Option<String>,
}
impl StorageBuilder {
#[must_use]
pub fn disk(mut self, name: impl Into<String>, config: StorageConfig) -> Self {
self.disks.push((name.into(), config));
self
}
#[must_use]
pub fn default_disk(mut self, name: impl Into<String>) -> Self {
self.default_name = Some(name.into());
self
}
pub async fn connect(self) -> Result<Storage, StorageConnectError> {
if self.disks.is_empty() {
return Err(StorageConnectError::Config {
source: crate::storage::error::StorageConfigError::EmptyRoot,
});
}
let default_name = self.default_name.unwrap_or_else(|| self.disks[0].0.clone());
let mut disks = HashMap::new();
for (name, config) in self.disks {
config.validate()?;
let operator = build_operator(&config).await?;
disks.insert(name, Disk { operator });
}
if !disks.contains_key(&default_name) {
return Err(StorageConnectError::Config {
source: crate::storage::error::StorageConfigError::EmptyRoot,
});
}
Ok(Storage {
inner: Arc::new(StorageInner {
disks,
default_name,
}),
})
}
}
#[cfg(all(test, feature = "uploads"))]
mod upload_tests {
use super::*;
use crate::storage::content::ContentAddress;
use crate::storage::filename::Extension;
fn png() -> Extension {
Extension::parse("png").expect("png is a valid extension")
}
async fn disk() -> (tempfile::TempDir, Disk) {
let root = tempfile::tempdir().expect("a temporary directory");
let config = StorageConfig::fs(root.path().to_string_lossy().into_owned())
.expect("the temporary path is a valid root");
let storage = Storage::connect(config).await.expect("the disk connects");
let disk = storage.default_disk();
(root, disk)
}
async fn count_under(disk: &Disk, prefix: &str) -> usize {
let path = StoragePath::new(prefix).expect("a valid list prefix");
match disk.list(&path).await {
Ok(entries) => entries
.iter()
.filter(|entry| !entry.path().ends_with('/'))
.count(),
Err(_) => 0,
}
}
#[tokio::test]
async fn a_streamed_upload_lands_on_its_content_addressed_key() {
let (_root, disk) = disk().await;
let mut upload = disk.begin_upload().await.expect("the upload opens");
for chunk in [&b"a"[..], b"b", b"c"] {
upload.write(chunk).await.expect("the chunk is written");
}
assert_eq!(upload.byte_len(), 3);
let address = upload.finish(png()).await.expect("the upload finishes");
assert_eq!(address, ContentAddress::of(b"abc", png()));
assert_eq!(
disk.get(&address.path())
.await
.expect("the object is there"),
bytes::Bytes::from_static(b"abc")
);
}
#[tokio::test]
async fn the_staging_object_does_not_survive_the_upload() {
let (_root, disk) = disk().await;
let mut upload = disk.begin_upload().await.expect("the upload opens");
let staging = upload.staging_path().clone();
upload
.write(&b"abc"[..])
.await
.expect("the chunk is written");
upload.finish(png()).await.expect("the upload finishes");
assert!(!disk.exists(&staging).await.expect("the check runs"));
assert_eq!(count_under(&disk, "_staging/").await, 0);
}
#[tokio::test]
async fn an_aborted_upload_leaves_nothing_behind() {
let (_root, disk) = disk().await;
let mut upload = disk.begin_upload().await.expect("the upload opens");
let staging = upload.staging_path().clone();
upload
.write(&b"partial"[..])
.await
.expect("the chunk is written");
upload.abort().await.expect("the abort succeeds");
assert!(!disk.exists(&staging).await.expect("the check runs"));
assert_eq!(count_under(&disk, "_staging/").await, 0);
}
#[tokio::test]
async fn the_same_bytes_twice_occupy_one_object() {
let (_root, disk) = disk().await;
let mut first = disk.begin_upload().await.expect("the upload opens");
first.write(&b"abc"[..]).await.expect("written");
let one = first.finish(png()).await.expect("the upload finishes");
let mut second = disk.begin_upload().await.expect("the upload opens");
second.write(&b"abc"[..]).await.expect("written");
let two = second.finish(png()).await.expect("the upload finishes");
assert_eq!(one, two);
assert_eq!(count_under(&disk, "_staging/").await, 0);
assert_eq!(
disk.get(&one.path()).await.expect("the object is there"),
bytes::Bytes::from_static(b"abc")
);
}
#[tokio::test]
async fn an_application_prefix_is_honoured() {
let (_root, disk) = disk().await;
let mut upload = disk
.begin_upload_under("avatars/")
.await
.expect("the upload opens");
upload.write(&b"abc"[..]).await.expect("written");
let address = upload.finish(png()).await.expect("the upload finishes");
let path = address.path_under("avatars").expect("a valid key");
assert!(path.as_str().starts_with("avatars/ba/78/"));
assert!(disk.exists(&path).await.expect("the check runs"));
}
#[tokio::test]
async fn a_hostile_prefix_is_refused_before_a_byte_is_written() {
let (_root, disk) = disk().await;
assert!(matches!(
disk.begin_upload_under("../../etc").await,
Err(StorageError::Path { .. })
));
assert!(matches!(
disk.begin_upload_under("/absolute").await,
Err(StorageError::Path { .. })
));
assert_eq!(count_under(&disk, "_staging/").await, 0);
}
#[tokio::test]
async fn two_uploads_in_flight_do_not_share_a_staging_key() {
let (_root, disk) = disk().await;
let first = disk.begin_upload().await.expect("the upload opens");
let second = disk.begin_upload().await.expect("the upload opens");
assert_ne!(first.staging_path(), second.staging_path());
first.abort().await.expect("the abort succeeds");
second.abort().await.expect("the abort succeeds");
}
const PNG: &[u8] = &[
0x89, b'P', b'N', b'G', 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0x0d, b'I', b'H', b'D', b'R',
];
const PHP: &[u8] = b"<?php system($_GET['c']); ?>";
#[tokio::test]
async fn a_verified_upload_of_real_png_bytes_is_kept() {
let (_root, disk) = disk().await;
let mut upload = disk.begin_upload().await.expect("the upload opens");
upload.write(PNG).await.expect("the write succeeds");
assert_eq!(
upload.sniffed().map(|kind| kind.mime()),
Some("image/png"),
"the bytes identify themselves"
);
let address = upload.finish_verified(png()).await.expect("a png is a png");
assert!(disk.exists(&address.path()).await.expect("the stat runs"));
}
#[tokio::test]
async fn a_php_script_renamed_to_png_is_refused_and_leaves_nothing() {
let (_root, disk) = disk().await;
let mut upload = disk.begin_upload().await.expect("the upload opens");
upload.write(PHP).await.expect("the write succeeds");
let error = upload
.finish_verified(png())
.await
.expect_err("a script is not an image");
assert!(matches!(
error,
crate::storage::error::UploadError::Content { .. }
));
assert_eq!(count_under(&disk, "_staging/").await, 0);
let rejected = ContentAddress::of(PHP, png());
assert!(
!disk.exists(&rejected.path()).await.expect("the stat runs"),
"the refused object must not have been promoted"
);
}
#[tokio::test]
async fn the_sniff_buffer_does_not_grow_with_the_object() {
let (_root, disk) = disk().await;
let mut upload = disk.begin_upload().await.expect("the upload opens");
upload.write(PNG).await.expect("the write succeeds");
for _ in 0..64 {
upload
.write(vec![0u8; 4096])
.await
.expect("the write succeeds");
}
assert_eq!(upload.head().len(), crate::storage::sniff::SNIFF_BYTES);
assert_eq!(upload.byte_len(), PNG.len() as u64 + 64 * 4096);
upload.abort().await.expect("the abort succeeds");
}
#[tokio::test]
async fn the_verdict_is_available_from_the_first_chunk() {
let (_root, disk) = disk().await;
let mut upload = disk.begin_upload().await.expect("the upload opens");
upload.write(&PHP[..8]).await.expect("the write succeeds");
assert!(upload.verify(&png()).is_err());
upload.abort().await.expect("the abort succeeds");
}
}