#[cfg(unix)]
use std::os::unix::fs::PermissionsExt;
use std::{
collections::HashMap,
io::Cursor,
path::{Path, PathBuf},
sync::{Arc, RwLock},
};
use tokio::{
fs::{self, File},
io::{AsyncRead, AsyncReadExt, AsyncWriteExt},
};
#[cfg(feature = "s3")]
mod s3;
#[derive(Clone, Debug, Eq, Hash, PartialEq)]
pub struct StorageKey(String);
impl StorageKey {
pub fn new(value: impl Into<String>) -> Result<Self> {
let value = value.into();
if value.is_empty()
|| value.len() > 1024
|| value.split('/').any(|segment| {
segment.is_empty()
|| segment == "."
|| segment == ".."
|| segment == ".capy-tmp"
|| segment.len() > 255
|| !segment
.bytes()
.all(|c| c.is_ascii_alphanumeric() || b"_-.".contains(&c))
})
{
return Err(StorageError::InvalidKey);
}
Ok(Self(value))
}
pub fn as_str(&self) -> &str {
&self.0
}
}
impl TryFrom<&str> for StorageKey {
type Error = StorageError;
fn try_from(value: &str) -> Result<Self> {
Self::new(value)
}
}
#[derive(Debug, thiserror::Error)]
pub enum StorageError {
#[error("Invalid storage key")]
InvalidKey,
#[error("Storage object not found")]
NotFound,
#[error("Storage object exceeds configured size limit")]
TooLarge,
#[error("Unsafe local storage path")]
UnsafePath,
#[error("Storage I/O failed")]
Io(#[source] std::io::Error),
#[error("Temporary download URLs are unsupported by this backend")]
Unsupported,
#[error("Invalid storage configuration: {0}")]
Configuration(&'static str),
#[error("S3 storage operation failed")]
Backend,
}
impl From<std::io::Error> for StorageError {
fn from(error: std::io::Error) -> Self {
Self::Io(error)
}
}
impl From<StorageError> for capy_core::AppError {
fn from(error: StorageError) -> Self {
match error {
StorageError::InvalidKey => Self::BadRequest("Invalid storage key.".into()),
StorageError::NotFound => Self::NotFound,
StorageError::TooLarge => Self::PayloadTooLarge,
StorageError::Configuration(message) => Self::Configuration(message.into()),
StorageError::Unsupported => {
Self::Configuration("Storage backend does not support temporary URLs.".into())
}
StorageError::UnsafePath | StorageError::Io(_) | StorageError::Backend => {
Self::Lifecycle
}
}
}
}
pub type Result<T> = std::result::Result<T, StorageError>;
#[derive(Clone, Debug, Eq, PartialEq)]
pub struct ObjectMetadata {
pub length: u64,
pub content_type: Option<String>,
}
pub struct StoredObject {
pub reader: Box<dyn AsyncRead + Send + Unpin>,
pub metadata: ObjectMetadata,
}
struct MemoryObject {
bytes: Vec<u8>,
content_type: Option<String>,
}
enum Backend {
Memory(RwLock<HashMap<StorageKey, MemoryObject>>),
Local(PathBuf),
#[cfg(feature = "s3")]
S3(s3::S3),
}
#[derive(Clone)]
pub struct Storage {
backend: Arc<Backend>,
max_object_bytes: u64,
}
impl Storage {
pub fn from_config(settings: &capy_core::StorageSettings) -> Result<Option<Self>> {
match settings.driver.as_str() {
"none" => Ok(None),
"memory" => Self::memory(settings.max_object_bytes).map(Some),
"local" => Self::local(&settings.root, settings.max_object_bytes).map(Some),
"s3" => {
#[cfg(feature = "s3")]
{
let backend = s3::S3::new(settings)?;
Ok(Some(Self {
backend: Arc::new(Backend::S3(backend)),
max_object_bytes: settings.max_object_bytes,
}))
}
#[cfg(not(feature = "s3"))]
{
Err(StorageError::Configuration(
"S3 support is not enabled in this build",
))
}
}
_ => Err(StorageError::Configuration("unknown storage driver")),
}
}
pub fn memory(max_object_bytes: u64) -> Result<Self> {
validate_limit(max_object_bytes)?;
Ok(Self {
backend: Arc::new(Backend::Memory(RwLock::new(HashMap::new()))),
max_object_bytes,
})
}
pub fn local(root: impl AsRef<Path>, max_object_bytes: u64) -> Result<Self> {
validate_limit(max_object_bytes)?;
let root = root.as_ref();
std::fs::create_dir_all(root)?;
let root = root.canonicalize()?;
if !root.is_dir() {
return Err(StorageError::Configuration(
"local root must be a directory",
));
}
let temporary = root.join(".capy-tmp");
reject_symlink(&temporary)?;
std::fs::create_dir_all(&temporary)?;
reject_symlink(&temporary)?;
#[cfg(unix)]
{
std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700))?;
std::fs::set_permissions(&temporary, std::fs::Permissions::from_mode(0o700))?;
}
Ok(Self {
backend: Arc::new(Backend::Local(root)),
max_object_bytes,
})
}
pub async fn put<R: AsyncRead + Unpin + Send>(
&self,
key: &StorageKey,
reader: &mut R,
content_type: Option<&str>,
) -> Result<ObjectMetadata> {
if content_type.is_some_and(|value| value.len() > 255 || value.contains(['\r', '\n'])) {
return Err(StorageError::Configuration("invalid content type hint"));
}
let metadata = ObjectMetadata {
length: 0,
content_type: content_type.map(str::to_owned),
};
match self.backend.as_ref() {
Backend::Memory(objects) => {
let mut bytes = Vec::new();
let length = read_bounded(reader, &mut bytes, self.max_object_bytes).await?;
objects.write().unwrap().insert(
key.clone(),
MemoryObject {
bytes,
content_type: metadata.content_type.clone(),
},
);
Ok(ObjectMetadata { length, ..metadata })
}
Backend::Local(root) => {
let parent = prepare_parent(root, key).await?;
let destination = parent.join(key.as_str().rsplit('/').next().unwrap());
reject_symlink(&destination)?;
let temporary = tempfile::NamedTempFile::new_in(root.join(".capy-tmp"))?;
let path = temporary.path().to_owned();
let mut file = File::from_std(temporary.reopen()?);
let length = read_bounded(reader, &mut file, self.max_object_bytes).await?;
file.flush().await?;
file.sync_all().await?;
drop(file);
let current_parent = prepare_parent(root, key).await?;
if current_parent != parent {
return Err(StorageError::UnsafePath);
}
reject_symlink(&destination)?;
fs::rename(path, destination).await?;
Ok(ObjectMetadata { length, ..metadata })
}
#[cfg(feature = "s3")]
Backend::S3(backend) => {
backend
.put(key, reader, content_type, self.max_object_bytes)
.await
}
}
}
pub async fn put_bytes(
&self,
key: &StorageKey,
bytes: &[u8],
content_type: Option<&str>,
) -> Result<ObjectMetadata> {
if bytes.len() as u64 > self.max_object_bytes {
return Err(StorageError::TooLarge);
}
let mut reader = Cursor::new(bytes.to_vec());
self.put(key, &mut reader, content_type).await
}
pub async fn get(&self, key: &StorageKey) -> Result<StoredObject> {
match self.backend.as_ref() {
Backend::Memory(objects) => {
let objects = objects.read().unwrap();
let object = objects.get(key).ok_or(StorageError::NotFound)?;
Ok(StoredObject {
reader: Box::new(Cursor::new(object.bytes.clone())),
metadata: ObjectMetadata {
length: object.bytes.len() as u64,
content_type: object.content_type.clone(),
},
})
}
Backend::Local(root) => {
let path = object_path(root, key)?;
let file = File::open(path).await.map_err(missing)?;
let metadata = file.metadata().await?;
if !metadata.is_file() {
return Err(StorageError::UnsafePath);
}
Ok(StoredObject {
reader: Box::new(file),
metadata: ObjectMetadata {
length: metadata.len(),
content_type: None,
},
})
}
#[cfg(feature = "s3")]
Backend::S3(backend) => backend.get(key).await,
}
}
pub async fn read(&self, key: &StorageKey) -> Result<Vec<u8>> {
let mut object = self.get(key).await?;
if object.metadata.length > self.max_object_bytes {
return Err(StorageError::TooLarge);
}
let mut bytes = Vec::new();
read_bounded(&mut object.reader, &mut bytes, self.max_object_bytes).await?;
Ok(bytes)
}
pub async fn head(&self, key: &StorageKey) -> Result<Option<ObjectMetadata>> {
#[cfg(feature = "s3")]
if let Backend::S3(backend) = self.backend.as_ref() {
return backend.head(key).await;
}
match self.get(key).await {
Ok(object) => Ok(Some(object.metadata)),
Err(StorageError::NotFound) => Ok(None),
Err(error) => Err(error),
}
}
pub async fn delete(&self, key: &StorageKey) -> Result<()> {
match self.backend.as_ref() {
Backend::Memory(objects) => {
objects.write().unwrap().remove(key);
Ok(())
}
Backend::Local(root) => {
let path = object_path(root, key)?;
match fs::remove_file(path).await {
Ok(()) => Ok(()),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(error) => Err(error.into()),
}
}
#[cfg(feature = "s3")]
Backend::S3(backend) => backend.delete(key).await,
}
}
pub async fn temporary_get_url(
&self,
key: &StorageKey,
duration: std::time::Duration,
) -> Result<String> {
match self.backend.as_ref() {
#[cfg(feature = "s3")]
Backend::S3(backend) => backend.temporary_get_url(key, duration),
_ => {
let _ = (key, duration);
Err(StorageError::Unsupported)
}
}
}
}
fn validate_limit(limit: u64) -> Result<()> {
if !(1..=1_073_741_824).contains(&limit) {
return Err(StorageError::Configuration(
"max_object_bytes must be 1..=1073741824",
));
}
Ok(())
}
async fn read_bounded<R: AsyncRead + Unpin, W: tokio::io::AsyncWrite + Unpin>(
reader: &mut R,
writer: &mut W,
limit: u64,
) -> Result<u64> {
let mut buffer = [0u8; 16 * 1024];
let mut total = 0;
loop {
let count = reader.read(&mut buffer).await?;
if count == 0 {
return Ok(total);
}
total += count as u64;
if total > limit {
return Err(StorageError::TooLarge);
}
writer.write_all(&buffer[..count]).await?;
}
}
async fn prepare_parent(root: &Path, key: &StorageKey) -> Result<PathBuf> {
let mut parent = root.to_owned();
let segments = key.as_str().split('/').collect::<Vec<_>>();
for segment in &segments[..segments.len() - 1] {
parent.push(segment);
reject_symlink(&parent)?;
fs::create_dir(&parent).await.or_else(|error| {
if error.kind() == std::io::ErrorKind::AlreadyExists {
Ok(())
} else {
Err(error)
}
})?;
reject_symlink(&parent)?;
#[cfg(unix)]
fs::set_permissions(&parent, std::fs::Permissions::from_mode(0o700)).await?;
}
let canonical = parent.canonicalize()?;
if !canonical.starts_with(root) {
return Err(StorageError::UnsafePath);
}
Ok(canonical)
}
fn object_path(root: &Path, key: &StorageKey) -> Result<PathBuf> {
let mut path = root.to_owned();
for segment in key.as_str().split('/') {
path.push(segment);
reject_symlink(&path)?;
}
Ok(path)
}
fn reject_symlink(path: &Path) -> Result<()> {
match std::fs::symlink_metadata(path) {
Ok(metadata) if metadata.file_type().is_symlink() => Err(StorageError::UnsafePath),
Ok(_) => Ok(()),
Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(error) => Err(error.into()),
}
}
fn missing(error: std::io::Error) -> StorageError {
if error.kind() == std::io::ErrorKind::NotFound {
StorageError::NotFound
} else {
StorageError::Io(error)
}
}
#[cfg(test)]
mod tests {
use super::*;
fn key(value: &str) -> StorageKey {
StorageKey::new(value).unwrap()
}
#[test]
fn keys_reject_traversal_and_path_confusion() {
for value in [
"",
"/absolute",
"../escape",
"a/../b",
"a//b",
"a/./b",
".capy-tmp/object",
"a\\b",
"a\0b",
"a%2fb",
] {
assert!(StorageKey::new(value).is_err(), "{value:?}");
}
assert_eq!(key("avatars/user-1.png").as_str(), "avatars/user-1.png");
}
async fn lifecycle(storage: Storage) {
let object = key("avatars/user-1.png");
assert!(storage.head(&object).await.unwrap().is_none());
assert_eq!(
storage
.put_bytes(&object, b"hello", Some("image/png"))
.await
.unwrap()
.length,
5
);
assert_eq!(storage.read(&object).await.unwrap(), b"hello");
assert_eq!(storage.head(&object).await.unwrap().unwrap().length, 5);
assert!(matches!(
storage
.temporary_get_url(&object, std::time::Duration::from_secs(60))
.await,
Err(StorageError::Unsupported)
));
storage.delete(&object).await.unwrap();
storage.delete(&object).await.unwrap();
assert!(storage.head(&object).await.unwrap().is_none());
}
#[tokio::test]
async fn memory_is_isolated_and_bounded() {
lifecycle(Storage::memory(16).unwrap()).await;
let first = Storage::memory(4).unwrap();
let second = Storage::memory(4).unwrap();
let object = key("export.txt");
first.put_bytes(&object, b"1234", None).await.unwrap();
assert!(matches!(
first.put_bytes(&object, b"12345", None).await,
Err(StorageError::TooLarge)
));
assert_eq!(first.read(&object).await.unwrap(), b"1234");
assert!(second.head(&object).await.unwrap().is_none());
}
#[tokio::test]
async fn local_is_private_atomic_and_rejects_symlinks() {
let root = tempfile::tempdir().unwrap();
let storage = Storage::local(root.path(), 8).unwrap();
#[cfg(unix)]
assert_eq!(
std::fs::metadata(root.path()).unwrap().permissions().mode() & 0o777,
0o700
);
lifecycle(storage.clone()).await;
let object = key("a/file.txt");
storage.put_bytes(&object, b"good", None).await.unwrap();
#[cfg(unix)]
assert_eq!(
std::fs::metadata(root.path().join("a"))
.unwrap()
.permissions()
.mode()
& 0o777,
0o700
);
assert!(matches!(
storage.put_bytes(&object, b"too-long!", None).await,
Err(StorageError::TooLarge)
));
assert_eq!(storage.read(&object).await.unwrap(), b"good");
#[cfg(unix)]
{
let outside = tempfile::tempdir().unwrap();
std::os::unix::fs::symlink(outside.path(), root.path().join("escape")).unwrap();
assert!(matches!(
storage.put_bytes(&key("escape/file"), b"x", None).await,
Err(StorageError::UnsafePath)
));
assert!(matches!(
storage.get(&key("escape/file")).await,
Err(StorageError::UnsafePath)
));
let unsafe_root = tempfile::tempdir().unwrap();
std::os::unix::fs::symlink(outside.path(), unsafe_root.path().join(".capy-tmp"))
.unwrap();
assert!(matches!(
Storage::local(unsafe_root.path(), 8),
Err(StorageError::UnsafePath)
));
}
}
}