use crate::error::Result;
use crate::types::Cursor;
use async_trait::async_trait;
use bytes::Bytes;
use std::collections::BTreeMap;
use std::pin::Pin;
use std::time::{Duration, SystemTime};
use tokio::io::{AsyncRead, AsyncReadExt};
pub const MAX_OBJECT_BYTES: usize = 50 * 1024 * 1024;
pub const MAX_KEY_BYTES: usize = 1024;
pub const MAX_CONTENT_TYPE_BYTES: usize = 256;
pub const MAX_METADATA_BYTES: usize = 2048;
pub const MAX_HTTP_METADATA_BYTES: usize = 1024;
pub const MAX_MULTIPART_PARTS: u32 = 10_000;
pub const MAX_PRESIGN_EXPIRES: Duration = Duration::from_secs(7 * 24 * 60 * 60);
pub const DEFAULT_CONTENT_TYPE: &str = "application/octet-stream";
pub type BlobReader = Pin<Box<dyn AsyncRead + Send>>;
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum PutPrecondition {
CreateOnly,
MatchVersion(String),
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ConditionalGet {
Missing,
NotModified { etag: String },
Found { body: Bytes, etag: String },
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum S3Encryption {
S3Managed,
Kms { key_id: Option<String> },
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MultipartUpload {
pub key: String,
pub upload_id: String,
pub precondition: Option<PutPrecondition>,
}
impl MultipartUpload {
pub fn new(
key: impl Into<String>,
upload_id: impl Into<String>,
precondition: Option<PutPrecondition>,
) -> Self {
Self {
key: key.into(),
upload_id: upload_id.into(),
precondition,
}
}
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct MultipartPart {
pub part_number: u32,
pub etag: String,
pub size: u64,
}
impl MultipartPart {
pub fn new(part_number: u32, etag: impl Into<String>, size: u64) -> Self {
Self {
part_number,
etag: etag.into(),
size,
}
}
}
#[non_exhaustive]
#[derive(Debug, Clone, Default)]
pub struct PutOpts {
pub content_type: Option<String>,
pub metadata: BTreeMap<String, String>,
pub cache_control: Option<String>,
pub content_disposition: Option<String>,
pub checksum_sha256: Option<String>,
pub s3_encryption: Option<S3Encryption>,
pub precondition: Option<PutPrecondition>,
}
impl PutOpts {
pub fn new() -> Self {
Self::default()
}
pub fn with_content_type(mut self, ct: impl Into<String>) -> Self {
self.content_type = Some(ct.into());
self
}
pub fn with_metadata(mut self, k: impl Into<String>, v: impl Into<String>) -> Self {
self.metadata.insert(k.into(), v.into());
self
}
pub fn with_cache_control(mut self, value: impl Into<String>) -> Self {
self.cache_control = Some(value.into());
self
}
pub fn with_content_disposition(mut self, value: impl Into<String>) -> Self {
self.content_disposition = Some(value.into());
self
}
pub fn with_checksum_sha256(mut self, value: impl Into<String>) -> Self {
self.checksum_sha256 = Some(value.into());
self
}
pub fn with_s3_encryption(mut self, value: S3Encryption) -> Self {
self.s3_encryption = Some(value);
self
}
pub fn create_only(mut self) -> Self {
self.precondition = Some(PutPrecondition::CreateOnly);
self
}
pub fn match_version(mut self, etag: impl Into<String>) -> Self {
self.precondition = Some(PutPrecondition::MatchVersion(etag.into()));
self
}
}
#[non_exhaustive]
#[derive(Debug, Clone)]
pub struct BlobInfo {
pub key: String,
pub size: u64,
pub content_type: String,
pub etag: String,
pub last_modified: SystemTime,
pub metadata: BTreeMap<String, String>,
pub cache_control: Option<String>,
pub content_disposition: Option<String>,
pub checksum_sha256: Option<String>,
pub server_side_encryption: Option<String>,
}
#[non_exhaustive]
#[derive(Debug, Clone)]
pub struct BlobSummary {
pub key: String,
pub size: u64,
pub etag: String,
pub last_modified: SystemTime,
}
impl BlobSummary {
pub fn new(key: String, size: u64, etag: String, last_modified: SystemTime) -> Self {
Self {
key,
size,
etag,
last_modified,
}
}
}
impl BlobInfo {
pub fn new(
key: String,
size: u64,
content_type: String,
etag: String,
last_modified: SystemTime,
metadata: BTreeMap<String, String>,
) -> Self {
Self {
key,
size,
content_type,
etag,
last_modified,
metadata,
cache_control: None,
content_disposition: None,
checksum_sha256: None,
server_side_encryption: None,
}
}
pub fn with_storage_metadata(
mut self,
cache_control: Option<String>,
content_disposition: Option<String>,
checksum_sha256: Option<String>,
server_side_encryption: Option<String>,
) -> Self {
self.cache_control = cache_control;
self.content_disposition = content_disposition;
self.checksum_sha256 = checksum_sha256;
self.server_side_encryption = server_side_encryption;
self
}
}
#[non_exhaustive]
#[derive(Debug, Clone)]
pub struct ListPage {
pub items: Vec<BlobSummary>,
pub next: Option<Cursor>,
}
impl ListPage {
pub fn new(items: Vec<BlobSummary>, next: Option<Cursor>) -> Self {
Self { items, next }
}
}
#[non_exhaustive]
#[derive(Debug, Clone)]
pub struct ProxyPresign {
pub url: String,
pub method: String,
pub key: String,
pub expires_epoch: i64,
pub max_bytes: u64,
pub signature: String,
pub required_headers: BTreeMap<String, String>,
}
#[non_exhaustive]
#[derive(Debug, Clone)]
pub struct NativePresign {
pub url: String,
pub method: String,
pub expires_epoch: i64,
pub required_headers: BTreeMap<String, String>,
pub constraints: BTreeMap<String, String>,
}
#[async_trait]
pub trait Blob: Send + Sync {
async fn put(&self, key: &str, data: Bytes, opts: PutOpts) -> Result<()>;
async fn put_stream(
&self,
key: &str,
reader: BlobReader,
content_length: u64,
opts: PutOpts,
) -> Result<()> {
if content_length > MAX_OBJECT_BYTES as u64 {
return Err(crate::error::ForgeError::limit(
"this backend only supports streaming objects up to the 50 MiB buffered limit",
));
}
let capacity = usize::try_from(content_length)
.map_err(|_| crate::error::ForgeError::limit("object length exceeds this platform"))?;
let mut body = Vec::with_capacity(capacity);
reader
.take(content_length.saturating_add(1))
.read_to_end(&mut body)
.await
.map_err(|error| {
crate::error::ForgeError::backend_with(
"could not read blob input stream",
false,
error,
)
})?;
if body.len() as u64 != content_length {
return Err(crate::error::ForgeError::invalid(
"blob stream length does not match content_length",
));
}
self.put(key, Bytes::from(body), opts).await
}
async fn get(&self, key: &str) -> Result<Option<Bytes>>;
async fn get_if(
&self,
key: &str,
if_match: Option<&str>,
if_none_match: Option<&str>,
) -> Result<ConditionalGet> {
crate::blob::common::check_get_conditions(if_match, if_none_match)?;
let Some(info) = self.head(key).await? else {
return Ok(ConditionalGet::Missing);
};
if if_match.is_some_and(|expected| expected != info.etag) {
return Err(crate::ForgeError::precondition(
"blob read version does not match",
));
}
if if_none_match.is_some_and(|version| version == info.etag) {
return Ok(ConditionalGet::NotModified { etag: info.etag });
}
let body = self.get(key).await?.ok_or_else(|| {
crate::ForgeError::precondition("blob changed during conditional read")
})?;
Ok(ConditionalGet::Found {
body,
etag: info.etag,
})
}
async fn open(&self, key: &str) -> Result<Option<BlobReader>> {
Ok(self
.get(key)
.await?
.map(|body| Box::pin(std::io::Cursor::new(body)) as BlobReader))
}
async fn get_range(&self, key: &str, start: u64, end: u64) -> Result<Option<Bytes>> {
if end < start {
return Err(crate::error::ForgeError::invalid(
"range end must be greater than or equal to start",
));
}
let Some(body) = self.get(key).await? else {
return Ok(None);
};
if start >= body.len() as u64 {
return Err(crate::error::ForgeError::precondition(
"range starts beyond the object",
));
}
let last = end.min(body.len() as u64 - 1);
let start = usize::try_from(start)
.map_err(|_| crate::error::ForgeError::limit("range exceeds this platform"))?;
let end = usize::try_from(last)
.map_err(|_| crate::error::ForgeError::limit("range exceeds this platform"))?;
Ok(Some(body.slice(start..=end)))
}
async fn head(&self, key: &str) -> Result<Option<BlobInfo>>;
async fn delete(&self, key: &str) -> Result<()>;
async fn list(&self, prefix: &str, cursor: Option<Cursor>, limit: u32) -> Result<ListPage>;
async fn copy(&self, source: &str, destination: &str, mut opts: PutOpts) -> Result<BlobInfo> {
let source_info = self
.head(source)
.await?
.ok_or(crate::ForgeError::NotFound)?;
let reader = self
.open(source)
.await?
.ok_or(crate::ForgeError::NotFound)?;
if opts.content_type.is_none() {
opts.content_type = Some(source_info.content_type);
}
if opts.metadata.is_empty() {
opts.metadata = source_info.metadata;
}
if opts.cache_control.is_none() {
opts.cache_control = source_info.cache_control;
}
if opts.content_disposition.is_none() {
opts.content_disposition = source_info.content_disposition;
}
self.put_stream(destination, reader, source_info.size, opts)
.await?;
self.head(destination)
.await?
.ok_or_else(|| crate::ForgeError::backend("copied blob is not readable"))
}
async fn create_multipart(&self, _key: &str, _opts: PutOpts) -> Result<MultipartUpload> {
Err(crate::ForgeError::not_configured(
"multipart handles require the S3 blob backend",
))
}
async fn upload_part(
&self,
_upload: &MultipartUpload,
_part_number: u32,
_body: Bytes,
) -> Result<MultipartPart> {
Err(crate::ForgeError::not_configured(
"multipart handles require the S3 blob backend",
))
}
async fn complete_multipart(
&self,
_upload: &MultipartUpload,
_parts: Vec<MultipartPart>,
) -> Result<BlobInfo> {
Err(crate::ForgeError::not_configured(
"multipart handles require the S3 blob backend",
))
}
async fn abort_multipart(&self, _upload: &MultipartUpload) -> Result<()> {
Err(crate::ForgeError::not_configured(
"multipart handles require the S3 blob backend",
))
}
async fn verify_checksum_sha256(&self, key: &str, expected_hex: &str) -> Result<bool> {
crate::blob::common::check_sha256(expected_hex)?;
let Some(mut reader) = self.open(key).await? else {
return Err(crate::ForgeError::NotFound);
};
use sha2::Digest as _;
let mut hasher = sha2::Sha256::new();
let mut buffer = vec![0u8; 64 * 1024];
loop {
let read = reader.read(&mut buffer).await.map_err(|error| {
crate::ForgeError::backend_with("could not read blob for checksum", false, error)
})?;
if read == 0 {
break;
}
let chunk = buffer.get(..read).ok_or_else(|| {
crate::ForgeError::backend("blob reader returned an invalid byte count")
})?;
hasher.update(chunk);
}
Ok(crate::util::hex(&hasher.finalize()) == expected_hex)
}
async fn presign_upload(
&self,
key: &str,
expires: Duration,
max_bytes: u64,
) -> Result<ProxyPresign>;
async fn presign_download(&self, key: &str, expires: Duration) -> Result<ProxyPresign>;
async fn presign_native_get(&self, _key: &str, _expires: Duration) -> Result<NativePresign> {
Err(crate::error::ForgeError::not_configured(
"native presigning requires the S3 blob backend",
))
}
async fn presign_native_put(
&self,
_key: &str,
_expires: Duration,
_opts: PutOpts,
) -> Result<NativePresign> {
Err(crate::error::ForgeError::not_configured(
"native presigning requires the S3 blob backend",
))
}
async fn verify_presigned(
&self,
method: &str,
key: &str,
expires_epoch: i64,
max_bytes: u64,
sig: &str,
) -> Result<bool>;
}
pub(crate) mod common;
pub(crate) mod sign;
mod fs;
mod memory;
mod postgres;
mod s3;
pub(crate) use fs::FsBlob;
pub(crate) use memory::MemBlob;
pub(crate) use postgres::PgBlob;
pub(crate) use s3::S3Blob;