use std::{
fmt,
pin::Pin,
sync::atomic::{AtomicU64, Ordering},
time::{SystemTime, UNIX_EPOCH},
};
use bytes::Bytes;
use futures_util::Stream;
use object_store::WriteMultipart;
use crate::ObjectKey;
use super::S3ObjectStoreError;
pub type S3ByteStream = Pin<Box<dyn Stream<Item = Result<Bytes, S3ObjectStoreError>> + Send>>;
pub enum BeginMultipartUploadResult {
AlreadyExists,
Upload(S3MultipartUploadWriter, ObjectKey),
}
pub struct S3MultipartUploadWriter {
pub(crate) writer: WriteMultipart,
}
impl S3MultipartUploadWriter {
pub fn write(&mut self, bytes: &[u8]) {
self.writer.write(bytes);
}
pub async fn wait_for_capacity(
&mut self,
max_in_flight_parts: usize,
) -> Result<(), S3ObjectStoreError> {
self.writer
.wait_for_capacity(max_in_flight_parts)
.await
.map_err(S3ObjectStoreError::External)
}
pub async fn finish(self) -> Result<(), S3ObjectStoreError> {
self.writer
.finish()
.await
.map(|_result| ())
.map_err(S3ObjectStoreError::External)
}
pub async fn abort(self) -> Result<(), S3ObjectStoreError> {
self.writer
.abort()
.await
.map_err(S3ObjectStoreError::External)
}
}
impl fmt::Debug for S3MultipartUploadWriter {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("S3MultipartUploadWriter")
.finish_non_exhaustive()
}
}
pub(crate) const STREAM_UPLOAD_CHUNK_BYTES: usize = 8 * 1024 * 1024;
pub(crate) const STREAM_COMPARE_CHUNK_BYTES: usize = 256 * 1024;
pub(crate) const MAX_SINGLE_COPY_BYTES: u64 = 5 * 1024 * 1024 * 1024;
pub(crate) const LARGE_COPY_CHUNK_BYTES: u64 = 64 * 1024 * 1024; pub(crate) static TEMP_UPLOAD_COUNTER: AtomicU64 = AtomicU64::new(0);
pub(crate) fn temp_key_for(key: &ObjectKey) -> Result<ObjectKey, S3ObjectStoreError> {
let counter = TEMP_UPLOAD_COUNTER.fetch_add(1, Ordering::Relaxed);
let now_nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_nanos();
let pid = std::process::id();
let suffix = format!("tmp.{counter}.{pid}.{now_nanos}");
ObjectKey::parse(&format!("{}.{suffix}", key.as_str()))
.map_err(|_err| S3ObjectStoreError::InvalidListedKey)
}
pub(crate) fn is_temp_upload_key(key: &str) -> bool {
key.find(".tmp.").is_some_and(|pos| {
key.as_bytes()
.get(pos.saturating_add(5))
.is_some_and(|b| b.is_ascii_digit())
})
}
pub(crate) fn normalize_prefix(value: &str) -> Option<String> {
let trimmed = value.trim_matches('/');
if trimmed.is_empty() {
None
} else {
Some(trimmed.to_owned())
}
}