use crate::{BlobStore, CasOutcome, MediaOpStats, ObjectLogError};
use async_trait::async_trait;
use aws_sdk_s3::Client;
use aws_sdk_s3::config::{
BehaviorVersion, Credentials, Region, RequestChecksumCalculation, ResponseChecksumValidation,
timeout::TimeoutConfig,
};
use aws_sdk_s3::error::SdkError;
use aws_sdk_s3::operation::put_object::PutObjectError;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CompletedMultipartUpload, CompletedPart};
use bytes::{Bytes, BytesMut};
use std::ops::Range;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;
mod authority;
#[derive(Debug)]
struct ResponseStatus {
status: std::sync::Arc<std::sync::atomic::AtomicU16>,
range: Range<u64>,
}
impl aws_sdk_s3::config::Intercept for ResponseStatus {
fn name(&self) -> &'static str {
"StrictRangeStatus"
}
fn read_before_transmit(
&self,
_context: &aws_sdk_s3::config::interceptors::BeforeTransmitInterceptorContextRef<'_>,
_runtime: &aws_sdk_s3::config::RuntimeComponents,
_config: &mut aws_sdk_s3::config::ConfigBag,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
self.status.store(0, Ordering::Relaxed);
Ok(())
}
fn read_before_deserialization(
&self,
context: &aws_sdk_s3::config::interceptors::BeforeDeserializationInterceptorContextRef<'_>,
_runtime: &aws_sdk_s3::config::RuntimeComponents,
_config: &mut aws_sdk_s3::config::ConfigBag,
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
let response = context.response();
let status = response.status().as_u16();
self.status.store(status, Ordering::Relaxed);
if status != 206 {
return Err(Box::new(unavailable(
"strict range response status is not 206",
)));
}
let length = response
.headers()
.get("content-length")
.and_then(|value| value.parse::<u64>().ok());
if length != Some(self.range.end - self.range.start) {
return Err(Box::new(unavailable(
"strict range response length differs from request",
)));
}
validate_content_range(response.headers().get("content-range"), &self.range)?;
Ok(())
}
}
fn validate_content_range(header: Option<&str>, range: &Range<u64>) -> Result<(), ObjectLogError> {
let invalid = || ObjectLogError::RangeOutOfBounds("invalid strict Content-Range".into());
let raw = header
.and_then(|header| header.strip_prefix("bytes "))
.ok_or_else(invalid)?;
let (bounds, total) = raw.split_once('/').ok_or_else(invalid)?;
let (start, end) = bounds.split_once('-').ok_or_else(invalid)?;
let number = |raw: &str| -> Result<u64, ObjectLogError> {
if raw.is_empty() || !raw.bytes().all(|byte| byte.is_ascii_digit()) {
return Err(invalid());
}
raw.parse().map_err(|_| invalid())
};
if number(start)? != range.start
|| number(end)?.checked_add(1) != Some(range.end)
|| number(total)? < range.end
{
return Err(invalid());
}
Ok(())
}
fn unavailable<E: std::fmt::Display>(e: E) -> ObjectLogError {
ObjectLogError::StorageUnavailable(e.to_string())
}
fn s3_precondition_failed(err: &SdkError<PutObjectError>) -> bool {
err.raw_response()
.is_some_and(|response| response.status().as_u16() == 412)
|| err.as_service_error().is_some_and(|service| {
matches!(
service.meta().code(),
Some("PreconditionFailed" | "ConditionalRequestConflict")
)
})
}
fn transient_s3(err: &impl std::fmt::Display) -> bool {
let text = err.to_string().to_ascii_lowercase();
text.contains("dispatch failure")
|| text.contains("service error")
|| text.contains("connection reset")
|| text.contains("broken pipe")
|| text.contains("connection refused")
|| text.contains("temporarily unavailable")
|| text.contains("slowdown")
}
fn legacy_s3_client(
endpoint_url: &str,
region: &str,
access_key_id: &str,
secret_access_key: &str,
) -> Client {
build_s3_client(
endpoint_url,
region,
access_key_id,
secret_access_key,
Duration::from_secs(env_u64("OBJECT_LOG_S3_CONNECT_TIMEOUT_SECS", 5)),
Duration::from_secs(env_u64("OBJECT_LOG_S3_READ_TIMEOUT_SECS", 10)),
Duration::from_secs(env_u64("OBJECT_LOG_S3_OPERATION_TIMEOUT_SECS", 30)),
)
}
mod owned_client;
use owned_client::OwnedClientConfiguration;
pub use owned_client::S3ClientOptions;
fn s3_client_config(
endpoint_url: &str,
region: &str,
access_key_id: &str,
secret_access_key: &str,
connect: Duration,
read: Duration,
operation: Duration,
) -> aws_sdk_s3::config::Builder {
let creds = Credentials::new(access_key_id, secret_access_key, None, None, "object-log");
aws_sdk_s3::config::Builder::new()
.behavior_version(BehaviorVersion::latest())
.endpoint_url(endpoint_url)
.region(Region::new(region.to_string()))
.credentials_provider(creds)
.force_path_style(true)
.request_checksum_calculation(RequestChecksumCalculation::WhenRequired)
.response_checksum_validation(ResponseChecksumValidation::WhenRequired)
.timeout_config(
TimeoutConfig::builder()
.connect_timeout(connect)
.read_timeout(read)
.operation_timeout(operation)
.build(),
)
}
fn build_s3_client(
endpoint: &str,
region: &str,
key: &str,
secret: &str,
connect: Duration,
read: Duration,
operation: Duration,
) -> Client {
Client::from_conf(
s3_client_config(endpoint, region, key, secret, connect, read, operation).build(),
)
}
pub struct S3BlobStore {
authority_identity: std::sync::Arc<()>,
#[cfg(test)]
authority_lost_reply: std::sync::Mutex<Option<(String, std::sync::Arc<AtomicU64>)>>,
client: Client,
_owned_configuration: Option<OwnedClientConfiguration>,
bucket: String,
multipart_threshold: usize,
part_size: usize,
disable_payload_signing: bool,
media_ops: AtomicU64,
bytes_written: AtomicU64,
}
impl S3BlobStore {
pub fn new(
endpoint_url: &str,
region: &str,
bucket: &str,
access_key_id: &str,
secret_access_key: &str,
) -> Self {
Self {
authority_identity: std::sync::Arc::new(()),
#[cfg(test)]
authority_lost_reply: std::sync::Mutex::new(None),
client: legacy_s3_client(endpoint_url, region, access_key_id, secret_access_key),
_owned_configuration: None,
bucket: bucket.to_string(),
multipart_threshold: 16 * 1024 * 1024,
part_size: 8 * 1024 * 1024,
disable_payload_signing: env_flag("OBJECT_LOG_S3_DISABLE_PAYLOAD_SIGNING"),
media_ops: AtomicU64::new(0),
bytes_written: AtomicU64::new(0),
}
}
pub fn new_owned_with_options(
endpoint_url: &str,
region: &str,
bucket: &str,
access_key_id: &str,
secret_access_key: &str,
options: S3ClientOptions,
) -> Result<Self, ObjectLogError> {
let (client, admitted) = owned_client::build_owned(
endpoint_url,
region,
bucket,
access_key_id,
secret_access_key,
&options,
)?;
Ok(Self {
authority_identity: std::sync::Arc::new(()),
#[cfg(test)]
authority_lost_reply: std::sync::Mutex::new(None),
client,
_owned_configuration: Some(admitted),
bucket: bucket.to_string(),
multipart_threshold: 16 * 1024 * 1024,
part_size: 8 * 1024 * 1024,
disable_payload_signing: options.disable_payload_signing,
media_ops: AtomicU64::new(0),
bytes_written: AtomicU64::new(0),
})
}
#[cfg(any(test, feature = "native"))]
pub(crate) fn native_page_source_scratch(
&self,
key_len: usize,
version_bytes: usize,
) -> Result<usize, ObjectLogError> {
Self::planned_native_source_scratch(key_len, version_bytes, self.bucket.capacity())
}
#[cfg(any(test, feature = "native"))]
pub(crate) fn planned_native_source_scratch(
key_len: usize,
version_bytes: usize,
bucket_capacity: usize,
) -> Result<usize, ObjectLogError> {
key_len
.checked_mul(8)
.and_then(|n| {
version_bytes
.checked_mul(8)
.and_then(|tokens| n.checked_add(tokens))
})
.and_then(|n| {
bucket_capacity
.checked_mul(8)
.and_then(|bucket| n.checked_add(bucket))
})
.and_then(|n| n.checked_add(4096))
.ok_or_else(|| {
ObjectLogError::InvalidConfig("native S3 source capacity overflow".into())
})
}
fn record_media(&self, ops: u64, bytes: u64) {
self.media_ops.fetch_add(ops, Ordering::Relaxed);
self.bytes_written.fetch_add(bytes, Ordering::Relaxed);
}
pub fn with_multipart(mut self, threshold: usize, part_size: usize) -> Self {
self.multipart_threshold = threshold;
self.part_size = part_size.max(5 * 1024 * 1024); self
}
pub fn with_payload_signing_disabled(mut self, disabled: bool) -> Self {
self.disable_payload_signing = disabled;
self
}
async fn put_multipart(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
let created = self
.client
.create_multipart_upload()
.bucket(&self.bucket)
.key(key)
.send()
.await
.map_err(unavailable)?;
let upload_id = created
.upload_id()
.ok_or_else(|| ObjectLogError::StorageUnavailable("missing upload id".into()))?
.to_string();
let byte_len = value.len() as u64;
match self.upload_parts(key, &upload_id, &value).await {
Ok(part_count) => {
self.record_media(2 + part_count, byte_len);
Ok(())
}
Err(e) => {
let _ = self
.client
.abort_multipart_upload()
.bucket(&self.bucket)
.key(key)
.upload_id(&upload_id)
.send()
.await;
self.record_media(2, 0);
Err(e)
}
}
}
async fn upload_parts(
&self,
key: &str,
upload_id: &str,
value: &Bytes,
) -> Result<u64, ObjectLogError> {
let mut parts = Vec::new();
let mut offset = 0usize;
let mut part_number = 1i32;
while offset < value.len() {
let end = (offset + self.part_size).min(value.len());
let chunk = value.slice(offset..end);
let request = self
.client
.upload_part()
.bucket(&self.bucket)
.key(key)
.upload_id(upload_id)
.part_number(part_number)
.body(ByteStream::from(chunk));
let uploaded = if self.disable_payload_signing {
request.customize().disable_payload_signing().send().await
} else {
request.send().await
}
.map_err(unavailable)?;
parts.push(
CompletedPart::builder()
.set_e_tag(uploaded.e_tag().map(str::to_string))
.part_number(part_number)
.build(),
);
part_number += 1;
offset = end;
}
let part_count = parts.len() as u64;
let completed = CompletedMultipartUpload::builder()
.set_parts(Some(parts))
.build();
self.client
.complete_multipart_upload()
.bucket(&self.bucket)
.key(key)
.upload_id(upload_id)
.multipart_upload(completed)
.send()
.await
.map_err(unavailable)?;
Ok(part_count)
}
async fn upload_one_part(
&self,
key: &str,
upload_id: &str,
part_number: i32,
chunk: Bytes,
) -> Result<CompletedPart, ObjectLogError> {
let request = self
.client
.upload_part()
.bucket(&self.bucket)
.key(key)
.upload_id(upload_id)
.part_number(part_number)
.body(ByteStream::from(chunk));
let uploaded = if self.disable_payload_signing {
request.customize().disable_payload_signing().send().await
} else {
request.send().await
}
.map_err(unavailable)?;
Ok(CompletedPart::builder()
.set_e_tag(uploaded.e_tag().map(str::to_string))
.part_number(part_number)
.build())
}
async fn put_multipart_chunks(
&self,
key: &str,
chunks: Vec<Bytes>,
) -> Result<(), ObjectLogError> {
let created = self
.client
.create_multipart_upload()
.bucket(&self.bucket)
.key(key)
.send()
.await
.map_err(unavailable)?;
let upload_id = created
.upload_id()
.ok_or_else(|| ObjectLogError::StorageUnavailable("missing upload id".into()))?
.to_string();
let byte_len: u64 = chunks.iter().map(|c| c.len() as u64).sum();
match self.upload_chunk_parts(key, &upload_id, chunks).await {
Ok(part_count) => {
self.record_media(2 + part_count, byte_len);
Ok(())
}
Err(e) => {
let _ = self
.client
.abort_multipart_upload()
.bucket(&self.bucket)
.key(key)
.upload_id(&upload_id)
.send()
.await;
self.record_media(2, 0);
Err(e)
}
}
}
async fn upload_chunk_parts(
&self,
key: &str,
upload_id: &str,
chunks: Vec<Bytes>,
) -> Result<u64, ObjectLogError> {
let mut parts = Vec::new();
let mut part_number = 1i32;
let mut pending = BytesMut::with_capacity(self.part_size);
for chunk in chunks {
let mut offset = 0usize;
while offset < chunk.len() {
if pending.is_empty() && chunk.len() - offset >= self.part_size {
let end = offset + self.part_size;
parts.push(
self.upload_one_part(key, upload_id, part_number, chunk.slice(offset..end))
.await?,
);
part_number += 1;
offset = end;
continue;
}
let take = (self.part_size - pending.len()).min(chunk.len() - offset);
pending.extend_from_slice(&chunk.slice(offset..offset + take));
offset += take;
if pending.len() == self.part_size {
parts.push(
self.upload_one_part(key, upload_id, part_number, pending.freeze())
.await?,
);
part_number += 1;
pending = BytesMut::with_capacity(self.part_size);
}
}
}
if !pending.is_empty() {
parts.push(
self.upload_one_part(key, upload_id, part_number, pending.freeze())
.await?,
);
}
let part_count = parts.len() as u64;
let completed = CompletedMultipartUpload::builder()
.set_parts(Some(parts))
.build();
self.client
.complete_multipart_upload()
.bucket(&self.bucket)
.key(key)
.upload_id(upload_id)
.multipart_upload(completed)
.send()
.await
.map_err(unavailable)?;
Ok(part_count)
}
async fn get_with_etag(
&self,
key: &str,
) -> Result<(Option<Bytes>, Option<String>), ObjectLogError> {
match self
.client
.get_object()
.bucket(&self.bucket)
.key(key)
.send()
.await
{
Ok(resp) => {
let etag = resp.e_tag().map(str::to_string);
let data = resp.body.collect().await.map_err(unavailable)?;
Ok((Some(data.into_bytes()), etag))
}
Err(error) => {
let svc = error.into_service_error();
if svc.is_no_such_key() {
Ok((None, None))
} else {
Err(unavailable(svc))
}
}
}
}
}
fn env_flag(name: &str) -> bool {
matches!(
std::env::var(name).as_deref(),
Ok("1") | Ok("true") | Ok("TRUE") | Ok("yes") | Ok("YES")
)
}
fn env_u64(name: &str, default: u64) -> u64 {
std::env::var(name)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
#[async_trait]
impl BlobStore for S3BlobStore {
async fn put(&self, key: &str, value: Bytes) -> Result<(), ObjectLogError> {
if value.len() > self.multipart_threshold {
return self.put_multipart(key, value).await;
}
let byte_len = value.len() as u64;
let mut delay = Duration::from_millis(25);
let mut last = None;
for attempt in 0..5u32 {
let request = self
.client
.put_object()
.bucket(&self.bucket)
.key(key)
.body(ByteStream::from(value.clone()));
let result = if self.disable_payload_signing {
request.customize().disable_payload_signing().send().await
} else {
request.send().await
};
match result {
Ok(_) => {
self.record_media(1, byte_len);
return Ok(());
}
Err(err) if attempt + 1 < 5 && transient_s3(&err) => {
tokio::time::sleep(delay).await;
delay = delay.saturating_mul(2);
last = Some(err);
}
Err(err) => return Err(unavailable(err)),
}
}
Err(unavailable(last.expect("s3 put retry exhausted")))
}
fn take_media_op_stats(&self) -> Option<MediaOpStats> {
Some(MediaOpStats {
media_ops: self.media_ops.swap(0, Ordering::Relaxed),
bytes: self.bytes_written.swap(0, Ordering::Relaxed),
})
}
async fn put_chunks(&self, key: &str, chunks: Vec<Bytes>) -> Result<(), ObjectLogError> {
let total: usize = chunks.iter().map(Bytes::len).sum();
if total > self.multipart_threshold {
return self.put_multipart_chunks(key, chunks).await;
}
match chunks.len() {
0 => self.put(key, Bytes::new()).await,
1 => self.put(key, chunks.into_iter().next().unwrap()).await,
_ => {
let mut value = BytesMut::with_capacity(total);
for chunk in chunks {
value.extend_from_slice(&chunk);
}
self.put(key, value.freeze()).await
}
}
}
async fn get(&self, key: &str) -> Result<Option<Bytes>, ObjectLogError> {
match self
.client
.get_object()
.bucket(&self.bucket)
.key(key)
.send()
.await
{
Ok(resp) => {
let data = resp.body.collect().await.map_err(unavailable)?;
Ok(Some(data.into_bytes()))
}
Err(e) => {
let svc = e.into_service_error();
if svc.is_no_such_key() {
Ok(None)
} else {
Err(unavailable(svc))
}
}
}
}
async fn get_range(
&self,
key: &str,
range: Range<u64>,
) -> Result<Option<Bytes>, ObjectLogError> {
if range.start > range.end {
return Err(ObjectLogError::RangeOutOfBounds(format!(
"start {} > end {}",
range.start, range.end
)));
}
if range.start == range.end {
return match self.get(key).await? {
Some(_) => Ok(Some(Bytes::new())),
None => Ok(None),
};
}
if std::env::var_os("OBJECT_LOG_S3_RANGE_FALLBACK").is_some() {
let Some(bytes) = self.get(key).await? else {
return Ok(None);
};
let start = range.start as usize;
let end = range.end as usize;
if start > bytes.len() || end > bytes.len() {
return Err(ObjectLogError::RangeOutOfBounds(format!(
"{}..{} outside object length {}",
range.start,
range.end,
bytes.len()
)));
}
return Ok(Some(bytes.slice(start..end)));
}
let header = format!("bytes={}-{}", range.start, range.end - 1);
match self
.client
.get_object()
.bucket(&self.bucket)
.key(key)
.range(header)
.send()
.await
{
Ok(resp) => {
let data = resp.body.collect().await.map_err(unavailable)?;
Ok(Some(data.into_bytes()))
}
Err(e) => {
let svc = e.into_service_error();
if svc.is_no_such_key() {
Ok(None)
} else {
Err(unavailable(svc))
}
}
}
}
async fn get_range_exact(
&self,
key: &str,
range: Range<u64>,
) -> Result<Option<Bytes>, ObjectLogError> {
if range.start > range.end {
return Err(ObjectLogError::RangeOutOfBounds(
"inverted strict range".into(),
));
}
let count = usize::try_from(range.end - range.start).map_err(|_| {
ObjectLogError::RangeOutOfBounds("strict range exceeds address space".into())
})?;
if count == 0 {
return match self
.client
.head_object()
.bucket(&self.bucket)
.key(key)
.send()
.await
{
Ok(output) => {
let len = output
.content_length()
.and_then(|n| u64::try_from(n).ok())
.ok_or_else(|| unavailable("HEAD omitted object length"))?;
if range.end > len {
return Err(ObjectLogError::RangeOutOfBounds(
"empty range beyond EOF".into(),
));
}
Ok(Some(Bytes::new()))
}
Err(error)
if error
.raw_response()
.is_some_and(|response| response.status().as_u16() == 404) =>
{
Ok(None)
}
Err(error) => Err(unavailable(error)),
};
}
let status = std::sync::Arc::new(std::sync::atomic::AtomicU16::new(0));
let result = self
.client
.get_object()
.bucket(&self.bucket)
.key(key)
.range(format!("bytes={}-{}", range.start, range.end - 1))
.customize()
.interceptor(ResponseStatus {
status: std::sync::Arc::clone(&status),
range: range.clone(),
})
.send()
.await;
let mut output = match result {
Ok(output) => output,
Err(_) if status.load(Ordering::Relaxed) == 404 => return Ok(None),
Err(error) => return Err(unavailable(error)),
};
if status.load(Ordering::Relaxed) != 206
|| output
.content_length()
.and_then(|n| usize::try_from(n).ok())
!= Some(count)
{
return Err(unavailable(
"strict range response must be 206 with exact length",
));
}
validate_content_range(output.content_range(), &range)?;
let mut bytes = Vec::with_capacity(count);
while let Some(chunk) = output.body.try_next().await.map_err(unavailable)? {
if bytes
.len()
.checked_add(chunk.len())
.is_none_or(|n| n > count)
{
return Err(unavailable("strict range body exceeds admitted length"));
}
bytes.extend_from_slice(&chunk);
}
if bytes.len() != count {
return Err(unavailable("strict range body is truncated"));
}
Ok(Some(Bytes::from(bytes)))
}
async fn compare_and_swap(
&self,
key: &str,
expected: Option<Bytes>,
new_value: Bytes,
) -> Result<CasOutcome, ObjectLogError> {
let (current, etag) = self.get_with_etag(key).await?;
let matches = match (&expected, ¤t) {
(None, None) => true,
(Some(expected), Some(current)) => expected == current,
_ => false,
};
if !matches {
return Ok(CasOutcome::Conflict { current });
}
let mut request = self
.client
.put_object()
.bucket(&self.bucket)
.key(key)
.body(ByteStream::from(new_value.clone()));
request = if let Some(etag) = etag {
request.if_match(etag)
} else {
request.if_none_match("*")
};
let request =
request
.customize()
.config_override(aws_sdk_s3::config::Builder::new().retry_config(
aws_sdk_s3::config::retry::RetryConfig::standard().with_max_attempts(1),
));
let request = if self.disable_payload_signing {
request.disable_payload_signing().send().await
} else {
request.send().await
};
match request {
Ok(_) => {
self.record_media(1, new_value.len() as u64);
Ok(CasOutcome::Stored)
}
Err(error) if s3_precondition_failed(&error) => {
let (current, _) = self.get_with_etag(key).await?;
Ok(CasOutcome::Conflict { current })
}
Err(error) => Err(unavailable(error)),
}
}
async fn list(&self, prefix: &str) -> Result<Vec<String>, ObjectLogError> {
let mut keys = Vec::new();
let mut token: Option<String> = None;
loop {
let mut req = self
.client
.list_objects_v2()
.bucket(&self.bucket)
.prefix(prefix);
if let Some(t) = &token {
req = req.continuation_token(t);
}
let resp = req.send().await.map_err(unavailable)?;
for obj in resp.contents() {
if let Some(k) = obj.key() {
keys.push(k.to_string());
}
}
match resp.next_continuation_token() {
Some(t) => token = Some(t.to_string()),
None => break,
}
}
Ok(keys)
}
async fn delete(&self, key: &str) -> Result<(), ObjectLogError> {
self.client
.delete_object()
.bucket(&self.bucket)
.key(key)
.send()
.await
.map_err(unavailable)?;
Ok(())
}
}
#[cfg(test)]
mod runtime_tests;