use alloc::boxed::Box;
use alloc::format;
use alloc::string::{String, ToString};
use alloc::vec::Vec;
use core::fmt::{Display, Formatter, Write as _};
use core::time::Duration;
use alloc::collections::BTreeSet;
use alloc::sync::Arc;
use core::sync::atomic::{AtomicU64, Ordering};
#[cfg(feature = "std")]
use std::io::{self, Read, Write};
#[cfg(feature = "std")]
use std::net::TcpStream;
#[cfg(feature = "std")]
use std::sync::{LazyLock, Mutex};
#[cfg(all(feature = "eko-s3", not(feature = "std")))]
use eko::thread::{LazyLock, Mutex};
#[cfg(feature = "std")]
use instant_xml::FromXml;
#[cfg(feature = "std")]
use rustls::pki_types::ServerName;
use rustls::{ClientConfig, RootCertStore};
#[cfg(feature = "std")]
use rustls::{ClientConnection, StreamOwned};
#[cfg(feature = "std")]
use rusty_s3::{Bucket, Credentials, S3Action, UrlStyle};
#[cfg(feature = "std")]
use ureq::{Agent, AgentBuilder, Error as UreqError, ReadWrite, TlsConnector};
use url::{Host, Url};
use super::{
CommittedGeneration, GenerationPlan, Head, MutationKind, ObjectRef, PageRef, PageStore,
StagedGeneration, StagedPage, StorageDomainId,
};
const HEAD_MAGIC: &[u8; 8] = b"DBS3H001";
const CATALOG_MAGIC: &[u8; 8] = b"DBCAT001";
const HEAD_FILE: &str = "head.v1";
const SEGMENT_TARGET: usize = 4 * 1024 * 1024;
const CATALOG_CHUNK_SIZE: usize = crate::PAGE_SIZE;
const SIGNED_URL_LIFETIME: Duration = Duration::from_secs(3600);
const MAX_RESPONSE_BYTES: usize = 64 * 1024 * 1024;
const MAX_RECORD_BYTES: usize = 4096;
type StoredObject = (Vec<u8>, Option<String>);
#[cfg(all(feature = "eko-s3", not(feature = "std")))]
#[derive(Debug)]
struct EkoS3TimeProvider;
#[cfg(all(feature = "eko-s3", not(feature = "std")))]
impl rustls::time_provider::TimeProvider for EkoS3TimeProvider {
fn current_time(&self) -> Option<rustls::pki_types::UnixTime> {
let secs = eko::time::epoch_secs();
if secs >= 0 {
return Some(rustls::pki_types::UnixTime::since_unix_epoch(
Duration::from_secs(secs as u64),
));
}
None
}
}
pub(super) fn default_s3_tls_config() -> Arc<ClientConfig> {
static TLS_CONFIG: LazyLock<Arc<ClientConfig>> = LazyLock::new(|| {
let provider = Arc::new(ps_rustcrypto::provider_with_rsa_pss());
let roots = RootCertStore {
roots: webpki_roots::TLS_SERVER_ROOTS.to_vec(),
};
#[cfg(feature = "std")]
let builder = ClientConfig::builder_with_provider(provider);
#[cfg(all(feature = "eko-s3", not(feature = "std")))]
let builder = ClientConfig::builder_with_details(provider, Arc::new(EkoS3TimeProvider));
let config = builder
.with_safe_default_protocol_versions()
.expect("S3 TLS provider supports the default protocol versions")
.with_root_certificates(roots)
.with_no_client_auth();
Arc::new(config)
});
Arc::clone(&TLS_CONFIG)
}
#[cfg(feature = "std")]
pub(super) fn s3_agent_builder() -> AgentBuilder {
static TLS: LazyLock<Arc<S3TlsConnector>> =
LazyLock::new(|| Arc::new(S3TlsConnector(default_s3_tls_config())));
AgentBuilder::new()
.tls_connector(Arc::clone(&TLS))
.redirects(0)
}
pub(super) struct S3TlsConnector(pub(super) Arc<ClientConfig>);
#[cfg(feature = "std")]
impl TlsConnector for S3TlsConnector {
fn connect(
&self,
host: &str,
mut io: Box<dyn ReadWrite>,
) -> Result<Box<dyn ReadWrite>, UreqError> {
let host = host
.strip_prefix('[')
.and_then(|address| address.strip_suffix(']'))
.unwrap_or(host);
let server_name = ServerName::try_from(host)
.map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "invalid S3 TLS server name"))?
.to_owned();
let mut connection =
ClientConnection::new(Arc::clone(&self.0), server_name).map_err(io::Error::other)?;
connection.complete_io(&mut io)?;
Ok(Box::new(S3TlsStream(StreamOwned::new(connection, io))))
}
}
#[cfg(feature = "std")]
struct S3TlsStream(StreamOwned<ClientConnection, Box<dyn ReadWrite>>);
#[cfg(feature = "std")]
impl core::fmt::Debug for S3TlsStream {
fn fmt(&self, f: &mut Formatter<'_>) -> core::fmt::Result {
f.write_str("S3TlsStream")
}
}
#[cfg(feature = "std")]
impl Read for S3TlsStream {
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
self.0.read(buf)
}
}
#[cfg(feature = "std")]
impl Write for S3TlsStream {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
self.0.write(buf)
}
fn flush(&mut self) -> io::Result<()> {
self.0.flush()
}
}
#[cfg(feature = "std")]
impl ReadWrite for S3TlsStream {
fn socket(&self) -> Option<&TcpStream> {
self.0.get_ref().socket()
}
}
#[cfg(all(feature = "eko-s3", not(feature = "std")))]
pub(super) mod eko_sigv4 {
use super::*;
use hmac::{Hmac, Mac};
use sha2::{Digest, Sha256};
type HmacSha256 = Hmac<Sha256>;
#[derive(Clone, Copy, PartialEq, Eq)]
pub enum UrlStyle {
Path,
VirtualHost,
}
#[derive(Clone)]
pub struct Credentials {
access_key: String,
secret_key: String,
session_token: Option<String>,
}
impl Credentials {
pub fn new(access_key: impl Into<String>, secret_key: impl Into<String>) -> Self {
Self {
access_key: access_key.into(),
secret_key: secret_key.into(),
session_token: None,
}
}
pub fn new_with_token(
access_key: impl Into<String>,
secret_key: impl Into<String>,
session_token: impl Into<String>,
) -> Self {
Self {
access_key: access_key.into(),
secret_key: secret_key.into(),
session_token: Some(session_token.into()),
}
}
}
#[derive(Clone)]
pub struct Bucket {
endpoint: Url,
style: UrlStyle,
name: String,
region: String,
}
impl Bucket {
pub fn new(
endpoint: Url,
style: UrlStyle,
name: String,
region: String,
) -> Result<Self, S3StoreError> {
if name.is_empty() || region.is_empty() || endpoint.host_str().is_none() {
return Err(S3StoreError::InvalidConfig);
}
Ok(Self {
endpoint,
style,
name,
region,
})
}
pub fn get_object<'a>(
&'a self,
credentials: Option<&'a Credentials>,
key: &'a str,
) -> SignedAction<'a> {
SignedAction::new(self, credentials, "GET", Some(key))
}
pub fn put_object<'a>(
&'a self,
credentials: Option<&'a Credentials>,
key: &'a str,
) -> SignedAction<'a> {
SignedAction::new(self, credentials, "PUT", Some(key))
}
pub fn delete_object<'a>(
&'a self,
credentials: Option<&'a Credentials>,
key: &'a str,
) -> SignedAction<'a> {
SignedAction::new(self, credentials, "DELETE", Some(key))
}
pub fn head_object<'a>(
&'a self,
credentials: Option<&'a Credentials>,
key: &'a str,
) -> SignedAction<'a> {
SignedAction::new(self, credentials, "HEAD", Some(key))
}
pub fn list_objects_v2<'a>(
&'a self,
credentials: Option<&'a Credentials>,
) -> ListObjectsV2Action<'a> {
let mut action = SignedAction::new(self, credentials, "GET", None);
action.query.push(("list-type".into(), "2".into()));
ListObjectsV2Action { inner: action }
}
}
pub struct SignedHeaders<'a> {
headers: &'a mut Vec<(String, String)>,
}
impl<'a> SignedHeaders<'a> {
pub fn insert(&mut self, name: &str, value: &str) {
let lower = name.to_ascii_lowercase();
if let Some(existing) = self.headers.iter_mut().find(|(k, _)| *k == lower) {
existing.1 = value.trim().to_string();
} else {
self.headers.push((lower, value.trim().to_string()));
}
}
}
pub struct SignedAction<'a> {
bucket: &'a Bucket,
credentials: Option<&'a Credentials>,
method: &'static str,
key: Option<&'a str>,
headers: Vec<(String, String)>,
query: Vec<(String, String)>,
}
impl<'a> SignedAction<'a> {
fn new(
bucket: &'a Bucket,
credentials: Option<&'a Credentials>,
method: &'static str,
key: Option<&'a str>,
) -> Self {
Self {
bucket,
credentials,
method,
key,
headers: Vec::new(),
query: Vec::new(),
}
}
pub fn headers_mut(&mut self) -> SignedHeaders<'_> {
SignedHeaders {
headers: &mut self.headers,
}
}
pub fn sign(&self, lifetime: Duration) -> Url {
let (base_url, host_header, canonical_uri) = self.build_target();
let Some(credentials) = self.credentials else {
return base_url;
};
let now_secs = eko::time::epoch_secs().max(0) as u64;
let (date_stamp, amz_date) = format_utc_timestamp(now_secs);
let credential_scope = format!("{date_stamp}/{}/s3/aws4_request", self.bucket.region);
let credential = format!("{}/{credential_scope}", credentials.access_key);
let mut signed_headers = self.headers.clone();
signed_headers.push(("host".into(), host_header));
signed_headers.sort_by(|a, b| a.0.cmp(&b.0));
let mut canonical_headers = String::new();
let mut signed_header_names = String::new();
for (index, (name, value)) in signed_headers.iter().enumerate() {
if index > 0 {
signed_header_names.push(';');
}
signed_header_names.push_str(name);
let _ = writeln!(&mut canonical_headers, "{name}:{}", value.trim());
}
let mut query_params = self.query.clone();
query_params.push(("X-Amz-Algorithm".into(), "AWS4-HMAC-SHA256".into()));
query_params.push(("X-Amz-Credential".into(), credential));
query_params.push(("X-Amz-Date".into(), amz_date.clone()));
query_params.push(("X-Amz-Expires".into(), lifetime.as_secs().to_string()));
if let Some(token) = &credentials.session_token {
query_params.push(("X-Amz-Security-Token".into(), token.clone()));
}
query_params.push(("X-Amz-SignedHeaders".into(), signed_header_names));
let mut encoded_query = query_params
.into_iter()
.map(|(k, v)| (aws_percent_encode(&k, true), aws_percent_encode(&v, true)))
.collect::<Vec<_>>();
encoded_query.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1)));
let mut canonical_query = String::new();
for (index, (k, v)) in encoded_query.iter().enumerate() {
if index > 0 {
canonical_query.push('&');
}
canonical_query.push_str(k);
canonical_query.push('=');
canonical_query.push_str(v);
}
let canonical_request = format!(
"{}\n{canonical_uri}\n{canonical_query}\n{canonical_headers}\n{}\nUNSIGNED-PAYLOAD",
self.method,
signed_headers
.iter()
.map(|(k, _)| k.as_str())
.collect::<Vec<_>>()
.join(";"),
);
let request_hash = sha256_hex(canonical_request.as_bytes());
let string_to_sign = format!(
"AWS4-HMAC-SHA256\n{amz_date}\n{credential_scope}\n{request_hash}"
);
let signing_key = derive_signing_key(
&credentials.secret_key,
&date_stamp,
&self.bucket.region,
"s3",
);
let signature = hmac_sha256_hex(&signing_key, string_to_sign.as_bytes());
let mut final_query = canonical_query;
if !final_query.is_empty() {
final_query.push('&');
}
final_query.push_str("X-Amz-Signature=");
final_query.push_str(&signature);
let mut url = base_url;
url.set_query(Some(&final_query));
url
}
fn build_target(&self) -> (Url, String, String) {
let mut url = self.bucket.endpoint.clone();
let base_path = url.path().trim_end_matches('/');
let key = self.key.map(|k| k.trim_start_matches('/'));
let raw_path = match (self.bucket.style, key) {
(UrlStyle::Path, Some(k)) if !k.is_empty() => {
if base_path.is_empty() {
format!("/{}/{k}", self.bucket.name)
} else {
format!("{base_path}/{}/{k}", self.bucket.name)
}
}
(UrlStyle::Path, _) => {
if base_path.is_empty() {
format!("/{}", self.bucket.name)
} else {
format!("{base_path}/{}", self.bucket.name)
}
}
(UrlStyle::VirtualHost, Some(k)) if !k.is_empty() => {
if base_path.is_empty() {
format!("/{k}")
} else {
format!("{base_path}/{k}")
}
}
(UrlStyle::VirtualHost, _) => {
if base_path.is_empty() {
"/".to_string()
} else {
format!("{base_path}/")
}
}
};
if self.bucket.style == UrlStyle::VirtualHost {
if let Some(host) = url.host_str().map(ToString::to_string) {
let vhost = format!("{}.{host}", self.bucket.name);
let _ = url.set_host(Some(&vhost));
}
}
let encoded_path = aws_percent_encode(&raw_path, false);
url.set_path(&encoded_path);
let host_str = url.host_str().unwrap_or("");
let host_header = match url.port() {
Some(port) => format!("{host_str}:{port}"),
None => host_str.to_string(),
};
(url, host_header, encoded_path)
}
}
pub struct ListObjectsV2Action<'a> {
inner: SignedAction<'a>,
}
impl<'a> ListObjectsV2Action<'a> {
pub fn with_prefix(&mut self, prefix: &str) {
self.inner.query.push(("prefix".into(), prefix.into()));
}
pub fn with_continuation_token(&mut self, token: &str) {
self.inner
.query
.push(("continuation-token".into(), token.into()));
}
pub fn sign(&self, lifetime: Duration) -> Url {
self.inner.sign(lifetime)
}
}
fn aws_percent_encode(input: &str, encode_slash: bool) -> String {
let mut out = String::with_capacity(input.len());
for byte in input.bytes() {
let unreserved = byte.is_ascii_alphanumeric()
|| matches!(byte, b'-' | b'_' | b'.' | b'~')
|| (!encode_slash && byte == b'/');
if unreserved {
out.push(byte as char);
} else {
let _ = write!(&mut out, "%{byte:02X}");
}
}
out
}
fn sha256_hex(bytes: &[u8]) -> String {
let digest = Sha256::digest(bytes);
let mut hex = String::with_capacity(64);
write_hex(&mut hex, digest.as_slice());
hex
}
fn hmac_sha256(key: &[u8], data: &[u8]) -> [u8; 32] {
let mut mac = HmacSha256::new_from_slice(key).expect("HMAC accepts any key length");
mac.update(data);
let bytes = mac.finalize().into_bytes();
let mut out = [0u8; 32];
out.copy_from_slice(&bytes);
out
}
fn hmac_sha256_hex(key: &[u8], data: &[u8]) -> String {
let digest = hmac_sha256(key, data);
let mut hex = String::with_capacity(64);
write_hex(&mut hex, &digest);
hex
}
fn derive_signing_key(secret: &str, date: &str, region: &str, service: &str) -> [u8; 32] {
let mut k_secret = Vec::with_capacity(4 + secret.len());
k_secret.extend_from_slice(b"AWS4");
k_secret.extend_from_slice(secret.as_bytes());
let k_date = hmac_sha256(&k_secret, date.as_bytes());
let k_region = hmac_sha256(&k_date, region.as_bytes());
let k_service = hmac_sha256(&k_region, service.as_bytes());
hmac_sha256(&k_service, b"aws4_request")
}
fn format_utc_timestamp(epoch_secs: u64) -> (String, String) {
let days = (epoch_secs / 86_400) as i64;
let rem = epoch_secs % 86_400;
let hour = rem / 3600;
let minute = (rem % 3600) / 60;
let second = rem % 60;
let z = days + 719_468;
let era = if z >= 0 { z } else { z - 146_096 } / 146_097;
let doe = (z - era * 146_097) as u64;
let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146_096) / 365;
let y = yoe as i64 + era * 400;
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
let mp = (5 * doy + 2) / 153;
let d = doy - (153 * mp + 2) / 5 + 1;
let m = if mp < 10 { mp + 3 } else { mp - 9 };
let year = if m <= 2 { y + 1 } else { y };
let date = format!("{year:04}{m:02}{d:02}");
let datetime = format!("{date}T{hour:02}{minute:02}{second:02}Z");
(date, datetime)
}
}
#[cfg(all(feature = "eko-s3", not(feature = "std")))]
pub(super) use eko_sigv4::{Bucket, Credentials, UrlStyle};
#[cfg(all(feature = "eko-s3", not(feature = "std")))]
#[derive(Clone, Debug)]
pub struct EkoHttpAgent {
tls_config: Arc<ClientConfig>,
timeout: Duration,
}
#[cfg(all(feature = "eko-s3", not(feature = "std")))]
impl EkoHttpAgent {
pub(super) fn new(tls_config: Arc<ClientConfig>, timeout: Duration) -> Self {
Self {
tls_config,
timeout,
}
}
pub(super) fn with_tls_config(mut self, tls_config: Arc<ClientConfig>) -> Self {
self.tls_config = tls_config;
self
}
pub(super) fn with_timeout(mut self, timeout: Duration) -> Self {
self.timeout = timeout;
self
}
pub(super) fn send(
&self,
method: &str,
url: &str,
headers: &[(&str, &str)],
body: Option<&[u8]>,
max_body: Option<usize>,
) -> Result<nago_http::Response, nago_http::Error> {
let parsed = Url::parse(url).map_err(|_| nago_http::Error::Url("invalid URL".into()))?;
let tls = match parsed.scheme() {
"https" => true,
"http" => false,
_ => {
return Err(nago_http::Error::Url(
"the scheme must be https:// or http://".into(),
))
}
};
let host = parsed
.host_str()
.ok_or_else(|| nago_http::Error::Url("the URL has no host".into()))?;
let host = host
.strip_prefix('[')
.and_then(|h| h.strip_suffix(']'))
.unwrap_or(host);
let port = parsed.port_or_known_default().unwrap_or(if tls { 443 } else { 80 });
let target = if tls {
nago_http::Target::resolve(host, port)?
} else {
nago_http::Target::resolve_plain(host, port)?
};
let mut path_and_query = String::from(parsed.path());
if path_and_query.is_empty() {
path_and_query.push('/');
}
if let Some(query) = parsed.query() {
path_and_query.push('?');
path_and_query.push_str(query);
}
let mut request = nago_http::Request::new(method, &path_and_query);
for (name, value) in headers {
request = request.header(name, value);
}
if let Some(bytes) = body {
request = request.body("application/octet-stream", bytes);
}
if let Some(limit) = max_body {
request = request.max_body(limit);
}
let tls_config = Arc::clone(&self.tls_config);
let timeout = self.timeout;
let reactor = nagoya::reactor::Reactor::local()
.map_err(|err| nago_http::Error::Io(format!("starting local reactor: {err:?}")))?;
let handle = reactor.handle();
let outcome = nagoya::reactor::block_on_with(&reactor, async {
let conn = nago_http::open_with_tls_config(&target, &handle, Some(tls_config)).await?;
let _ = timeout;
conn.send(request).await
});
reactor.shutdown();
outcome
}
}
#[cfg(feature = "std")]
type S3HttpClient = Agent;
#[cfg(all(feature = "eko-s3", not(feature = "std")))]
type S3HttpClient = EkoHttpAgent;
#[derive(Clone)]
pub struct RecordVersion {
key: String,
etag: String,
}
#[cfg(feature = "std")]
#[derive(instant_xml::FromXml)]
#[xml(rename = "Error")]
struct RecordReadError {
#[xml(rename = "Code")]
code: String,
}
#[cfg(all(feature = "eko-s3", not(feature = "std")))]
fn xml_is_no_such_key(xml: &str) -> bool {
use xmlparser::{ElementEnd, Token, Tokenizer};
let mut stack: Vec<&str> = Vec::new();
let mut code: Option<String> = None;
for tok in Tokenizer::from(xml) {
let Ok(tok) = tok else { return false };
match tok {
Token::ElementStart { local, .. } => stack.push(local.as_str()),
Token::ElementEnd { end: ElementEnd::Empty | ElementEnd::Close(..), .. } => {
stack.pop();
}
Token::Text { text } if stack.as_slice() == ["Error", "Code"] => {
code.get_or_insert_with(String::new).push_str(text.as_str());
}
_ => {}
}
}
code.as_deref().map(str::trim) == Some("NoSuchKey")
}
pub trait S3Transport: Send + Sync {
fn send(
&self,
method: &str,
url: &str,
headers: &[(&str, &str)],
body: &[u8],
) -> Result<S3Response, S3StoreError>;
}
pub struct S3Response {
pub status: u16,
pub etag: Option<String>,
pub body: Vec<u8>,
}
#[derive(Clone)]
pub struct S3Config {
pub bucket_name: String,
pub endpoint: String,
pub access_key: String,
pub secret_key: String,
pub session_token: Option<String>,
pub region: String,
pub prefix: Option<String>,
pub virtual_host_style: bool,
}
impl core::fmt::Debug for S3Config {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_struct("S3Config").finish_non_exhaustive()
}
}
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum S3StoreError {
InvalidConfig,
Transport,
Status(u16),
Conflict,
MissingObject,
CorruptObject,
InvalidHead,
LengthOverflow,
}
impl Display for S3StoreError {
fn fmt(&self, formatter: &mut Formatter<'_>) -> core::fmt::Result {
match self {
Self::InvalidConfig => write!(formatter, "invalid S3 page-store configuration"),
Self::Transport => write!(formatter, "S3 page-store transport failed"),
Self::Status(status) => write!(formatter, "S3 page-store returned HTTP {status}"),
Self::Conflict => write!(formatter, "S3 bootstrap generation changed concurrently"),
Self::MissingObject => write!(formatter, "S3 generation references a missing object"),
Self::CorruptObject => {
write!(formatter, "S3 object failed length or checksum validation")
}
Self::InvalidHead => write!(formatter, "S3 bootstrap head is invalid"),
Self::LengthOverflow => {
write!(
formatter,
"S3 page, segment, response, or catalog length exceeds supported bounds"
)
}
}
}
}
impl core::error::Error for S3StoreError {}
pub struct S3PageStore {
config: S3Config,
bucket: Bucket,
credentials: Credentials,
client: S3HttpClient,
record_client: S3HttpClient,
transport: Option<Arc<dyn S3Transport>>,
known_objects: Mutex<BTreeSet<String>>,
put_count: AtomicU64,
put_bytes: AtomicU64,
}
impl S3PageStore {
pub fn new(config: S3Config) -> Result<Self, S3StoreError> {
let endpoint: Url = config
.endpoint
.parse()
.map_err(|_| S3StoreError::InvalidConfig)?;
if endpoint.scheme() != "https" && !is_loopback_endpoint(&endpoint) {
return Err(S3StoreError::InvalidConfig);
}
let style = if config.virtual_host_style {
UrlStyle::VirtualHost
} else {
UrlStyle::Path
};
let bucket = Bucket::new(
endpoint,
style,
config.bucket_name.clone(),
config.region.clone(),
)
.map_err(|_| S3StoreError::InvalidConfig)?;
let credentials = match &config.session_token {
Some(token) => Credentials::new_with_token(
config.access_key.clone(),
config.secret_key.clone(),
token.clone(),
),
None => Credentials::new(&config.access_key, &config.secret_key),
};
#[cfg(feature = "std")]
let (client, record_client) = (
s3_agent_builder()
.timeout(Duration::from_secs(120))
.build(),
s3_agent_builder()
.timeout(Duration::from_secs(5))
.build(),
);
#[cfg(all(feature = "eko-s3", not(feature = "std")))]
let (client, record_client) = (
EkoHttpAgent::new(default_s3_tls_config(), Duration::from_secs(120)),
EkoHttpAgent::new(default_s3_tls_config(), Duration::from_secs(5)),
);
Ok(Self {
config,
bucket,
credentials,
client,
record_client,
transport: None,
known_objects: Mutex::new(BTreeSet::new()),
put_count: AtomicU64::new(0),
put_bytes: AtomicU64::new(0),
})
}
pub fn with_transport(
config: S3Config,
transport: Arc<dyn S3Transport>,
) -> Result<Self, S3StoreError> {
let mut store = Self::new(config)?;
store.transport = Some(transport);
Ok(store)
}
fn send(
&self,
method: &str,
url: &str,
headers: &[(&str, &str)],
body: &[u8],
) -> Result<S3Response, S3StoreError> {
self.send_bounded(&self.client, method, url, headers, body, MAX_RESPONSE_BYTES)
}
fn send_bounded(
&self,
client: &S3HttpClient,
method: &str,
url: &str,
headers: &[(&str, &str)],
body: &[u8],
max_response_bytes: usize,
) -> Result<S3Response, S3StoreError> {
if method == "PUT" {
self.put_count.fetch_add(1, Ordering::Relaxed);
self.put_bytes
.fetch_add(body.len() as u64, Ordering::Relaxed);
}
if let Some(transport) = &self.transport {
let response = transport.send(method, url, headers, body)?;
if response.body.len() > max_response_bytes {
return Err(S3StoreError::LengthOverflow);
}
return Ok(response);
}
#[cfg(feature = "std")]
{
let mut request = client.request(method, url);
for (name, value) in headers {
request = request.set(name, value);
}
let response = match request.send_bytes(body) {
Ok(response) | Err(UreqError::Status(_, response)) => response,
Err(error) => return Err(map_ureq(error)),
};
let status = response.status();
let etag = response.header("etag").map(ToString::to_string);
let mut bytes = Vec::new();
response
.into_reader()
.take(max_response_bytes as u64 + 1)
.read_to_end(&mut bytes)
.map_err(|_| S3StoreError::Transport)?;
if bytes.len() > max_response_bytes {
return Err(S3StoreError::LengthOverflow);
}
Ok(S3Response {
status,
etag,
body: bytes,
})
}
#[cfg(all(feature = "eko-s3", not(feature = "std")))]
{
let req_body = if method == "PUT" || !body.is_empty() {
Some(body)
} else {
None
};
let response = client
.send(method, url, headers, req_body, Some(max_response_bytes + 1))
.map_err(|err| match err {
nago_http::Error::TooLarge => S3StoreError::LengthOverflow,
_ => S3StoreError::Transport,
})?;
if response.body.len() > max_response_bytes {
return Err(S3StoreError::LengthOverflow);
}
let etag = response.header("etag").map(ToString::to_string);
Ok(S3Response {
status: response.status,
etag,
body: response.body,
})
}
}
fn record_key(&self, namespace: &str, key: &str) -> Result<String, S3StoreError> {
let valid = |part: &str| {
!part.is_empty() && part.len() <= 96
&& part.bytes().all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_'))
};
if !valid(namespace) || !valid(key) {
return Err(S3StoreError::InvalidConfig);
}
let prefix = self.config.prefix.as_deref().unwrap_or("").trim_matches('/');
if prefix.is_empty() {
Ok(format!("records/{namespace}/{key}"))
} else {
Ok(format!("{prefix}/records/{namespace}/{key}"))
}
}
pub fn read_record(
&self,
namespace: &str,
key: &str,
) -> Result<Option<(Vec<u8>, RecordVersion)>, S3StoreError> {
let key = self.record_key(namespace, key)?;
let url = self.bucket.get_object(Some(&self.credentials), &key).sign(SIGNED_URL_LIFETIME);
let response = self.send_bounded(&self.record_client, "GET", url.as_str(), &[], &[], MAX_RECORD_BYTES)?;
#[cfg(feature = "std")]
let is_missing = response.status == 404
&& core::str::from_utf8(&response.body)
.ok()
.and_then(|xml| instant_xml::from_str::<RecordReadError>(xml).ok())
.is_some_and(|error| error.code == "NoSuchKey");
#[cfg(all(feature = "eko-s3", not(feature = "std")))]
let is_missing = response.status == 404
&& core::str::from_utf8(&response.body)
.ok()
.is_some_and(xml_is_no_such_key);
if is_missing {
return Ok(None);
}
if response.status != 200 {
return Err(S3StoreError::Status(response.status));
}
let version = response.etag.filter(|etag| {
etag.len() <= 128 && etag.strip_prefix('"').and_then(|value| value.strip_suffix('"'))
.is_some_and(|value| !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_graphic() && byte != b'"'))
}).ok_or(S3StoreError::CorruptObject)?;
Ok(Some((response.body, RecordVersion { key, etag: version })))
}
pub fn compare_and_put_record(
&self,
namespace: &str,
key: &str,
expected: Option<&RecordVersion>,
bytes: &[u8],
) -> Result<bool, S3StoreError> {
if bytes.is_empty() || bytes.len() > MAX_RECORD_BYTES {
return Err(S3StoreError::LengthOverflow);
}
let key = self.record_key(namespace, key)?;
let mut action = self.bucket.put_object(Some(&self.credentials), &key);
let condition = match expected {
Some(version) if version.key == key => ("if-match", version.etag.as_str()),
Some(_) => return Err(S3StoreError::InvalidConfig),
None => ("if-none-match", "*"),
};
action.headers_mut().insert(condition.0, condition.1);
let url = action.sign(SIGNED_URL_LIFETIME);
let response = self.send_bounded(
&self.record_client, "PUT", url.as_str(), &[condition], bytes, MAX_RECORD_BYTES,
)?;
match response.status {
200 => Ok(true),
412 => Ok(false),
status => Err(S3StoreError::Status(status)),
}
}
pub fn put_totals(&self) -> (u64, u64) {
(
self.put_count.load(Ordering::Relaxed),
self.put_bytes.load(Ordering::Relaxed),
)
}
fn object_key(&self, domain: StorageDomainId, suffix: &str) -> String {
let mut domain_hex = String::with_capacity(32);
write_hex(&mut domain_hex, &domain.0);
let prefix = self
.config
.prefix
.as_deref()
.unwrap_or("")
.trim_matches('/');
if prefix.is_empty() {
format!("{domain_hex}/{suffix}")
} else {
format!("{prefix}/{domain_hex}/{suffix}")
}
}
fn segment_key(&self, domain: StorageDomainId, object: &[u8; 32]) -> String {
let mut hash = String::with_capacity(64);
write_hex(&mut hash, object);
self.object_key(domain, &format!("segments/{hash}"))
}
fn catalog_key(&self, domain: StorageDomainId, object: &[u8; 32]) -> String {
let mut hash = String::with_capacity(64);
write_hex(&mut hash, object);
self.object_key(domain, &format!("catalog/{hash}"))
}
fn catalog_chunk_key(&self, domain: StorageDomainId, object: &[u8; 32]) -> String {
let mut hash = String::with_capacity(64);
write_hex(&mut hash, object);
self.object_key(domain, &format!("catalog-pages/{hash}"))
}
fn get(&self, key: &str) -> Result<Option<StoredObject>, S3StoreError> {
let url = self
.bucket
.get_object(Some(&self.credentials), key)
.sign(SIGNED_URL_LIFETIME);
let response = self.send("GET", url.as_str(), &[], &[])?;
if response.status == 404 {
return Ok(None);
}
if response.status != 200 {
return Err(S3StoreError::Status(response.status));
}
let etag = response.etag;
let bytes = response.body;
Ok(Some((bytes, etag)))
}
fn get_range(&self, key: &str, offset: u64, length: u32) -> Result<Vec<u8>, S3StoreError> {
let end = offset
.checked_add(u64::from(length))
.and_then(|value| value.checked_sub(1))
.ok_or(S3StoreError::LengthOverflow)?;
let url = self
.bucket
.get_object(Some(&self.credentials), key)
.sign(SIGNED_URL_LIFETIME);
let range = format!("bytes={offset}-{end}");
let response = self.send("GET", url.as_str(), &[("range", &range)], &[])?;
if response.status != 206 {
return Err(S3StoreError::Status(response.status));
}
Ok(response.body)
}
fn put(&self, key: &str, bytes: &[u8]) -> Result<(), S3StoreError> {
let url = self
.bucket
.put_object(Some(&self.credentials), key)
.sign(SIGNED_URL_LIFETIME);
let response = self.send("PUT", url.as_str(), &[("if-none-match", "*")], bytes)?;
if !(200..300).contains(&response.status) {
return Err(S3StoreError::Status(response.status));
}
Ok(())
}
fn lock_known_objects(&self) -> Result<impl core::ops::DerefMut<Target = BTreeSet<String>> + '_, S3StoreError> {
#[cfg(feature = "std")]
{
self.known_objects.lock().map_err(|_| S3StoreError::Transport)
}
#[cfg(all(feature = "eko-s3", not(feature = "std")))]
{
Ok(self.known_objects.lock())
}
}
fn put_verified(&self, key: &str, bytes: &[u8]) -> Result<(), S3StoreError> {
if self.lock_known_objects()?.contains(key) {
return Ok(());
}
let result = match self.put(key, bytes) {
Ok(_) => Ok(()),
Err(error) => match self.get(key)? {
Some((stored, _)) if stored == bytes => Ok(()),
_ => Err(error),
},
};
if result.is_ok() {
self.lock_known_objects()?.insert(key.to_string());
}
result
}
fn put_head(
&self,
domain: StorageDomainId,
bytes: &[u8],
current_etag: Option<&str>,
) -> Result<(), S3StoreError> {
let key = self.object_key(domain, HEAD_FILE);
let url = self
.bucket
.put_object(Some(&self.credentials), &key)
.sign(SIGNED_URL_LIFETIME);
let headers = match current_etag {
Some(etag) => [("if-match", etag)],
None => [("if-none-match", "*")],
};
match self.send("PUT", url.as_str(), &headers, bytes) {
Ok(response) if (200..300).contains(&response.status) => Ok(()),
Ok(response) if response.status == 412 => Err(S3StoreError::Conflict),
Ok(response) => Err(S3StoreError::Status(response.status)),
Err(error) => match self.get(&key)? {
Some((stored, _)) if stored == bytes => Ok(()),
_ => Err(error),
},
}
}
fn current_head(
&self,
domain: StorageDomainId,
) -> Result<Option<(Head, Option<String>)>, S3StoreError> {
let key = self.object_key(domain, HEAD_FILE);
let Some((bytes, etag)) = self.get(&key)? else {
return Ok(None);
};
Ok(Some((decode_head(&bytes)?, etag)))
}
pub fn retire(&self, domain: StorageDomainId) -> Result<(), S3StoreError> {
let Some((mut head, etag)) = self.current_head(domain)? else {
let head = Head {
domain,
generation: 1,
parent: 0,
writer_epoch: super::RETIRED_WRITER_EPOCH,
catalog: ObjectRef {
object: [0; 32],
offset: 0,
encoded_length: 0,
decoded_length: 0,
checksum: [0; 32],
},
};
return self.put_head(domain, &encode_head(&head), None);
};
if head.writer_epoch == super::RETIRED_WRITER_EPOCH {
return Ok(());
}
let etag = etag.ok_or(S3StoreError::InvalidHead)?;
head.writer_epoch = super::RETIRED_WRITER_EPOCH;
self.put_head(domain, &encode_head(&head), Some(&etag))
}
}
impl PageStore for S3PageStore {
type Error = S3StoreError;
fn load_head(&self, domain: StorageDomainId) -> Result<Option<Head>, Self::Error> {
self.current_head(domain)
.map(|head| head.map(|(head, _)| head))
}
fn load_catalog(&self, head: &Head) -> Result<Vec<u8>, Self::Error> {
if head.catalog.offset != 0 {
return Err(S3StoreError::InvalidHead);
}
let key = self.catalog_key(head.domain, &head.catalog.object);
let (manifest, _) = self.get(&key)?.ok_or(S3StoreError::MissingObject)?;
if manifest.len() != head.catalog.encoded_length as usize
|| blake3::hash(&manifest).as_bytes() != &head.catalog.object
{
return Err(S3StoreError::CorruptObject);
}
self.lock_known_objects()?.insert(key);
let catalog = decode_catalog_manifest(&manifest, |object| {
let key = self.catalog_chunk_key(head.domain, object);
let bytes = self
.get(&key)?
.map(|(bytes, _)| bytes)
.ok_or(S3StoreError::MissingObject)?;
if blake3::hash(&bytes).as_bytes() != object {
return Err(S3StoreError::CorruptObject);
}
self.lock_known_objects()?.insert(key);
Ok(bytes)
})?;
if catalog.len() != head.catalog.decoded_length as usize
|| blake3::hash(&catalog).as_bytes() != &head.catalog.checksum
{
return Err(S3StoreError::CorruptObject);
}
Ok(catalog)
}
fn read_page(&self, page: &PageRef) -> Result<Vec<u8>, Self::Error> {
let key = self.segment_key(page.address.domain, &page.object.object);
let bytes = self.get_range(&key, page.object.offset, page.object.encoded_length)?;
if bytes.len() != page.object.decoded_length as usize
|| blake3::hash(&bytes).as_bytes() != &page.object.checksum
{
return Err(S3StoreError::CorruptObject);
}
Ok(bytes)
}
fn stage(&self, plan: &GenerationPlan) -> Result<StagedGeneration, Self::Error> {
let domain = plan.domain;
if plan.pages.iter().any(|page| page.address.domain != domain) {
return Err(S3StoreError::InvalidHead);
}
let mut puts = plan
.pages
.iter()
.filter_map(|page| match &page.kind {
MutationKind::Put {
image,
live_rows,
live_bytes,
} if !image.is_empty() => {
Some((page.address, image.as_slice(), *live_rows, *live_bytes))
}
MutationKind::Put { .. } => None,
MutationKind::Delete => None,
})
.collect::<Vec<_>>();
puts.sort_by_key(|(address, _, _, _)| {
(
address.table_id.0,
address.space_id.0,
address.page_kind as u8,
usize::from(address.page_id),
)
});
if puts.len()
!= plan
.pages
.iter()
.filter(|page| matches!(&page.kind, MutationKind::Put { .. }))
.count()
{
return Err(S3StoreError::LengthOverflow);
}
let mut pages = Vec::with_capacity(puts.len());
let mut at = 0;
while at < puts.len() {
let first = at;
let mut length = puts[at].1.len();
at += 1;
while at < puts.len()
&& adjacent(puts[at - 1].0, puts[at].0)
&& length.saturating_add(puts[at].1.len()) <= SEGMENT_TARGET
{
length += puts[at].1.len();
at += 1;
}
let mut segment = Vec::with_capacity(length);
for (_, image, _, _) in &puts[first..at] {
segment.extend_from_slice(image);
}
let object = *blake3::hash(&segment).as_bytes();
let key = self.segment_key(domain, &object);
self.put_verified(&key, &segment)?;
let mut offset = 0_u64;
for (address, image, live_rows, live_bytes) in &puts[first..at] {
let encoded_length =
u32::try_from(image.len()).map_err(|_| S3StoreError::LengthOverflow)?;
pages.push(StagedPage {
address: *address,
object: ObjectRef {
object,
offset,
encoded_length,
decoded_length: encoded_length,
checksum: *blake3::hash(image).as_bytes(),
},
live_rows: *live_rows,
live_bytes: *live_bytes,
});
offset = offset
.checked_add(u64::from(encoded_length))
.ok_or(S3StoreError::LengthOverflow)?;
}
}
Ok(StagedGeneration {
domain,
generation: plan.id,
parent: plan.parent,
writer_epoch: plan.writer_epoch,
pages,
catalog: None,
})
}
fn stage_catalog(
&self,
staged: &mut StagedGeneration,
checkpoint: &[u8],
) -> Result<(), Self::Error> {
let mut chunks = Vec::new();
for bytes in checkpoint.chunks(CATALOG_CHUNK_SIZE) {
let object = *blake3::hash(bytes).as_bytes();
let length = u32::try_from(bytes.len()).map_err(|_| S3StoreError::LengthOverflow)?;
let key = self.catalog_chunk_key(staged.domain, &object);
self.put_verified(&key, bytes)?;
chunks.push((object, length));
}
let manifest = encode_catalog_manifest(checkpoint, &chunks)?;
let object = *blake3::hash(&manifest).as_bytes();
let key = self.catalog_key(staged.domain, &object);
self.put_verified(&key, &manifest)?;
let encoded_length =
u32::try_from(manifest.len()).map_err(|_| S3StoreError::LengthOverflow)?;
let decoded_length =
u32::try_from(checkpoint.len()).map_err(|_| S3StoreError::LengthOverflow)?;
staged.catalog = Some(ObjectRef {
object,
offset: 0,
encoded_length,
decoded_length,
checksum: *blake3::hash(checkpoint).as_bytes(),
});
Ok(())
}
fn commit(&self, staged: StagedGeneration) -> Result<CommittedGeneration, Self::Error> {
let catalog = staged.catalog.ok_or(S3StoreError::InvalidHead)?;
let current = self.current_head(staged.domain)?;
match ¤t {
Some((head, _)) if head.writer_epoch == super::RETIRED_WRITER_EPOCH => {
return Err(S3StoreError::Conflict)
}
Some((head, _)) if head.generation != staged.parent => {
return Err(S3StoreError::Conflict)
}
Some((_, None)) => return Err(S3StoreError::InvalidHead),
None if staged.parent != 0 => return Err(S3StoreError::Conflict),
_ => {}
}
let head = Head {
domain: staged.domain,
generation: staged.generation,
parent: staged.parent,
writer_epoch: staged.writer_epoch,
catalog,
};
let bytes = encode_head(&head);
self.put_head(
staged.domain,
&bytes,
current.as_ref().and_then(|(_, etag)| etag.as_deref()),
)?;
Ok(CommittedGeneration { head })
}
}
fn is_loopback_endpoint(endpoint: &Url) -> bool {
match endpoint.host() {
Some(Host::Domain(host)) => host == "localhost",
Some(Host::Ipv4(address)) => address.is_loopback(),
Some(Host::Ipv6(address)) => address.is_loopback(),
None => false,
}
}
fn adjacent(left: super::PageAddress, right: super::PageAddress) -> bool {
left.domain == right.domain
&& left.table_id == right.table_id
&& left.space_id == right.space_id
&& left.page_kind == right.page_kind
&& usize::from(left.page_id).checked_add(1) == Some(usize::from(right.page_id))
}
fn encode_catalog_manifest(
checkpoint: &[u8],
chunks: &[([u8; 32], u32)],
) -> Result<Vec<u8>, S3StoreError> {
let count = u32::try_from(chunks.len()).map_err(|_| S3StoreError::LengthOverflow)?;
let length = u64::try_from(checkpoint.len()).map_err(|_| S3StoreError::LengthOverflow)?;
let mut bytes = Vec::with_capacity(20 + chunks.len() * 36 + 32);
bytes.extend_from_slice(CATALOG_MAGIC);
bytes.extend_from_slice(&length.to_le_bytes());
bytes.extend_from_slice(&count.to_le_bytes());
for (object, chunk_length) in chunks {
bytes.extend_from_slice(object);
bytes.extend_from_slice(&chunk_length.to_le_bytes());
}
bytes.extend_from_slice(blake3::hash(checkpoint).as_bytes());
Ok(bytes)
}
fn decode_catalog_manifest<F>(manifest: &[u8], mut load: F) -> Result<Vec<u8>, S3StoreError>
where
F: FnMut(&[u8; 32]) -> Result<Vec<u8>, S3StoreError>,
{
if manifest.len() < 52 || manifest.get(..8) != Some(CATALOG_MAGIC) {
return Err(S3StoreError::CorruptObject);
}
let length = read_catalog_u64(manifest, 8)?;
let count = read_catalog_u32(manifest, 16)? as usize;
let entries_length = count
.checked_mul(36)
.and_then(|value| value.checked_add(20))
.ok_or(S3StoreError::LengthOverflow)?;
let expected_length = entries_length
.checked_add(32)
.ok_or(S3StoreError::LengthOverflow)?;
if manifest.len() != expected_length {
return Err(S3StoreError::CorruptObject);
}
let mut total = 0u64;
for index in 0..count {
let chunk_length = read_catalog_u32(manifest, 20 + index * 36 + 32)?;
if chunk_length == 0 || chunk_length as usize > CATALOG_CHUNK_SIZE {
return Err(S3StoreError::CorruptObject);
}
total = total
.checked_add(u64::from(chunk_length))
.ok_or(S3StoreError::LengthOverflow)?;
}
if total != length || total > MAX_RESPONSE_BYTES as u64 {
return Err(S3StoreError::CorruptObject);
}
let mut checkpoint = Vec::new();
for index in 0..count {
let at = 20 + index * 36;
let mut object = [0; 32];
object.copy_from_slice(&manifest[at..at + 32]);
let chunk_length = read_catalog_u32(manifest, at + 32)? as usize;
if chunk_length == 0 || chunk_length > CATALOG_CHUNK_SIZE {
return Err(S3StoreError::CorruptObject);
}
let chunk = load(&object)?;
if chunk.len() != chunk_length || blake3::hash(&chunk).as_bytes() != &object {
return Err(S3StoreError::CorruptObject);
}
checkpoint.extend_from_slice(&chunk);
}
let checksum = manifest
.get(entries_length..expected_length)
.ok_or(S3StoreError::CorruptObject)?;
if u64::try_from(checkpoint.len()) != Ok(length)
|| blake3::hash(&checkpoint).as_bytes() != checksum
{
return Err(S3StoreError::CorruptObject);
}
Ok(checkpoint)
}
fn read_catalog_u32(bytes: &[u8], at: usize) -> Result<u32, S3StoreError> {
let value = bytes
.get(at..at + 4)
.ok_or(S3StoreError::CorruptObject)?
.try_into()
.map_err(|_| S3StoreError::CorruptObject)?;
Ok(u32::from_le_bytes(value))
}
fn read_catalog_u64(bytes: &[u8], at: usize) -> Result<u64, S3StoreError> {
let value = bytes
.get(at..at + 8)
.ok_or(S3StoreError::CorruptObject)?
.try_into()
.map_err(|_| S3StoreError::CorruptObject)?;
Ok(u64::from_le_bytes(value))
}
fn encode_head(head: &Head) -> Vec<u8> {
let mut bytes = Vec::with_capacity(160);
bytes.extend_from_slice(HEAD_MAGIC);
bytes.extend_from_slice(&head.domain.0);
bytes.extend_from_slice(&head.generation.to_le_bytes());
bytes.extend_from_slice(&head.parent.to_le_bytes());
bytes.extend_from_slice(&head.writer_epoch.to_le_bytes());
bytes.extend_from_slice(&head.catalog.object);
bytes.extend_from_slice(&head.catalog.offset.to_le_bytes());
bytes.extend_from_slice(&head.catalog.encoded_length.to_le_bytes());
bytes.extend_from_slice(&head.catalog.decoded_length.to_le_bytes());
bytes.extend_from_slice(&head.catalog.checksum);
let checksum = blake3::hash(&bytes);
bytes.extend_from_slice(checksum.as_bytes());
bytes
}
fn decode_head(bytes: &[u8]) -> Result<Head, S3StoreError> {
const PAYLOAD: usize = 128;
if bytes.len() != PAYLOAD + 32 || &bytes[..8] != HEAD_MAGIC {
return Err(S3StoreError::InvalidHead);
}
if blake3::hash(&bytes[..PAYLOAD]).as_bytes() != &bytes[PAYLOAD..] {
return Err(S3StoreError::InvalidHead);
}
let mut domain = [0; 16];
domain.copy_from_slice(&bytes[8..24]);
let generation = read_u64(bytes, 24)?;
let parent = read_u64(bytes, 32)?;
let writer_epoch = read_u64(bytes, 40)?;
let mut object = [0; 32];
object.copy_from_slice(&bytes[48..80]);
let offset = read_u64(bytes, 80)?;
let encoded_length = read_u32(bytes, 88)?;
let decoded_length = read_u32(bytes, 92)?;
let mut checksum = [0; 32];
checksum.copy_from_slice(&bytes[96..128]);
Ok(Head {
domain: StorageDomainId(domain),
generation,
parent,
writer_epoch,
catalog: ObjectRef {
object,
offset,
encoded_length,
decoded_length,
checksum,
},
})
}
fn read_u32(bytes: &[u8], offset: usize) -> Result<u32, S3StoreError> {
let value = bytes
.get(offset..offset + 4)
.ok_or(S3StoreError::InvalidHead)?
.try_into()
.map_err(|_| S3StoreError::InvalidHead)?;
Ok(u32::from_le_bytes(value))
}
fn read_u64(bytes: &[u8], offset: usize) -> Result<u64, S3StoreError> {
let value = bytes
.get(offset..offset + 8)
.ok_or(S3StoreError::InvalidHead)?
.try_into()
.map_err(|_| S3StoreError::InvalidHead)?;
Ok(u64::from_le_bytes(value))
}
fn write_hex(output: &mut String, bytes: &[u8]) {
for byte in bytes {
write!(output, "{byte:02x}").expect("writing to String cannot fail");
}
}
#[cfg(feature = "std")]
fn map_ureq(error: UreqError) -> S3StoreError {
match error {
UreqError::Status(status, _) => S3StoreError::Status(status),
UreqError::Transport(_) => S3StoreError::Transport,
}
}
#[cfg(all(test, feature = "std"))]
mod tests {
use super::*;
use rustls::{server::Acceptor, CipherSuite};
use std::net::TcpListener;
use std::time::Instant;
#[test]
fn default_s3_agent_sends_chacha_tls_and_requires_a_handshake() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
listener.set_nonblocking(true).unwrap();
let endpoint = format!("https://{}", listener.local_addr().unwrap());
let store = S3PageStore::new(S3Config {
bucket_name: "test".into(),
endpoint,
access_key: "test".into(),
secret_key: "test".into(),
session_token: None,
region: "auto".into(),
prefix: None,
virtual_host_style: false,
})
.unwrap();
let server = std::thread::spawn(move || {
let deadline = Instant::now() + Duration::from_secs(5);
let mut socket = loop {
match listener.accept() {
Ok((socket, _)) => break socket,
Err(error) if error.kind() == io::ErrorKind::WouldBlock => {
assert!(Instant::now() < deadline, "S3 client did not connect");
std::thread::sleep(Duration::from_millis(1));
}
Err(error) => panic!("accept: {error}"),
}
};
socket.set_nonblocking(false).unwrap();
socket
.set_read_timeout(Some(Duration::from_secs(5)))
.unwrap();
let mut acceptor = Acceptor::default();
let accepted = loop {
assert_ne!(acceptor.read_tls(&mut socket).unwrap(), 0);
if let Some(accepted) = acceptor.accept().unwrap() {
break accepted;
}
};
let hello = accepted.client_hello();
assert!(hello
.cipher_suites()
.contains(&CipherSuite::TLS13_CHACHA20_POLY1305_SHA256));
assert!(hello.cipher_suites().iter().all(|suite| matches!(
suite,
CipherSuite::TLS13_CHACHA20_POLY1305_SHA256
| CipherSuite::TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256
| CipherSuite::TLS_EMPTY_RENEGOTIATION_INFO_SCSV
)));
});
let error = store.read_record("test", "record").err();
server.join().unwrap();
assert_eq!(error, Some(S3StoreError::Transport));
}
}