use std::fs::File;
use std::io::Read;
use std::path::Path;
use std::time::Duration;
use ureq::Agent;
use crate::durable::backend::{Backend, PutOutcome, ReplaceOutcome, Tagged};
use crate::durable::digest::Digest;
use crate::durable::errors::{err, Category, Error, Result};
use crate::durable::keys::is_valid_object_key;
use super::sigv4::{sha256_hex, sign, Credentials, Signable};
pub const MAX_SINGLE_PUT_BYTES: u64 = 5 * 1024 * 1024 * 1024;
#[derive(Debug, Clone, Default)]
pub struct S3Options {
pub bucket: String,
pub prefix: String,
pub region: Option<String>,
pub endpoint: Option<String>,
pub path_style: Option<bool>,
pub credentials: Option<Credentials>,
pub timeout: Option<Duration>,
}
pub struct S3Backend {
bucket: String,
prefix: String,
region: String,
endpoint: Option<(String, String, String)>,
path_style: bool,
credentials: Credentials,
agent: Agent,
describe: String,
}
impl std::fmt::Debug for S3Backend {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("S3Backend")
.field("location", &self.describe)
.field("region", &self.region)
.finish_non_exhaustive()
}
}
fn backend_error(message: impl Into<String>) -> Error {
err(Category::Backend, message)
}
fn split_endpoint(endpoint: &str) -> Result<(String, String, String)> {
let (scheme, rest) = endpoint.split_once("://").ok_or_else(|| {
backend_error(format!(
"durable: an S3 endpoint needs a scheme, got {endpoint:?}"
))
})?;
let (host, path) = match rest.split_once('/') {
Some((host, path)) => (host, format!("/{}", path.trim_end_matches('/'))),
None => (rest, String::new()),
};
if scheme.is_empty() || host.is_empty() {
return Err(backend_error(format!(
"durable: an S3 endpoint needs a scheme and a host, got {endpoint:?}"
)));
}
Ok((
scheme.to_string(),
host.to_string(),
path.trim_end_matches('/').to_string(),
))
}
fn build_agent(timeout: Duration) -> Result<Agent> {
let config = Agent::config_builder()
.timeout_global(Some(timeout))
.http_status_as_error(false);
let Ok(bundle) = std::env::var("AWS_CA_BUNDLE") else {
return Ok(config.build().into());
};
if bundle.is_empty() {
return Ok(config.build().into());
}
let pem = std::fs::read(&bundle).map_err(|e| {
Error::wrap(
Category::Backend,
e,
format!("durable: cannot read the AWS_CA_BUNDLE at {bundle}"),
)
})?;
let mut roots = Vec::new();
for item in ureq::tls::parse_pem(&pem) {
if let Ok(ureq::tls::PemItem::Certificate(certificate)) = item {
roots.push(certificate);
}
}
if roots.is_empty() {
return Err(backend_error(format!(
"durable: the AWS_CA_BUNDLE at {bundle} holds no certificates"
)));
}
let tls = ureq::tls::TlsConfig::builder()
.root_certs(ureq::tls::RootCerts::Specific(std::sync::Arc::new(roots)))
.build();
Ok(config.tls_config(tls).build().into())
}
impl S3Backend {
pub fn new(options: S3Options) -> Result<Self> {
if options.bucket.is_empty() {
return Err(backend_error("durable: the S3 backend needs a bucket"));
}
let credentials = Credentials::resolve(options.credentials)?;
let region = options
.region
.filter(|region| !region.is_empty())
.or_else(|| std::env::var("AWS_REGION").ok().filter(|r| !r.is_empty()))
.or_else(|| {
std::env::var("AWS_DEFAULT_REGION")
.ok()
.filter(|r| !r.is_empty())
})
.unwrap_or_else(|| "us-east-1".to_string());
let endpoint = match &options.endpoint {
Some(endpoint) if !endpoint.is_empty() => Some(split_endpoint(endpoint)?),
_ => None,
};
let path_style = options.path_style.unwrap_or(endpoint.is_some());
let prefix = options.prefix.trim_matches('/').to_string();
Ok(Self {
describe: format!("s3://{}/{}", options.bucket, prefix),
bucket: options.bucket,
prefix,
region,
endpoint,
path_style,
credentials,
agent: build_agent(options.timeout.unwrap_or(Duration::from_secs(300)))?,
})
}
fn key_for(&self, key: &str) -> Result<String> {
if !is_valid_object_key(key) {
return Err(backend_error(format!(
"durable: refusing to resolve invalid key {key:?}"
)));
}
if self.prefix.is_empty() {
Ok(key.to_string())
} else {
Ok(format!("{}/{key}", self.prefix))
}
}
fn url_for(&self, key: &str) -> Result<(String, String, String)> {
let full = self.key_for(key)?;
let (scheme, host, path) = match (&self.endpoint, self.path_style) {
(Some((scheme, host, base)), true) => (
scheme.clone(),
host.clone(),
format!("{base}/{}/{full}", self.bucket),
),
(Some((scheme, host, base)), false) => (
scheme.clone(),
format!("{}.{host}", self.bucket),
format!("{base}/{full}"),
),
(None, true) => (
"https".to_string(),
format!("s3.{}.amazonaws.com", self.region),
format!("/{}/{full}", self.bucket),
),
(None, false) => (
"https".to_string(),
format!("{}.s3.{}.amazonaws.com", self.bucket, self.region),
format!("/{full}"),
),
};
let encoded = super::sigv4::uri_encode_path(&path);
Ok((format!("{scheme}://{host}{encoded}"), host, path))
}
fn send(
&self,
key: &str,
headers: &[(String, String)],
payload_hash: &str,
body: Body<'_>,
) -> Result<Outcome> {
let method = body.method();
let (url, host, path) = self.url_for(key)?;
let signed = sign(
Signable {
method,
host: &host,
path: &path,
query: "",
headers,
payload_hash,
},
&self.credentials,
&self.region,
std::time::SystemTime::now(),
);
let all = || headers.iter().chain(signed.iter());
let sent = match body {
Body::None => {
let mut request = self.agent.get(&url);
for (name, value) in all() {
request = request.header(name.as_str(), value.as_str());
}
request.call()
}
Body::Bytes(bytes) => {
let mut request = self.agent.put(&url);
for (name, value) in all() {
request = request.header(name.as_str(), value.as_str());
}
request.send(bytes)
}
Body::File(file) => {
let mut request = self.agent.put(&url);
for (name, value) in all() {
request = request.header(name.as_str(), value.as_str());
}
request.send(file)
}
};
match sent {
Ok(response) => Ok(Outcome::Responded(response)),
Err(e) if reached_the_service(&e) => Ok(Outcome::Indeterminate),
Err(e) => Err(Error::wrap(
Category::Backend,
e,
format!("durable: {method} {key} failed against {}", self.describe),
)),
}
}
fn assert_within_single_put(&self, key: &str, size: u64) -> Result<()> {
if size > MAX_SINGLE_PUT_BYTES {
return Err(err(
Category::LimitExceeded,
format!(
"durable: {key} is {size} bytes, over the {MAX_SINGLE_PUT_BYTES}-byte ceiling \
for a single PutObject; multipart upload is not implemented, so checkpoint \
more often or reduce the database"
),
)
.with_limit(MAX_SINGLE_PUT_BYTES, size));
}
Ok(())
}
fn status_error(&self, what: &str, mut response: Response) -> Error {
let status = response.status().as_u16();
let detail = response
.body_mut()
.with_config()
.limit(2048)
.read_to_string()
.unwrap_or_default();
let detail = detail.trim();
let suffix = if detail.is_empty() {
String::new()
} else {
format!(": {detail}")
};
backend_error(format!(
"durable: {what} failed against {} (HTTP {status}){suffix}",
self.describe
))
}
}
type Response = ureq::http::Response<ureq::Body>;
enum Outcome {
Responded(Response),
Indeterminate,
}
enum Body<'a> {
None,
Bytes(&'a [u8]),
File(File),
}
impl Body<'_> {
fn method(&self) -> &'static str {
match self {
Self::None => "GET",
Self::Bytes(_) | Self::File(_) => "PUT",
}
}
}
fn reached_the_service(error: &ureq::Error) -> bool {
match error {
ureq::Error::HostNotFound | ureq::Error::ConnectionFailed => false,
ureq::Error::Io(io) => !matches!(
io.kind(),
std::io::ErrorKind::ConnectionRefused | std::io::ErrorKind::NotFound
),
ureq::Error::BadUri(_)
| ureq::Error::Http(_)
| ureq::Error::InvalidProxyUrl
| ureq::Error::TooManyRedirects
| ureq::Error::BodyExceedsLimit(_) => false,
_ => true,
}
}
fn is_precondition_failure(status: u16) -> bool {
status == 412 || status == 409
}
fn etag_of(response: &Response) -> Option<String> {
response
.headers()
.get("etag")
.and_then(|value| value.to_str().ok())
.map(str::to_string)
}
impl Backend for S3Backend {
fn describe(&self) -> String {
self.describe.clone()
}
fn get_bytes(&self, key: &str) -> Result<Option<Vec<u8>>> {
Ok(self.get_bytes_with_etag(key)?.map(|tagged| tagged.data))
}
fn get_bytes_with_etag(&self, key: &str) -> Result<Option<Tagged>> {
let Outcome::Responded(mut response) = self.send(key, &[], &sha256_hex(b""), Body::None)?
else {
return Err(backend_error(format!(
"durable: reading {key} from {} did not complete",
self.describe
)));
};
let status = response.status().as_u16();
if status == 404 {
return Ok(None);
}
if status != 200 {
return Err(self.status_error(&format!("reading {key}"), response));
}
let etag = etag_of(&response).ok_or_else(|| {
backend_error(format!(
"durable: {key} came back from {} without an ETag; this provider cannot support \
compare-and-swap",
self.describe
))
})?;
let data = response
.body_mut()
.with_config()
.limit(crate::durable::MAX_WAL_SEGMENT_BYTES)
.read_to_vec()
.map_err(|e| {
Error::wrap(
Category::Backend,
e,
format!("durable: reading the body of {key} from {}", self.describe),
)
})?;
Ok(Some(Tagged { data, etag }))
}
fn open_reader(&self, key: &str) -> Result<Option<Box<dyn Read + Send>>> {
let Outcome::Responded(response) = self.send(key, &[], &sha256_hex(b""), Body::None)?
else {
return Err(backend_error(format!(
"durable: opening {key} from {} did not complete",
self.describe
)));
};
let status = response.status().as_u16();
if status == 404 {
return Ok(None);
}
if status != 200 {
return Err(self.status_error(&format!("opening {key}"), response));
}
Ok(Some(Box::new(response.into_body().into_reader())))
}
fn put_bytes_if_absent(&self, key: &str, data: &[u8]) -> Result<PutOutcome> {
self.assert_within_single_put(key, data.len() as u64)?;
let headers = vec![("if-none-match".to_string(), "*".to_string())];
match self.send(key, &headers, &sha256_hex(data), Body::Bytes(data))? {
Outcome::Indeterminate => Ok(PutOutcome::Ambiguous),
Outcome::Responded(response) => Ok(self.classify_put(response)),
}
}
fn put_file_if_absent(
&self,
key: &str,
local_path: &Path,
digest: &Digest,
) -> Result<PutOutcome> {
self.assert_within_single_put(key, digest.size)?;
let file = File::open(local_path).map_err(|e| {
Error::wrap(
Category::Backend,
e,
format!("durable: cannot read {}", local_path.display()),
)
})?;
let headers = vec![("if-none-match".to_string(), "*".to_string())];
match self.send(key, &headers, &digest.sha256, Body::File(file))? {
Outcome::Indeterminate => Ok(PutOutcome::Ambiguous),
Outcome::Responded(response) => Ok(self.classify_put(response)),
}
}
fn replace_if_match(&self, key: &str, data: &[u8], etag: &str) -> Result<ReplaceOutcome> {
self.assert_within_single_put(key, data.len() as u64)?;
let headers = vec![("if-match".to_string(), etag.to_string())];
let response = match self.send(key, &headers, &sha256_hex(data), Body::Bytes(data))? {
Outcome::Indeterminate => return Ok(ReplaceOutcome::Ambiguous),
Outcome::Responded(response) => response,
};
let status = response.status().as_u16();
if is_precondition_failure(status) {
return Ok(ReplaceOutcome::NotMatched);
}
if status == 404 {
return Ok(ReplaceOutcome::NotMatched);
}
if status >= 500 {
return Ok(ReplaceOutcome::Ambiguous);
}
if status != 200 {
return Err(self.status_error(&format!("replacing {key}"), response));
}
match etag_of(&response) {
Some(etag) => Ok(ReplaceOutcome::Done { etag }),
None => Ok(ReplaceOutcome::Ambiguous),
}
}
}
impl S3Backend {
fn classify_put(&self, response: Response) -> PutOutcome {
let status = response.status().as_u16();
match status {
_ if is_precondition_failure(status) => PutOutcome::AlreadyExists,
200 => PutOutcome::Created,
_ => PutOutcome::Ambiguous,
}
}
}
#[cfg(test)]
mod test {
use super::*;
fn options(bucket: &str) -> S3Options {
S3Options {
bucket: bucket.to_string(),
prefix: "durable/tenant-1".to_string(),
region: Some("eu-central-1".to_string()),
credentials: Some(Credentials {
access_key_id: "AKIDEXAMPLE".to_string(),
secret_access_key: "secret".to_string(),
session_token: None,
}),
..S3Options::default()
}
}
#[test]
fn aws_gets_a_virtual_hosted_url_and_a_gateway_gets_a_path_style_one() -> Result<()> {
let aws = S3Backend::new(options("my-bucket"))?;
let (url, host, path) = aws.url_for("head.json")?;
assert_eq!(host, "my-bucket.s3.eu-central-1.amazonaws.com");
assert_eq!(path, "/durable/tenant-1/head.json");
assert_eq!(
url,
"https://my-bucket.s3.eu-central-1.amazonaws.com/durable/tenant-1/head.json"
);
let minio = S3Backend::new(S3Options {
endpoint: Some("http://127.0.0.1:9000".to_string()),
..options("my-bucket")
})?;
let (url, host, _) = minio.url_for("wal/1-1-aa.jsonl")?;
assert_eq!(
host, "127.0.0.1:9000",
"a gateway keeps the bucket in the path"
);
assert_eq!(
url,
"http://127.0.0.1:9000/my-bucket/durable/tenant-1/wal/1-1-aa.jsonl"
);
Ok(())
}
#[test]
fn a_key_out_of_an_untrusted_head_cannot_leave_the_prefix() -> Result<()> {
let backend = S3Backend::new(options("my-bucket"))?;
for key in ["../elsewhere", "/absolute", "wal/../../escape", ""] {
assert_eq!(
backend.key_for(key).unwrap_err().category(),
Category::Backend,
"{key:?}"
);
}
assert_eq!(backend.key_for("head.json")?, "durable/tenant-1/head.json");
Ok(())
}
#[test]
fn the_description_carries_no_credential() -> Result<()> {
let backend = S3Backend::new(options("my-bucket"))?;
let described = format!("{} {:?}", backend.describe(), backend);
assert!(described.contains("s3://my-bucket/durable/tenant-1"));
assert!(!described.contains("secret"), "{described}");
assert!(!described.contains("AKIDEXAMPLE"), "{described}");
Ok(())
}
#[test]
fn an_object_beyond_one_put_is_a_limit_rather_than_a_truncation() -> Result<()> {
let backend = S3Backend::new(options("my-bucket"))?;
let error = backend
.assert_within_single_put("checkpoints/1-1-aa.tar.gz", MAX_SINGLE_PUT_BYTES + 1)
.expect_err("over one PutObject");
assert_eq!(error.category(), Category::LimitExceeded);
assert_eq!(
error.limit(),
Some((MAX_SINGLE_PUT_BYTES, MAX_SINGLE_PUT_BYTES + 1))
);
Ok(())
}
#[test]
fn a_precondition_failure_is_a_race_rather_than_an_error() {
assert!(is_precondition_failure(412), "If-Match / If-None-Match");
assert!(is_precondition_failure(409), "ConditionalRequestConflict");
assert!(!is_precondition_failure(404));
assert!(!is_precondition_failure(200));
assert!(!is_precondition_failure(500));
}
#[test]
fn only_a_request_that_provably_never_left_is_ruled_out() {
assert!(!reached_the_service(&ureq::Error::HostNotFound));
assert!(!reached_the_service(&ureq::Error::ConnectionFailed));
assert!(!reached_the_service(&ureq::Error::Io(
std::io::Error::from(std::io::ErrorKind::ConnectionRefused)
)));
assert!(reached_the_service(&ureq::Error::Io(std::io::Error::from(
std::io::ErrorKind::ConnectionReset
))));
assert!(reached_the_service(&ureq::Error::Io(std::io::Error::from(
std::io::ErrorKind::BrokenPipe
))));
}
#[test]
fn an_endpoint_needs_a_scheme_and_a_host() {
assert!(split_endpoint("127.0.0.1:9000").is_err());
assert!(split_endpoint("http://").is_err());
assert_eq!(
split_endpoint("https://x.r2.cloudflarestorage.com").unwrap(),
(
"https".to_string(),
"x.r2.cloudflarestorage.com".to_string(),
String::new()
)
);
assert_eq!(
split_endpoint("http://gateway.example/base/").unwrap(),
(
"http".to_string(),
"gateway.example".to_string(),
"/base".to_string()
)
);
}
}