use crate::{ObjectMetadata, Result, StorageError, StorageKey, StoredObject};
use chrono::Utc;
use futures_util::TryStreamExt;
use hmac::{Hmac, Mac};
use reqwest::{Method, Url};
use sha2::{Digest, Sha256};
use std::{io::Seek, time::Duration};
use tokio::{
fs::File,
io::{AsyncRead, AsyncReadExt, AsyncWriteExt},
};
use tokio_util::io::{ReaderStream, StreamReader};
type HmacSha256 = Hmac<Sha256>;
pub(crate) struct S3 {
client: reqwest::Client,
base: Url,
bucket: String,
region: String,
path_style: bool,
credentials: Credentials,
}
struct Credentials {
access: String,
secret: String,
token: Option<String>,
}
impl S3 {
pub(crate) fn new(settings: &capy_core::StorageSettings) -> Result<Self> {
let _ = rustls::crypto::ring::default_provider().install_default();
settings
.validate(false)
.map_err(|_| StorageError::Configuration("invalid S3 settings"))?;
let bucket = settings
.bucket
.clone()
.ok_or(StorageError::Configuration("missing S3 bucket"))?;
let region = settings
.region
.clone()
.ok_or(StorageError::Configuration("missing S3 region"))?;
if bucket.len() < 3
|| bucket.len() > 63
|| !bucket
.bytes()
.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || b"-.".contains(&c))
|| !region
.bytes()
.all(|c| c.is_ascii_alphanumeric() || c == b'-')
{
return Err(StorageError::Configuration("invalid S3 bucket or region"));
}
let access = settings
.access_key_id
.clone()
.or_else(|| std::env::var("AWS_ACCESS_KEY_ID").ok())
.ok_or(StorageError::Configuration("S3 credentials unavailable"))?;
let secret = settings
.secret_access_key
.clone()
.or_else(|| std::env::var("AWS_SECRET_ACCESS_KEY").ok())
.ok_or(StorageError::Configuration("S3 credentials unavailable"))?;
if access.is_empty() || secret.is_empty() {
return Err(StorageError::Configuration("S3 credentials unavailable"));
}
let token = settings
.session_token
.clone()
.or_else(|| std::env::var("AWS_SESSION_TOKEN").ok());
let endpoint = settings
.endpoint
.clone()
.unwrap_or_else(|| format!("https://s3.{region}.amazonaws.com"));
let base = Url::parse(&endpoint)
.map_err(|_| StorageError::Configuration("invalid S3 endpoint"))?;
if base.query().is_some()
|| base.fragment().is_some()
|| base.username() != ""
|| base.password().is_some()
|| !matches!(base.scheme(), "http" | "https")
{
return Err(StorageError::Configuration("invalid S3 endpoint"));
}
let client = reqwest::Client::builder()
.redirect(reqwest::redirect::Policy::none())
.connect_timeout(Duration::from_secs(10))
.build()
.map_err(|_| StorageError::Configuration("could not initialize S3 transport"))?;
Ok(Self {
client,
base,
bucket,
region,
path_style: settings.path_style,
credentials: Credentials {
access,
secret,
token,
},
})
}
fn object_url(&self, key: &StorageKey) -> Result<Url> {
let mut url = self.base.clone();
let path = self.base.path().trim_end_matches('/');
if self.path_style {
url.set_path(&format!("{path}/{}/{}", self.bucket, key.as_str()));
} else {
let host = self
.base
.host_str()
.ok_or(StorageError::Configuration("S3 endpoint needs host"))?;
url.set_host(Some(&format!("{}.{}", self.bucket, host)))
.map_err(|_| StorageError::Configuration("invalid S3 virtual host"))?;
url.set_path(&format!("{path}/{}", key.as_str()));
}
Ok(url)
}
fn signed_request(
&self,
method: Method,
key: &StorageKey,
payload_hash: &str,
) -> Result<reqwest::RequestBuilder> {
let url = self.object_url(key)?;
let now = Utc::now();
let date = now.format("%Y%m%d").to_string();
let timestamp = now.format("%Y%m%dT%H%M%SZ").to_string();
let host = host(&url)?;
let token_header = self
.credentials
.token
.as_ref()
.map(|token| format!("x-amz-security-token:{token}\n"))
.unwrap_or_default();
let signed_headers = if self.credentials.token.is_some() {
"host;x-amz-content-sha256;x-amz-date;x-amz-security-token"
} else {
"host;x-amz-content-sha256;x-amz-date"
};
let canonical = format!(
"{}\n{}\n\nhost:{}\nx-amz-content-sha256:{}\nx-amz-date:{}\n{}\n{}\n{}",
method.as_str(),
url.path(),
host,
payload_hash,
timestamp,
token_header,
signed_headers,
payload_hash
);
let signature = self.signature(&date, ×tamp, &canonical);
let scope = format!("{date}/{}/s3/aws4_request", self.region);
let authorization = format!("AWS4-HMAC-SHA256 Credential={}/{scope},SignedHeaders={signed_headers},Signature={signature}", self.credentials.access);
let mut request = self
.client
.request(method, url)
.header("host", host)
.header("x-amz-content-sha256", payload_hash)
.header("x-amz-date", timestamp)
.header("authorization", authorization);
if let Some(token) = &self.credentials.token {
request = request.header("x-amz-security-token", token);
}
Ok(request)
}
fn signature(&self, date: &str, timestamp: &str, canonical: &str) -> String {
let scope = format!("{date}/{}/s3/aws4_request", self.region);
let string_to_sign = format!(
"AWS4-HMAC-SHA256\n{timestamp}\n{scope}\n{}",
hex::encode(Sha256::digest(canonical.as_bytes()))
);
let key = hmac(
format!("AWS4{}", self.credentials.secret).as_bytes(),
date.as_bytes(),
);
let key = hmac(&key, self.region.as_bytes());
let key = hmac(&key, b"s3");
let key = hmac(&key, b"aws4_request");
hex::encode(hmac(&key, string_to_sign.as_bytes()))
}
pub(crate) async fn put<R: AsyncRead + Unpin + Send>(
&self,
key: &StorageKey,
reader: &mut R,
content_type: Option<&str>,
limit: u64,
) -> Result<ObjectMetadata> {
let mut temporary = tempfile::tempfile()?;
let mut writer = File::from_std(temporary.try_clone()?);
let mut digest = Sha256::new();
let mut length = 0u64;
let mut buffer = [0u8; 16 * 1024];
loop {
let count = reader.read(&mut buffer).await?;
if count == 0 {
break;
}
length += count as u64;
if length > limit {
return Err(StorageError::TooLarge);
}
writer.write_all(&buffer[..count]).await?;
digest.update(&buffer[..count]);
}
writer.flush().await?;
drop(writer);
temporary.rewind()?;
let hash = hex::encode(digest.finalize());
let body = reqwest::Body::wrap_stream(ReaderStream::new(File::from_std(temporary)));
let mut request = self
.signed_request(Method::PUT, key, &hash)?
.header("content-length", length)
.body(body);
if let Some(content_type) = content_type {
request = request.header("content-type", content_type);
}
let response = request.send().await.map_err(|_| StorageError::Backend)?;
if !response.status().is_success() {
return Err(StorageError::Backend);
}
Ok(ObjectMetadata {
length,
content_type: content_type.map(str::to_owned),
})
}
pub(crate) async fn get(&self, key: &StorageKey) -> Result<StoredObject> {
let response = self
.signed_request(Method::GET, key, &empty_hash())?
.send()
.await
.map_err(|_| StorageError::Backend)?;
if response.status() == reqwest::StatusCode::NOT_FOUND {
return Err(StorageError::NotFound);
}
if !response.status().is_success() {
return Err(StorageError::Backend);
}
let metadata = response_metadata(&response)?;
let stream = Box::pin(response.bytes_stream().map_err(std::io::Error::other));
Ok(StoredObject {
reader: Box::new(StreamReader::new(stream)),
metadata,
})
}
pub(crate) async fn head(&self, key: &StorageKey) -> Result<Option<ObjectMetadata>> {
let response = self
.signed_request(Method::HEAD, key, &empty_hash())?
.send()
.await
.map_err(|_| StorageError::Backend)?;
if response.status() == reqwest::StatusCode::NOT_FOUND {
return Ok(None);
}
if !response.status().is_success() {
return Err(StorageError::Backend);
}
response_metadata(&response).map(Some)
}
pub(crate) async fn delete(&self, key: &StorageKey) -> Result<()> {
let response = self
.signed_request(Method::DELETE, key, &empty_hash())?
.send()
.await
.map_err(|_| StorageError::Backend)?;
if response.status().is_success() || response.status() == reqwest::StatusCode::NOT_FOUND {
Ok(())
} else {
Err(StorageError::Backend)
}
}
pub(crate) fn temporary_get_url(&self, key: &StorageKey, duration: Duration) -> Result<String> {
let expires = duration.as_secs();
if !(1..=604_800).contains(&expires) {
return Err(StorageError::Configuration(
"S3 URL expiry must be 1..=604800 seconds",
));
}
let mut url = self.object_url(key)?;
let now = Utc::now();
let date = now.format("%Y%m%d").to_string();
let timestamp = now.format("%Y%m%dT%H%M%SZ").to_string();
let scope = format!("{date}/{}/s3/aws4_request", self.region);
let host = host(&url)?;
let mut values = vec![
("X-Amz-Algorithm", "AWS4-HMAC-SHA256".to_owned()),
(
"X-Amz-Credential",
format!("{}/{}", self.credentials.access, scope),
),
("X-Amz-Date", timestamp.clone()),
("X-Amz-Expires", expires.to_string()),
("X-Amz-SignedHeaders", "host".into()),
];
if let Some(token) = &self.credentials.token {
values.push(("X-Amz-Security-Token", token.clone()));
}
values.sort_by_key(|(name, _)| *name);
let query = values
.iter()
.map(|(name, value)| format!("{}={}", uri_encode(name), uri_encode(value)))
.collect::<Vec<_>>()
.join("&");
let canonical = format!(
"GET\n{}\n{}\nhost:{}\n\nhost\nUNSIGNED-PAYLOAD",
url.path(),
query,
host
);
let signature = self.signature(&date, ×tamp, &canonical);
url.set_query(Some(&format!("{query}&X-Amz-Signature={signature}")));
Ok(url.into())
}
}
fn response_metadata(response: &reqwest::Response) -> Result<ObjectMetadata> {
let length = response
.headers()
.get("content-length")
.and_then(|value| value.to_str().ok())
.and_then(|value| value.parse::<u64>().ok())
.ok_or(StorageError::Backend)?;
let content_type = response
.headers()
.get("content-type")
.and_then(|value| value.to_str().ok())
.map(str::to_owned);
Ok(ObjectMetadata {
length,
content_type,
})
}
fn empty_hash() -> String {
hex::encode(Sha256::digest([]))
}
fn hmac(key: &[u8], data: &[u8]) -> Vec<u8> {
let mut mac = HmacSha256::new_from_slice(key).expect("HMAC accepts arbitrary key length");
mac.update(data);
mac.finalize().into_bytes().to_vec()
}
fn host(url: &Url) -> Result<String> {
let name = url
.host_str()
.ok_or(StorageError::Configuration("S3 endpoint needs host"))?;
Ok(match url.port() {
Some(port) => format!("{name}:{port}"),
None => name.into(),
})
}
fn uri_encode(value: &str) -> String {
let mut result = String::new();
for byte in value.bytes() {
if byte.is_ascii_alphanumeric() || b"-._~".contains(&byte) {
result.push(char::from(byte));
} else {
result.push_str(&format!("%{byte:02X}"));
}
}
result
}
#[cfg(test)]
mod tests {
use super::*;
fn settings() -> capy_core::StorageSettings {
capy_core::StorageSettings {
driver: "s3".into(),
bucket: Some("examplebucket".into()),
region: Some("us-east-1".into()),
access_key_id: Some("AKIAIOSFODNN7EXAMPLE".into()),
secret_access_key: Some("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".into()),
..Default::default()
}
}
#[test]
fn signature_matches_aws_published_vector() {
let s3 = S3::new(&settings()).unwrap();
let empty = empty_hash();
let canonical = format!("GET\n/test.txt\n\nhost:examplebucket.s3.amazonaws.com\nrange:bytes=0-9\nx-amz-content-sha256:{empty}\nx-amz-date:20130524T000000Z\n\nhost;range;x-amz-content-sha256;x-amz-date\n{empty}");
assert_eq!(
hex::encode(Sha256::digest(canonical.as_bytes())),
"7344ae5b7ee6c3e7e6b0fe0640412a37625d1fbfff95c48bbb2dc43964946972"
);
assert_eq!(
s3.signature("20130524", "20130524T000000Z", &canonical),
"f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41"
);
assert_eq!(
s3.object_url(&StorageKey::new("photos/test.png").unwrap())
.unwrap()
.as_str(),
"https://s3.us-east-1.amazonaws.com/examplebucket/photos/test.png"
);
let url = s3
.temporary_get_url(
&StorageKey::new("photos/test.png").unwrap(),
Duration::from_secs(600),
)
.unwrap();
assert!(url.contains("X-Amz-Signature="));
assert!(url.contains("X-Amz-Expires=600"));
assert!(!url.contains("EXAMPLEKEY"));
}
#[tokio::test]
#[ignore = "requires CAPY_S3_TEST_ENDPOINT, CAPY_S3_TEST_BUCKET and disposable S3-compatible service"]
async fn isolated_compatible_service_round_trip() {
let mut settings = settings();
settings.endpoint = Some(std::env::var("CAPY_S3_TEST_ENDPOINT").unwrap());
settings.bucket = Some(std::env::var("CAPY_S3_TEST_BUCKET").unwrap());
settings.access_key_id = Some(std::env::var("CAPY_S3_TEST_ACCESS_KEY").unwrap());
settings.secret_access_key = Some(std::env::var("CAPY_S3_TEST_SECRET_KEY").unwrap());
settings.path_style = true;
let storage = crate::Storage::from_config(&settings).unwrap().unwrap();
let key = StorageKey::new(format!(
"capy-tests/{}-{}.txt",
std::process::id(),
Utc::now().timestamp_nanos_opt().unwrap()
))
.unwrap();
assert!(storage.head(&key).await.unwrap().is_none());
storage
.put_bytes(&key, b"private S3 round trip", Some("text/plain"))
.await
.unwrap();
assert_eq!(storage.head(&key).await.unwrap().unwrap().length, 21);
assert_eq!(storage.read(&key).await.unwrap(), b"private S3 round trip");
let url = storage
.temporary_get_url(&key, Duration::from_secs(60))
.await
.unwrap();
assert!(url.contains("X-Amz-Signature="));
let signed_download = reqwest::Client::new().get(url).send().await.unwrap();
assert!(signed_download.status().is_success());
assert_eq!(
signed_download.bytes().await.unwrap().as_ref(),
b"private S3 round trip"
);
storage.delete(&key).await.unwrap();
assert!(storage.head(&key).await.unwrap().is_none());
}
}