mod config;
mod sigv4;
pub use config::S3Config;
use std::time::{Duration, SystemTime};
use async_trait::async_trait;
use bytes::Bytes;
use reqwest::header::{CONTENT_LENGTH, CONTENT_TYPE, ETAG};
use crate::error::{Error, Result};
use crate::storage::{ListOptions, ObjectStorage};
use crate::types::{ListOutcome, ObjectMetadata, PutBody, PutOptions};
use sigv4::{
amz_date_now, canonical_query, sha256_hex, uri_encode, Credentials, RequestToSign,
EMPTY_PAYLOAD_SHA256,
};
const S3_MIN_PART_SIZE: usize = 5 * 1024 * 1024;
const S3_DEFAULT_PART_SIZE: usize = 8 * 1024 * 1024;
const S3_MAX_PARTS: u32 = 10_000;
const S3_MAX_EXPIRES: u64 = 604_800;
#[derive(Debug, Clone)]
pub struct S3Client {
config: S3Config,
http: reqwest::Client,
}
impl S3Client {
pub fn new(config: S3Config) -> Self {
S3Client {
config,
http: reqwest::Client::new(),
}
}
pub fn with_http(mut self, http: reqwest::Client) -> Self {
self.http = http;
self
}
pub fn config(&self) -> &S3Config {
&self.config
}
fn creds(&self) -> Credentials<'_> {
Credentials {
access_key: &self.config.access_key,
secret_key: &self.config.secret_key,
session_token: self.config.session_token.as_deref(),
region: &self.config.region,
service: "s3",
}
}
fn object_parts(&self, key: &str) -> (String, String) {
let endpoint_host = self.config.endpoint_host();
let ek = encode_key(key);
if self.config.force_path_style {
(
endpoint_host.clone(),
format!("/{}/{}", self.config.bucket, ek),
)
} else {
(
format!("{}.{}", self.config.bucket, endpoint_host),
format!("/{ek}"),
)
}
}
fn bucket_parts(&self) -> (String, String) {
let endpoint_host = self.config.endpoint_host();
if self.config.force_path_style {
(endpoint_host.clone(), format!("/{}", self.config.bucket))
} else {
(
format!("{}.{}", self.config.bucket, endpoint_host),
"/".to_string(),
)
}
}
fn signed(
&self,
method: &str,
host: &str,
path: &str,
query: &[(String, String)],
extra_signed: &[(&str, &str)],
payload_sha256: &str,
) -> (String, String, String) {
let amz_date = amz_date_now(SystemTime::now());
let mut headers: Vec<(String, String)> = vec![
("host".to_string(), host.to_string()),
(
"x-amz-content-sha256".to_string(),
payload_sha256.to_string(),
),
("x-amz-date".to_string(), amz_date.clone()),
];
if let Some(token) = &self.config.session_token {
headers.push(("x-amz-security-token".to_string(), token.clone()));
}
for (n, v) in extra_signed {
headers.push((n.to_string(), v.to_string()));
}
let req = RequestToSign {
method,
canonical_uri: path,
query,
headers,
payload_sha256,
amz_date: &amz_date,
};
let auth = req.authorization(&self.creds());
let cq = canonical_query(query);
let url = if cq.is_empty() {
format!("{}://{}{}", self.config.scheme(), host, path)
} else {
format!("{}://{}{}?{}", self.config.scheme(), host, path, cq)
};
(url, amz_date, auth)
}
async fn check(&self, resp: reqwest::Response) -> Result<reqwest::Response> {
let status = resp.status();
if status.is_success() {
return Ok(resp);
}
let body = resp.text().await.unwrap_or_default();
let short: String = body.chars().take(512).collect();
let friendly = s3_error_message(&short).unwrap_or_else(|| short.clone());
Err(Error::api(status.as_u16(), None, friendly))
}
pub async fn create_multipart_upload(
&self,
key: &str,
opts: &PutOptions,
) -> Result<MultipartUpload> {
let (host, path) = self.bucket_parts();
let query = vec![("uploads".to_string(), String::new())];
let extra = match &opts.content_type {
Some(ct) => vec![("content-type", ct.as_str())],
None => vec![],
};
let (url, amz_date, auth) =
self.signed("POST", &host, &path, &query, &extra, EMPTY_PAYLOAD_SHA256);
let mut reqb = self
.http
.post(&url)
.header(reqwest::header::AUTHORIZATION, &auth)
.header("x-amz-content-sha256", EMPTY_PAYLOAD_SHA256)
.header("x-amz-date", &amz_date);
if let Some(ct) = &opts.content_type {
reqb = reqb.header(CONTENT_TYPE, ct);
}
let resp = reqb.send().await?;
let resp = self.check(resp).await?;
let xml = resp.text().await?;
let upload_id = between_tags(&xml, "UploadId")
.into_iter()
.next()
.ok_or_else(|| Error::InvalidInput("multipart init returned no UploadId".into()))?;
Ok(MultipartUpload {
client: self.clone(),
upload_id: upload_id.trim().to_string(),
key: key.to_string(),
})
}
pub fn multipart_upload(&self, key: &str, upload_id: impl Into<String>) -> MultipartUpload {
MultipartUpload {
client: self.clone(),
upload_id: upload_id.into(),
key: key.to_string(),
}
}
async fn upload_resumable(
&self,
key: &str,
data: Bytes,
part_size: usize,
opts: &PutOptions,
) -> Result<ObjectMetadata> {
if data.is_empty() || data.len() <= S3_MIN_PART_SIZE {
return self.put(key, PutBody::Bytes(data), opts).await;
}
let mp = self.create_multipart_upload(key, opts).await?;
let part_size = if part_size == 0 {
S3_DEFAULT_PART_SIZE
} else {
part_size.max(S3_MIN_PART_SIZE)
};
let total = data.len();
let total_parts = (total + part_size - 1) / part_size;
if total_parts > S3_MAX_PARTS as usize {
let _ = mp.abort().await;
return Err(Error::InvalidInput(format!(
"need {total_parts} parts (> {S3_MAX_PARTS}); use a larger part_size"
)));
}
let mut parts: Vec<CompletedPart> = Vec::with_capacity(total_parts);
for idx in 0..total_parts {
let number = (idx + 1) as u32;
let start = idx * part_size;
let end = (start + part_size).min(total);
let chunk = data.slice(start..end);
let etag = mp.upload_part(number, chunk).await?;
parts.push(CompletedPart { number, etag });
}
mp.complete(&parts).await
}
}
fn encode_key(key: &str) -> String {
let key = key.strip_prefix('/').unwrap_or(key);
key.split('/')
.map(|seg| uri_encode(seg, false))
.collect::<Vec<_>>()
.join("/")
}
#[async_trait]
impl ObjectStorage for S3Client {
fn backend(&self) -> &'static str {
"s3"
}
fn bucket(&self) -> &str {
&self.config.bucket
}
async fn put(&self, key: &str, body: PutBody, opts: &PutOptions) -> Result<ObjectMetadata> {
let data = body.into_bytes();
let size = data.len() as u64;
let payload_sha = sha256_hex(&data);
let (host, path) = self.object_parts(key);
let extra = match &opts.content_type {
Some(ct) => vec![("content-type", ct.as_str())],
None => vec![],
};
let (url, amz_date, auth) = self.signed("PUT", &host, &path, &[], &extra, &payload_sha);
let mut reqb = self
.http
.put(&url)
.header(reqwest::header::AUTHORIZATION, &auth)
.header("x-amz-content-sha256", &payload_sha)
.header("x-amz-date", &amz_date);
if let Some(ct) = &opts.content_type {
reqb = reqb.header(CONTENT_TYPE, ct);
}
if opts.insert_only {
reqb = reqb.header("if-none-match", "*");
}
let resp = reqb.body(data).send().await?;
let resp = self.check(resp).await?;
let etag = read_etag(&resp);
Ok(ObjectMetadata {
key: key.to_string(),
size,
content_type: opts.content_type.clone(),
etag,
last_modified: None,
})
}
async fn put_resumable(
&self,
key: &str,
body: PutBody,
part_size: usize,
opts: &PutOptions,
) -> Result<ObjectMetadata> {
self.upload_resumable(key, body.into_bytes(), part_size, opts)
.await
}
async fn get(&self, key: &str) -> Result<Bytes> {
let (host, path) = self.object_parts(key);
let (url, amz_date, auth) =
self.signed("GET", &host, &path, &[], &[], EMPTY_PAYLOAD_SHA256);
let resp = self
.http
.get(&url)
.header(reqwest::header::AUTHORIZATION, &auth)
.header("x-amz-content-sha256", EMPTY_PAYLOAD_SHA256)
.header("x-amz-date", &amz_date)
.send()
.await?;
let resp = self.check(resp).await?;
Ok(resp.bytes().await?)
}
async fn delete(&self, key: &str) -> Result<()> {
let (host, path) = self.object_parts(key);
let (url, amz_date, auth) =
self.signed("DELETE", &host, &path, &[], &[], EMPTY_PAYLOAD_SHA256);
let resp = self
.http
.delete(&url)
.header(reqwest::header::AUTHORIZATION, &auth)
.header("x-amz-content-sha256", EMPTY_PAYLOAD_SHA256)
.header("x-amz-date", &amz_date)
.send()
.await?;
self.check(resp).await?;
Ok(())
}
async fn exists(&self, key: &str) -> Result<bool> {
match self.stat(key).await {
Ok(_) => Ok(true),
Err(Error::Api { status: 404, .. }) => Ok(false),
Err(other) => Err(other),
}
}
async fn stat(&self, key: &str) -> Result<ObjectMetadata> {
let (host, path) = self.object_parts(key);
let (url, amz_date, auth) =
self.signed("HEAD", &host, &path, &[], &[], EMPTY_PAYLOAD_SHA256);
let resp = self
.http
.head(&url)
.header(reqwest::header::AUTHORIZATION, &auth)
.header("x-amz-content-sha256", EMPTY_PAYLOAD_SHA256)
.header("x-amz-date", &amz_date)
.send()
.await?;
let status = resp.status();
if status.as_u16() == 404 {
return Err(Error::api(404, None, "NoSuchKey"));
}
let resp = self.check(resp).await?;
let headers = resp.headers();
let size = headers
.get(CONTENT_LENGTH)
.and_then(|v| v.to_str().ok())
.and_then(|s| s.parse::<u64>().ok())
.unwrap_or(0);
let content_type = headers
.get(CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.map(str::to_string);
let etag = headers
.get(ETAG)
.and_then(|v| v.to_str().ok())
.map(strip_quotes);
let last_modified = headers
.get("last-modified")
.and_then(|v| v.to_str().ok())
.and_then(parse_iso8601);
Ok(ObjectMetadata {
key: key.to_string(),
size,
content_type,
etag,
last_modified,
})
}
async fn list(&self, prefix: &str, opts: &ListOptions) -> Result<ListOutcome> {
let (host, path) = self.bucket_parts();
let mut q: Vec<(String, String)> = vec![("list-type".to_string(), "2".to_string())];
if !prefix.is_empty() {
q.push(("prefix".to_string(), prefix.to_string()));
}
if let Some(d) = &opts.delimiter {
q.push(("delimiter".to_string(), d.clone()));
}
if let Some(m) = &opts.marker {
q.push(("continuation-token".to_string(), m.clone()));
}
q.push((
"max-keys".to_string(),
opts.limit.unwrap_or(1000).to_string(),
));
let (url, amz_date, auth) = self.signed("GET", &host, &path, &q, &[], EMPTY_PAYLOAD_SHA256);
let resp = self
.http
.get(&url)
.header(reqwest::header::AUTHORIZATION, &auth)
.header("x-amz-content-sha256", EMPTY_PAYLOAD_SHA256)
.header("x-amz-date", &amz_date)
.send()
.await?;
let resp = self.check(resp).await?;
let xml = resp.text().await?;
parse_list(&xml)
}
fn presign_get(&self, key: &str, expires: Duration) -> Result<String> {
let (host, path) = self.object_parts(key);
let amz_date = amz_date_now(SystemTime::now());
let headers = vec![("host".to_string(), host.clone())];
let req = RequestToSign {
method: "GET",
canonical_uri: &path,
query: &[],
headers,
payload_sha256: "UNSIGNED-PAYLOAD",
amz_date: &amz_date,
};
let expires = expires.as_secs().min(S3_MAX_EXPIRES);
let (appended, _sig) = req.presign(&self.creds(), expires);
Ok(format!(
"{}://{}{}?{}",
self.config.scheme(),
host,
path,
appended
))
}
}
#[derive(Debug, Clone)]
pub struct MultipartUpload {
client: S3Client,
upload_id: String,
key: String,
}
impl MultipartUpload {
pub fn upload_id(&self) -> &str {
&self.upload_id
}
pub fn key(&self) -> &str {
&self.key
}
pub async fn upload_part(&self, part_number: u32, data: Bytes) -> Result<String> {
if part_number == 0 || part_number > S3_MAX_PARTS {
return Err(Error::InvalidInput(
"part_number must be within 1..=10000".into(),
));
}
let (host, path) = self.client.object_parts(&self.key);
let query = vec![
("partNumber".to_string(), part_number.to_string()),
("uploadId".to_string(), self.upload_id.clone()),
];
let payload_sha = sha256_hex(&data);
let (url, amz_date, auth) =
self.client
.signed("PUT", &host, &path, &query, &[], &payload_sha);
let resp = self
.client
.http
.put(&url)
.header(reqwest::header::AUTHORIZATION, &auth)
.header("x-amz-content-sha256", &payload_sha)
.header("x-amz-date", &amz_date)
.body(data)
.send()
.await?;
let resp = self.client.check(resp).await?;
let etag = read_etag(&resp)
.ok_or_else(|| Error::InvalidInput("upload-part response missing ETag".into()))?;
Ok(etag)
}
pub async fn complete(&self, parts: &[CompletedPart]) -> Result<ObjectMetadata> {
if parts.is_empty() {
return Err(Error::InvalidInput("no parts to complete".into()));
}
let mut ordered: Vec<&CompletedPart> = parts.iter().collect();
ordered.sort_by_key(|p| p.number);
let body = build_complete_xml(&ordered);
let (host, path) = self.client.object_parts(&self.key);
let query = vec![("uploadId".to_string(), self.upload_id.clone())];
let payload_sha = sha256_hex(body.as_bytes());
let (url, amz_date, auth) =
self.client
.signed("POST", &host, &path, &query, &[], &payload_sha);
let resp = self
.client
.http
.post(&url)
.header(reqwest::header::AUTHORIZATION, &auth)
.header("x-amz-content-sha256", &payload_sha)
.header("x-amz-date", &amz_date)
.header(CONTENT_TYPE, "application/xml")
.body(body)
.send()
.await?;
let resp = self.client.check(resp).await?;
let xml = resp.text().await?;
let etag = between_tags(&xml, "ETag")
.into_iter()
.next()
.map(|s| strip_quotes(&s));
let key = between_tags(&xml, "Key")
.into_iter()
.next()
.map(|s| unescape(&s))
.unwrap_or_else(|| self.key.clone());
Ok(ObjectMetadata {
key,
size: 0,
content_type: None,
etag,
last_modified: None,
})
}
pub async fn abort(&self) -> Result<()> {
let (host, path) = self.client.object_parts(&self.key);
let query = vec![("uploadId".to_string(), self.upload_id.clone())];
let (url, amz_date, auth) =
self.client
.signed("DELETE", &host, &path, &query, &[], EMPTY_PAYLOAD_SHA256);
let resp = self
.client
.http
.delete(&url)
.header(reqwest::header::AUTHORIZATION, &auth)
.header("x-amz-content-sha256", EMPTY_PAYLOAD_SHA256)
.header("x-amz-date", &amz_date)
.send()
.await?;
self.client.check(resp).await?;
Ok(())
}
}
#[derive(Debug, Clone)]
pub struct CompletedPart {
pub number: u32,
pub etag: String,
}
fn read_etag(resp: &reqwest::Response) -> Option<String> {
resp.headers()
.get(ETAG)
.and_then(|v| v.to_str().ok())
.map(strip_quotes)
}
fn strip_quotes(s: &str) -> String {
let s = s.trim();
if s.len() >= 2 && s.starts_with('"') && s.ends_with('"') {
s[1..s.len() - 1].to_string()
} else {
s.to_string()
}
}
fn between_tags(xml: &str, tag: &str) -> Vec<String> {
let open = format!("<{tag}>");
let close = format!("</{tag}>");
let mut out = Vec::new();
let mut i = 0;
while let Some(o) = xml[i..].find(&open) {
let start = i + o + open.len();
if let Some(c) = xml[start..].find(&close) {
out.push(xml[start..start + c].to_string());
i = start + c + close.len();
} else {
break;
}
}
out
}
fn build_complete_xml(parts: &[&CompletedPart]) -> String {
let mut body = String::from("<CompleteMultipartUpload>");
for p in parts {
body.push_str(&format!(
"<Part><PartNumber>{}</PartNumber><ETag>{}</ETag></Part>",
p.number,
escape_xml(&p.etag)
));
}
body.push_str("</CompleteMultipartUpload>");
body
}
fn escape_xml(s: &str) -> String {
let mut out = String::with_capacity(s.len());
for c in s.chars() {
match c {
'&' => out.push_str("&"),
'<' => out.push_str("<"),
'>' => out.push_str(">"),
'"' => out.push_str("""),
_ => out.push(c),
}
}
out
}
fn unescape(s: &str) -> String {
s.replace("<", "<")
.replace(">", ">")
.replace(""", "\"")
.replace("'", "'")
.replace("&", "&")
}
fn s3_error_message(body: &str) -> Option<String> {
let code = between_tags(body, "Code").into_iter().next()?;
let msg = between_tags(body, "Message")
.into_iter()
.next()
.unwrap_or_default();
Some(format!("{code}: {msg}"))
}
fn parse_list(xml: &str) -> Result<ListOutcome> {
let items = between_tags(xml, "Contents")
.into_iter()
.map(|c| {
let key = between_tags(&c, "Key")
.into_iter()
.next()
.unwrap_or_default();
let key = unescape(&key);
let size = between_tags(&c, "Size")
.into_iter()
.next()
.and_then(|s| s.trim().parse::<u64>().ok())
.unwrap_or(0);
let etag = between_tags(&c, "ETag")
.into_iter()
.next()
.map(|s| strip_quotes(&s));
let last_modified = between_tags(&c, "LastModified")
.into_iter()
.next()
.and_then(|s| parse_iso8601(&s));
ObjectMetadata {
key,
size,
content_type: None,
etag,
last_modified,
}
})
.collect();
let next_marker = between_tags(xml, "NextContinuationToken")
.into_iter()
.next()
.filter(|s| !s.is_empty());
Ok(ListOutcome { items, next_marker })
}
fn parse_iso8601(s: &str) -> Option<SystemTime> {
let s = s.trim();
let (body, offset): (&str, i64) = if let Some(stripped) = s.strip_suffix('Z') {
(stripped, 0)
} else if let Some(idx) = s.rfind(['+', '-']) {
let (head, tail) = s.split_at(idx);
if head.contains('T') {
let sign = if tail.starts_with('-') { -1 } else { 1 };
let off = parse_offset(&tail[1..])? as i64;
(head, sign * off)
} else {
(s, 0)
}
} else {
(s, 0)
};
let (d, t) = body.split_once('T')?;
let mut ymd = d.splitn(3, '-');
let y: i64 = ymd.next()?.parse().ok()?;
let m: i64 = ymd.next()?.parse().ok()?;
let day: i64 = ymd.next()?.parse().ok()?;
let t = t.trim_end_matches('Z');
let mut hms = t.splitn(3, ':');
let h: i64 = hms.next()?.parse().ok()?;
let min: i64 = hms.next()?.parse().ok()?;
let sec_str = hms.next()?.trim_end_matches('Z');
let sec: f64 = sec_str.parse().ok()?; let days = days_from_civil(y, m, day);
let total = days * 86_400 + h * 3_600 + min * 60 + sec as i64 - offset;
let secs = total.max(0) as u64;
Some(std::time::UNIX_EPOCH + Duration::from_secs(secs))
}
fn parse_offset(s: &str) -> Option<i64> {
let s = s.trim_end_matches('Z').trim_end_matches('z');
let s = s.trim();
if s.is_empty() {
return Some(0);
}
let (h, m) = s.split_once(':').unwrap_or((s, "0"));
let h: i64 = h.parse().ok()?;
let m: i64 = m.parse().ok()?;
Some(h * 3_600 + m * 60)
}
fn days_from_civil(y: i64, m: i64, d: i64) -> i64 {
let y = if m <= 2 { y - 1 } else { y };
let era = if y >= 0 { y } else { y - 399 } / 400;
let yoe = y - era * 400;
let doy = (153 * (if m > 2 { m - 3 } else { m + 9 }) + 2) / 5 + d - 1;
let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
era * 146_097 + doe - 719_468
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_list_objects_v2() {
let xml = r#"<?xml version="1.0" encoding="UTF-8"?>
<ListBucketResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Name>example-bucket</Name>
<Prefix>docs/</Prefix>
<KeyCount>2</KeyCount>
<MaxKeys>1000</MaxKeys>
<IsTruncated>false</IsTruncated>
<Contents>
<Key>docs/a.txt</Key>
<LastModified>2021-03-12T09:00:00.000Z</LastModified>
<ETag>"abc123"</ETag>
<Size>1024</Size>
<StorageClass>STANDARD</StorageClass>
</Contents>
<Contents>
<Key>docs/b & c.txt</Key>
<LastModified>2022-01-02T03:04:05.000Z</LastModified>
<ETag>"def456"</ETag>
<Size>2048</Size>
</Contents>
<NextContinuationToken>token123</NextContinuationToken>
</ListBucketResult>"#;
let out = parse_list(xml).unwrap();
assert_eq!(out.items.len(), 2);
assert_eq!(out.items[0].key, "docs/a.txt");
assert_eq!(out.items[0].size, 1024);
assert_eq!(out.items[0].etag.as_deref(), Some("abc123"));
assert_eq!(out.items[1].key, "docs/b & c.txt"); assert_eq!(out.items[1].size, 2048);
assert_eq!(out.next_marker.as_deref(), Some("token123"));
}
#[test]
fn encodes_object_keys_per_rfc3986() {
assert_eq!(encode_key("a/b c.txt"), "a/b%20c.txt");
assert_eq!(encode_key("photos/me+you.png"), "photos/me%2Byou.png");
assert_eq!(encode_key("/leading/slash"), "leading/slash");
}
}