mbx_cache_core/
sigv4.rs

1//! AWS Signature Version 4 request signing.
2//!
3//! Only what an S3 object store needs: `GET`, `HEAD`, and `PUT` against a
4//! single object, signed with a fixed header set. There is no request
5//! execution here and no I/O -- [`sign`] takes a request's shape and returns
6//! the headers that authenticate it, which is what makes it testable against
7//! published vectors.
8
9use eyre::Result;
10use reqwest::header::{HeaderName, HeaderValue};
11use sha2::{Digest as _, Sha256};
12use std::time::{SystemTime, UNIX_EPOCH};
13use url::Url;
14
15/// The signing algorithm this module implements.
16const ALGORITHM: &str = "AWS4-HMAC-SHA256";
17/// SigV4 signs per service; an object store is always `s3`.
18const SERVICE: &str = "s3";
19/// SHA-256 of the empty string, the payload hash of a body-less request.
20const EMPTY_PAYLOAD_SHA256: &str =
21    "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
22/// What S3 accepts in place of a payload hash when the body is not read twice.
23const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
24
25pub(crate) const X_AMZ_DATE: &str = "x-amz-date";
26pub(crate) const X_AMZ_CONTENT_SHA256: &str = "x-amz-content-sha256";
27pub(crate) const X_AMZ_SECURITY_TOKEN: &str = "x-amz-security-token";
28
29/// Long-lived or session credentials for an S3-compatible service.
30///
31/// Deliberately not `Debug`: the secret would otherwise reach any log line or
32/// panic message that formats a config.
33#[derive(Clone)]
34pub struct S3Credentials {
35    /// Access key identifier.
36    pub access_key_id: String,
37    /// Secret access key, never logged or formatted.
38    pub secret_access_key: String,
39    /// Session token accompanying temporary credentials.
40    pub session_token: Option<String>,
41}
42
43impl S3Credentials {
44    /// Read credentials from the environment variables the AWS tools set.
45    ///
46    /// `None` means the environment does not carry any. These come first in
47    /// the credential chain; [`InstanceRoleCredentials`](crate::InstanceRoleCredentials)
48    /// is the fallback for a machine that has an EC2 instance role instead.
49    /// Anything else that produces temporary credentials, such as an OIDC
50    /// exchange, is expected to have exported them here, which is what
51    /// `aws-actions/configure-aws-credentials` does on GitHub Actions.
52    pub fn from_env() -> Option<Self> {
53        Some(Self {
54            access_key_id: non_empty_var("AWS_ACCESS_KEY_ID")?,
55            secret_access_key: non_empty_var("AWS_SECRET_ACCESS_KEY")?,
56            session_token: non_empty_var("AWS_SESSION_TOKEN"),
57        })
58    }
59}
60
61fn non_empty_var(name: &str) -> Option<String> {
62    std::env::var(name)
63        .ok()
64        .map(|value| value.trim().to_string())
65        .filter(|value| !value.is_empty())
66}
67
68/// How the payload of a signed request is covered by its signature.
69pub(crate) enum PayloadHash {
70    /// Hex SHA-256 of the exact bytes being sent.
71    Sha256Hex(String),
72    /// The body is streamed from a file and deliberately not hashed.
73    ///
74    /// Hashing would mean reading an artifact twice, up to the 5 GiB single-PUT
75    /// ceiling, to protect bytes TLS already covers in transit and the content
76    /// address already covers at rest.
77    Unsigned,
78}
79
80impl PayloadHash {
81    /// The hash of a request with no body.
82    pub(crate) fn empty() -> Self {
83        Self::Sha256Hex(EMPTY_PAYLOAD_SHA256.to_string())
84    }
85
86    /// The hash of a body held in memory.
87    pub(crate) fn of(bytes: &[u8]) -> Self {
88        Self::Sha256Hex(hex::encode(Sha256::digest(bytes)))
89    }
90
91    fn as_str(&self) -> &str {
92        match self {
93            Self::Sha256Hex(hash) => hash,
94            Self::Unsigned => UNSIGNED_PAYLOAD,
95        }
96    }
97}
98
99/// Everything about a request that its signature covers besides the payload.
100pub(crate) struct SigningContext<'a> {
101    pub(crate) credentials: &'a S3Credentials,
102    pub(crate) region: &'a str,
103    /// Request time. Injected rather than read from the clock so a signature
104    /// can be compared against a fixed expected value in tests.
105    pub(crate) timestamp: SystemTime,
106}
107
108/// Sign a request, returning the headers that authenticate it.
109///
110/// The signed header set is fixed at `host`, `x-amz-content-sha256`, and
111/// `x-amz-date`, plus `x-amz-security-token` when the credentials are
112/// temporary. Conditional headers and content types are deliberately left
113/// unsigned: S3 requires only `host` and the `x-amz-*` headers to be covered,
114/// and a fixed set keeps the canonical request predictable.
115pub(crate) fn sign(
116    method: &str,
117    url: &Url,
118    context: &SigningContext<'_>,
119    payload: &PayloadHash,
120) -> Result<Vec<(HeaderName, HeaderValue)>> {
121    let host = url
122        .host_str()
123        .ok_or_else(|| eyre::eyre!("an S3 endpoint must have a host"))?;
124    let host = match url.port() {
125        Some(port) => format!("{host}:{port}"),
126        None => host.to_string(),
127    };
128    let (date, date_time) = format_timestamp(context.timestamp)?;
129    let payload_hash = payload.as_str();
130
131    let mut headers = vec![
132        ("host".to_string(), host),
133        (X_AMZ_CONTENT_SHA256.to_string(), payload_hash.to_string()),
134        (X_AMZ_DATE.to_string(), date_time.clone()),
135    ];
136    if let Some(token) = &context.credentials.session_token {
137        headers.push((X_AMZ_SECURITY_TOKEN.to_string(), token.clone()));
138    }
139    headers.sort_by(|left, right| left.0.cmp(&right.0));
140
141    let signed_headers = headers
142        .iter()
143        .map(|(name, _)| name.as_str())
144        .collect::<Vec<_>>()
145        .join(";");
146    let canonical_headers = headers
147        .iter()
148        .map(|(name, value)| format!("{name}:{}\n", value.trim()))
149        .collect::<String>();
150
151    let canonical_request = format!(
152        "{method}\n{}\n{}\n{canonical_headers}\n{signed_headers}\n{payload_hash}",
153        canonical_uri(url.path()),
154        canonical_query(url),
155    );
156    let scope = format!("{date}/{}/{SERVICE}/aws4_request", context.region);
157    let string_to_sign = format!(
158        "{ALGORITHM}\n{date_time}\n{scope}\n{}",
159        hex::encode(Sha256::digest(canonical_request.as_bytes()))
160    );
161
162    let signature = hex::encode(hmac_sha256(
163        &signing_key(
164            &context.credentials.secret_access_key,
165            &date,
166            context.region,
167        ),
168        string_to_sign.as_bytes(),
169    ));
170    let authorization = format!(
171        "{ALGORITHM} Credential={}/{scope}, SignedHeaders={signed_headers}, Signature={signature}",
172        context.credentials.access_key_id
173    );
174
175    let mut signed = Vec::with_capacity(headers.len());
176    for (name, value) in headers {
177        // `host` is set by the HTTP layer from the URL; sending it again would
178        // duplicate it in the request.
179        if name == "host" {
180            continue;
181        }
182        signed.push((
183            HeaderName::from_bytes(name.as_bytes())?,
184            header_value(&name, &value)?,
185        ));
186    }
187    signed.push((
188        reqwest::header::AUTHORIZATION,
189        header_value("authorization", &authorization)?,
190    ));
191    Ok(signed)
192}
193
194fn header_value(name: &str, value: &str) -> Result<HeaderValue> {
195    let mut header = HeaderValue::from_str(value)?;
196    if name == X_AMZ_SECURITY_TOKEN || name == "authorization" {
197        header.set_sensitive(true);
198    }
199    Ok(header)
200}
201
202/// Derive the request's signing key from the secret, scoped to date and region.
203///
204/// The scoping is what keeps a leaked signature from being replayed against
205/// another day or another region.
206fn signing_key(secret: &str, date: &str, region: &str) -> Vec<u8> {
207    let date_key = hmac_sha256(format!("AWS4{secret}").as_bytes(), date.as_bytes());
208    let region_key = hmac_sha256(&date_key, region.as_bytes());
209    let service_key = hmac_sha256(&region_key, SERVICE.as_bytes());
210    hmac_sha256(&service_key, b"aws4_request")
211}
212
213/// HMAC-SHA256 (RFC 2104).
214///
215/// Hand-written over the SHA-256 this crate already depends on. Only signatures
216/// are produced here, never verified, so there is nothing to compare in
217/// constant time.
218fn hmac_sha256(key: &[u8], message: &[u8]) -> Vec<u8> {
219    const BLOCK_BYTES: usize = 64;
220    let mut block = [0_u8; BLOCK_BYTES];
221    if key.len() > BLOCK_BYTES {
222        block[..32].copy_from_slice(&Sha256::digest(key));
223    } else {
224        block[..key.len()].copy_from_slice(key);
225    }
226    let mut inner = Sha256::new();
227    inner.update(block.map(|byte| byte ^ 0x36));
228    inner.update(message);
229    let mut outer = Sha256::new();
230    outer.update(block.map(|byte| byte ^ 0x5c));
231    outer.update(inner.finalize());
232    outer.finalize().to_vec()
233}
234
235/// The request path as the canonical request states it.
236///
237/// S3 is the one service that encodes the path exactly once rather than twice,
238/// and the caller passes [`Url::path`], which is already encoded. Encoding it
239/// again here would sign `%2520` for a key the request line spells `%20`, and
240/// every such key would fail to authenticate.
241fn canonical_uri(path: &str) -> String {
242    if path.is_empty() {
243        "/".to_string()
244    } else {
245        path.to_string()
246    }
247}
248
249/// Encode a query string in the canonical form: sorted, and encoded per value.
250///
251/// mbx addresses objects by path alone, so this is empty in practice. It exists
252/// so that adding a query parameter later cannot silently break signing.
253fn canonical_query(url: &Url) -> String {
254    let mut pairs = url
255        .query_pairs()
256        .map(|(key, value)| (uri_encode(&key), uri_encode(&value)))
257        .collect::<Vec<_>>();
258    pairs.sort();
259    pairs
260        .iter()
261        .map(|(key, value)| format!("{key}={value}"))
262        .collect::<Vec<_>>()
263        .join("&")
264}
265
266/// Percent-encode everything outside AWS's unreserved set, with uppercase hex.
267fn uri_encode(value: &str) -> String {
268    let mut encoded = String::with_capacity(value.len());
269    for byte in value.bytes() {
270        if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
271            encoded.push(byte as char);
272        } else {
273            encoded.push_str(&format!("%{byte:02X}"));
274        }
275    }
276    encoded
277}
278
279/// Format a request time as the `YYYYMMDD` scope date and `YYYYMMDDTHHMMSSZ`
280/// stamp SigV4 requires.
281fn format_timestamp(timestamp: SystemTime) -> Result<(String, String)> {
282    let seconds = timestamp
283        .duration_since(UNIX_EPOCH)
284        .map_err(|_| eyre::eyre!("system clock is before the Unix epoch"))?
285        .as_secs();
286    let days = (seconds / 86_400) as i64;
287    let time_of_day = seconds % 86_400;
288    let (year, month, day) = civil_from_days(days);
289    Ok((
290        format!("{year:04}{month:02}{day:02}"),
291        format!(
292            "{year:04}{month:02}{day:02}T{:02}{:02}{:02}Z",
293            time_of_day / 3_600,
294            (time_of_day % 3_600) / 60,
295            time_of_day % 60
296        ),
297    ))
298}
299
300/// Convert a count of days since the Unix epoch to a civil date.
301///
302/// Howard Hinnant's `civil_from_days`, which shifts the year to start in March
303/// so that the leap day lands at the end of a 400-year era and needs no special
304/// case. Used instead of a date library because SigV4 needs exactly this and
305/// nothing else about calendars.
306fn civil_from_days(days: i64) -> (i64, u32, u32) {
307    let z = days + 719_468;
308    let era = if z >= 0 { z } else { z - 146_096 } / 146_097;
309    let day_of_era = z - era * 146_097;
310    let year_of_era =
311        (day_of_era - day_of_era / 1_460 + day_of_era / 36_524 - day_of_era / 146_096) / 365;
312    let year = year_of_era + era * 400;
313    let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100);
314    let shifted_month = (5 * day_of_year + 2) / 153;
315    let day = (day_of_year - (153 * shifted_month + 2) / 5 + 1) as u32;
316    let month = if shifted_month < 10 {
317        shifted_month + 3
318    } else {
319        shifted_month - 9
320    } as u32;
321    (if month <= 2 { year + 1 } else { year }, month, day)
322}
323
324/// A `SystemTime` a fixed number of seconds after the Unix epoch.
325#[cfg(test)]
326fn epoch_plus(seconds: u64) -> SystemTime {
327    UNIX_EPOCH + std::time::Duration::from_secs(seconds)
328}
329
330#[cfg(test)]
331mod tests {
332    use super::*;
333
334    fn credentials() -> S3Credentials {
335        S3Credentials {
336            access_key_id: "AKIDEXAMPLE".into(),
337            secret_access_key: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY".into(),
338            session_token: None,
339        }
340    }
341
342    /// RFC 4231 test case 1.
343    #[test]
344    fn hmac_matches_rfc_4231() {
345        assert_eq!(
346            hex::encode(hmac_sha256(&[0x0b; 20], b"Hi There")),
347            "b0344c61d8db38535ca8afceaf0bf12b881dc200c9833da726e9376c2e32cff7"
348        );
349    }
350
351    /// RFC 4231 test case 3, whose message is longer than one hash block.
352    #[test]
353    fn hmac_matches_rfc_4231_multi_block_message() {
354        assert_eq!(
355            hex::encode(hmac_sha256(&[0xaa; 20], &[0xdd; 50])),
356            "773ea91e36800e46854db8ebd09181a72959098b3ef8c122d9635514ced565fe"
357        );
358    }
359
360    /// RFC 4231 test case 6, whose key is longer than one hash block and is
361    /// therefore replaced by its own digest.
362    #[test]
363    fn hmac_hashes_keys_longer_than_a_block() {
364        assert_eq!(
365            hex::encode(hmac_sha256(
366                &[0xaa; 131],
367                b"Test Using Larger Than Block-Size Key - Hash Key First"
368            )),
369            "60e431591ee0b67f0d8a26aacbf5b77f8e0bc6213728c5140546040f0ee37f54"
370        );
371    }
372
373    #[test]
374    fn empty_payload_constant_is_the_hash_of_no_bytes() {
375        let PayloadHash::Sha256Hex(hash) = PayloadHash::empty() else {
376            panic!("empty payload is hashed");
377        };
378        assert_eq!(hash, hex::encode(Sha256::digest(b"")));
379    }
380
381    #[test]
382    fn timestamps_format_as_sigv4_expects() {
383        assert_eq!(
384            format_timestamp(epoch_plus(0)).unwrap(),
385            ("19700101".to_string(), "19700101T000000Z".to_string())
386        );
387        // 2015-08-30T12:36:00Z, the instant AWS's own signing examples use.
388        assert_eq!(
389            format_timestamp(epoch_plus(1_440_938_160)).unwrap(),
390            ("20150830".to_string(), "20150830T123600Z".to_string())
391        );
392        // 2024-02-29T23:59:59Z: a leap day, at the end of the day.
393        assert_eq!(
394            format_timestamp(epoch_plus(1_709_251_199)).unwrap(),
395            ("20240229".to_string(), "20240229T235959Z".to_string())
396        );
397        // 2100-03-01T00:00:00Z: past a century that is not a leap year.
398        assert_eq!(
399            format_timestamp(epoch_plus(4_107_542_400)).unwrap(),
400            ("21000301".to_string(), "21000301T000000Z".to_string())
401        );
402    }
403
404    #[test]
405    fn paths_are_signed_exactly_as_the_request_spells_them() {
406        assert_eq!(
407            canonical_uri("/ns/v1/blobs/blake3/ab/12"),
408            "/ns/v1/blobs/blake3/ab/12"
409        );
410        assert_eq!(canonical_uri(""), "/");
411        // `Url::path` has already encoded the path. Encoding it a second time
412        // would sign a different key than the one the request asks for.
413        assert_eq!(canonical_uri("/a%20b"), "/a%20b");
414    }
415
416    #[test]
417    fn query_values_are_encoded_with_the_unreserved_set() {
418        assert_eq!(uri_encode("-._~"), "-._~");
419        assert_eq!(uri_encode("a b"), "a%20b");
420        assert_eq!(uri_encode("c+d/e"), "c%2Bd%2Fe");
421    }
422
423    #[test]
424    fn query_strings_are_sorted_and_encoded() {
425        let url: Url = "https://bucket.example.com/key?b=2&a=1&c=with%20space"
426            .parse()
427            .unwrap();
428        assert_eq!(canonical_query(&url), "a=1&b=2&c=with%20space");
429        let bare: Url = "https://bucket.example.com/key".parse().unwrap();
430        assert_eq!(canonical_query(&bare), "");
431    }
432
433    /// Pinned against an independent implementation of SigV4 (Python's `hmac`
434    /// and `hashlib` driving the same canonical request), so a change in this
435    /// module's own arithmetic cannot move the expected value with it.
436    #[test]
437    fn signs_a_get_the_way_the_specification_does() {
438        let url: Url = "https://examplebucket.s3.amazonaws.com/test.txt"
439            .parse()
440            .unwrap();
441        let context = SigningContext {
442            credentials: &credentials(),
443            region: "us-east-1",
444            timestamp: epoch_plus(1_440_938_160),
445        };
446
447        let headers = sign("GET", &url, &context, &PayloadHash::empty()).unwrap();
448
449        let authorization = header(&headers, "authorization");
450        assert_eq!(
451            authorization,
452            "AWS4-HMAC-SHA256 \
453             Credential=AKIDEXAMPLE/20150830/us-east-1/s3/aws4_request, \
454             SignedHeaders=host;x-amz-content-sha256;x-amz-date, \
455             Signature=bbfdf4d3c3eab24da182f8f790e0c7d8e2a20658191717a6546076effa9f5a5e"
456        );
457        assert_eq!(header(&headers, X_AMZ_DATE), "20150830T123600Z");
458        assert_eq!(header(&headers, X_AMZ_CONTENT_SHA256), EMPTY_PAYLOAD_SHA256);
459        assert!(!headers.iter().any(|(name, _)| name.as_str() == "host"));
460    }
461
462    #[test]
463    fn temporary_credentials_sign_and_send_their_session_token() {
464        let url: Url = "https://examplebucket.s3.amazonaws.com/test.txt"
465            .parse()
466            .unwrap();
467        let credentials = S3Credentials {
468            session_token: Some("session-token".into()),
469            ..credentials()
470        };
471        let context = SigningContext {
472            credentials: &credentials,
473            region: "us-east-1",
474            timestamp: epoch_plus(1_440_938_160),
475        };
476
477        let headers = sign("PUT", &url, &context, &PayloadHash::Unsigned).unwrap();
478
479        assert_eq!(
480            header(&headers, "authorization"),
481            "AWS4-HMAC-SHA256 \
482             Credential=AKIDEXAMPLE/20150830/us-east-1/s3/aws4_request, \
483             SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, \
484             Signature=37e8be08e60e6a292f591b6a462e7fb96436b4c1925f6978e8869a7c4577e12f"
485        );
486        assert_eq!(header(&headers, X_AMZ_SECURITY_TOKEN), "session-token");
487        assert_eq!(header(&headers, X_AMZ_CONTENT_SHA256), UNSIGNED_PAYLOAD);
488    }
489
490    #[test]
491    fn a_signature_covers_the_payload_the_method_and_the_key() {
492        let url: Url = "https://examplebucket.s3.amazonaws.com/test.txt"
493            .parse()
494            .unwrap();
495        let other: Url = "https://examplebucket.s3.amazonaws.com/other.txt"
496            .parse()
497            .unwrap();
498        let credentials = credentials();
499        let signature = |method: &str, url: &Url, region: &str, payload: PayloadHash| {
500            let context = SigningContext {
501                credentials: &credentials,
502                region,
503                timestamp: epoch_plus(1_440_938_160),
504            };
505            header(
506                &sign(method, url, &context, &payload).unwrap(),
507                "authorization",
508            )
509            .to_string()
510        };
511
512        let baseline = signature("GET", &url, "us-east-1", PayloadHash::empty());
513        for (what, other) in [
514            (
515                "method",
516                signature("PUT", &url, "us-east-1", PayloadHash::empty()),
517            ),
518            (
519                "key",
520                signature("GET", &other, "us-east-1", PayloadHash::empty()),
521            ),
522            (
523                "region",
524                signature("GET", &url, "eu-west-1", PayloadHash::empty()),
525            ),
526            (
527                "payload",
528                signature("GET", &url, "us-east-1", PayloadHash::of(b"body")),
529            ),
530            (
531                "unsigned payload",
532                signature("GET", &url, "us-east-1", PayloadHash::Unsigned),
533            ),
534        ] {
535            assert_ne!(baseline, other, "signature ignores the request's {what}");
536        }
537    }
538
539    fn header<'a>(headers: &'a [(HeaderName, HeaderValue)], name: &str) -> &'a str {
540        headers
541            .iter()
542            .find(|(header, _)| header.as_str() == name)
543            .map(|(_, value)| value.to_str().unwrap())
544            .unwrap_or_else(|| panic!("missing header {name}"))
545    }
546}
mbx_cache_core/sigv4.rs

mbx_cache_core/
sigv4.rs
