mbx_cache_core/
sigv4.rs

1//! AWS Signature Version 4 request signing.
2//!
3//! Only what an S3 object store needs: `GET`, `HEAD`, and `PUT` against a
4//! single object, signed with a fixed header set. There is no request
5//! execution here and no I/O -- [`sign`] takes a request's shape and returns
6//! the headers that authenticate it, which is what makes it testable against
7//! published vectors.
8
9use eyre::Result;
10use reqwest::header::{HeaderName, HeaderValue};
11use sha2::{Digest as _, Sha256};
12use std::time::{SystemTime, UNIX_EPOCH};
13use url::Url;
14
15/// The signing algorithm this module implements.
16const ALGORITHM: &str = "AWS4-HMAC-SHA256";
17/// SigV4 signs per service; an object store is always `s3`.
18const SERVICE: &str = "s3";
19/// SHA-256 of the empty string, the payload hash of a body-less request.
20const EMPTY_PAYLOAD_SHA256: &str =
21    "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
22/// What S3 accepts in place of a payload hash when the body is not read twice.
23const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
24
25pub(crate) const X_AMZ_DATE: &str = "x-amz-date";
26pub(crate) const X_AMZ_CONTENT_SHA256: &str = "x-amz-content-sha256";
27pub(crate) const X_AMZ_SECURITY_TOKEN: &str = "x-amz-security-token";
28
29/// Long-lived or session credentials for an S3-compatible service.
30///
31/// Deliberately not `Debug`: the secret would otherwise reach any log line or
32/// panic message that formats a config.
33#[derive(Clone)]
34pub struct S3Credentials {
35    /// Access key identifier.
36    pub access_key_id: String,
37    /// Secret access key, never logged or formatted.
38    pub secret_access_key: String,
39    /// Session token accompanying temporary credentials.
40    pub session_token: Option<String>,
41}
42
43impl S3Credentials {
44    /// Read credentials from the environment variables the AWS tools set.
45    ///
46    /// This is the whole credential chain mbx implements, and `None` means the
47    /// environment does not carry one. Anything that produces temporary
48    /// credentials -- an OIDC exchange, an instance role -- is expected to have
49    /// exported them here first, which is what
50    /// `aws-actions/configure-aws-credentials` does on GitHub Actions.
51    pub fn from_env() -> Option<Self> {
52        Some(Self {
53            access_key_id: non_empty_var("AWS_ACCESS_KEY_ID")?,
54            secret_access_key: non_empty_var("AWS_SECRET_ACCESS_KEY")?,
55            session_token: non_empty_var("AWS_SESSION_TOKEN"),
56        })
57    }
58}
59
60fn non_empty_var(name: &str) -> Option<String> {
61    std::env::var(name)
62        .ok()
63        .map(|value| value.trim().to_string())
64        .filter(|value| !value.is_empty())
65}
66
67/// How the payload of a signed request is covered by its signature.
68pub(crate) enum PayloadHash {
69    /// Hex SHA-256 of the exact bytes being sent.
70    Sha256Hex(String),
71    /// The body is streamed from a file and deliberately not hashed.
72    ///
73    /// Hashing would mean reading an artifact twice, up to the 5 GiB single-PUT
74    /// ceiling, to protect bytes TLS already covers in transit and the content
75    /// address already covers at rest.
76    Unsigned,
77}
78
79impl PayloadHash {
80    /// The hash of a request with no body.
81    pub(crate) fn empty() -> Self {
82        Self::Sha256Hex(EMPTY_PAYLOAD_SHA256.to_string())
83    }
84
85    /// The hash of a body held in memory.
86    pub(crate) fn of(bytes: &[u8]) -> Self {
87        Self::Sha256Hex(hex::encode(Sha256::digest(bytes)))
88    }
89
90    fn as_str(&self) -> &str {
91        match self {
92            Self::Sha256Hex(hash) => hash,
93            Self::Unsigned => UNSIGNED_PAYLOAD,
94        }
95    }
96}
97
98/// Everything about a request that its signature covers besides the payload.
99pub(crate) struct SigningContext<'a> {
100    pub(crate) credentials: &'a S3Credentials,
101    pub(crate) region: &'a str,
102    /// Request time. Injected rather than read from the clock so a signature
103    /// can be compared against a fixed expected value in tests.
104    pub(crate) timestamp: SystemTime,
105}
106
107/// Sign a request, returning the headers that authenticate it.
108///
109/// The signed header set is fixed at `host`, `x-amz-content-sha256`, and
110/// `x-amz-date`, plus `x-amz-security-token` when the credentials are
111/// temporary. Conditional headers and content types are deliberately left
112/// unsigned: S3 requires only `host` and the `x-amz-*` headers to be covered,
113/// and a fixed set keeps the canonical request predictable.
114pub(crate) fn sign(
115    method: &str,
116    url: &Url,
117    context: &SigningContext<'_>,
118    payload: &PayloadHash,
119) -> Result<Vec<(HeaderName, HeaderValue)>> {
120    let host = url
121        .host_str()
122        .ok_or_else(|| eyre::eyre!("an S3 endpoint must have a host"))?;
123    let host = match url.port() {
124        Some(port) => format!("{host}:{port}"),
125        None => host.to_string(),
126    };
127    let (date, date_time) = format_timestamp(context.timestamp)?;
128    let payload_hash = payload.as_str();
129
130    let mut headers = vec![
131        ("host".to_string(), host),
132        (X_AMZ_CONTENT_SHA256.to_string(), payload_hash.to_string()),
133        (X_AMZ_DATE.to_string(), date_time.clone()),
134    ];
135    if let Some(token) = &context.credentials.session_token {
136        headers.push((X_AMZ_SECURITY_TOKEN.to_string(), token.clone()));
137    }
138    headers.sort_by(|left, right| left.0.cmp(&right.0));
139
140    let signed_headers = headers
141        .iter()
142        .map(|(name, _)| name.as_str())
143        .collect::<Vec<_>>()
144        .join(";");
145    let canonical_headers = headers
146        .iter()
147        .map(|(name, value)| format!("{name}:{}\n", value.trim()))
148        .collect::<String>();
149
150    let canonical_request = format!(
151        "{method}\n{}\n{}\n{canonical_headers}\n{signed_headers}\n{payload_hash}",
152        canonical_uri(url.path()),
153        canonical_query(url),
154    );
155    let scope = format!("{date}/{}/{SERVICE}/aws4_request", context.region);
156    let string_to_sign = format!(
157        "{ALGORITHM}\n{date_time}\n{scope}\n{}",
158        hex::encode(Sha256::digest(canonical_request.as_bytes()))
159    );
160
161    let signature = hex::encode(hmac_sha256(
162        &signing_key(
163            &context.credentials.secret_access_key,
164            &date,
165            context.region,
166        ),
167        string_to_sign.as_bytes(),
168    ));
169    let authorization = format!(
170        "{ALGORITHM} Credential={}/{scope}, SignedHeaders={signed_headers}, Signature={signature}",
171        context.credentials.access_key_id
172    );
173
174    let mut signed = Vec::with_capacity(headers.len());
175    for (name, value) in headers {
176        // `host` is set by the HTTP layer from the URL; sending it again would
177        // duplicate it in the request.
178        if name == "host" {
179            continue;
180        }
181        signed.push((
182            HeaderName::from_bytes(name.as_bytes())?,
183            header_value(&name, &value)?,
184        ));
185    }
186    signed.push((
187        reqwest::header::AUTHORIZATION,
188        header_value("authorization", &authorization)?,
189    ));
190    Ok(signed)
191}
192
193fn header_value(name: &str, value: &str) -> Result<HeaderValue> {
194    let mut header = HeaderValue::from_str(value)?;
195    if name == X_AMZ_SECURITY_TOKEN || name == "authorization" {
196        header.set_sensitive(true);
197    }
198    Ok(header)
199}
200
201/// Derive the request's signing key from the secret, scoped to date and region.
202///
203/// The scoping is what keeps a leaked signature from being replayed against
204/// another day or another region.
205fn signing_key(secret: &str, date: &str, region: &str) -> Vec<u8> {
206    let date_key = hmac_sha256(format!("AWS4{secret}").as_bytes(), date.as_bytes());
207    let region_key = hmac_sha256(&date_key, region.as_bytes());
208    let service_key = hmac_sha256(&region_key, SERVICE.as_bytes());
209    hmac_sha256(&service_key, b"aws4_request")
210}
211
212/// HMAC-SHA256 (RFC 2104).
213///
214/// Hand-written over the SHA-256 this crate already depends on. Only signatures
215/// are produced here, never verified, so there is nothing to compare in
216/// constant time.
217fn hmac_sha256(key: &[u8], message: &[u8]) -> Vec<u8> {
218    const BLOCK_BYTES: usize = 64;
219    let mut block = [0_u8; BLOCK_BYTES];
220    if key.len() > BLOCK_BYTES {
221        block[..32].copy_from_slice(&Sha256::digest(key));
222    } else {
223        block[..key.len()].copy_from_slice(key);
224    }
225    let mut inner = Sha256::new();
226    inner.update(block.map(|byte| byte ^ 0x36));
227    inner.update(message);
228    let mut outer = Sha256::new();
229    outer.update(block.map(|byte| byte ^ 0x5c));
230    outer.update(inner.finalize());
231    outer.finalize().to_vec()
232}
233
234/// The request path as the canonical request states it.
235///
236/// S3 is the one service that encodes the path exactly once rather than twice,
237/// and the caller passes [`Url::path`], which is already encoded. Encoding it
238/// again here would sign `%2520` for a key the request line spells `%20`, and
239/// every such key would fail to authenticate.
240fn canonical_uri(path: &str) -> String {
241    if path.is_empty() {
242        "/".to_string()
243    } else {
244        path.to_string()
245    }
246}
247
248/// Encode a query string in the canonical form: sorted, and encoded per value.
249///
250/// mbx addresses objects by path alone, so this is empty in practice. It exists
251/// so that adding a query parameter later cannot silently break signing.
252fn canonical_query(url: &Url) -> String {
253    let mut pairs = url
254        .query_pairs()
255        .map(|(key, value)| (uri_encode(&key), uri_encode(&value)))
256        .collect::<Vec<_>>();
257    pairs.sort();
258    pairs
259        .iter()
260        .map(|(key, value)| format!("{key}={value}"))
261        .collect::<Vec<_>>()
262        .join("&")
263}
264
265/// Percent-encode everything outside AWS's unreserved set, with uppercase hex.
266fn uri_encode(value: &str) -> String {
267    let mut encoded = String::with_capacity(value.len());
268    for byte in value.bytes() {
269        if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
270            encoded.push(byte as char);
271        } else {
272            encoded.push_str(&format!("%{byte:02X}"));
273        }
274    }
275    encoded
276}
277
278/// Format a request time as the `YYYYMMDD` scope date and `YYYYMMDDTHHMMSSZ`
279/// stamp SigV4 requires.
280fn format_timestamp(timestamp: SystemTime) -> Result<(String, String)> {
281    let seconds = timestamp
282        .duration_since(UNIX_EPOCH)
283        .map_err(|_| eyre::eyre!("system clock is before the Unix epoch"))?
284        .as_secs();
285    let days = (seconds / 86_400) as i64;
286    let time_of_day = seconds % 86_400;
287    let (year, month, day) = civil_from_days(days);
288    Ok((
289        format!("{year:04}{month:02}{day:02}"),
290        format!(
291            "{year:04}{month:02}{day:02}T{:02}{:02}{:02}Z",
292            time_of_day / 3_600,
293            (time_of_day % 3_600) / 60,
294            time_of_day % 60
295        ),
296    ))
297}
298
299/// Convert a count of days since the Unix epoch to a civil date.
300///
301/// Howard Hinnant's `civil_from_days`, which shifts the year to start in March
302/// so that the leap day lands at the end of a 400-year era and needs no special
303/// case. Used instead of a date library because SigV4 needs exactly this and
304/// nothing else about calendars.
305fn civil_from_days(days: i64) -> (i64, u32, u32) {
306    let z = days + 719_468;
307    let era = if z >= 0 { z } else { z - 146_096 } / 146_097;
308    let day_of_era = z - era * 146_097;
309    let year_of_era =
310        (day_of_era - day_of_era / 1_460 + day_of_era / 36_524 - day_of_era / 146_096) / 365;
311    let year = year_of_era + era * 400;
312    let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100);
313    let shifted_month = (5 * day_of_year + 2) / 153;
314    let day = (day_of_year - (153 * shifted_month + 2) / 5 + 1) as u32;
315    let month = if shifted_month < 10 {
316        shifted_month + 3
317    } else {
318        shifted_month - 9
319    } as u32;
320    (if month <= 2 { year + 1 } else { year }, month, day)
321}
322
323/// A `SystemTime` a fixed number of seconds after the Unix epoch.
324#[cfg(test)]
325fn epoch_plus(seconds: u64) -> SystemTime {
326    UNIX_EPOCH + std::time::Duration::from_secs(seconds)
327}
328
329#[cfg(test)]
330mod tests {
331    use super::*;
332
333    fn credentials() -> S3Credentials {
334        S3Credentials {
335            access_key_id: "AKIDEXAMPLE".into(),
336            secret_access_key: "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY".into(),
337            session_token: None,
338        }
339    }
340
341    /// RFC 4231 test case 1.
342    #[test]
343    fn hmac_matches_rfc_4231() {
344        assert_eq!(
345            hex::encode(hmac_sha256(&[0x0b; 20], b"Hi There")),
346            "b0344c61d8db38535ca8afceaf0bf12b881dc200c9833da726e9376c2e32cff7"
347        );
348    }
349
350    /// RFC 4231 test case 3, whose message is longer than one hash block.
351    #[test]
352    fn hmac_matches_rfc_4231_multi_block_message() {
353        assert_eq!(
354            hex::encode(hmac_sha256(&[0xaa; 20], &[0xdd; 50])),
355            "773ea91e36800e46854db8ebd09181a72959098b3ef8c122d9635514ced565fe"
356        );
357    }
358
359    /// RFC 4231 test case 6, whose key is longer than one hash block and is
360    /// therefore replaced by its own digest.
361    #[test]
362    fn hmac_hashes_keys_longer_than_a_block() {
363        assert_eq!(
364            hex::encode(hmac_sha256(
365                &[0xaa; 131],
366                b"Test Using Larger Than Block-Size Key - Hash Key First"
367            )),
368            "60e431591ee0b67f0d8a26aacbf5b77f8e0bc6213728c5140546040f0ee37f54"
369        );
370    }
371
372    #[test]
373    fn empty_payload_constant_is_the_hash_of_no_bytes() {
374        let PayloadHash::Sha256Hex(hash) = PayloadHash::empty() else {
375            panic!("empty payload is hashed");
376        };
377        assert_eq!(hash, hex::encode(Sha256::digest(b"")));
378    }
379
380    #[test]
381    fn timestamps_format_as_sigv4_expects() {
382        assert_eq!(
383            format_timestamp(epoch_plus(0)).unwrap(),
384            ("19700101".to_string(), "19700101T000000Z".to_string())
385        );
386        // 2015-08-30T12:36:00Z, the instant AWS's own signing examples use.
387        assert_eq!(
388            format_timestamp(epoch_plus(1_440_938_160)).unwrap(),
389            ("20150830".to_string(), "20150830T123600Z".to_string())
390        );
391        // 2024-02-29T23:59:59Z: a leap day, at the end of the day.
392        assert_eq!(
393            format_timestamp(epoch_plus(1_709_251_199)).unwrap(),
394            ("20240229".to_string(), "20240229T235959Z".to_string())
395        );
396        // 2100-03-01T00:00:00Z: past a century that is not a leap year.
397        assert_eq!(
398            format_timestamp(epoch_plus(4_107_542_400)).unwrap(),
399            ("21000301".to_string(), "21000301T000000Z".to_string())
400        );
401    }
402
403    #[test]
404    fn paths_are_signed_exactly_as_the_request_spells_them() {
405        assert_eq!(
406            canonical_uri("/ns/v1/blobs/blake3/ab/12"),
407            "/ns/v1/blobs/blake3/ab/12"
408        );
409        assert_eq!(canonical_uri(""), "/");
410        // `Url::path` has already encoded the path. Encoding it a second time
411        // would sign a different key than the one the request asks for.
412        assert_eq!(canonical_uri("/a%20b"), "/a%20b");
413    }
414
415    #[test]
416    fn query_values_are_encoded_with_the_unreserved_set() {
417        assert_eq!(uri_encode("-._~"), "-._~");
418        assert_eq!(uri_encode("a b"), "a%20b");
419        assert_eq!(uri_encode("c+d/e"), "c%2Bd%2Fe");
420    }
421
422    #[test]
423    fn query_strings_are_sorted_and_encoded() {
424        let url: Url = "https://bucket.example.com/key?b=2&a=1&c=with%20space"
425            .parse()
426            .unwrap();
427        assert_eq!(canonical_query(&url), "a=1&b=2&c=with%20space");
428        let bare: Url = "https://bucket.example.com/key".parse().unwrap();
429        assert_eq!(canonical_query(&bare), "");
430    }
431
432    /// Pinned against an independent implementation of SigV4 (Python's `hmac`
433    /// and `hashlib` driving the same canonical request), so a change in this
434    /// module's own arithmetic cannot move the expected value with it.
435    #[test]
436    fn signs_a_get_the_way_the_specification_does() {
437        let url: Url = "https://examplebucket.s3.amazonaws.com/test.txt"
438            .parse()
439            .unwrap();
440        let context = SigningContext {
441            credentials: &credentials(),
442            region: "us-east-1",
443            timestamp: epoch_plus(1_440_938_160),
444        };
445
446        let headers = sign("GET", &url, &context, &PayloadHash::empty()).unwrap();
447
448        let authorization = header(&headers, "authorization");
449        assert_eq!(
450            authorization,
451            "AWS4-HMAC-SHA256 \
452             Credential=AKIDEXAMPLE/20150830/us-east-1/s3/aws4_request, \
453             SignedHeaders=host;x-amz-content-sha256;x-amz-date, \
454             Signature=bbfdf4d3c3eab24da182f8f790e0c7d8e2a20658191717a6546076effa9f5a5e"
455        );
456        assert_eq!(header(&headers, X_AMZ_DATE), "20150830T123600Z");
457        assert_eq!(header(&headers, X_AMZ_CONTENT_SHA256), EMPTY_PAYLOAD_SHA256);
458        assert!(!headers.iter().any(|(name, _)| name.as_str() == "host"));
459    }
460
461    #[test]
462    fn temporary_credentials_sign_and_send_their_session_token() {
463        let url: Url = "https://examplebucket.s3.amazonaws.com/test.txt"
464            .parse()
465            .unwrap();
466        let credentials = S3Credentials {
467            session_token: Some("session-token".into()),
468            ..credentials()
469        };
470        let context = SigningContext {
471            credentials: &credentials,
472            region: "us-east-1",
473            timestamp: epoch_plus(1_440_938_160),
474        };
475
476        let headers = sign("PUT", &url, &context, &PayloadHash::Unsigned).unwrap();
477
478        assert_eq!(
479            header(&headers, "authorization"),
480            "AWS4-HMAC-SHA256 \
481             Credential=AKIDEXAMPLE/20150830/us-east-1/s3/aws4_request, \
482             SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, \
483             Signature=37e8be08e60e6a292f591b6a462e7fb96436b4c1925f6978e8869a7c4577e12f"
484        );
485        assert_eq!(header(&headers, X_AMZ_SECURITY_TOKEN), "session-token");
486        assert_eq!(header(&headers, X_AMZ_CONTENT_SHA256), UNSIGNED_PAYLOAD);
487    }
488
489    #[test]
490    fn a_signature_covers_the_payload_the_method_and_the_key() {
491        let url: Url = "https://examplebucket.s3.amazonaws.com/test.txt"
492            .parse()
493            .unwrap();
494        let other: Url = "https://examplebucket.s3.amazonaws.com/other.txt"
495            .parse()
496            .unwrap();
497        let credentials = credentials();
498        let signature = |method: &str, url: &Url, region: &str, payload: PayloadHash| {
499            let context = SigningContext {
500                credentials: &credentials,
501                region,
502                timestamp: epoch_plus(1_440_938_160),
503            };
504            header(
505                &sign(method, url, &context, &payload).unwrap(),
506                "authorization",
507            )
508            .to_string()
509        };
510
511        let baseline = signature("GET", &url, "us-east-1", PayloadHash::empty());
512        for (what, other) in [
513            (
514                "method",
515                signature("PUT", &url, "us-east-1", PayloadHash::empty()),
516            ),
517            (
518                "key",
519                signature("GET", &other, "us-east-1", PayloadHash::empty()),
520            ),
521            (
522                "region",
523                signature("GET", &url, "eu-west-1", PayloadHash::empty()),
524            ),
525            (
526                "payload",
527                signature("GET", &url, "us-east-1", PayloadHash::of(b"body")),
528            ),
529            (
530                "unsigned payload",
531                signature("GET", &url, "us-east-1", PayloadHash::Unsigned),
532            ),
533        ] {
534            assert_ne!(baseline, other, "signature ignores the request's {what}");
535        }
536    }
537
538    fn header<'a>(headers: &'a [(HeaderName, HeaderValue)], name: &str) -> &'a str {
539        headers
540            .iter()
541            .find(|(header, _)| header.as_str() == name)
542            .map(|(_, value)| value.to_str().unwrap())
543            .unwrap_or_else(|| panic!("missing header {name}"))
544    }
545}
mbx_cache_core/sigv4.rs

mbx_cache_core/
sigv4.rs
