object_store/aws/
credential.rs

1// Licensed to the Apache Software Foundation (ASF) under one
2// or more contributor license agreements.  See the NOTICE file
3// distributed with this work for additional information
4// regarding copyright ownership.  The ASF licenses this file
5// to you under the Apache License, Version 2.0 (the
6// "License"); you may not use this file except in compliance
7// with the License.  You may obtain a copy of the License at
8//
9//   http://www.apache.org/licenses/LICENSE-2.0
10//
11// Unless required by applicable law or agreed to in writing,
12// software distributed under the License is distributed on an
13// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14// KIND, either express or implied.  See the License for the
15// specific language governing permissions and limitations
16// under the License.
17
18use crate::aws::{AwsCredentialProvider, STORE, STRICT_ENCODE_SET, STRICT_PATH_ENCODE_SET};
19use crate::client::builder::HttpRequestBuilder;
20use crate::client::retry::RetryExt;
21use crate::client::token::{TemporaryToken, TokenCache};
22use crate::client::{
23    CryptoProvider, DigestAlgorithm, HttpClient, HttpError, HttpRequest, TokenProvider,
24    crypto_provider,
25};
26use crate::util::{append_strict_query_pairs, hex_digest, hex_encode};
27use crate::{CredentialProvider, Result, RetryConfig};
28use async_trait::async_trait;
29use bytes::Buf;
30use chrono::{DateTime, Utc};
31use http::header::{AUTHORIZATION, HOST, HeaderMap, HeaderName, HeaderValue};
32use http::{Method, StatusCode};
33use percent_encoding::utf8_percent_encode;
34use serde::Deserialize;
35use std::collections::BTreeMap;
36use std::sync::Arc;
37use std::time::{Duration, Instant};
38use tracing::warn;
39use url::Url;
40
41#[derive(Debug, thiserror::Error)]
42#[allow(clippy::enum_variant_names)]
43enum Error {
44    #[error("Error performing CreateSession request: {source}")]
45    CreateSessionRequest {
46        source: crate::client::retry::RetryError,
47    },
48
49    #[error("Error getting CreateSession response: {source}")]
50    CreateSessionResponse { source: HttpError },
51
52    #[error("Invalid CreateSessionOutput response: {source}")]
53    CreateSessionOutput { source: quick_xml::DeError },
54}
55
56impl From<Error> for crate::Error {
57    fn from(value: Error) -> Self {
58        Self::Generic {
59            store: STORE,
60            source: Box::new(value),
61        }
62    }
63}
64
65type StdError = Box<dyn std::error::Error + Send + Sync>;
66
67/// SHA256 hash of empty string
68static EMPTY_SHA256_HASH: &str = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
69static UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
70static STREAMING_PAYLOAD: &str = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD";
71
72/// A set of AWS security credentials
73#[derive(Eq, PartialEq)]
74pub struct AwsCredential {
75    /// AWS_ACCESS_KEY_ID
76    pub key_id: String,
77    /// AWS_SECRET_ACCESS_KEY
78    pub secret_key: String,
79    /// AWS_SESSION_TOKEN
80    pub token: Option<String>,
81}
82
83impl std::fmt::Debug for AwsCredential {
84    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
85        f.debug_struct("AwsCredential")
86            .field("key_id", &self.key_id)
87            .field("secret_key", &"******")
88            .field("token", &self.token.as_ref().map(|_| "******"))
89            .finish()
90    }
91}
92
93impl AwsCredential {
94    /// Signs a string
95    ///
96    /// <https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html>
97    fn sign(
98        &self,
99        crypto: &dyn CryptoProvider,
100        to_sign: &str,
101        date: DateTime<Utc>,
102        region: &str,
103        service: &str,
104    ) -> Result<String> {
105        let date_string = date.format("%Y%m%d").to_string();
106        let secret_key = format!("AWS4{}", self.secret_key);
107
108        let mut ctx = crypto.hmac(DigestAlgorithm::Sha256, secret_key.as_bytes())?;
109        ctx.update(date_string.as_bytes());
110
111        let mut ctx = crypto.hmac(DigestAlgorithm::Sha256, ctx.finish()?)?;
112        ctx.update(region.as_bytes());
113
114        let mut ctx = crypto.hmac(DigestAlgorithm::Sha256, ctx.finish()?)?;
115        ctx.update(service.as_bytes());
116
117        let mut ctx = crypto.hmac(DigestAlgorithm::Sha256, ctx.finish()?)?;
118        ctx.update(b"aws4_request");
119
120        let mut ctx = crypto.hmac(DigestAlgorithm::Sha256, ctx.finish()?)?;
121        ctx.update(to_sign.as_bytes());
122
123        Ok(hex_encode(ctx.finish()?))
124    }
125}
126
127/// Authorize a [`HttpRequest`] with an [`AwsCredential`] using [AWS SigV4]
128///
129/// [AWS SigV4]: https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
130#[derive(Debug)]
131pub struct AwsAuthorizer<'a> {
132    date: Option<DateTime<Utc>>,
133    credential: &'a AwsCredential,
134    crypto: Option<&'a dyn CryptoProvider>,
135    service: &'a str,
136    region: &'a str,
137    token_header: Option<HeaderName>,
138    sign_payload: bool,
139    request_payer: bool,
140}
141
142static DATE_HEADER: HeaderName = HeaderName::from_static("x-amz-date");
143static HASH_HEADER: HeaderName = HeaderName::from_static("x-amz-content-sha256");
144static TOKEN_HEADER: HeaderName = HeaderName::from_static("x-amz-security-token");
145static REQUEST_PAYER_HEADER: HeaderName = HeaderName::from_static("x-amz-request-payer");
146static REQUEST_PAYER_HEADER_VALUE: HeaderValue = HeaderValue::from_static("requester");
147const ALGORITHM: &str = "AWS4-HMAC-SHA256";
148
149impl<'a> AwsAuthorizer<'a> {
150    /// Create a new [`AwsAuthorizer`]
151    pub fn new(credential: &'a AwsCredential, service: &'a str, region: &'a str) -> Self {
152        Self {
153            credential,
154            service,
155            region,
156            crypto: None,
157            date: None,
158            sign_payload: true,
159            token_header: None,
160            request_payer: false,
161        }
162    }
163
164    /// Controls whether this [`AwsAuthorizer`] will attempt to sign the request payload,
165    /// the default is `true`
166    pub fn with_sign_payload(mut self, signed: bool) -> Self {
167        self.sign_payload = signed;
168        self
169    }
170
171    /// Overrides the header name for security tokens, defaults to `x-amz-security-token`
172    pub(crate) fn with_token_header(mut self, header: HeaderName) -> Self {
173        self.token_header = Some(header);
174        self
175    }
176
177    /// Set whether to include requester pays headers
178    ///
179    /// <https://docs.aws.amazon.com/AmazonS3/latest/userguide/ObjectsinRequesterPaysBuckets.html>
180    pub fn with_request_payer(mut self, request_payer: bool) -> Self {
181        self.request_payer = request_payer;
182        self
183    }
184
185    /// Specify the crypto provider
186    pub fn with_crypto(mut self, crypto: &'a dyn CryptoProvider) -> Self {
187        self.crypto = Some(crypto);
188        self
189    }
190
191    /// Authorize `request` with an optional pre-calculated SHA256 digest by attaching
192    /// the relevant [AWS SigV4] headers
193    ///
194    /// # Panics
195    ///
196    /// Panics on cryptography error
197    ///
198    #[deprecated(note = "use AwsAuthorized::try_authorize")]
199    pub fn authorize(&self, request: &mut HttpRequest, pre_calculated_digest: Option<&[u8]>) {
200        self.try_authorize(request, pre_calculated_digest).unwrap()
201    }
202
203    /// Authorize `request` with an optional pre-calculated SHA256 digest by attaching
204    /// the relevant [AWS SigV4] headers
205    ///
206    /// # Payload Signature
207    ///
208    /// AWS SigV4 requests must contain the `x-amz-content-sha256` header, it is set as follows:
209    ///
210    /// * If not configured to sign payloads, it is set to `UNSIGNED-PAYLOAD`
211    /// * If a `pre_calculated_digest` is provided, it is set to the hex encoding of it
212    /// * If it is a streaming request, it is set to `STREAMING-AWS4-HMAC-SHA256-PAYLOAD`
213    /// * Otherwise it is set to the hex encoded SHA256 of the request body
214    ///
215    /// [AWS SigV4]: https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html
216    pub fn try_authorize(
217        &self,
218        request: &mut HttpRequest,
219        pre_calculated_digest: Option<&[u8]>,
220    ) -> Result<()> {
221        let crypto = crypto_provider(self.crypto)?;
222        let url = Url::parse(&request.uri().to_string()).unwrap();
223
224        if let Some(ref token) = self.credential.token {
225            let token_val = HeaderValue::from_str(token).unwrap();
226            let header = self.token_header.as_ref().unwrap_or(&TOKEN_HEADER);
227            request.headers_mut().insert(header, token_val);
228        }
229
230        let date = self.date.unwrap_or_else(Utc::now);
231        let date_str = date.format("%Y%m%dT%H%M%SZ").to_string();
232        let date_val = HeaderValue::from_str(&date_str).unwrap();
233        request.headers_mut().insert(&DATE_HEADER, date_val);
234
235        let digest = match self.sign_payload {
236            false => UNSIGNED_PAYLOAD.to_string(),
237            true => match pre_calculated_digest {
238                Some(digest) => hex_encode(digest),
239                None => match request.body().is_empty() {
240                    true => EMPTY_SHA256_HASH.to_string(),
241                    false => match request.body().as_bytes() {
242                        Some(bytes) => hex_digest(crypto, bytes)?,
243                        None => STREAMING_PAYLOAD.to_string(),
244                    },
245                },
246            },
247        };
248
249        let header_digest = HeaderValue::from_str(&digest).unwrap();
250        request.headers_mut().insert(&HASH_HEADER, header_digest);
251
252        if self.request_payer {
253            // For DELETE, GET, HEAD, POST, and PUT requests, include x-amz-request-payer :
254            // requester in the header
255            // https://docs.aws.amazon.com/AmazonS3/latest/userguide/ObjectsinRequesterPaysBuckets.html
256            request
257                .headers_mut()
258                .insert(&REQUEST_PAYER_HEADER, REQUEST_PAYER_HEADER_VALUE.clone());
259        }
260
261        // the SigV4 must include a value for `host`, but the actual Host header may need to change
262        // due to a redirect. Therefore do not override the Host header for the http
263        // request, let the client set it.
264        let host = &url[url::Position::BeforeHost..url::Position::AfterPort];
265        request
266            .headers_mut()
267            .insert(&HOST, HeaderValue::from_str(host).unwrap());
268        let (signed_headers, canonical_headers) = canonicalize_headers(request.headers());
269        request.headers_mut().remove(&HOST);
270
271        let scope = self.scope(date);
272
273        let string_to_sign = self.string_to_sign(
274            crypto,
275            date,
276            &scope,
277            request.method(),
278            &url,
279            &canonical_headers,
280            &signed_headers,
281            &digest,
282        )?;
283
284        // sign the string
285        let signature =
286            self.credential
287                .sign(crypto, &string_to_sign, date, self.region, self.service)?;
288
289        // build the actual auth header
290        let authorisation = format!(
291            "{} Credential={}/{}, SignedHeaders={}, Signature={}",
292            ALGORITHM, self.credential.key_id, scope, signed_headers, signature
293        );
294
295        let authorization_val = HeaderValue::from_str(&authorisation).unwrap();
296        request
297            .headers_mut()
298            .insert(&AUTHORIZATION, authorization_val);
299        Ok(())
300    }
301
302    /// Generate a presigned URL, additionally folding `extra_query` parameters and
303    /// `signed_headers` into the SigV4 signature.
304    ///
305    /// `extra_query` lets callers sign query parameters that the recipient must send (e.g.
306    /// `partNumber` and `uploadId` for a multipart `UploadPart`, or a `versionId`).
307    ///
308    /// `signed_headers` lets callers bind specific request headers to the signature (e.g.
309    /// `x-amz-checksum-sha256`, `content-type`, or SSE headers). For a presigned URL these
310    /// values are fixed at signing time and the recipient must send exactly these headers and
311    /// values. The mandatory `host` header is always signed.
312    pub(crate) fn sign_with(
313        &self,
314        method: Method,
315        url: &mut Url,
316        extra_query: &[(String, String)],
317        signed_headers: &HeaderMap,
318        expires_in: Duration,
319    ) -> Result<()> {
320        let crypto = crypto_provider(self.crypto)?;
321
322        let date = self.date.unwrap_or_else(Utc::now);
323        let scope = self.scope(date);
324
325        // Append any caller-provided query parameters before signing so they are folded into
326        // `canonicalize_query` and become part of the signature.
327        append_strict_query_pairs(url, extra_query);
328
329        // The `host` header is always signed; callers may bind additional headers whose values
330        // are committed to the signature at signing time.
331        let host = &url[url::Position::BeforeHost..url::Position::AfterPort].to_string();
332        let mut headers = HeaderMap::with_capacity(1 + signed_headers.len());
333        let host_val = HeaderValue::from_str(host).unwrap();
334        headers.insert("host", host_val);
335        for (name, value) in signed_headers {
336            headers.append(name.clone(), value.clone());
337        }
338
339        let (signed_headers, canonical_headers) = canonicalize_headers(&headers);
340
341        // https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
342        url.query_pairs_mut()
343            .append_pair("X-Amz-Algorithm", ALGORITHM)
344            .append_pair(
345                "X-Amz-Credential",
346                &format!("{}/{}", self.credential.key_id, scope),
347            )
348            .append_pair("X-Amz-Date", &date.format("%Y%m%dT%H%M%SZ").to_string())
349            .append_pair("X-Amz-Expires", &expires_in.as_secs().to_string())
350            .append_pair("X-Amz-SignedHeaders", &signed_headers);
351
352        if self.request_payer {
353            // For signed URLs, include x-amz-request-payer=requester in the request
354            // https://docs.aws.amazon.com/AmazonS3/latest/userguide/ObjectsinRequesterPaysBuckets.html
355            url.query_pairs_mut()
356                .append_pair("x-amz-request-payer", "requester");
357        }
358
359        // For S3, you must include the X-Amz-Security-Token query parameter in the URL if
360        // using credentials sourced from the STS service.
361        if let Some(ref token) = self.credential.token {
362            url.query_pairs_mut()
363                .append_pair("X-Amz-Security-Token", token);
364        }
365
366        // We don't have a payload; the user is going to send the payload directly themselves.
367        let digest = UNSIGNED_PAYLOAD;
368
369        let string_to_sign = self.string_to_sign(
370            crypto,
371            date,
372            &scope,
373            &method,
374            url,
375            &canonical_headers,
376            &signed_headers,
377            digest,
378        )?;
379
380        let signature =
381            self.credential
382                .sign(crypto, &string_to_sign, date, self.region, self.service)?;
383
384        url.query_pairs_mut()
385            .append_pair("X-Amz-Signature", &signature);
386        Ok(())
387    }
388
389    #[allow(clippy::too_many_arguments)]
390    fn string_to_sign(
391        &self,
392        crypto: &dyn CryptoProvider,
393        date: DateTime<Utc>,
394        scope: &str,
395        request_method: &Method,
396        url: &Url,
397        canonical_headers: &str,
398        signed_headers: &str,
399        digest: &str,
400    ) -> Result<String> {
401        // Each path segment must be URI-encoded twice (except for Amazon S3 which only gets
402        // URI-encoded once).
403        // see https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
404        let canonical_uri = match self.service {
405            "s3" => url.path().to_string(),
406            _ => utf8_percent_encode(url.path(), &STRICT_PATH_ENCODE_SET).to_string(),
407        };
408
409        let canonical_query = canonicalize_query(url);
410
411        // https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
412        let canonical_request = format!(
413            "{}\n{}\n{}\n{}\n{}\n{}",
414            request_method.as_str(),
415            canonical_uri,
416            canonical_query,
417            canonical_headers,
418            signed_headers,
419            digest
420        );
421
422        let hashed_canonical_request = hex_digest(crypto, canonical_request.as_bytes())?;
423
424        Ok(format!(
425            "{}\n{}\n{}\n{}",
426            ALGORITHM,
427            date.format("%Y%m%dT%H%M%SZ"),
428            scope,
429            hashed_canonical_request
430        ))
431    }
432
433    fn scope(&self, date: DateTime<Utc>) -> String {
434        format!(
435            "{}/{}/{}/aws4_request",
436            date.format("%Y%m%d"),
437            self.region,
438            self.service
439        )
440    }
441}
442
443pub(crate) trait CredentialExt: Sized {
444    /// Sign a request <https://docs.aws.amazon.com/general/latest/gr/sigv4_signing.html>
445    fn with_aws_sigv4(
446        self,
447        authorizer: Option<AwsAuthorizer<'_>>,
448        payload_sha256: Option<&[u8]>,
449    ) -> Result<Self>;
450}
451
452impl CredentialExt for HttpRequestBuilder {
453    fn with_aws_sigv4(
454        self,
455        authorizer: Option<AwsAuthorizer<'_>>,
456        payload_sha256: Option<&[u8]>,
457    ) -> Result<Self> {
458        match authorizer {
459            Some(authorizer) => {
460                let (client, request) = self.into_parts();
461                let mut request = request.expect("request valid");
462                authorizer.try_authorize(&mut request, payload_sha256)?;
463
464                Ok(Self::from_parts(client, request))
465            }
466            None => Ok(self),
467        }
468    }
469}
470
471/// Canonicalizes query parameters into the AWS canonical form
472///
473/// Parameters are sorted by encoded name, with ties broken by encoded value, as required by the
474/// canonical request specification — sorting by the decoded name (or ignoring the value for
475/// duplicate names) can order parameters differently from how the server verifies them.
476///
477/// <https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html>
478fn canonicalize_query(url: &Url) -> String {
479    use std::fmt::Write;
480
481    let capacity = match url.query() {
482        Some(q) if !q.is_empty() => q.len(),
483        _ => return String::new(),
484    };
485
486    let mut params = url
487        .query_pairs()
488        .map(|(k, v)| {
489            (
490                utf8_percent_encode(k.as_ref(), &STRICT_ENCODE_SET).to_string(),
491                utf8_percent_encode(v.as_ref(), &STRICT_ENCODE_SET).to_string(),
492            )
493        })
494        .collect::<Vec<_>>();
495    params.sort_unstable();
496
497    let mut encoded = String::with_capacity(capacity + 1);
498    for (i, (k, v)) in params.iter().enumerate() {
499        if i > 0 {
500            encoded.push('&');
501        }
502        let _ = write!(encoded, "{k}={v}");
503    }
504    encoded
505}
506
507fn append_normalized_whitespace_value(headers: &'_ mut String, input: &str) {
508    let mut iter = input.split_whitespace();
509
510    if let Some(first) = iter.next() {
511        headers.push_str(first);
512        for word in iter {
513            headers.push(' ');
514            headers.push_str(word);
515        }
516    }
517}
518
519/// Canonicalizes headers into the AWS Canonical Form.
520///
521/// <https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html>
522fn canonicalize_headers(header_map: &HeaderMap) -> (String, String) {
523    let mut headers = BTreeMap::<&str, Vec<&str>>::new();
524    let mut value_count = 0;
525    let mut value_bytes = 0;
526    let mut key_bytes = 0;
527
528    for (key, value) in header_map {
529        let key = key.as_str();
530        if ["authorization", "content-length", "user-agent"].contains(&key) {
531            continue;
532        }
533
534        let value = std::str::from_utf8(value.as_bytes()).unwrap();
535        key_bytes += key.len();
536        value_bytes += value.len();
537        value_count += 1;
538        headers.entry(key).or_default().push(value);
539    }
540
541    let mut signed_headers = String::with_capacity(key_bytes + headers.len());
542    let mut canonical_headers =
543        String::with_capacity(key_bytes + value_bytes + headers.len() + value_count);
544
545    for (header_idx, (name, values)) in headers.into_iter().enumerate() {
546        if header_idx != 0 {
547            signed_headers.push(';');
548        }
549
550        signed_headers.push_str(name);
551        canonical_headers.push_str(name);
552        canonical_headers.push(':');
553        for (value_idx, value) in values.into_iter().enumerate() {
554            if value_idx != 0 {
555                canonical_headers.push(',');
556            }
557            append_normalized_whitespace_value(&mut canonical_headers, value.trim());
558        }
559        canonical_headers.push('\n');
560    }
561
562    (signed_headers, canonical_headers)
563}
564
565/// Credentials sourced from the instance metadata service
566///
567/// <https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html>
568#[derive(Debug)]
569pub(crate) struct InstanceCredentialProvider {
570    pub imdsv1_fallback: bool,
571    pub metadata_endpoint: String,
572}
573
574#[async_trait]
575impl TokenProvider for InstanceCredentialProvider {
576    type Credential = AwsCredential;
577
578    async fn fetch_token(
579        &self,
580        client: &HttpClient,
581        retry: &RetryConfig,
582    ) -> Result<TemporaryToken<Arc<AwsCredential>>> {
583        instance_creds(client, retry, &self.metadata_endpoint, self.imdsv1_fallback)
584            .await
585            .map_err(|source| crate::Error::Generic {
586                store: STORE,
587                source,
588            })
589    }
590}
591
592/// Credentials sourced using AssumeRoleWithWebIdentity
593///
594/// <https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts-technical-overview.html>
595#[derive(Debug)]
596pub(crate) struct WebIdentityProvider {
597    pub token_path: String,
598    pub role_arn: String,
599    pub session_name: String,
600    pub endpoint: String,
601}
602
603#[async_trait]
604impl TokenProvider for WebIdentityProvider {
605    type Credential = AwsCredential;
606
607    async fn fetch_token(
608        &self,
609        client: &HttpClient,
610        retry: &RetryConfig,
611    ) -> Result<TemporaryToken<Arc<AwsCredential>>> {
612        web_identity(
613            client,
614            retry,
615            &self.token_path,
616            &self.role_arn,
617            &self.session_name,
618            &self.endpoint,
619        )
620        .await
621        .map_err(|source| crate::Error::Generic {
622            store: STORE,
623            source,
624        })
625    }
626}
627
628#[derive(Debug, Deserialize)]
629#[serde(rename_all = "PascalCase")]
630struct InstanceCredentials {
631    access_key_id: String,
632    secret_access_key: String,
633    token: String,
634    expiration: DateTime<Utc>,
635}
636
637impl From<InstanceCredentials> for AwsCredential {
638    fn from(s: InstanceCredentials) -> Self {
639        Self {
640            key_id: s.access_key_id,
641            secret_key: s.secret_access_key,
642            token: Some(s.token),
643        }
644    }
645}
646
647/// <https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#instance-metadata-security-credentials>
648async fn instance_creds(
649    client: &HttpClient,
650    retry_config: &RetryConfig,
651    endpoint: &str,
652    imdsv1_fallback: bool,
653) -> Result<TemporaryToken<Arc<AwsCredential>>, StdError> {
654    const CREDENTIALS_PATH: &str = "latest/meta-data/iam/security-credentials";
655    const AWS_EC2_METADATA_TOKEN_HEADER: &str = "X-aws-ec2-metadata-token";
656
657    let token_url = format!("{endpoint}/latest/api/token");
658
659    let token_result = client
660        .request(Method::PUT, token_url)
661        .header("X-aws-ec2-metadata-token-ttl-seconds", "600") // 10 minute TTL
662        .retryable(retry_config)
663        .idempotent(true)
664        .send()
665        .await;
666
667    let token = match token_result {
668        Ok(t) => Some(t.into_body().text().await?),
669        Err(e) if imdsv1_fallback && matches!(e.status(), Some(StatusCode::FORBIDDEN)) => {
670            warn!("received 403 from metadata endpoint, falling back to IMDSv1");
671            None
672        }
673        Err(e) => return Err(e.into()),
674    };
675
676    let role_url = format!("{endpoint}/{CREDENTIALS_PATH}/");
677    let mut role_request = client.request(Method::GET, role_url);
678
679    if let Some(token) = &token {
680        role_request = role_request.header(AWS_EC2_METADATA_TOKEN_HEADER, token);
681    }
682
683    let role = role_request
684        .send_retry(retry_config)
685        .await?
686        .into_body()
687        .text()
688        .await?;
689
690    let creds_url = format!("{endpoint}/{CREDENTIALS_PATH}/{role}");
691    let mut creds_request = client.request(Method::GET, creds_url);
692    if let Some(token) = &token {
693        creds_request = creds_request.header(AWS_EC2_METADATA_TOKEN_HEADER, token);
694    }
695
696    let creds: InstanceCredentials = creds_request
697        .send_retry(retry_config)
698        .await?
699        .into_body()
700        .json()
701        .await?;
702
703    let now = Utc::now();
704    let ttl = (creds.expiration - now).to_std().unwrap_or_default();
705    Ok(TemporaryToken {
706        token: Arc::new(creds.into()),
707        expiry: Some(Instant::now() + ttl),
708    })
709}
710
711#[derive(Debug, Deserialize)]
712#[serde(rename_all = "PascalCase")]
713struct AssumeRoleResponse {
714    assume_role_with_web_identity_result: AssumeRoleResult,
715}
716
717#[derive(Debug, Deserialize)]
718#[serde(rename_all = "PascalCase")]
719struct AssumeRoleResult {
720    credentials: SessionCredentials,
721}
722
723#[derive(Debug, Deserialize)]
724#[serde(rename_all = "PascalCase")]
725struct SessionCredentials {
726    session_token: String,
727    secret_access_key: String,
728    access_key_id: String,
729    expiration: DateTime<Utc>,
730}
731
732impl From<SessionCredentials> for AwsCredential {
733    fn from(s: SessionCredentials) -> Self {
734        Self {
735            key_id: s.access_key_id,
736            secret_key: s.secret_access_key,
737            token: Some(s.session_token),
738        }
739    }
740}
741
742/// <https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts-technical-overview.html>
743async fn web_identity(
744    client: &HttpClient,
745    retry_config: &RetryConfig,
746    token_path: &str,
747    role_arn: &str,
748    session_name: &str,
749    endpoint: &str,
750) -> Result<TemporaryToken<Arc<AwsCredential>>, StdError> {
751    let token = std::fs::read_to_string(token_path)
752        .map_err(|e| format!("Failed to read token file '{token_path}': {e}"))?;
753
754    let bytes = client
755        .post(endpoint)
756        .query(&[
757            ("Action", "AssumeRoleWithWebIdentity"),
758            ("DurationSeconds", "3600"),
759            ("RoleArn", role_arn),
760            ("RoleSessionName", session_name),
761            ("Version", "2011-06-15"),
762            ("WebIdentityToken", &token),
763        ])
764        .retryable(retry_config)
765        .idempotent(true)
766        .sensitive(true)
767        .send()
768        .await?
769        .into_body()
770        .bytes()
771        .await?;
772
773    let resp: AssumeRoleResponse = quick_xml::de::from_reader(bytes.reader())
774        .map_err(|e| format!("Invalid AssumeRoleWithWebIdentity response: {e}"))?;
775
776    let creds = resp.assume_role_with_web_identity_result.credentials;
777    let now = Utc::now();
778    let ttl = (creds.expiration - now).to_std().unwrap_or_default();
779
780    Ok(TemporaryToken {
781        token: Arc::new(creds.into()),
782        expiry: Some(Instant::now() + ttl),
783    })
784}
785
786/// Credentials sourced from a task IAM role
787///
788/// <https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html>
789#[derive(Debug)]
790pub(crate) struct TaskCredentialProvider {
791    pub url: String,
792    pub retry: RetryConfig,
793    pub client: HttpClient,
794    pub cache: TokenCache<Arc<AwsCredential>>,
795}
796
797#[async_trait]
798impl CredentialProvider for TaskCredentialProvider {
799    type Credential = AwsCredential;
800
801    async fn get_credential(&self) -> Result<Arc<AwsCredential>> {
802        self.cache
803            .get_or_insert_with(|| task_credential(&self.client, &self.retry, &self.url))
804            .await
805            .map_err(|source| crate::Error::Generic {
806                store: STORE,
807                source,
808            })
809    }
810}
811
812/// <https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-iam-roles.html>
813async fn task_credential(
814    client: &HttpClient,
815    retry: &RetryConfig,
816    url: &str,
817) -> Result<TemporaryToken<Arc<AwsCredential>>, StdError> {
818    let creds: InstanceCredentials = client
819        .get(url)
820        .send_retry(retry)
821        .await?
822        .into_body()
823        .json()
824        .await?;
825
826    let now = Utc::now();
827    let ttl = (creds.expiration - now).to_std().unwrap_or_default();
828    Ok(TemporaryToken {
829        token: Arc::new(creds.into()),
830        expiry: Some(Instant::now() + ttl),
831    })
832}
833
834/// EKS Pod Identity credential provider.
835///
836/// Uses the endpoint in `AWS_CONTAINER_CREDENTIALS_FULL_URI`
837/// and the bearer token in `AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE`
838/// to fetch ephemeral AWS credentials from an EKS pod.
839#[derive(Debug)]
840pub(crate) struct EKSPodCredentialProvider {
841    pub url: String,
842    pub token_file: String,
843    pub retry: RetryConfig,
844    pub client: HttpClient,
845    pub cache: TokenCache<Arc<AwsCredential>>,
846}
847
848#[async_trait]
849impl CredentialProvider for EKSPodCredentialProvider {
850    type Credential = AwsCredential;
851
852    async fn get_credential(&self) -> Result<Arc<AwsCredential>> {
853        self.cache
854            .get_or_insert_with(|| {
855                eks_credential(&self.client, &self.retry, &self.url, &self.token_file)
856            })
857            .await
858            .map_err(|source| crate::Error::Generic {
859                store: STORE,
860                source,
861            })
862    }
863}
864
865/// Performs the actual credential retrieval and parsing for `EKSPodCredentialProvider`.
866///
867/// <https://mozillazg.com/2023/12/security-deep-dive-into-aws-eks-pod-identity-feature-en.html>
868async fn eks_credential(
869    client: &HttpClient,
870    retry: &RetryConfig,
871    url: &str,
872    token_file: &str,
873) -> Result<TemporaryToken<Arc<AwsCredential>>, StdError> {
874    // Spawn IO to blocking tokio pool if running in tokio context
875    let token = match tokio::runtime::Handle::try_current() {
876        Ok(runtime) => {
877            let path = token_file.to_string();
878            runtime
879                .spawn_blocking(move || std::fs::read_to_string(&path))
880                .await?
881        }
882        Err(_) => std::fs::read_to_string(token_file),
883    }
884    .map_err(|e| format!("Failed to read EKS token file '{token_file}': {e}"))?;
885
886    let mut req = client.request(Method::GET, url);
887    req = req.header("Authorization", token);
888
889    // The JSON from the EKS credential endpoint has the same shape as ECS task credentials
890    let creds: InstanceCredentials = req.send_retry(retry).await?.into_body().json().await?;
891
892    let now = Utc::now();
893    let ttl = (creds.expiration - now).to_std().unwrap_or_default();
894
895    Ok(TemporaryToken {
896        token: Arc::new(creds.into()),
897        expiry: Some(Instant::now() + ttl),
898    })
899}
900
901/// A session provider as used by S3 Express One Zone
902///
903/// <https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateSession.html>
904#[derive(Debug)]
905pub(crate) struct SessionProvider {
906    pub endpoint: String,
907    pub region: String,
908    pub credentials: AwsCredentialProvider,
909    pub crypto: Option<Arc<dyn CryptoProvider>>,
910}
911
912#[async_trait]
913impl TokenProvider for SessionProvider {
914    type Credential = AwsCredential;
915
916    async fn fetch_token(
917        &self,
918        client: &HttpClient,
919        retry: &RetryConfig,
920    ) -> Result<TemporaryToken<Arc<Self::Credential>>> {
921        let crypto = crypto_provider(self.crypto.as_deref())?;
922        let creds = self.credentials.get_credential().await?;
923        let authorizer = AwsAuthorizer::new(&creds, "s3", &self.region).with_crypto(crypto);
924
925        let bytes = client
926            .get(format!("{}?session", self.endpoint))
927            .with_aws_sigv4(Some(authorizer), None)?
928            .send_retry(retry)
929            .await
930            .map_err(|source| Error::CreateSessionRequest { source })?
931            .into_body()
932            .bytes()
933            .await
934            .map_err(|source| Error::CreateSessionResponse { source })?;
935
936        let resp: CreateSessionOutput = quick_xml::de::from_reader(bytes.reader())
937            .map_err(|source| Error::CreateSessionOutput { source })?;
938
939        let creds = resp.credentials;
940        Ok(TemporaryToken {
941            token: Arc::new(creds.into()),
942            // Credentials last 5 minutes - https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateSession.html
943            expiry: Some(Instant::now() + Duration::from_secs(5 * 60)),
944        })
945    }
946}
947
948#[derive(Debug, Deserialize)]
949#[serde(rename_all = "PascalCase")]
950struct CreateSessionOutput {
951    credentials: SessionCredentials,
952}
953
954#[cfg(test)]
955mod tests {
956    use super::*;
957    use crate::aws::{AmazonS3Builder, AmazonS3ConfigKey};
958    use crate::client::HttpClient;
959    use crate::client::mock_server::MockServer;
960    use http::{Method, Response};
961    #[cfg(feature = "reqwest")]
962    use reqwest::Client;
963    use std::collections::HashMap;
964    use std::env;
965
966    // Test generated using https://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
967    #[cfg(feature = "reqwest")]
968    #[test]
969    fn test_sign_with_signed_payload() {
970        let client = HttpClient::new(Client::new());
971
972        // Test credentials from https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTAuthentication.html
973        let credential = AwsCredential {
974            key_id: "AKIAIOSFODNN7EXAMPLE".to_string(),
975            secret_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(),
976            token: None,
977        };
978
979        // method = 'GET'
980        // service = 'ec2'
981        // host = 'ec2.amazonaws.com'
982        // region = 'us-east-1'
983        // endpoint = 'https://ec2.amazonaws.com'
984        // request_parameters = ''
985        let date = DateTime::parse_from_rfc3339("2022-08-06T18:01:34Z")
986            .unwrap()
987            .with_timezone(&Utc);
988
989        let mut request = client
990            .request(Method::GET, "https://ec2.amazon.com/")
991            .into_parts()
992            .1
993            .unwrap();
994
995        let signer = AwsAuthorizer {
996            date: Some(date),
997            crypto: None,
998            credential: &credential,
999            service: "ec2",
1000            region: "us-east-1",
1001            sign_payload: true,
1002            token_header: None,
1003            request_payer: false,
1004        };
1005
1006        signer.try_authorize(&mut request, None).unwrap();
1007        assert_eq!(
1008            request.headers().get(&AUTHORIZATION).unwrap(),
1009            "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20220806/us-east-1/ec2/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=a3c787a7ed37f7fdfbfd2d7056a3d7c9d85e6d52a2bfbec73793c0be6e7862d4"
1010        )
1011    }
1012
1013    /// `host` must be part of the SigV4 signature, but must NOT be pinned as a header on the
1014    /// request itself — the transport (hyper) sets Host from the URL, so a pinned Host header
1015    /// would be carried onto a cross-host redirect (e.g. an S3-compatible gateway 302 to a CDN)
1016    /// and break the redirected request.
1017    #[cfg(feature = "reqwest")]
1018    #[test]
1019    fn test_host_is_signed_but_not_pinned_on_request() {
1020        let client = HttpClient::new(Client::new());
1021        let credential = AwsCredential {
1022            key_id: "AKIAIOSFODNN7EXAMPLE".to_string(),
1023            secret_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(),
1024            token: None,
1025        };
1026        let date = DateTime::parse_from_rfc3339("2022-08-06T18:01:34Z")
1027            .unwrap()
1028            .with_timezone(&Utc);
1029        let mut request = client
1030            .request(Method::GET, "https://ec2.amazon.com/")
1031            .into_parts()
1032            .1
1033            .unwrap();
1034        let signer = AwsAuthorizer {
1035            date: Some(date),
1036            crypto: None,
1037            credential: &credential,
1038            service: "ec2",
1039            region: "us-east-1",
1040            sign_payload: true,
1041            token_header: None,
1042            request_payer: false,
1043        };
1044
1045        signer.try_authorize(&mut request, None).unwrap();
1046
1047        // host is signed ...
1048        let auth = request
1049            .headers()
1050            .get(&AUTHORIZATION)
1051            .unwrap()
1052            .to_str()
1053            .unwrap();
1054        assert!(auth.contains("SignedHeaders=host;"), "auth was: {auth}");
1055        // ... but not pinned as a request header (so it is regenerated across redirects).
1056        assert!(request.headers().get(&HOST).is_none());
1057    }
1058
1059    #[cfg(feature = "reqwest")]
1060    #[test]
1061    fn test_sign_with_signed_payload_request_payer() {
1062        let client = HttpClient::new(Client::new());
1063
1064        // Test credentials from https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTAuthentication.html
1065        let credential = AwsCredential {
1066            key_id: "AKIAIOSFODNN7EXAMPLE".to_string(),
1067            secret_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(),
1068            token: None,
1069        };
1070
1071        // method = 'GET'
1072        // service = 'ec2'
1073        // host = 'ec2.amazonaws.com'
1074        // region = 'us-east-1'
1075        // endpoint = 'https://ec2.amazonaws.com'
1076        // request_parameters = ''
1077        let date = DateTime::parse_from_rfc3339("2022-08-06T18:01:34Z")
1078            .unwrap()
1079            .with_timezone(&Utc);
1080
1081        let mut request = client
1082            .request(Method::GET, "https://ec2.amazon.com/")
1083            .into_parts()
1084            .1
1085            .unwrap();
1086
1087        let signer = AwsAuthorizer {
1088            date: Some(date),
1089            crypto: None,
1090            credential: &credential,
1091            service: "ec2",
1092            region: "us-east-1",
1093            sign_payload: true,
1094            token_header: None,
1095            request_payer: true,
1096        };
1097
1098        signer.try_authorize(&mut request, None).unwrap();
1099        assert_eq!(
1100            request.headers().get(&AUTHORIZATION).unwrap(),
1101            "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20220806/us-east-1/ec2/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-request-payer, Signature=7030625a9e9b57ed2a40e63d749f4a4b7714b6e15004cab026152f870dd8565d"
1102        )
1103    }
1104
1105    #[cfg(feature = "reqwest")]
1106    #[test]
1107    fn test_sign_with_unsigned_payload() {
1108        let client = HttpClient::new(Client::new());
1109
1110        // Test credentials from https://docs.aws.amazon.com/AmazonS3/latest/userguide/RESTAuthentication.html
1111        let credential = AwsCredential {
1112            key_id: "AKIAIOSFODNN7EXAMPLE".to_string(),
1113            secret_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(),
1114            token: None,
1115        };
1116
1117        // method = 'GET'
1118        // service = 'ec2'
1119        // host = 'ec2.amazonaws.com'
1120        // region = 'us-east-1'
1121        // endpoint = 'https://ec2.amazonaws.com'
1122        // request_parameters = ''
1123        let date = DateTime::parse_from_rfc3339("2022-08-06T18:01:34Z")
1124            .unwrap()
1125            .with_timezone(&Utc);
1126
1127        let mut request = client
1128            .request(Method::GET, "https://ec2.amazon.com/")
1129            .into_parts()
1130            .1
1131            .unwrap();
1132
1133        let authorizer = AwsAuthorizer {
1134            date: Some(date),
1135            crypto: None,
1136            credential: &credential,
1137            service: "ec2",
1138            region: "us-east-1",
1139            token_header: None,
1140            sign_payload: false,
1141            request_payer: false,
1142        };
1143
1144        authorizer.try_authorize(&mut request, None).unwrap();
1145        assert_eq!(
1146            request.headers().get(&AUTHORIZATION).unwrap(),
1147            "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20220806/us-east-1/ec2/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=653c3d8ea261fd826207df58bc2bb69fbb5003e9eb3c0ef06e4a51f2a81d8699"
1148        );
1149    }
1150
1151    #[test]
1152    fn signed_get_url() {
1153        // Values from https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
1154        let credential = AwsCredential {
1155            key_id: "AKIAIOSFODNN7EXAMPLE".to_string(),
1156            secret_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(),
1157            token: None,
1158        };
1159
1160        let date = DateTime::parse_from_rfc3339("2013-05-24T00:00:00Z")
1161            .unwrap()
1162            .with_timezone(&Utc);
1163
1164        let authorizer = AwsAuthorizer {
1165            date: Some(date),
1166            crypto: None,
1167            credential: &credential,
1168            service: "s3",
1169            region: "us-east-1",
1170            token_header: None,
1171            sign_payload: false,
1172            request_payer: false,
1173        };
1174
1175        let mut url = Url::parse("https://examplebucket.s3.amazonaws.com/test.txt").unwrap();
1176        authorizer
1177            .sign_with(
1178                Method::GET,
1179                &mut url,
1180                &[],
1181                &HeaderMap::new(),
1182                Duration::from_secs(86400),
1183            )
1184            .unwrap();
1185
1186        assert_eq!(
1187            url,
1188            Url::parse(
1189                "https://examplebucket.s3.amazonaws.com/test.txt?\
1190                X-Amz-Algorithm=AWS4-HMAC-SHA256&\
1191                X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&\
1192                X-Amz-Date=20130524T000000Z&\
1193                X-Amz-Expires=86400&\
1194                X-Amz-SignedHeaders=host&\
1195                X-Amz-Signature=aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404"
1196            )
1197            .unwrap()
1198        );
1199    }
1200
1201    #[test]
1202    fn signed_get_url_request_payer() {
1203        // Values from https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
1204        let credential = AwsCredential {
1205            key_id: "AKIAIOSFODNN7EXAMPLE".to_string(),
1206            secret_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(),
1207            token: None,
1208        };
1209
1210        let date = DateTime::parse_from_rfc3339("2013-05-24T00:00:00Z")
1211            .unwrap()
1212            .with_timezone(&Utc);
1213
1214        let authorizer = AwsAuthorizer {
1215            date: Some(date),
1216            crypto: None,
1217            credential: &credential,
1218            service: "s3",
1219            region: "us-east-1",
1220            token_header: None,
1221            sign_payload: false,
1222            request_payer: true,
1223        };
1224
1225        let mut url = Url::parse("https://examplebucket.s3.amazonaws.com/test.txt").unwrap();
1226        authorizer
1227            .sign_with(
1228                Method::GET,
1229                &mut url,
1230                &[],
1231                &HeaderMap::new(),
1232                Duration::from_secs(86400),
1233            )
1234            .unwrap();
1235
1236        assert_eq!(
1237            url,
1238            Url::parse(
1239                "https://examplebucket.s3.amazonaws.com/test.txt?\
1240                X-Amz-Algorithm=AWS4-HMAC-SHA256&\
1241                X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&\
1242                X-Amz-Date=20130524T000000Z&\
1243                X-Amz-Expires=86400&\
1244                X-Amz-SignedHeaders=host&\
1245                x-amz-request-payer=requester&\
1246                X-Amz-Signature=9ad7c781cc30121f199b47d35ed3528473e4375b63c5d91cd87c927803e4e00a"
1247            )
1248            .unwrap()
1249        );
1250    }
1251
1252    #[test]
1253    fn signed_url_with_query_value_containing_space() {
1254        // A space in a query value must be encoded as `%20` in the URL (not `+`, as
1255        // application/x-www-form-urlencoded would produce) so that the URL bytes match the
1256        // canonical query string that is signed. The expected signature was computed with an
1257        // independent SigV4 implementation.
1258        let credential = AwsCredential {
1259            key_id: "AKIAIOSFODNN7EXAMPLE".to_string(),
1260            secret_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(),
1261            token: None,
1262        };
1263
1264        let date = DateTime::parse_from_rfc3339("2013-05-24T00:00:00Z")
1265            .unwrap()
1266            .with_timezone(&Utc);
1267
1268        let authorizer = AwsAuthorizer {
1269            date: Some(date),
1270            crypto: None,
1271            credential: &credential,
1272            service: "s3",
1273            region: "us-east-1",
1274            token_header: None,
1275            sign_payload: false,
1276            request_payer: false,
1277        };
1278
1279        let mut url = Url::parse("https://examplebucket.s3.amazonaws.com/test.txt").unwrap();
1280        authorizer
1281            .sign_with(
1282                Method::PUT,
1283                &mut url,
1284                &[("prefix".to_string(), "a b".to_string())],
1285                &HeaderMap::new(),
1286                Duration::from_secs(86400),
1287            )
1288            .unwrap();
1289
1290        // The URL carries `%20`, never `+`.
1291        assert!(url.query().unwrap().contains("prefix=a%20b"));
1292        assert!(!url.query().unwrap().contains('+'));
1293
1294        let signature = url
1295            .query_pairs()
1296            .find(|(k, _)| k == "X-Amz-Signature")
1297            .unwrap()
1298            .1
1299            .into_owned();
1300        assert_eq!(
1301            signature,
1302            "e392940dfa4480f872625b35ef1224bfa1ae989c73874df01022e6d1fa4c2f00"
1303        );
1304    }
1305
1306    #[test]
1307    fn signed_url_with_duplicate_query_keys_sorted_by_value() {
1308        // Duplicate query parameter names must be sorted by encoded value in the canonical
1309        // request, regardless of the order they are supplied. The expected signature was computed
1310        // with an independent SigV4 implementation for the sorted form `partNumber=1&partNumber=2`.
1311        let credential = AwsCredential {
1312            key_id: "AKIAIOSFODNN7EXAMPLE".to_string(),
1313            secret_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(),
1314            token: None,
1315        };
1316
1317        let date = DateTime::parse_from_rfc3339("2013-05-24T00:00:00Z")
1318            .unwrap()
1319            .with_timezone(&Utc);
1320
1321        let authorizer = AwsAuthorizer {
1322            date: Some(date),
1323            crypto: None,
1324            credential: &credential,
1325            service: "s3",
1326            region: "us-east-1",
1327            token_header: None,
1328            sign_payload: false,
1329            request_payer: false,
1330        };
1331
1332        let mut url = Url::parse("https://examplebucket.s3.amazonaws.com/test.txt").unwrap();
1333        authorizer
1334            .sign_with(
1335                Method::PUT,
1336                &mut url,
1337                // Supplied in reverse value order; canonicalization must sort them.
1338                &[
1339                    ("partNumber".to_string(), "2".to_string()),
1340                    ("partNumber".to_string(), "1".to_string()),
1341                ],
1342                &HeaderMap::new(),
1343                Duration::from_secs(86400),
1344            )
1345            .unwrap();
1346
1347        let signature = url
1348            .query_pairs()
1349            .find(|(k, _)| k == "X-Amz-Signature")
1350            .unwrap()
1351            .1
1352            .into_owned();
1353        assert_eq!(
1354            signature,
1355            "d3d68662bef0097adce4878c453f22ce9449c7084789b03d9abe9e8c05db38e7"
1356        );
1357    }
1358
1359    #[test]
1360    fn signed_url_with_query_params() {
1361        // Presign a multipart `UploadPart` request, where `partNumber` and `uploadId` must be
1362        // folded into the signature.
1363        let credential = AwsCredential {
1364            key_id: "AKIAIOSFODNN7EXAMPLE".to_string(),
1365            secret_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(),
1366            token: None,
1367        };
1368
1369        let date = DateTime::parse_from_rfc3339("2013-05-24T00:00:00Z")
1370            .unwrap()
1371            .with_timezone(&Utc);
1372
1373        let authorizer = AwsAuthorizer {
1374            date: Some(date),
1375            crypto: None,
1376            credential: &credential,
1377            service: "s3",
1378            region: "us-east-1",
1379            token_header: None,
1380            sign_payload: false,
1381            request_payer: false,
1382        };
1383
1384        let mut url = Url::parse("https://examplebucket.s3.amazonaws.com/test.txt").unwrap();
1385        authorizer
1386            .sign_with(
1387                Method::PUT,
1388                &mut url,
1389                &[
1390                    ("partNumber".to_string(), "1".to_string()),
1391                    ("uploadId".to_string(), "abc123".to_string()),
1392                ],
1393                &HeaderMap::new(),
1394                Duration::from_secs(86400),
1395            )
1396            .unwrap();
1397
1398        let pairs: HashMap<_, _> = url.query_pairs().into_owned().collect();
1399        // Caller-provided params are present and signed (only `host` is a signed header).
1400        assert_eq!(pairs.get("partNumber").map(String::as_str), Some("1"));
1401        assert_eq!(pairs.get("uploadId").map(String::as_str), Some("abc123"));
1402        assert_eq!(
1403            pairs.get("X-Amz-SignedHeaders").map(String::as_str),
1404            Some("host")
1405        );
1406        assert!(pairs.contains_key("X-Amz-Signature"));
1407
1408        // Full golden URL locks the signature; the query params change it relative to a plain
1409        // presigned PUT, proving they were folded into the canonical request.
1410        assert_eq!(
1411            url,
1412            Url::parse(
1413                "https://examplebucket.s3.amazonaws.com/test.txt?\
1414                partNumber=1&\
1415                uploadId=abc123&\
1416                X-Amz-Algorithm=AWS4-HMAC-SHA256&\
1417                X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&\
1418                X-Amz-Date=20130524T000000Z&\
1419                X-Amz-Expires=86400&\
1420                X-Amz-SignedHeaders=host&\
1421                X-Amz-Signature=4fb562e4da520986ae14e0e0259e2a2355625743452bb3c0ab8a6b50d8f0fcd4"
1422            )
1423            .unwrap()
1424        );
1425    }
1426
1427    #[test]
1428    fn signed_url_with_signed_headers() {
1429        // Bind a `content-type` header to the signature; the recipient must send exactly this
1430        // header and value.
1431        let credential = AwsCredential {
1432            key_id: "AKIAIOSFODNN7EXAMPLE".to_string(),
1433            secret_key: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string(),
1434            token: None,
1435        };
1436
1437        let date = DateTime::parse_from_rfc3339("2013-05-24T00:00:00Z")
1438            .unwrap()
1439            .with_timezone(&Utc);
1440
1441        let authorizer = AwsAuthorizer {
1442            date: Some(date),
1443            crypto: None,
1444            credential: &credential,
1445            service: "s3",
1446            region: "us-east-1",
1447            token_header: None,
1448            sign_payload: false,
1449            request_payer: false,
1450        };
1451
1452        let mut signed_headers = HeaderMap::new();
1453        signed_headers.insert(
1454            HeaderName::from_static("content-type"),
1455            HeaderValue::from_static("text/plain"),
1456        );
1457
1458        let mut url = Url::parse("https://examplebucket.s3.amazonaws.com/test.txt").unwrap();
1459        authorizer
1460            .sign_with(
1461                Method::PUT,
1462                &mut url,
1463                &[],
1464                &signed_headers,
1465                Duration::from_secs(86400),
1466            )
1467            .unwrap();
1468
1469        let pairs: HashMap<_, _> = url.query_pairs().into_owned().collect();
1470        // The extra header is reflected in X-Amz-SignedHeaders, sorted alongside `host`.
1471        assert_eq!(
1472            pairs.get("X-Amz-SignedHeaders").map(String::as_str),
1473            Some("content-type;host")
1474        );
1475
1476        assert_eq!(
1477            url,
1478            Url::parse(
1479                "https://examplebucket.s3.amazonaws.com/test.txt?\
1480                X-Amz-Algorithm=AWS4-HMAC-SHA256&\
1481                X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request&\
1482                X-Amz-Date=20130524T000000Z&\
1483                X-Amz-Expires=86400&\
1484                X-Amz-SignedHeaders=content-type%3Bhost&\
1485                X-Amz-Signature=653d5b202899172830dfe331b817de16c682b8b1e366de1b251ae709a20e20f9"
1486            )
1487            .unwrap()
1488        );
1489    }
1490
1491    #[cfg(feature = "reqwest")]
1492    #[test]
1493    fn test_sign_port() {
1494        let client = HttpClient::new(Client::new());
1495
1496        let credential = AwsCredential {
1497            key_id: "H20ABqCkLZID4rLe".to_string(),
1498            secret_key: "jMqRDgxSsBqqznfmddGdu1TmmZOJQxdM".to_string(),
1499            token: None,
1500        };
1501
1502        let date = DateTime::parse_from_rfc3339("2022-08-09T13:05:25Z")
1503            .unwrap()
1504            .with_timezone(&Utc);
1505
1506        let mut request = client
1507            .request(Method::GET, "http://localhost:9000/tsm-schemas")
1508            .query(&[
1509                ("delimiter", "/"),
1510                ("encoding-type", "url"),
1511                ("list-type", "2"),
1512                ("prefix", ""),
1513            ])
1514            .into_parts()
1515            .1
1516            .unwrap();
1517
1518        let authorizer = AwsAuthorizer {
1519            date: Some(date),
1520            crypto: None,
1521            credential: &credential,
1522            service: "s3",
1523            region: "us-east-1",
1524            token_header: None,
1525            sign_payload: true,
1526            request_payer: false,
1527        };
1528
1529        authorizer.try_authorize(&mut request, None).unwrap();
1530        assert_eq!(
1531            request.headers().get(&AUTHORIZATION).unwrap(),
1532            "AWS4-HMAC-SHA256 Credential=H20ABqCkLZID4rLe/20220809/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=9ebf2f92872066c99ac94e573b4e1b80f4dbb8a32b1e8e23178318746e7d1b4d"
1533        )
1534    }
1535
1536    #[cfg(feature = "reqwest")]
1537    #[tokio::test]
1538    async fn test_instance_metadata() {
1539        if env::var("TEST_INTEGRATION").is_err() {
1540            eprintln!("skipping AWS integration test");
1541            return;
1542        }
1543
1544        // For example https://github.com/aws/amazon-ec2-metadata-mock
1545        let endpoint = env::var("EC2_METADATA_ENDPOINT").unwrap();
1546        let client = HttpClient::new(Client::new());
1547        let retry_config = RetryConfig::default();
1548
1549        // Verify only allows IMDSv2
1550        let (client, req) = client
1551            .request(Method::GET, format!("{endpoint}/latest/meta-data/ami-id"))
1552            .into_parts();
1553
1554        let resp = client.execute(req.unwrap()).await.unwrap();
1555
1556        assert_eq!(
1557            resp.status(),
1558            StatusCode::UNAUTHORIZED,
1559            "Ensure metadata endpoint is set to only allow IMDSv2"
1560        );
1561
1562        let creds = instance_creds(&client, &retry_config, &endpoint, false)
1563            .await
1564            .unwrap();
1565
1566        let id = &creds.token.key_id;
1567        let secret = &creds.token.secret_key;
1568        let token = creds.token.token.as_ref().unwrap();
1569
1570        assert!(!id.is_empty());
1571        assert!(!secret.is_empty());
1572        assert!(!token.is_empty())
1573    }
1574
1575    #[cfg(feature = "reqwest")]
1576    #[tokio::test]
1577    async fn test_mock() {
1578        let server = MockServer::new().await;
1579
1580        const IMDSV2_HEADER: &str = "X-aws-ec2-metadata-token";
1581
1582        let secret_access_key = "SECRET";
1583        let access_key_id = "KEYID";
1584        let token = "TOKEN";
1585
1586        let endpoint = server.url();
1587        let client = HttpClient::new(Client::new());
1588        let retry_config = RetryConfig::default();
1589
1590        // Test IMDSv2
1591        server.push_fn(|req| {
1592            assert_eq!(req.uri().path(), "/latest/api/token");
1593            assert_eq!(req.method(), &Method::PUT);
1594            Response::new("cupcakes".to_string())
1595        });
1596        server.push_fn(|req| {
1597            assert_eq!(
1598                req.uri().path(),
1599                "/latest/meta-data/iam/security-credentials/"
1600            );
1601            assert_eq!(req.method(), &Method::GET);
1602            let t = req.headers().get(IMDSV2_HEADER).unwrap().to_str().unwrap();
1603            assert_eq!(t, "cupcakes");
1604            Response::new("myrole".to_string())
1605        });
1606        server.push_fn(|req| {
1607            assert_eq!(req.uri().path(), "/latest/meta-data/iam/security-credentials/myrole");
1608            assert_eq!(req.method(), &Method::GET);
1609            let t = req.headers().get(IMDSV2_HEADER).unwrap().to_str().unwrap();
1610            assert_eq!(t, "cupcakes");
1611            Response::new(r#"{"AccessKeyId":"KEYID","Code":"Success","Expiration":"2022-08-30T10:51:04Z","LastUpdated":"2022-08-30T10:21:04Z","SecretAccessKey":"SECRET","Token":"TOKEN","Type":"AWS-HMAC"}"#.to_string())
1612        });
1613
1614        let creds = instance_creds(&client, &retry_config, endpoint, true)
1615            .await
1616            .unwrap();
1617
1618        assert_eq!(creds.token.token.as_deref().unwrap(), token);
1619        assert_eq!(&creds.token.key_id, access_key_id);
1620        assert_eq!(&creds.token.secret_key, secret_access_key);
1621
1622        // Test IMDSv1 fallback
1623        server.push_fn(|req| {
1624            assert_eq!(req.uri().path(), "/latest/api/token");
1625            assert_eq!(req.method(), &Method::PUT);
1626            Response::builder()
1627                .status(StatusCode::FORBIDDEN)
1628                .body(String::new())
1629                .unwrap()
1630        });
1631        server.push_fn(|req| {
1632            assert_eq!(
1633                req.uri().path(),
1634                "/latest/meta-data/iam/security-credentials/"
1635            );
1636            assert_eq!(req.method(), &Method::GET);
1637            assert!(req.headers().get(IMDSV2_HEADER).is_none());
1638            Response::new("myrole".to_string())
1639        });
1640        server.push_fn(|req| {
1641            assert_eq!(req.uri().path(), "/latest/meta-data/iam/security-credentials/myrole");
1642            assert_eq!(req.method(), &Method::GET);
1643            assert!(req.headers().get(IMDSV2_HEADER).is_none());
1644            Response::new(r#"{"AccessKeyId":"KEYID","Code":"Success","Expiration":"2022-08-30T10:51:04Z","LastUpdated":"2022-08-30T10:21:04Z","SecretAccessKey":"SECRET","Token":"TOKEN","Type":"AWS-HMAC"}"#.to_string())
1645        });
1646
1647        let creds = instance_creds(&client, &retry_config, endpoint, true)
1648            .await
1649            .unwrap();
1650
1651        assert_eq!(creds.token.token.as_deref().unwrap(), token);
1652        assert_eq!(&creds.token.key_id, access_key_id);
1653        assert_eq!(&creds.token.secret_key, secret_access_key);
1654
1655        // Test IMDSv1 fallback disabled
1656        server.push(
1657            Response::builder()
1658                .status(StatusCode::FORBIDDEN)
1659                .body(String::new())
1660                .unwrap(),
1661        );
1662
1663        // Should fail
1664        instance_creds(&client, &retry_config, endpoint, false)
1665            .await
1666            .unwrap_err();
1667    }
1668
1669    #[cfg(feature = "reqwest")]
1670    #[tokio::test]
1671    async fn test_eks_pod_credential_provider() {
1672        use crate::client::mock_server::MockServer;
1673        use http::Response;
1674        use std::fs::File;
1675        use std::io::Write;
1676
1677        let mock_server = MockServer::new().await;
1678
1679        mock_server.push(Response::new(
1680            r#"{
1681            "AccessKeyId": "TEST_KEY",
1682            "SecretAccessKey": "TEST_SECRET",
1683            "Token": "TEST_SESSION_TOKEN",
1684            "Expiration": "2100-01-01T00:00:00Z"
1685        }"#
1686            .to_string(),
1687        ));
1688
1689        let token_file = tempfile::NamedTempFile::new().expect("cannot create temp file");
1690        let path = token_file.path().to_string_lossy().into_owned();
1691        let mut f = File::create(token_file.path()).unwrap();
1692        write!(f, "TEST_BEARER_TOKEN").unwrap();
1693
1694        let builder = AmazonS3Builder::new()
1695            .with_bucket_name("some-bucket")
1696            .with_config(
1697                AmazonS3ConfigKey::ContainerCredentialsFullUri,
1698                mock_server.url(),
1699            )
1700            .with_config(AmazonS3ConfigKey::ContainerAuthorizationTokenFile, &path);
1701
1702        let s3 = builder.build().unwrap();
1703
1704        let cred = s3.client.config.credentials.get_credential().await.unwrap();
1705
1706        assert_eq!(cred.key_id, "TEST_KEY");
1707        assert_eq!(cred.secret_key, "TEST_SECRET");
1708        assert_eq!(cred.token.as_deref(), Some("TEST_SESSION_TOKEN"));
1709    }
1710
1711    #[test]
1712    fn test_output_masks_all_fields() {
1713        let cred = AwsCredential {
1714            key_id: "AKIAXXX".to_string(),
1715            secret_key: "super_secret".to_string(),
1716            token: Some("temp_token".to_string()),
1717        };
1718
1719        let debug_output = format!("{cred:?}");
1720
1721        assert!(debug_output.contains("key_id: \"AKIAXXX\""));
1722        assert!(debug_output.contains("secret_key: \"******\""));
1723        assert!(debug_output.contains("token: Some(\"******\")"));
1724
1725        assert!(!debug_output.contains("super_secret"));
1726        assert!(!debug_output.contains("temp_token"));
1727    }
1728
1729    #[test]
1730    fn test_normalize_whitespace() {
1731        // test cases from minio: https://github.com/minio/minio/blob/05e569960ac584c8927a9af76755708d20f16129/cmd/signature-v4-utils_test.go#L307-L324
1732        let test_cases = vec![
1733            // input, expected
1734            ("本語", "本語"),
1735            (" abc ", "abc"),
1736            (" a b ", "a b"),
1737            ("a b ", "a b"),
1738            ("a  b", "a b"),
1739            ("a   b", "a b"),
1740            ("   a   b  c   ", "a b c"),
1741            ("a \t b  c   ", "a b c"),
1742            ("\"a \t b  c   ", "\"a b c"),
1743            (
1744                " \t\n\u{000b}\r\u{000c}a \t\n\u{000b}\r\u{000c} b \t\n\u{000b}\r\u{000c} c \t\n\u{000b}\r\u{000c}",
1745                "a b c",
1746            ),
1747        ];
1748
1749        for (input, expected) in test_cases {
1750            let mut headers = String::new();
1751
1752            append_normalized_whitespace_value(&mut headers, input);
1753            assert_eq!(headers, expected);
1754        }
1755    }
1756
1757    #[test]
1758    fn test_canonicalize_headers_whitespace_normalization() {
1759        use http::header::HeaderMap;
1760
1761        let mut headers = HeaderMap::new();
1762        headers.insert("x-amz-meta-example", "  foo   bar  ".parse().unwrap());
1763        headers.insert(
1764            "x-amz-meta-another",
1765            "  multiple                  spaces here  ".parse().unwrap(),
1766        );
1767        headers.insert(
1768            "x-amz-meta-and-another-one",
1769            "foo\t\t\t bar".parse().unwrap(),
1770        );
1771        // ignored headers
1772        headers.insert("authorization", "SHOULD_BE_IGNORED".parse().unwrap());
1773        headers.insert("content-length", "1337".parse().unwrap());
1774
1775        let (signed_headers, canonical_headers) = super::canonicalize_headers(&headers);
1776
1777        assert_eq!(
1778            signed_headers,
1779            "x-amz-meta-and-another-one;x-amz-meta-another;x-amz-meta-example"
1780        );
1781
1782        let expected_canonical_headers = "x-amz-meta-and-another-one:foo bar\n\
1783            x-amz-meta-another:multiple spaces here\n\
1784            x-amz-meta-example:foo bar\n";
1785        assert_eq!(canonical_headers, expected_canonical_headers);
1786    }
1787}
object_store/aws/credential.rs

object_store/aws/
credential.rs
