Skip to main content

aws_s3_transfer/config/
loader.rs

1/*
2 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3 * SPDX-License-Identifier: Apache-2.0
4 */
5
6use aws_config::BehaviorVersion;
7
8use crate::config::RuntimeConfig;
9use crate::config::{Builder, Config, S3ClientConfig};
10use crate::config::{ConcurrencyMode, MemoryConfig, PartSize, ReadAhead};
11
12/// Loads transfer manager [`Config`] from the environment.
13///
14/// Created by [`from_env`](crate::from_env). [`load`](Self::load) resolves the S3
15/// client configuration from the shared AWS configuration sources (environment
16/// variables, shared config and credentials files, and the default credential and
17/// region provider chains) unless [`s3_config`](Self::s3_config) supplies one. It
18/// also detects the host's EC2 instance type, vCPU count, and memory, which size
19/// the automatic defaults. Settings not configured on the loader keep their
20/// defaults.
21///
22/// Use [`Config::builder`] instead to construct a configuration from explicit
23/// values without loading or instance-metadata detection.
24///
25/// ```no_run
26/// # async fn example() {
27/// use aws_s3_transfer::config::PartSize;
28/// use aws_s3_transfer::types::ByteSize;
29///
30/// let config = aws_s3_transfer::from_env()
31///     .part_size(PartSize::Target(ByteSize::mib(16)))
32///     .load()
33///     .await;
34/// let client = aws_s3_transfer::Client::new(config);
35/// # }
36/// ```
37#[derive(Default, Debug)]
38pub struct ConfigLoader {
39    builder: Builder,
40    s3_config: Option<S3ClientConfig>,
41}
42
43impl ConfigLoader {
44    /// Minimum object size that should trigger a multipart upload.
45    ///
46    /// The minimum part size is 5 MiB, any part size less than that will be rounded up.
47    /// Default is [PartSize::Auto]
48    pub fn multipart_threshold(mut self, threshold: PartSize) -> Self {
49        self.builder = self.builder.multipart_threshold(threshold);
50        self
51    }
52
53    /// The target size of each part when using a multipart upload to complete the request.
54    ///
55    /// When a request's content length is less than [`multipart_threshold`],
56    /// this setting is ignored and a single [`PutObject`] request will be made instead.
57    ///
58    /// NOTE: The actual part size used may be larger than the configured part size if
59    /// the current value would result in more than 10,000 parts for an upload request.
60    ///
61    /// Default is [PartSize::Auto]
62    ///
63    /// [`multipart_threshold`]: method@Self::multipart_threshold
64    /// [`PutObject`]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html
65    pub fn part_size(mut self, part_size: PartSize) -> Self {
66        self.builder = self.builder.part_size(part_size);
67        self
68    }
69
70    /// Set the concurrency mode this client should use.
71    ///
72    /// This sets the mode used for concurrent in-flight requests across _all_ operations.
73    /// Default is [ConcurrencyMode::Auto].
74    pub fn concurrency(mut self, mode: ConcurrencyMode) -> Self {
75        self.builder = self.builder.concurrency(mode);
76        self
77    }
78
79    /// Sets execution for transfer work.
80    ///
81    /// Defaults to owned managed execution.
82    pub fn runtime_config(mut self, config: RuntimeConfig) -> Self {
83        self.builder = self.builder.runtime_config(config);
84        self
85    }
86
87    /// Set how far downloads may prefetch ahead of the consumer.
88    ///
89    /// This is the client default for all downloads; a request may override it via
90    /// [`DownloadInput`](crate::operation::download::DownloadInput), or a running
91    /// transfer may adjust it via
92    /// [`DownloadIoCtl`](crate::operation::download::DownloadIoCtl).
93    /// Default is [ReadAhead::Auto].
94    pub fn read_ahead(mut self, mode: ReadAhead) -> Self {
95        self.builder = self.builder.read_ahead(mode);
96        self
97    }
98
99    /// Set the payload-memory configuration.
100    ///
101    /// Default is [`MemoryConfig::Auto`].
102    pub fn memory(mut self, memory: MemoryConfig) -> Self {
103        self.builder = self.builder.memory(memory);
104        self
105    }
106
107    /// Use this S3 client configuration instead of loading shared AWS
108    /// configuration.
109    ///
110    /// When set, [`load`](Self::load) reads no shared AWS configuration
111    /// sources; it still detects the machine profile.
112    /// [`Builder::s3_config`] describes how the transfer manager uses the
113    /// configuration.
114    ///
115    /// ```no_run
116    /// # async fn example(sdk_config: aws_config::SdkConfig) {
117    /// let config = aws_s3_transfer::from_env()
118    ///     .s3_config(&sdk_config)
119    ///     .load()
120    ///     .await;
121    /// # }
122    /// ```
123    pub fn s3_config(mut self, config: impl Into<S3ClientConfig>) -> Self {
124        self.s3_config = Some(config.into());
125        self
126    }
127
128    /// Load the configuration.
129    ///
130    /// Without [`s3_config`](Self::s3_config), the S3 client configuration is
131    /// loaded from the shared AWS configuration sources with the latest
132    /// behavior version. Machine detection always runs. Where local DMI can't
133    /// identify the host, as on every non-Linux host and in containers without
134    /// DMI, `load` makes one EC2 instance metadata (IMDS) lookup: a single
135    /// attempt with 500 ms connect and read timeouts. Setting
136    /// `AWS_EC2_METADATA_DISABLED` to `true`, `1`, `yes` or `on`
137    /// (case-insensitive) suppresses it.
138    pub async fn load(self) -> Config {
139        let s3_config = match self.s3_config {
140            Some(config) => config,
141            None => {
142                S3ClientConfig::new(&aws_config::defaults(BehaviorVersion::latest()).load().await)
143            }
144        };
145
146        // Detect machine facts here, on the async load path, so client
147        // construction stays free of blocking DMI reads and network IMDS calls.
148        let profile = detect_machine_profile().await;
149
150        self.builder
151            .machine_profile(Some(profile))
152            .s3_config(s3_config)
153            .build()
154            .expect("the loader always sets s3_config, the only required field")
155    }
156}
157
158/// Detect the machine profile for auto-sizing: instance type via local DMI, then
159/// IMDS *only* when DMI is inconclusive, plus the usable vCPU count. Detection
160/// failure yields `None` for the instance type and consumers fall back to
161/// vCPU-scaled defaults.
162///
163/// The IMDS fallback is gated on DMI being inconclusive. On **Linux**, DMI
164/// answers "is this EC2" locally: a readable `NotEc2` reading skips IMDS (no
165/// network call on a Linux laptop, on-prem host, or another cloud); only an
166/// `Unknown` (a container without DMI) probes IMDS. On **non-Linux** there is no
167/// DMI path, so detection is always `Unknown` and IMDS is always attempted —
168/// this is deliberate (EC2 does run Windows/macOS, and IMDS is the only detection
169/// path there), the cost being that a non-EC2 non-Linux host makes one IMDS probe
170/// per client build. In all cases the probe honors `AWS_EC2_METADATA_DISABLED`
171/// and is bounded to a single short attempt (~500ms), so a miss is cheap.
172async fn detect_machine_profile() -> crate::runtime::platform::MachineProfile {
173    use crate::runtime::platform::{self, DmiDetection};
174    // `detect_instance_type_dmi` reads DMI attributes with synchronous `std::fs`,
175    // which runs inline on the async worker rather than via `spawn_blocking`. This
176    // is intentional: the reads target `/sys/.../dmi/id/*`, which are RAM-backed
177    // sysfs pseudo-files served by the kernel from the SMBIOS table parsed at boot
178    // (no disk I/O), and this runs once at client construction. The `spawn_blocking`
179    // hop would cost more than the reads it guards. (Ref: sysfs(5); the crate's
180    // hot-path fs is handled separately by the blocking-fs work.)
181    let instance_type = match platform::detect_instance_type_dmi() {
182        DmiDetection::Instance(ty) => Some(ty),
183        DmiDetection::NotEc2 => None,
184        DmiDetection::Unknown => imds_instance_type().await,
185    };
186    platform::MachineProfile {
187        instance_type,
188        vcpus: platform::local_vcpus(),
189        ram_bytes: platform::available_ram(),
190    }
191}
192
193/// Whether the user disabled EC2 IMDS via `AWS_EC2_METADATA_DISABLED=true`. The
194/// raw `imds::Client` does not honor this itself (only the region/credentials
195/// providers do), so we check it before constructing one.
196fn imds_disabled() -> bool {
197    imds_disabled_value(std::env::var("AWS_EC2_METADATA_DISABLED").ok().as_deref())
198}
199
200/// Pure predicate for the disable flag, split out so it is testable without
201/// mutating the process environment (which would race concurrent tests).
202///
203/// Lenient: accepts the common truthy spellings (`true`, `1`, `yes`, `on`,
204/// case-insensitive). Anything else — including unset, `false`, `0`, or an
205/// unrecognized value — leaves IMDS enabled.
206fn imds_disabled_value(raw: Option<&str>) -> bool {
207    matches!(
208        raw.map(|v| v.trim().to_ascii_lowercase()).as_deref(),
209        Some("true" | "1" | "yes" | "on")
210    )
211}
212
213/// Query the EC2 instance type from IMDS. `None` on any failure (IMDS disabled,
214/// not on EC2, hop-limited, timeout). IMDS returns only the instance-type string
215/// — no bandwidth data — so it feeds the family table like the DMI path.
216///
217/// Bounded to a single attempt with short connect/read timeouts: this only runs
218/// when DMI was inconclusive, and a non-EC2 host must not pay the library default
219/// (4 attempts, 1s connect each). On EC2 the endpoint is link-local and answers
220/// in microseconds, so one short attempt is ample.
221async fn imds_instance_type() -> Option<String> {
222    if imds_disabled() {
223        tracing::debug!(
224            target: crate::telemetry::TARGET_CONCURRENCY,
225            "IMDS disabled via AWS_EC2_METADATA_DISABLED; using vCPU-scaled fallback",
226        );
227        return None;
228    }
229    let client = aws_config::imds::Client::builder()
230        .max_attempts(1)
231        .connect_timeout(std::time::Duration::from_millis(500))
232        .read_timeout(std::time::Duration::from_millis(500))
233        .build();
234    match client.get("/latest/meta-data/instance-type").await {
235        Ok(ty) => {
236            let ty = ty.as_ref().trim().to_string();
237            (!ty.is_empty()).then_some(ty)
238        }
239        Err(err) => {
240            tracing::debug!(
241                target: crate::telemetry::TARGET_CONCURRENCY,
242                error = %err,
243                "IMDS instance-type lookup failed; using vCPU-scaled fallback",
244            );
245            None
246        }
247    }
248}
249
250#[cfg(test)]
251mod tests {
252    use crate::config::PartSize;
253    use crate::types::ByteSize;
254    use aws_config::{BehaviorVersion, Region};
255    use aws_sdk_s3::config::{Credentials, Intercept, SharedCredentialsProvider};
256    use aws_smithy_http_client::test_util::capture_request;
257
258    #[cfg_attr(miri, ignore)]
259    #[tokio::test]
260    async fn load_with_interceptor() {
261        let config = crate::from_env()
262            .part_size(PartSize::Target(ByteSize::new(8)))
263            .load()
264            .await;
265        let tm_client = crate::Client::new(config);
266        let sdk_s3_config = tm_client.handle.s3_client.config();
267        let tm_interceptor_exists = sdk_s3_config
268            .interceptors()
269            .any(|item| item.name() == "S3TransferManager");
270        assert!(tm_interceptor_exists);
271    }
272
273    // `load()` always populates a MachineProfile with a real vCPU count. Instance
274    // type is environment-dependent (Some on EC2 with DMI, None off-EC2); we only
275    // assert the profile exists and vCPU is sane, so the test is host-independent.
276    //
277    // Off-EC2 this exercises the bounded IMDS fallback (1 attempt, 500ms), which
278    // fast-fails; that path is intentionally short so this stays quick in CI.
279    #[cfg_attr(miri, ignore)]
280    #[tokio::test]
281    async fn load_populates_machine_profile() {
282        let config = crate::from_env().load().await;
283        let profile = config
284            .machine_profile()
285            .expect("loader populates a machine profile");
286        assert!(profile.vcpus >= 1);
287    }
288
289    /// A supplied configuration replaces the shared-configuration load, so the request is
290    /// sent through its HTTP client to its region. Loaded defaults would carry neither: the
291    /// capture client exists only in the supplied configuration, and the test environment
292    /// does not set this region.
293    #[cfg_attr(miri, ignore)]
294    #[tokio::test]
295    async fn load_uses_supplied_s3_config() {
296        let (http_client, captured_request) = capture_request(None);
297        let sdk_config = aws_types::SdkConfig::builder()
298            .http_client(http_client)
299            .region(Region::from_static("ap-southeast-4"))
300            .credentials_provider(SharedCredentialsProvider::new(Credentials::for_tests()))
301            .behavior_version(BehaviorVersion::latest())
302            .build();
303        let config = crate::from_env()
304            .s3_config(&sdk_config)
305            .runtime_config(crate::config::RuntimeConfig::multi_thread_tokio())
306            .load()
307            .await;
308        assert!(
309            config.machine_profile().is_some(),
310            "machine detection runs with a supplied configuration"
311        );
312
313        let tm = crate::Client::new(config);
314        let mut handle = tm.download().bucket("foo").key("bar").initiate().unwrap();
315        // The capture client returns a canned response, so the transfer is expected to fail.
316        let _ = handle.body_mut().next().await;
317        let uri = captured_request.expect_request().uri().to_string();
318        assert!(
319            uri.contains("ap-southeast-4"),
320            "request did not use the supplied region: {uri}"
321        );
322    }
323
324    /// A supplied configuration without an HTTP client keeps the managed transport under
325    /// managed execution, as loaded defaults do; one with an HTTP client uses it instead.
326    #[cfg_attr(miri, ignore)]
327    #[tokio::test]
328    async fn supplied_s3_config_selects_managed_transport_unless_it_has_http() {
329        use aws_sdk_s3::config::IntoShared;
330
331        let sdk_config = |http_client: Option<_>| {
332            let mut builder = aws_types::SdkConfig::builder()
333                .region(Region::from_static("us-west-2"))
334                .credentials_provider(SharedCredentialsProvider::new(Credentials::for_tests()))
335                .behavior_version(BehaviorVersion::latest());
336            builder.set_http_client(http_client);
337            builder.build()
338        };
339
340        let config = crate::from_env().s3_config(&sdk_config(None)).load().await;
341        assert!(
342            config.runtime_http(64).is_some(),
343            "no HTTP client supplied: the managed transport is installed"
344        );
345
346        let (http_client, _captured) = capture_request(None);
347        let config = crate::from_env()
348            .s3_config(&sdk_config(Some(http_client.into_shared())))
349            .load()
350            .await;
351        assert!(
352            config.runtime_http(64).is_none(),
353            "an HTTP client is supplied: the managed transport is not installed"
354        );
355    }
356
357    // Tests the pure parser, so it never touches the process environment and
358    // cannot race the concurrent `load()`-based tests that read the same var.
359    #[test]
360    fn imds_disabled_parses_flag() {
361        use super::imds_disabled_value;
362        // Truthy spellings, case- and whitespace-insensitive.
363        for truthy in ["true", "TRUE", "  true  ", "1", "yes", "YES", "on"] {
364            assert!(imds_disabled_value(Some(truthy)), "{truthy:?} disables");
365        }
366        // Everything else leaves IMDS enabled.
367        for falsy in ["false", "0", "no", "off", "", "  ", "enabled", "2"] {
368            assert!(!imds_disabled_value(Some(falsy)), "{falsy:?} stays enabled");
369        }
370        assert!(!imds_disabled_value(None), "unset defaults to enabled");
371    }
372
373    /// The loader builds its own S3 configuration, so a captured request can only be had by
374    /// extracting that configuration and rebuilding a client from it. What this covers is the
375    /// loader path reaching the wire with our attribution; the sections themselves are owned by
376    /// the tests in [`crate::config::user_agent`].
377    #[cfg_attr(miri, ignore)]
378    #[tokio::test]
379    async fn load_path_emits_attribution() {
380        let (http_client, captured_request) = capture_request(None);
381        let config = crate::from_env()
382            .part_size(PartSize::Target(ByteSize::new(8)))
383            .load()
384            .await;
385        // Build the TM client so we can extract the S3 client's config with interceptors.
386        let tm_client = crate::Client::new(config);
387        let sdk_s3_config = tm_client
388            .handle
389            .s3_client
390            .config()
391            .to_builder()
392            .http_client(http_client)
393            .region(Region::from_static("us-west-2"))
394            .with_test_defaults()
395            .build();
396
397        let capture_request_config = crate::Config::builder()
398            .sdk_client(aws_sdk_s3::Client::from_conf(sdk_s3_config))
399            .part_size(PartSize::Target(ByteSize::new(8)))
400            .build()
401            .expect("valid config");
402
403        let transfer_manager = crate::Client::new(capture_request_config);
404
405        let mut handle = transfer_manager
406            .download()
407            .bucket("foo")
408            .key("bar")
409            .initiate()
410            .unwrap();
411        // Expect to fail
412        let _ = handle.body_mut().next().await;
413        let expected_req = captured_request.expect_request();
414        let user_agent = expected_req.headers().get("x-amz-user-agent").unwrap();
415        let ours = concat!("md/rust-tm#", env!("CARGO_PKG_VERSION"));
416        assert!(
417            user_agent.contains(ours),
418            "loader path carries no attribution: {user_agent:?}"
419        );
420    }
421}