aws_s3_transfer/config/loader.rs
1/*
2 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3 * SPDX-License-Identifier: Apache-2.0
4 */
5
6use aws_config::BehaviorVersion;
7
8use crate::config::RuntimeConfig;
9use crate::config::{Builder, Config, S3ClientConfig};
10use crate::config::{ConcurrencyMode, MemoryConfig, PartSize, ReadAhead};
11
12/// Loads transfer manager [`Config`] from the environment.
13///
14/// Created by [`from_env`](crate::from_env). [`load`](Self::load) resolves the S3
15/// client configuration from the shared AWS configuration sources (environment
16/// variables, shared config and credentials files, and the default credential and
17/// region provider chains) unless [`s3_config`](Self::s3_config) supplies one. It
18/// also detects the host's EC2 instance type, vCPU count, and memory, which size
19/// the automatic defaults. Settings not configured on the loader keep their
20/// defaults.
21///
22/// Use [`Config::builder`] instead to construct a configuration from explicit
23/// values without loading or instance-metadata detection.
24///
25/// ```no_run
26/// # async fn example() {
27/// use aws_s3_transfer::config::PartSize;
28/// use aws_s3_transfer::types::ByteSize;
29///
30/// let config = aws_s3_transfer::from_env()
31/// .part_size(PartSize::Target(ByteSize::mib(16)))
32/// .load()
33/// .await;
34/// let client = aws_s3_transfer::Client::new(config);
35/// # }
36/// ```
37#[derive(Default, Debug)]
38pub struct ConfigLoader {
39 builder: Builder,
40 s3_config: Option<S3ClientConfig>,
41}
42
43impl ConfigLoader {
44 /// Minimum object size that should trigger a multipart upload.
45 ///
46 /// The minimum part size is 5 MiB, any part size less than that will be rounded up.
47 /// Default is [PartSize::Auto]
48 pub fn multipart_threshold(mut self, threshold: PartSize) -> Self {
49 self.builder = self.builder.multipart_threshold(threshold);
50 self
51 }
52
53 /// The target size of each part when using a multipart upload to complete the request.
54 ///
55 /// When a request's content length is less than [`multipart_threshold`],
56 /// this setting is ignored and a single [`PutObject`] request will be made instead.
57 ///
58 /// NOTE: The actual part size used may be larger than the configured part size if
59 /// the current value would result in more than 10,000 parts for an upload request.
60 ///
61 /// Default is [PartSize::Auto]
62 ///
63 /// [`multipart_threshold`]: method@Self::multipart_threshold
64 /// [`PutObject`]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html
65 pub fn part_size(mut self, part_size: PartSize) -> Self {
66 self.builder = self.builder.part_size(part_size);
67 self
68 }
69
70 /// Set the concurrency mode this client should use.
71 ///
72 /// This sets the mode used for concurrent in-flight requests across _all_ operations.
73 /// Default is [ConcurrencyMode::Auto].
74 pub fn concurrency(mut self, mode: ConcurrencyMode) -> Self {
75 self.builder = self.builder.concurrency(mode);
76 self
77 }
78
79 /// Sets execution for transfer work.
80 ///
81 /// Defaults to owned managed execution.
82 pub fn runtime_config(mut self, config: RuntimeConfig) -> Self {
83 self.builder = self.builder.runtime_config(config);
84 self
85 }
86
87 /// Set how far downloads may prefetch ahead of the consumer.
88 ///
89 /// This is the client default for all downloads; a request may override it via
90 /// [`DownloadInput`](crate::operation::download::DownloadInput), or a running
91 /// transfer may adjust it via
92 /// [`DownloadIoCtl`](crate::operation::download::DownloadIoCtl).
93 /// Default is [ReadAhead::Auto].
94 pub fn read_ahead(mut self, mode: ReadAhead) -> Self {
95 self.builder = self.builder.read_ahead(mode);
96 self
97 }
98
99 /// Set the payload-memory configuration.
100 ///
101 /// Default is [`MemoryConfig::Auto`].
102 pub fn memory(mut self, memory: MemoryConfig) -> Self {
103 self.builder = self.builder.memory(memory);
104 self
105 }
106
107 /// Use this S3 client configuration instead of loading shared AWS
108 /// configuration.
109 ///
110 /// When set, [`load`](Self::load) reads no shared AWS configuration
111 /// sources; it still detects the machine profile.
112 /// [`Builder::s3_config`] describes how the transfer manager uses the
113 /// configuration.
114 ///
115 /// ```no_run
116 /// # async fn example(sdk_config: aws_config::SdkConfig) {
117 /// let config = aws_s3_transfer::from_env()
118 /// .s3_config(&sdk_config)
119 /// .load()
120 /// .await;
121 /// # }
122 /// ```
123 pub fn s3_config(mut self, config: impl Into<S3ClientConfig>) -> Self {
124 self.s3_config = Some(config.into());
125 self
126 }
127
128 /// Load the configuration.
129 ///
130 /// Without [`s3_config`](Self::s3_config), the S3 client configuration is
131 /// loaded from the shared AWS configuration sources with the latest
132 /// behavior version. Machine detection always runs. Where local DMI can't
133 /// identify the host, as on every non-Linux host and in containers without
134 /// DMI, `load` makes one EC2 instance metadata (IMDS) lookup: a single
135 /// attempt with 500 ms connect and read timeouts. Setting
136 /// `AWS_EC2_METADATA_DISABLED` to `true`, `1`, `yes` or `on`
137 /// (case-insensitive) suppresses it.
138 pub async fn load(self) -> Config {
139 let s3_config = match self.s3_config {
140 Some(config) => config,
141 None => {
142 S3ClientConfig::new(&aws_config::defaults(BehaviorVersion::latest()).load().await)
143 }
144 };
145
146 // Detect machine facts here, on the async load path, so client
147 // construction stays free of blocking DMI reads and network IMDS calls.
148 let profile = detect_machine_profile().await;
149
150 self.builder
151 .machine_profile(Some(profile))
152 .s3_config(s3_config)
153 .build()
154 .expect("the loader always sets s3_config, the only required field")
155 }
156}
157
158/// Detect the machine profile for auto-sizing: instance type via local DMI, then
159/// IMDS *only* when DMI is inconclusive, plus the usable vCPU count. Detection
160/// failure yields `None` for the instance type and consumers fall back to
161/// vCPU-scaled defaults.
162///
163/// The IMDS fallback is gated on DMI being inconclusive. On **Linux**, DMI
164/// answers "is this EC2" locally: a readable `NotEc2` reading skips IMDS (no
165/// network call on a Linux laptop, on-prem host, or another cloud); only an
166/// `Unknown` (a container without DMI) probes IMDS. On **non-Linux** there is no
167/// DMI path, so detection is always `Unknown` and IMDS is always attempted —
168/// this is deliberate (EC2 does run Windows/macOS, and IMDS is the only detection
169/// path there), the cost being that a non-EC2 non-Linux host makes one IMDS probe
170/// per client build. In all cases the probe honors `AWS_EC2_METADATA_DISABLED`
171/// and is bounded to a single short attempt (~500ms), so a miss is cheap.
172async fn detect_machine_profile() -> crate::runtime::platform::MachineProfile {
173 use crate::runtime::platform::{self, DmiDetection};
174 // `detect_instance_type_dmi` reads DMI attributes with synchronous `std::fs`,
175 // which runs inline on the async worker rather than via `spawn_blocking`. This
176 // is intentional: the reads target `/sys/.../dmi/id/*`, which are RAM-backed
177 // sysfs pseudo-files served by the kernel from the SMBIOS table parsed at boot
178 // (no disk I/O), and this runs once at client construction. The `spawn_blocking`
179 // hop would cost more than the reads it guards. (Ref: sysfs(5); the crate's
180 // hot-path fs is handled separately by the blocking-fs work.)
181 let instance_type = match platform::detect_instance_type_dmi() {
182 DmiDetection::Instance(ty) => Some(ty),
183 DmiDetection::NotEc2 => None,
184 DmiDetection::Unknown => imds_instance_type().await,
185 };
186 platform::MachineProfile {
187 instance_type,
188 vcpus: platform::local_vcpus(),
189 ram_bytes: platform::available_ram(),
190 }
191}
192
193/// Whether the user disabled EC2 IMDS via `AWS_EC2_METADATA_DISABLED=true`. The
194/// raw `imds::Client` does not honor this itself (only the region/credentials
195/// providers do), so we check it before constructing one.
196fn imds_disabled() -> bool {
197 imds_disabled_value(std::env::var("AWS_EC2_METADATA_DISABLED").ok().as_deref())
198}
199
200/// Pure predicate for the disable flag, split out so it is testable without
201/// mutating the process environment (which would race concurrent tests).
202///
203/// Lenient: accepts the common truthy spellings (`true`, `1`, `yes`, `on`,
204/// case-insensitive). Anything else — including unset, `false`, `0`, or an
205/// unrecognized value — leaves IMDS enabled.
206fn imds_disabled_value(raw: Option<&str>) -> bool {
207 matches!(
208 raw.map(|v| v.trim().to_ascii_lowercase()).as_deref(),
209 Some("true" | "1" | "yes" | "on")
210 )
211}
212
213/// Query the EC2 instance type from IMDS. `None` on any failure (IMDS disabled,
214/// not on EC2, hop-limited, timeout). IMDS returns only the instance-type string
215/// — no bandwidth data — so it feeds the family table like the DMI path.
216///
217/// Bounded to a single attempt with short connect/read timeouts: this only runs
218/// when DMI was inconclusive, and a non-EC2 host must not pay the library default
219/// (4 attempts, 1s connect each). On EC2 the endpoint is link-local and answers
220/// in microseconds, so one short attempt is ample.
221async fn imds_instance_type() -> Option<String> {
222 if imds_disabled() {
223 tracing::debug!(
224 target: crate::telemetry::TARGET_CONCURRENCY,
225 "IMDS disabled via AWS_EC2_METADATA_DISABLED; using vCPU-scaled fallback",
226 );
227 return None;
228 }
229 let client = aws_config::imds::Client::builder()
230 .max_attempts(1)
231 .connect_timeout(std::time::Duration::from_millis(500))
232 .read_timeout(std::time::Duration::from_millis(500))
233 .build();
234 match client.get("/latest/meta-data/instance-type").await {
235 Ok(ty) => {
236 let ty = ty.as_ref().trim().to_string();
237 (!ty.is_empty()).then_some(ty)
238 }
239 Err(err) => {
240 tracing::debug!(
241 target: crate::telemetry::TARGET_CONCURRENCY,
242 error = %err,
243 "IMDS instance-type lookup failed; using vCPU-scaled fallback",
244 );
245 None
246 }
247 }
248}
249
250#[cfg(test)]
251mod tests {
252 use crate::config::PartSize;
253 use crate::types::ByteSize;
254 use aws_config::{BehaviorVersion, Region};
255 use aws_sdk_s3::config::{Credentials, Intercept, SharedCredentialsProvider};
256 use aws_smithy_http_client::test_util::capture_request;
257
258 #[cfg_attr(miri, ignore)]
259 #[tokio::test]
260 async fn load_with_interceptor() {
261 let config = crate::from_env()
262 .part_size(PartSize::Target(ByteSize::new(8)))
263 .load()
264 .await;
265 let tm_client = crate::Client::new(config);
266 let sdk_s3_config = tm_client.handle.s3_client.config();
267 let tm_interceptor_exists = sdk_s3_config
268 .interceptors()
269 .any(|item| item.name() == "S3TransferManager");
270 assert!(tm_interceptor_exists);
271 }
272
273 // `load()` always populates a MachineProfile with a real vCPU count. Instance
274 // type is environment-dependent (Some on EC2 with DMI, None off-EC2); we only
275 // assert the profile exists and vCPU is sane, so the test is host-independent.
276 //
277 // Off-EC2 this exercises the bounded IMDS fallback (1 attempt, 500ms), which
278 // fast-fails; that path is intentionally short so this stays quick in CI.
279 #[cfg_attr(miri, ignore)]
280 #[tokio::test]
281 async fn load_populates_machine_profile() {
282 let config = crate::from_env().load().await;
283 let profile = config
284 .machine_profile()
285 .expect("loader populates a machine profile");
286 assert!(profile.vcpus >= 1);
287 }
288
289 /// A supplied configuration replaces the shared-configuration load, so the request is
290 /// sent through its HTTP client to its region. Loaded defaults would carry neither: the
291 /// capture client exists only in the supplied configuration, and the test environment
292 /// does not set this region.
293 #[cfg_attr(miri, ignore)]
294 #[tokio::test]
295 async fn load_uses_supplied_s3_config() {
296 let (http_client, captured_request) = capture_request(None);
297 let sdk_config = aws_types::SdkConfig::builder()
298 .http_client(http_client)
299 .region(Region::from_static("ap-southeast-4"))
300 .credentials_provider(SharedCredentialsProvider::new(Credentials::for_tests()))
301 .behavior_version(BehaviorVersion::latest())
302 .build();
303 let config = crate::from_env()
304 .s3_config(&sdk_config)
305 .runtime_config(crate::config::RuntimeConfig::multi_thread_tokio())
306 .load()
307 .await;
308 assert!(
309 config.machine_profile().is_some(),
310 "machine detection runs with a supplied configuration"
311 );
312
313 let tm = crate::Client::new(config);
314 let mut handle = tm.download().bucket("foo").key("bar").initiate().unwrap();
315 // The capture client returns a canned response, so the transfer is expected to fail.
316 let _ = handle.body_mut().next().await;
317 let uri = captured_request.expect_request().uri().to_string();
318 assert!(
319 uri.contains("ap-southeast-4"),
320 "request did not use the supplied region: {uri}"
321 );
322 }
323
324 /// A supplied configuration without an HTTP client keeps the managed transport under
325 /// managed execution, as loaded defaults do; one with an HTTP client uses it instead.
326 #[cfg_attr(miri, ignore)]
327 #[tokio::test]
328 async fn supplied_s3_config_selects_managed_transport_unless_it_has_http() {
329 use aws_sdk_s3::config::IntoShared;
330
331 let sdk_config = |http_client: Option<_>| {
332 let mut builder = aws_types::SdkConfig::builder()
333 .region(Region::from_static("us-west-2"))
334 .credentials_provider(SharedCredentialsProvider::new(Credentials::for_tests()))
335 .behavior_version(BehaviorVersion::latest());
336 builder.set_http_client(http_client);
337 builder.build()
338 };
339
340 let config = crate::from_env().s3_config(&sdk_config(None)).load().await;
341 assert!(
342 config.runtime_http(64).is_some(),
343 "no HTTP client supplied: the managed transport is installed"
344 );
345
346 let (http_client, _captured) = capture_request(None);
347 let config = crate::from_env()
348 .s3_config(&sdk_config(Some(http_client.into_shared())))
349 .load()
350 .await;
351 assert!(
352 config.runtime_http(64).is_none(),
353 "an HTTP client is supplied: the managed transport is not installed"
354 );
355 }
356
357 // Tests the pure parser, so it never touches the process environment and
358 // cannot race the concurrent `load()`-based tests that read the same var.
359 #[test]
360 fn imds_disabled_parses_flag() {
361 use super::imds_disabled_value;
362 // Truthy spellings, case- and whitespace-insensitive.
363 for truthy in ["true", "TRUE", " true ", "1", "yes", "YES", "on"] {
364 assert!(imds_disabled_value(Some(truthy)), "{truthy:?} disables");
365 }
366 // Everything else leaves IMDS enabled.
367 for falsy in ["false", "0", "no", "off", "", " ", "enabled", "2"] {
368 assert!(!imds_disabled_value(Some(falsy)), "{falsy:?} stays enabled");
369 }
370 assert!(!imds_disabled_value(None), "unset defaults to enabled");
371 }
372
373 /// The loader builds its own S3 configuration, so a captured request can only be had by
374 /// extracting that configuration and rebuilding a client from it. What this covers is the
375 /// loader path reaching the wire with our attribution; the sections themselves are owned by
376 /// the tests in [`crate::config::user_agent`].
377 #[cfg_attr(miri, ignore)]
378 #[tokio::test]
379 async fn load_path_emits_attribution() {
380 let (http_client, captured_request) = capture_request(None);
381 let config = crate::from_env()
382 .part_size(PartSize::Target(ByteSize::new(8)))
383 .load()
384 .await;
385 // Build the TM client so we can extract the S3 client's config with interceptors.
386 let tm_client = crate::Client::new(config);
387 let sdk_s3_config = tm_client
388 .handle
389 .s3_client
390 .config()
391 .to_builder()
392 .http_client(http_client)
393 .region(Region::from_static("us-west-2"))
394 .with_test_defaults()
395 .build();
396
397 let capture_request_config = crate::Config::builder()
398 .sdk_client(aws_sdk_s3::Client::from_conf(sdk_s3_config))
399 .part_size(PartSize::Target(ByteSize::new(8)))
400 .build()
401 .expect("valid config");
402
403 let transfer_manager = crate::Client::new(capture_request_config);
404
405 let mut handle = transfer_manager
406 .download()
407 .bucket("foo")
408 .key("bar")
409 .initiate()
410 .unwrap();
411 // Expect to fail
412 let _ = handle.body_mut().next().await;
413 let expected_req = captured_request.expect_request();
414 let user_agent = expected_req.headers().get("x-amz-user-agent").unwrap();
415 let ours = concat!("md/rust-tm#", env!("CARGO_PKG_VERSION"));
416 assert!(
417 user_agent.contains(ours),
418 "loader path carries no attribution: {user_agent:?}"
419 );
420 }
421}