aws-s3-transfer 1.0.0

Transfer Manager for Amazon S3 - a high-performance data plane client for uploading and downloading objects and directories.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
/*
 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 * SPDX-License-Identifier: Apache-2.0
 */

use aws_config::BehaviorVersion;

use crate::config::RuntimeConfig;
use crate::config::{Builder, Config, S3ClientConfig};
use crate::config::{ConcurrencyMode, MemoryConfig, PartSize, ReadAhead};

/// Loads transfer manager [`Config`] from the environment.
///
/// Created by [`from_env`](crate::from_env). [`load`](Self::load) resolves the S3
/// client configuration from the shared AWS configuration sources (environment
/// variables, shared config and credentials files, and the default credential and
/// region provider chains) unless [`s3_config`](Self::s3_config) supplies one. It
/// also detects the host's EC2 instance type, vCPU count, and memory, which size
/// the automatic defaults. Settings not configured on the loader keep their
/// defaults.
///
/// Use [`Config::builder`] instead to construct a configuration from explicit
/// values without loading or instance-metadata detection.
///
/// ```no_run
/// # async fn example() {
/// use aws_s3_transfer::config::PartSize;
/// use aws_s3_transfer::types::ByteSize;
///
/// let config = aws_s3_transfer::from_env()
///     .part_size(PartSize::Target(ByteSize::mib(16)))
///     .load()
///     .await;
/// let client = aws_s3_transfer::Client::new(config);
/// # }
/// ```
#[derive(Default, Debug)]
pub struct ConfigLoader {
    builder: Builder,
    s3_config: Option<S3ClientConfig>,
}

impl ConfigLoader {
    /// Minimum object size that should trigger a multipart upload.
    ///
    /// The minimum part size is 5 MiB, any part size less than that will be rounded up.
    /// Default is [PartSize::Auto]
    pub fn multipart_threshold(mut self, threshold: PartSize) -> Self {
        self.builder = self.builder.multipart_threshold(threshold);
        self
    }

    /// The target size of each part when using a multipart upload to complete the request.
    ///
    /// When a request's content length is less than [`multipart_threshold`],
    /// this setting is ignored and a single [`PutObject`] request will be made instead.
    ///
    /// NOTE: The actual part size used may be larger than the configured part size if
    /// the current value would result in more than 10,000 parts for an upload request.
    ///
    /// Default is [PartSize::Auto]
    ///
    /// [`multipart_threshold`]: method@Self::multipart_threshold
    /// [`PutObject`]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html
    pub fn part_size(mut self, part_size: PartSize) -> Self {
        self.builder = self.builder.part_size(part_size);
        self
    }

    /// Set the concurrency mode this client should use.
    ///
    /// This sets the mode used for concurrent in-flight requests across _all_ operations.
    /// Default is [ConcurrencyMode::Auto].
    pub fn concurrency(mut self, mode: ConcurrencyMode) -> Self {
        self.builder = self.builder.concurrency(mode);
        self
    }

    /// Sets execution for transfer work.
    ///
    /// Defaults to owned managed execution.
    pub fn runtime_config(mut self, config: RuntimeConfig) -> Self {
        self.builder = self.builder.runtime_config(config);
        self
    }

    /// Set how far downloads may prefetch ahead of the consumer.
    ///
    /// This is the client default for all downloads; a request may override it via
    /// [`DownloadInput`](crate::operation::download::DownloadInput), or a running
    /// transfer may adjust it via
    /// [`DownloadIoCtl`](crate::operation::download::DownloadIoCtl).
    /// Default is [ReadAhead::Auto].
    pub fn read_ahead(mut self, mode: ReadAhead) -> Self {
        self.builder = self.builder.read_ahead(mode);
        self
    }

    /// Set the payload-memory configuration.
    ///
    /// Default is [`MemoryConfig::Auto`].
    pub fn memory(mut self, memory: MemoryConfig) -> Self {
        self.builder = self.builder.memory(memory);
        self
    }

    /// Use this S3 client configuration instead of loading shared AWS
    /// configuration.
    ///
    /// When set, [`load`](Self::load) reads no shared AWS configuration
    /// sources; it still detects the machine profile.
    /// [`Builder::s3_config`] describes how the transfer manager uses the
    /// configuration.
    ///
    /// ```no_run
    /// # async fn example(sdk_config: aws_config::SdkConfig) {
    /// let config = aws_s3_transfer::from_env()
    ///     .s3_config(&sdk_config)
    ///     .load()
    ///     .await;
    /// # }
    /// ```
    pub fn s3_config(mut self, config: impl Into<S3ClientConfig>) -> Self {
        self.s3_config = Some(config.into());
        self
    }

    /// Load the configuration.
    ///
    /// Without [`s3_config`](Self::s3_config), the S3 client configuration is
    /// loaded from the shared AWS configuration sources with the latest
    /// behavior version. Machine detection always runs. Where local DMI can't
    /// identify the host, as on every non-Linux host and in containers without
    /// DMI, `load` makes one EC2 instance metadata (IMDS) lookup: a single
    /// attempt with 500 ms connect and read timeouts. Setting
    /// `AWS_EC2_METADATA_DISABLED` to `true`, `1`, `yes` or `on`
    /// (case-insensitive) suppresses it.
    pub async fn load(self) -> Config {
        let s3_config = match self.s3_config {
            Some(config) => config,
            None => {
                S3ClientConfig::new(&aws_config::defaults(BehaviorVersion::latest()).load().await)
            }
        };

        // Detect machine facts here, on the async load path, so client
        // construction stays free of blocking DMI reads and network IMDS calls.
        let profile = detect_machine_profile().await;

        self.builder
            .machine_profile(Some(profile))
            .s3_config(s3_config)
            .build()
            .expect("the loader always sets s3_config, the only required field")
    }
}

/// Detect the machine profile for auto-sizing: instance type via local DMI, then
/// IMDS *only* when DMI is inconclusive, plus the usable vCPU count. Detection
/// failure yields `None` for the instance type and consumers fall back to
/// vCPU-scaled defaults.
///
/// The IMDS fallback is gated on DMI being inconclusive. On **Linux**, DMI
/// answers "is this EC2" locally: a readable `NotEc2` reading skips IMDS (no
/// network call on a Linux laptop, on-prem host, or another cloud); only an
/// `Unknown` (a container without DMI) probes IMDS. On **non-Linux** there is no
/// DMI path, so detection is always `Unknown` and IMDS is always attempted —
/// this is deliberate (EC2 does run Windows/macOS, and IMDS is the only detection
/// path there), the cost being that a non-EC2 non-Linux host makes one IMDS probe
/// per client build. In all cases the probe honors `AWS_EC2_METADATA_DISABLED`
/// and is bounded to a single short attempt (~500ms), so a miss is cheap.
async fn detect_machine_profile() -> crate::runtime::platform::MachineProfile {
    use crate::runtime::platform::{self, DmiDetection};
    // `detect_instance_type_dmi` reads DMI attributes with synchronous `std::fs`,
    // which runs inline on the async worker rather than via `spawn_blocking`. This
    // is intentional: the reads target `/sys/.../dmi/id/*`, which are RAM-backed
    // sysfs pseudo-files served by the kernel from the SMBIOS table parsed at boot
    // (no disk I/O), and this runs once at client construction. The `spawn_blocking`
    // hop would cost more than the reads it guards. (Ref: sysfs(5); the crate's
    // hot-path fs is handled separately by the blocking-fs work.)
    let instance_type = match platform::detect_instance_type_dmi() {
        DmiDetection::Instance(ty) => Some(ty),
        DmiDetection::NotEc2 => None,
        DmiDetection::Unknown => imds_instance_type().await,
    };
    platform::MachineProfile {
        instance_type,
        vcpus: platform::local_vcpus(),
        ram_bytes: platform::available_ram(),
    }
}

/// Whether the user disabled EC2 IMDS via `AWS_EC2_METADATA_DISABLED=true`. The
/// raw `imds::Client` does not honor this itself (only the region/credentials
/// providers do), so we check it before constructing one.
fn imds_disabled() -> bool {
    imds_disabled_value(std::env::var("AWS_EC2_METADATA_DISABLED").ok().as_deref())
}

/// Pure predicate for the disable flag, split out so it is testable without
/// mutating the process environment (which would race concurrent tests).
///
/// Lenient: accepts the common truthy spellings (`true`, `1`, `yes`, `on`,
/// case-insensitive). Anything else — including unset, `false`, `0`, or an
/// unrecognized value — leaves IMDS enabled.
fn imds_disabled_value(raw: Option<&str>) -> bool {
    matches!(
        raw.map(|v| v.trim().to_ascii_lowercase()).as_deref(),
        Some("true" | "1" | "yes" | "on")
    )
}

/// Query the EC2 instance type from IMDS. `None` on any failure (IMDS disabled,
/// not on EC2, hop-limited, timeout). IMDS returns only the instance-type string
/// — no bandwidth data — so it feeds the family table like the DMI path.
///
/// Bounded to a single attempt with short connect/read timeouts: this only runs
/// when DMI was inconclusive, and a non-EC2 host must not pay the library default
/// (4 attempts, 1s connect each). On EC2 the endpoint is link-local and answers
/// in microseconds, so one short attempt is ample.
async fn imds_instance_type() -> Option<String> {
    if imds_disabled() {
        tracing::debug!(
            target: crate::telemetry::TARGET_CONCURRENCY,
            "IMDS disabled via AWS_EC2_METADATA_DISABLED; using vCPU-scaled fallback",
        );
        return None;
    }
    let client = aws_config::imds::Client::builder()
        .max_attempts(1)
        .connect_timeout(std::time::Duration::from_millis(500))
        .read_timeout(std::time::Duration::from_millis(500))
        .build();
    match client.get("/latest/meta-data/instance-type").await {
        Ok(ty) => {
            let ty = ty.as_ref().trim().to_string();
            (!ty.is_empty()).then_some(ty)
        }
        Err(err) => {
            tracing::debug!(
                target: crate::telemetry::TARGET_CONCURRENCY,
                error = %err,
                "IMDS instance-type lookup failed; using vCPU-scaled fallback",
            );
            None
        }
    }
}

#[cfg(test)]
mod tests {
    use crate::config::PartSize;
    use crate::types::ByteSize;
    use aws_config::{BehaviorVersion, Region};
    use aws_sdk_s3::config::{Credentials, Intercept, SharedCredentialsProvider};
    use aws_smithy_http_client::test_util::capture_request;

    #[cfg_attr(miri, ignore)]
    #[tokio::test]
    async fn load_with_interceptor() {
        let config = crate::from_env()
            .part_size(PartSize::Target(ByteSize::new(8)))
            .load()
            .await;
        let tm_client = crate::Client::new(config);
        let sdk_s3_config = tm_client.handle.s3_client.config();
        let tm_interceptor_exists = sdk_s3_config
            .interceptors()
            .any(|item| item.name() == "S3TransferManager");
        assert!(tm_interceptor_exists);
    }

    // `load()` always populates a MachineProfile with a real vCPU count. Instance
    // type is environment-dependent (Some on EC2 with DMI, None off-EC2); we only
    // assert the profile exists and vCPU is sane, so the test is host-independent.
    //
    // Off-EC2 this exercises the bounded IMDS fallback (1 attempt, 500ms), which
    // fast-fails; that path is intentionally short so this stays quick in CI.
    #[cfg_attr(miri, ignore)]
    #[tokio::test]
    async fn load_populates_machine_profile() {
        let config = crate::from_env().load().await;
        let profile = config
            .machine_profile()
            .expect("loader populates a machine profile");
        assert!(profile.vcpus >= 1);
    }

    /// A supplied configuration replaces the shared-configuration load, so the request is
    /// sent through its HTTP client to its region. Loaded defaults would carry neither: the
    /// capture client exists only in the supplied configuration, and the test environment
    /// does not set this region.
    #[cfg_attr(miri, ignore)]
    #[tokio::test]
    async fn load_uses_supplied_s3_config() {
        let (http_client, captured_request) = capture_request(None);
        let sdk_config = aws_types::SdkConfig::builder()
            .http_client(http_client)
            .region(Region::from_static("ap-southeast-4"))
            .credentials_provider(SharedCredentialsProvider::new(Credentials::for_tests()))
            .behavior_version(BehaviorVersion::latest())
            .build();
        let config = crate::from_env()
            .s3_config(&sdk_config)
            .runtime_config(crate::config::RuntimeConfig::multi_thread_tokio())
            .load()
            .await;
        assert!(
            config.machine_profile().is_some(),
            "machine detection runs with a supplied configuration"
        );

        let tm = crate::Client::new(config);
        let mut handle = tm.download().bucket("foo").key("bar").initiate().unwrap();
        // The capture client returns a canned response, so the transfer is expected to fail.
        let _ = handle.body_mut().next().await;
        let uri = captured_request.expect_request().uri().to_string();
        assert!(
            uri.contains("ap-southeast-4"),
            "request did not use the supplied region: {uri}"
        );
    }

    /// A supplied configuration without an HTTP client keeps the managed transport under
    /// managed execution, as loaded defaults do; one with an HTTP client uses it instead.
    #[cfg_attr(miri, ignore)]
    #[tokio::test]
    async fn supplied_s3_config_selects_managed_transport_unless_it_has_http() {
        use aws_sdk_s3::config::IntoShared;

        let sdk_config = |http_client: Option<_>| {
            let mut builder = aws_types::SdkConfig::builder()
                .region(Region::from_static("us-west-2"))
                .credentials_provider(SharedCredentialsProvider::new(Credentials::for_tests()))
                .behavior_version(BehaviorVersion::latest());
            builder.set_http_client(http_client);
            builder.build()
        };

        let config = crate::from_env().s3_config(&sdk_config(None)).load().await;
        assert!(
            config.runtime_http(64).is_some(),
            "no HTTP client supplied: the managed transport is installed"
        );

        let (http_client, _captured) = capture_request(None);
        let config = crate::from_env()
            .s3_config(&sdk_config(Some(http_client.into_shared())))
            .load()
            .await;
        assert!(
            config.runtime_http(64).is_none(),
            "an HTTP client is supplied: the managed transport is not installed"
        );
    }

    // Tests the pure parser, so it never touches the process environment and
    // cannot race the concurrent `load()`-based tests that read the same var.
    #[test]
    fn imds_disabled_parses_flag() {
        use super::imds_disabled_value;
        // Truthy spellings, case- and whitespace-insensitive.
        for truthy in ["true", "TRUE", "  true  ", "1", "yes", "YES", "on"] {
            assert!(imds_disabled_value(Some(truthy)), "{truthy:?} disables");
        }
        // Everything else leaves IMDS enabled.
        for falsy in ["false", "0", "no", "off", "", "  ", "enabled", "2"] {
            assert!(!imds_disabled_value(Some(falsy)), "{falsy:?} stays enabled");
        }
        assert!(!imds_disabled_value(None), "unset defaults to enabled");
    }

    /// The loader builds its own S3 configuration, so a captured request can only be had by
    /// extracting that configuration and rebuilding a client from it. What this covers is the
    /// loader path reaching the wire with our attribution; the sections themselves are owned by
    /// the tests in [`crate::config::user_agent`].
    #[cfg_attr(miri, ignore)]
    #[tokio::test]
    async fn load_path_emits_attribution() {
        let (http_client, captured_request) = capture_request(None);
        let config = crate::from_env()
            .part_size(PartSize::Target(ByteSize::new(8)))
            .load()
            .await;
        // Build the TM client so we can extract the S3 client's config with interceptors.
        let tm_client = crate::Client::new(config);
        let sdk_s3_config = tm_client
            .handle
            .s3_client
            .config()
            .to_builder()
            .http_client(http_client)
            .region(Region::from_static("us-west-2"))
            .with_test_defaults()
            .build();

        let capture_request_config = crate::Config::builder()
            .sdk_client(aws_sdk_s3::Client::from_conf(sdk_s3_config))
            .part_size(PartSize::Target(ByteSize::new(8)))
            .build()
            .expect("valid config");

        let transfer_manager = crate::Client::new(capture_request_config);

        let mut handle = transfer_manager
            .download()
            .bucket("foo")
            .key("bar")
            .initiate()
            .unwrap();
        // Expect to fail
        let _ = handle.body_mut().next().await;
        let expected_req = captured_request.expect_request();
        let user_agent = expected_req.headers().get("x-amz-user-agent").unwrap();
        let ours = concat!("md/rust-tm#", env!("CARGO_PKG_VERSION"));
        assert!(
            user_agent.contains(ours),
            "loader path carries no attribution: {user_agent:?}"
        );
    }
}