Skip to main content

aws_s3_transfer/
config.rs

1/*
2 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3 * SPDX-License-Identifier: Apache-2.0
4 */
5
6use aws_smithy_types::error::operation::BuildError;
7use std::cmp;
8
9use crate::types::ByteSize;
10
11mod runtime;
12pub use runtime::{
13    ConcurrencyMode, Managed, MemoryBudgetConfig, MemoryConfig, RuntimeConfig, TargetThroughput,
14};
15
16mod diagnostics;
17pub(crate) use diagnostics::{
18    DiagnosticsConfig, MemoryDiagnosticsConfig, TransferDiagnosticsConfig,
19};
20
21mod loader;
22pub use loader::ConfigLoader;
23pub(crate) mod user_agent;
24
25/// Minimum upload part size
26pub(crate) const MIN_MULTIPART_PART_SIZE: ByteSize = ByteSize::mib(5);
27
28/// The target part size for an upload or download request.
29#[non_exhaustive]
30#[derive(Debug, Clone, Default)]
31pub enum PartSize {
32    /// Automatically configure an optimal target part size based on the execution environment.
33    #[default]
34    Auto,
35
36    /// Target part size explicitly given.
37    ///
38    /// NOTE: This is a suggestion and will be used if possible but may be adjusted for an individual request
39    /// as required by the underlying API.
40    Target(ByteSize),
41}
42
43/// How far a download fetches ahead of the data its consumer has taken.
44///
45/// A download fetches parts concurrently and holds each one until the consumer
46/// takes it: a streamed body reads it, or a file destination writes it.
47/// Read-ahead limits how many parts one download fetches and holds before the
48/// consumer takes them. When the consumer falls that far behind, the download
49/// stops fetching new parts until the consumer takes more. The client's memory
50/// budget applies across all transfers in addition to this limit.
51#[non_exhaustive]
52#[derive(Debug, Clone, PartialEq, Eq, Default)]
53pub enum ReadAhead {
54    /// A limit that doesn't slow a consumer that keeps pace, so the memory
55    /// budget is the bound in practice. This is the default.
56    #[default]
57    Auto,
58
59    /// At most `n` parts beyond the one the consumer is waiting for; `0`
60    /// fetches only that part, one at a time.
61    ///
62    /// Use a small value when a consumer may stop reading early or reads
63    /// slowly. Change it while a download runs with
64    /// [`DownloadIoCtl::set_read_ahead`](crate::operation::download::DownloadIoCtl::set_read_ahead).
65    Parts(usize),
66}
67
68/// S3 client configuration for the transfer manager.
69///
70/// The transfer manager builds its S3 client from this configuration and keeps
71/// any HTTP client the configuration sets. With no HTTP client configured, managed execution
72/// supplies the transfer manager's transport and
73/// [`RuntimeConfig::multi_thread_tokio`] uses the SDK default; see
74/// [`RuntimeConfig`](RuntimeConfig#http-transport).
75///
76/// With the `sdk-v1` feature, this also converts from
77/// `aws_sdk_s3::config::Builder` for S3-specific configuration.
78///
79/// ```no_run
80/// # async fn example() -> Result<(), aws_smithy_types::error::operation::BuildError> {
81/// use aws_config::BehaviorVersion;
82///
83/// let sdk_config = aws_config::load_defaults(BehaviorVersion::latest()).await;
84/// let config = aws_s3_transfer::Config::builder()
85///     .s3_config(&sdk_config)
86///     .build()?;
87/// # Ok(())
88/// # }
89/// ```
90pub struct S3ClientConfig {
91    pub(crate) config: aws_sdk_s3::Config,
92}
93
94impl S3ClientConfig {
95    /// Create S3 client configuration from shared AWS configuration.
96    ///
97    /// With `sdk-v1` enabled, an `aws_sdk_s3::config::Builder` is also accepted.
98    pub fn new(config: impl Into<Self>) -> Self {
99        config.into()
100    }
101
102    pub(crate) fn from_sdk_builder(builder: aws_sdk_s3::config::Builder) -> Self {
103        Self {
104            config: builder.build(),
105        }
106    }
107}
108
109impl From<&aws_types::SdkConfig> for S3ClientConfig {
110    fn from(sdk_config: &aws_types::SdkConfig) -> Self {
111        Self::from_sdk_builder(aws_sdk_s3::config::Builder::from(sdk_config))
112    }
113}
114
115#[cfg(feature = "sdk-v1")]
116impl From<aws_sdk_s3::config::Builder> for S3ClientConfig {
117    fn from(builder: aws_sdk_s3::config::Builder) -> Self {
118        Self::from_sdk_builder(builder)
119    }
120}
121
122impl std::fmt::Debug for S3ClientConfig {
123    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
124        f.debug_struct("S3ClientConfig").finish_non_exhaustive()
125    }
126}
127
128/// How the S3 client is provided to the transfer manager.
129#[derive(Debug)]
130pub(crate) enum S3ClientSource {
131    /// User provided a finished S3 client. Use as-is.
132    Provided(aws_sdk_s3::Client),
133    /// Build the S3 client from config, injecting runtime components.
134    FromConfig(Box<S3ClientConfig>),
135}
136
137/// Configuration for a [`Client`](crate::client::Client)
138#[derive(Debug)]
139pub struct Config {
140    multipart_threshold: PartSize,
141    target_part_size: PartSize,
142    concurrency: ConcurrencyMode,
143    runtime_config: RuntimeConfig,
144    read_ahead: ReadAhead,
145    memory: MemoryConfig,
146    diagnostics: DiagnosticsConfig,
147    s3_client_source: Option<S3ClientSource>,
148    /// Machine facts detected once by the async config loader, off the
149    /// `Client::new` hot path. `None` when the config was built directly
150    /// (bypassing the loader); consumers then fall back to cheap local detection.
151    machine_profile: Option<crate::runtime::platform::MachineProfile>,
152    /// Interfaces the runtime-provided transport binds its connections to;
153    /// empty leaves interface selection to OS routing.
154    pub(crate) network_interfaces: Vec<String>,
155}
156
157impl Config {
158    /// Create a new `Config` builder.
159    ///
160    /// The builder uses only the values it is given: it loads no shared AWS
161    /// configuration and does not query EC2 instance metadata. Use
162    /// [`from_env`](crate::from_env) to load configuration from the environment.
163    pub fn builder() -> Builder {
164        Builder::default()
165    }
166
167    /// Returns a reference to the multipart upload threshold part size
168    pub fn multipart_threshold(&self) -> &PartSize {
169        &self.multipart_threshold
170    }
171
172    /// Returns a reference to the target part size to use for transfer operations
173    pub fn part_size(&self) -> &PartSize {
174        &self.target_part_size
175    }
176
177    /// Returns the concurrency mode to use for transfer operations.
178    ///
179    /// This is the mode used for concurrent in-flight requests across _all_ operations.
180    pub fn concurrency(&self) -> &ConcurrencyMode {
181        &self.concurrency
182    }
183
184    /// The [`ConcurrencyMode::TargetThroughput`] target in Gbps, if configured.
185    pub(crate) fn throughput_target_gbps(&self) -> Option<f64> {
186        match &self.concurrency {
187            ConcurrencyMode::TargetThroughput(target) => Some(target.as_gbps()),
188            _ => None,
189        }
190    }
191
192    /// Returns the execution configuration for transfer work.
193    pub fn runtime_config(&self) -> &RuntimeConfig {
194        &self.runtime_config
195    }
196
197    #[cfg(not(target_family = "wasm"))]
198    pub(crate) fn managed_config_mut(&mut self) -> Option<&mut Managed> {
199        self.runtime_config.managed_config_mut()
200    }
201
202    /// Returns the read-ahead mode used for downloads.
203    ///
204    /// This is the client default; a download may override it per request via
205    /// [`DownloadInput`](crate::operation::download::DownloadInput).
206    pub fn read_ahead(&self) -> &ReadAhead {
207        &self.read_ahead
208    }
209
210    /// Returns the payload-memory configuration.
211    pub fn memory(&self) -> &MemoryConfig {
212        &self.memory
213    }
214
215    /// Returns the diagnostic policy fixed when this configuration was built.
216    pub(crate) fn diagnostics(&self) -> DiagnosticsConfig {
217        self.diagnostics
218    }
219
220    /// Machine facts detected by the async config loader, if this config was
221    /// built through it. `None` when built directly.
222    pub(crate) fn machine_profile(&self) -> Option<&crate::runtime::platform::MachineProfile> {
223        self.machine_profile.as_ref()
224    }
225
226    /// HTTP options for the runtime-provided transport, or `None` when the
227    /// managed transport would not be installed: HTTP is already configured,
228    /// a finished S3 client was supplied, execution uses the caller's runtime,
229    /// or the target is wasm, which has no managed transport.
230    pub(crate) fn runtime_http(
231        &self,
232        max_connections_per_host: usize,
233    ) -> Option<crate::runtime::RuntimeHttpOptions> {
234        if cfg!(target_family = "wasm") {
235            return None;
236        }
237        match self.s3_client_source.as_ref()? {
238            S3ClientSource::FromConfig(config)
239                if self.runtime_config.managed_config().is_some()
240                    && config.config.http_client().is_none() =>
241            {
242                Some(crate::runtime::RuntimeHttpOptions {
243                    network_interfaces: self.network_interfaces.clone(),
244                    max_connections_per_host,
245                })
246            }
247            _ => None,
248        }
249    }
250
251    /// Consume the S3 client source, returning it.
252    pub(crate) fn take_s3_client_source(&mut self) -> S3ClientSource {
253        self.s3_client_source
254            .take()
255            .expect("s3 client source already taken")
256    }
257}
258
259/// Fluent style builder for [Config]
260#[derive(Debug, Default)]
261pub struct Builder {
262    multipart_threshold_part_size: PartSize,
263    target_part_size: PartSize,
264    concurrency: ConcurrencyMode,
265    runtime_config: RuntimeConfig,
266    read_ahead: ReadAhead,
267    memory: MemoryConfig,
268    diagnostics: Option<DiagnosticsConfig>,
269    client: Option<aws_sdk_s3::Client>,
270    s3_client_config: Option<S3ClientConfig>,
271    machine_profile: Option<crate::runtime::platform::MachineProfile>,
272}
273
274impl Builder {
275    /// Minimum object size that should trigger a multipart upload.
276    ///
277    /// The minimum part size is 5 MiB, any part size less than that will be rounded up.
278    /// Default is [PartSize::Auto]
279    pub fn multipart_threshold(self, threshold: PartSize) -> Self {
280        let threshold = match threshold {
281            PartSize::Target(part_size) => {
282                PartSize::Target(cmp::max(part_size, MIN_MULTIPART_PART_SIZE))
283            }
284            tps => tps,
285        };
286
287        self.set_multipart_threshold(threshold)
288    }
289
290    /// The target size of each part when using a multipart upload to complete the request.
291    ///
292    /// When a request's content length is less than [`multipart_threshold`],
293    /// this setting is ignored and a single [`PutObject`] request will be made instead.
294    ///
295    /// NOTE: The actual part size used may be larger than the configured part size if
296    /// the current value would result in more than 10,000 parts for an upload request.
297    ///
298    /// Default is [PartSize::Auto]
299    ///
300    /// [`multipart_threshold`]: method@Self::multipart_threshold
301    /// [`PutObject`]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html
302    pub fn part_size(self, part_size: PartSize) -> Self {
303        let threshold = match part_size {
304            PartSize::Target(part_size) => {
305                PartSize::Target(cmp::max(part_size, MIN_MULTIPART_PART_SIZE))
306            }
307            tps => tps,
308        };
309
310        self.set_target_part_size(threshold)
311    }
312
313    /// Minimum object size that should trigger a multipart upload.
314    ///
315    /// NOTE: This does not validate the setting and is meant for internal use only.
316    pub(crate) fn set_multipart_threshold(mut self, threshold: PartSize) -> Self {
317        self.multipart_threshold_part_size = threshold;
318        self
319    }
320
321    /// Target part size for a multipart upload.
322    ///
323    /// NOTE: This does not validate the setting and is meant for internal use only.
324    pub(crate) fn set_target_part_size(mut self, threshold: PartSize) -> Self {
325        self.target_part_size = threshold;
326        self
327    }
328
329    /// Set the concurrency mode this client should use.
330    ///
331    /// This sets the mode used for concurrent in-flight requests across _all_ operations.
332    /// Default is [ConcurrencyMode::Auto].
333    pub fn concurrency(mut self, mode: ConcurrencyMode) -> Self {
334        self.concurrency = mode;
335        self
336    }
337
338    /// Sets execution for transfer work.
339    ///
340    /// Defaults to owned managed execution.
341    pub fn runtime_config(mut self, config: RuntimeConfig) -> Self {
342        self.set_runtime_config(Some(config));
343        self
344    }
345
346    /// Sets execution configuration, or restores the default with `None`.
347    pub fn set_runtime_config(&mut self, config: Option<RuntimeConfig>) -> &mut Self {
348        self.runtime_config = config.unwrap_or_default();
349        self
350    }
351
352    /// Set how far downloads may prefetch ahead of the consumer.
353    ///
354    /// This is the client default for all downloads; a request may override it via
355    /// [`DownloadInput`](crate::operation::download::DownloadInput), or a running
356    /// transfer may adjust it via
357    /// [`DownloadIoCtl`](crate::operation::download::DownloadIoCtl).
358    /// Default is [ReadAhead::Auto].
359    pub fn read_ahead(mut self, mode: ReadAhead) -> Self {
360        self.read_ahead = mode;
361        self
362    }
363
364    /// Set the payload-memory configuration.
365    ///
366    /// Default is [`MemoryConfig::Auto`].
367    pub fn memory(mut self, memory: MemoryConfig) -> Self {
368        self.memory = memory;
369        self
370    }
371
372    /// Installs deterministic diagnostics for internal tests.
373    #[cfg(test)]
374    pub(crate) fn diagnostics_for_test(
375        mut self,
376        memory: MemoryDiagnosticsConfig,
377        transfer_detail_level: u64,
378    ) -> Self {
379        self.diagnostics = Some(DiagnosticsConfig::for_test(memory, transfer_detail_level));
380        self
381    }
382
383    /// Inject a finished SDK client for library unit fixtures.
384    #[cfg(test)]
385    pub(crate) fn sdk_client(mut self, client: aws_sdk_s3::Client) -> Self {
386        self.client = Some(client);
387        self
388    }
389
390    /// Set the S3 client configuration.
391    ///
392    /// The transfer manager preserves a configured HTTP client. With no
393    /// configured HTTP, managed execution installs its default transport;
394    /// caller-runtime execution uses the SDK default.
395    ///
396    /// Required: [`build`](Self::build) fails without it.
397    pub fn s3_config(mut self, config: impl Into<S3ClientConfig>) -> Self {
398        self.s3_client_config = Some(config.into());
399        self
400    }
401
402    /// Set the machine profile detected by the async config loader. Internal:
403    /// populated by [`ConfigLoader`], not a public builder method.
404    pub(crate) fn machine_profile(
405        mut self,
406        profile: Option<crate::runtime::platform::MachineProfile>,
407    ) -> Self {
408        self.machine_profile = profile;
409        self
410    }
411
412    /// Consumes the builder and constructs a [`Config`].
413    ///
414    /// # Errors
415    ///
416    /// Returns a [`BuildError`] if [`s3_config`](Self::s3_config) was not set.
417    pub fn build(self) -> Result<Config, BuildError> {
418        let s3_client_source = match (self.client, self.s3_client_config) {
419            (Some(client), _) => S3ClientSource::Provided(client),
420            (None, Some(config)) => S3ClientSource::FromConfig(Box::new(config)),
421            (None, None) => {
422                return Err(BuildError::missing_field(
423                    "s3_config",
424                    "S3 client configuration is required",
425                ))
426            }
427        };
428        Ok(Config {
429            multipart_threshold: self.multipart_threshold_part_size,
430            target_part_size: self.target_part_size,
431            concurrency: self.concurrency,
432            runtime_config: self.runtime_config,
433            read_ahead: self.read_ahead,
434            memory: self.memory,
435            diagnostics: self.diagnostics.unwrap_or_else(DiagnosticsConfig::from_env),
436            s3_client_source: Some(s3_client_source),
437            machine_profile: self.machine_profile,
438            network_interfaces: Vec::new(),
439        })
440    }
441}
442
443#[cfg(test)]
444mod tests {
445    use super::*;
446
447    fn s3_config_builder() -> aws_sdk_s3::config::Builder {
448        aws_sdk_s3::config::Builder::new()
449    }
450
451    #[test]
452    fn s3_client_config_converts_from_sdk_config_and_builder() {
453        let region = aws_types::region::Region::from_static("us-west-2");
454        let sdk_config = aws_types::SdkConfig::builder()
455            .region(region.clone())
456            .build();
457        let from_sdk = S3ClientConfig::from(&sdk_config);
458        assert_eq!(from_sdk.config.region(), Some(&region));
459
460        let builder = s3_config_builder().region(region.clone());
461        #[cfg(feature = "sdk-v1")]
462        let from_builder = S3ClientConfig::new(builder);
463        #[cfg(not(feature = "sdk-v1"))]
464        let from_builder = S3ClientConfig::from_sdk_builder(builder);
465        assert_eq!(from_builder.config.region(), Some(&region));
466    }
467
468    #[test]
469    fn runtime_http_present_for_s3_config() {
470        let config = Config::builder()
471            .s3_config(S3ClientConfig::from_sdk_builder(s3_config_builder()))
472            .build()
473            .expect("valid config");
474        let http = config.runtime_http(64).expect("runtime HTTP enabled");
475        assert!(http.network_interfaces.is_empty());
476        assert_eq!(http.max_connections_per_host, 64);
477    }
478
479    #[test]
480    fn runtime_http_absent_without_runtime_transport() {
481        let shared = Config::builder()
482            .s3_config(S3ClientConfig::from_sdk_builder(s3_config_builder()))
483            .runtime_config(RuntimeConfig::multi_thread_tokio())
484            .build()
485            .expect("valid config");
486        assert!(shared.runtime_http(64).is_none());
487
488        let configured = Config::builder()
489            .s3_config(S3ClientConfig::from_sdk_builder(
490                aws_smithy_mocks::mock_client!(aws_sdk_s3, [])
491                    .config()
492                    .to_builder(),
493            ))
494            .build()
495            .expect("valid config");
496        assert!(configured.runtime_http(64).is_none());
497
498        // A mock client: a real one would build the default HTTPS client, whose
499        // TLS initialization is foreign code that miri cannot run.
500        let provided = Config::builder()
501            .sdk_client(aws_smithy_mocks::mock_client!(aws_sdk_s3, []))
502            .build()
503            .expect("valid config");
504        assert!(provided.runtime_http(64).is_none());
505    }
506
507    #[test]
508    fn build_without_s3_config_is_an_error() {
509        let error = Config::builder()
510            .build()
511            .expect_err("s3_config is required");
512        assert!(
513            error.to_string().contains("s3_config"),
514            "error does not name the missing field: {error}"
515        );
516    }
517}