aws_s3_transfer/lib.rs
1/*
2 * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3 * SPDX-License-Identifier: Apache-2.0
4 */
5
6/* Automatically managed default lints */
7#![cfg_attr(docsrs, feature(doc_cfg))]
8/* End of automatically managed default lints */
9#![warn(
10 missing_debug_implementations,
11 missing_docs,
12 rustdoc::missing_crate_level_docs,
13 unreachable_pub,
14 rust_2018_idioms
15)]
16
17//! Transfer Manager for Amazon S3 - a high-performance data plane client for uploading and downloading
18//! objects and directories.
19//!
20//! The transfer manager splits large transfers into concurrent part requests over the Amazon S3
21//! [service API], retries each failed part individually, and sizes its concurrency and memory use to the
22//! host. [`Client`] starts transfers; each returns a handle to wait for, control or abort the transfer.
23//!
24//! [service API]: https://docs.aws.amazon.com/AmazonS3/latest/API/API_Operations_Amazon_Simple_Storage_Service.html
25//!
26//! # Examples
27//!
28//! Load the default configuration:
29//!
30//! ```no_run
31//! # async fn example() {
32//! let config = aws_s3_transfer::from_env().load().await;
33//! let client = aws_s3_transfer::Client::new(config);
34//! # }
35//! ```
36//!
37//! Download a bucket to a local directory:
38//!
39//! ```no_run
40//! # async fn example() -> Result<(), aws_s3_transfer::error::Error> {
41//! let config = aws_s3_transfer::from_env().load().await;
42//! let client = aws_s3_transfer::Client::new(config);
43//!
44//! let handle = client
45//! .download_objects()
46//! .bucket("my-bucket")
47//! .destination("/tmp/my-bucket")
48//! .initiate()?;
49//!
50//! // wait for transfer to complete
51//! handle.join().await?;
52//!
53//! # Ok(())
54//! # }
55//!
56//! ```
57//!
58//! # Operations
59//!
60//! See the documentation for each client operation for more information:
61//!
62//! * [`download`](crate::Client::download) - download a single object
63//! * [`upload`](crate::Client::upload) - upload a single object
64//! * [`download_objects`](crate::Client::download_objects) - download a bucket, or the objects
65//! under a prefix set with [`S3WalkerBuilder::prefix`](crate::io::walk::S3WalkerBuilder::prefix),
66//! to a local directory
67//! * [`upload_objects`](crate::Client::upload_objects) - upload the files in a local directory to a bucket.
68//! By default only the directory's immediate files are uploaded; supply
69//! [`FsWalker::builder().recursive(true)`](crate::io::walk::FsWalkerBuilder::recursive)
70//! to upload the whole tree.
71//!
72//! # Configuration
73//!
74//! [`from_env`] returns a [`ConfigLoader`], which loads
75//! S3 client configuration from the shared AWS configuration sources, detects
76//! the host to size the automatic defaults, and applies any settings made on
77//! it. [`ConfigLoader::s3_config`](config::ConfigLoader::s3_config) replaces
78//! the shared-configuration load with an existing [`aws_types::SdkConfig`] or
79//! [`S3ClientConfig`](config::S3ClientConfig).
80//!
81//! [`Config::builder`] constructs a configuration from explicit values
82//! instead. It requires an [`S3ClientConfig`](config::S3ClientConfig), loads no shared configuration,
83//! and makes no EC2 instance metadata request; the client detects the host
84//! from local system information only. Use it when the S3 configuration is
85//! already complete and neither environment loading nor network detection is
86//! wanted.
87//!
88//! [`RuntimeConfig`](config::RuntimeConfig) selects where transfer work runs,
89//! which also determines the HTTP transport used when no HTTP client is
90//! configured; see [Runtime and HTTP](#runtime-and-http).
91//!
92//! # Integrity
93//!
94//! Uploads send the checksums selected by
95//! [`ChecksumStrategy`](crate::types::ChecksumStrategy), which S3
96//! verifies against the bytes it receives. An expected value set with
97//! [`ChecksumStrategy::with_expected_value`](crate::types::ChecksumStrategy::with_expected_value)
98//! has S3 check the uploaded object against its source.
99//!
100//! A completed download reports whether the bytes it delivered were validated
101//! against S3's checksums:
102//! [`DownloadOutput::integrity_checks`](crate::operation::download::DownloadOutput::integrity_checks)
103//! returns [`IntegrityChecks`](crate::types::IntegrityChecks), whose
104//! [`ChecksumValidation`](crate::types::ChecksumValidation) is `Validated` or
105//! says why the bytes were not validated. Validation covers the bytes as
106//! received from S3, not the destination they were written to. A checksum
107//! mismatch fails `join()` with
108//! [`ErrorKind::IntegrityError`](crate::error::ErrorKind::IntegrityError).
109//! Streamed bytes can arrive before the final check, so a download has
110//! succeeded only when `join()` returns `Ok`.
111//!
112//! A successful download to a path means the operating system accepted every
113//! byte and the file was renamed into place; the transfer manager does not sync
114//! the data to stable storage. The [`download`](crate::operation::download)
115//! module describes when each check applies.
116//!
117//! # Runtime and HTTP
118//!
119//! By default, transfer work runs on threads the transfer manager creates and
120//! owns ([`RuntimeConfig::managed`](config::RuntimeConfig::managed)), and
121//! requests use the transfer manager's own HTTP transport, which drives each
122//! connection on the thread that opened it.
123//! [`RuntimeConfig::multi_thread_tokio`](config::RuntimeConfig::multi_thread_tokio)
124//! runs transfer work on the caller's Tokio runtime instead, and requests use
125//! the AWS SDK's default HTTP client.
126//!
127//! An HTTP client set on the [`aws_types::SdkConfig`] or S3 configuration is
128//! used with either runtime, in place of the transfer manager's transport or
129//! the SDK default. To change TLS, DNS or connection settings, configure an
130//! HTTP client. [`RuntimeConfig`](config::RuntimeConfig#http-transport)
131//! describes what each transport provides.
132//!
133//! `aws-s3-transfer-ext` is an experimental crate that adopts a
134//! managed runtime built by the caller; its API may change in any release.
135//!
136//! # Modeled types
137//!
138//! S3 request values, response metadata, and enums live in [`types`], and
139//! their builders in [`types::builders`]. Operation modules re-export the
140//! modeled types and builders they use.
141//!
142//! The `sdk-v1` feature enables conversions between modeled values and AWS SDK
143//! for Rust S3 v1 types, conversion from an S3 configuration builder, and
144//! [`S3WalkContextBuilder`](crate::io::walk::S3WalkContextBuilder) construction
145//! from an SDK client.
146//!
147//! # WebAssembly
148//!
149//! The crate builds for `wasm32-wasip1`, `wasm32-wasip2`, and `wasm32-wasip3`.
150//! Other wasm targets are not supported. On these targets:
151//!
152//! - Build with `RUSTFLAGS="--cfg tokio_unstable"`. The S3 client's tokio
153//! integration enables tokio's `fs` feature, which tokio accepts on wasm only
154//! with that flag.
155//! - The AWS SDK's default HTTPS client is not built. Requests fail unless the
156//! [`aws_types::SdkConfig`] or S3 configuration supplies an HTTP client, for
157//! example one built on `wasi:http`. `wasm32-wasip1` has no networking
158//! interface for such a client.
159//! - [`Managed`](config::Managed) execution behaves as
160//! [`RuntimeConfig::multi_thread_tokio`](config::RuntimeConfig::multi_thread_tokio), and a current-thread tokio runtime
161//! is supported. [`Client::shutdown_completion`] returns `None`. File reads
162//! and writes run inline on the thread that polls the transfer.
163//! - The `__unstable_ext` feature, which adopts a prebuilt managed runtime, is
164//! unavailable.
165//! - Memory detection is unavailable, so
166//! [`MemoryBudgetConfig::Auto`](crate::config::MemoryBudgetConfig::Auto) uses
167//! its 2 GiB fallback. Use
168//! [`MemoryBudgetConfig::Limit`](crate::config::MemoryBudgetConfig::Limit) to
169//! fit the module's linear memory.
170//!
171//! # Runtime diagnostics
172//!
173//! Tracing output and `AWS_S3_TM_DIAGNOSTICS` are diagnostics, not stable API:
174//! tracing targets, event and span names, their fields, and the settings the
175//! environment variable accepts may change in minor releases. Events are
176//! grouped under these subsystem targets:
177//!
178//! - `aws_s3_transfer::concurrency`: concurrency resolution and
179//! control.
180//! - `aws_s3_transfer::scheduling`: scheduler capacity decisions,
181//! worker pool growth, and memory-budget admission.
182//! - `aws_s3_transfer::memory`: buffer-pool admission, physical
183//! storage, and reclamation.
184//! - `aws_s3_transfer::execution`: per-work-item dispatch and
185//! completion.
186//! - `aws_s3_transfer::transfer`: transfer lifecycle and state
187//! transitions.
188//!
189//! Other events use the target of the module that emits them.
190//!
191//! `AWS_S3_TM_DIAGNOSTICS` enables optional diagnostic collection without
192//! changing the client configuration API. Settings are namespaced by subsystem
193//! and resolved once when a client configuration is built. A standalone
194//! [`memory::BufferPool`] resolves only the memory settings.
195//!
196//! The value is a comma-separated list of case-sensitive `key=value` settings:
197//!
198//! ```text
199//! AWS_S3_TM_DIAGNOSTICS=memory.snapshot=1000ms,memory.detail=1,transfer.detail=1
200//! ```
201//!
202//! Whitespace around entries, keys, and values is ignored. Later valid
203//! assignments replace earlier ones. Unknown keys are ignored for forward
204//! compatibility. A malformed recognized setting produces a warning and keeps
205//! the preceding value. Unsupported detail levels use the highest level
206//! understood by this version and produce a warning.
207//!
208//! The environment variable controls collection cost. Tracing filters
209//! independently control whether collected records are emitted.
210//!
211//! ## Memory pool
212//!
213//! Memory diagnostics report pool capacity, admission pressure, allocation
214//! fallback, preparation, and reclamation.
215//!
216//! - `memory.snapshot=off` disables periodic reports. A positive integer
217//! followed by `ms` enables reports; values below `100ms` use `100ms`.
218//! - `memory.detail=0` uses counters updated only at pressure and lifecycle
219//! boundaries.
220//! - `memory.detail=1` also counts every optimistic allocation attempt and
221//! bitmap word inspected, adding relaxed atomic updates to the acquisition
222//! path.
223//!
224//! Memory snapshots use the `aws_s3_transfer::memory` tracing target
225//! at `DEBUG`:
226//!
227//! ```text
228//! AWS_S3_TM_DIAGNOSTICS=memory.snapshot=1000ms
229//! RUST_LOG=aws_s3_transfer::memory=debug
230//! ```
231//!
232//! Snapshot reporting reuses the memory pool's maintenance thread. It does not
233//! create a diagnostics-only thread. Detailed counters and periodic reporting
234//! are independent; both are disabled by default.
235//!
236//! ## Transfer state machines
237//!
238//! Transfer diagnostics report where an operation spends time and how work
239//! moves through its direction-specific state machine.
240//!
241//! - `transfer.detail=0` disables optional transfer-state collection.
242//! - `transfer.detail=1` collects an aggregate summary for each upload,
243//! download, `upload_objects` and `download_objects` transfer.
244//! - `transfer.detail=2` also collects individual state transitions.
245//!
246//! Transfer summaries use the operation's tracing target at `DEBUG`;
247//! level-2 transitions use the same target at `TRACE`. Each operation logs
248//! under its own target below `aws_s3_transfer::transfer`, so the
249//! `transfer` target enables all of them:
250//!
251//! ```text
252//! AWS_S3_TM_DIAGNOSTICS=transfer.detail=1
253//! RUST_LOG=aws_s3_transfer::transfer=debug
254//! ```
255
256/// Errors returned by transfers, and how to classify them.
257pub mod error;
258
259/// S3 values and response metadata, transfer policies and outcomes, and byte-size and
260/// throughput units.
261pub mod types;
262
263/// Modeled S3 values and builders.
264pub(crate) mod model;
265pub(crate) mod sdk_v1;
266
267/// Upload bodies, custom part streams, and walkers for local directories and S3 buckets.
268pub mod io;
269
270/// The transfer manager client and its operations.
271pub mod client;
272
273/// Inputs, builders, handles and outputs for each operation.
274pub mod operation;
275
276/// Client configuration and loading: S3 settings, part size, concurrency, read-ahead, runtime
277/// and memory.
278pub mod config;
279
280/// Experimental integrations with version-coupled managed execution.
281#[cfg(feature = "__unstable_ext")]
282pub mod unstable;
283
284#[cfg(all(feature = "__unstable_ext", target_family = "wasm"))]
285compile_error!("`__unstable_ext` adopts a managed runtime, which wasm targets cannot run");
286
287/// Payload-memory configuration and shared pooled storage.
288///
289/// Transfer-manager clients construct a pool automatically by default.
290/// Applications that need to share one memory budget across clients or another
291/// component can construct a [`BufferPool`](crate::memory::BufferPool) and
292/// install it through
293/// [`MemoryConfig::Explicit`](crate::config::MemoryConfig::Explicit).
294///
295/// [`BufferPool::metrics`](crate::memory::BufferPool::metrics) returns current
296/// accounting gauges without enabling diagnostic sampling. Buffer-pool events
297/// use the `aws_s3_transfer::memory` tracing target.
298pub mod memory {
299 pub use crate::runtime::buffer_pool::{
300 BufferPool, BufferPoolBuildError, BufferPoolBuilder, SegmentedBytes,
301 };
302}
303
304/// Test-only entry points used by external fuzz targets.
305#[cfg(s3_tm_fuzz)]
306#[doc(hidden)]
307pub mod __fuzz {
308 /// Runs one encoded buffer-pool operation sequence.
309 pub fn buffer_pool_operations(data: &[u8]) {
310 crate::runtime::buffer_pool::run_fuzz_input(data);
311 }
312
313 /// Runs one encoded buffer-pool placement sequence.
314 pub fn buffer_pool_placement(data: &[u8]) {
315 crate::runtime::buffer_pool::run_placement_fuzz_input(data);
316 }
317}
318
319/// Allocation-API access for out-of-crate benchmarks, built only under `cfg(s3_tm_bench)`.
320#[cfg(s3_tm_bench)]
321#[doc(hidden)]
322pub mod __bench {
323 pub use crate::runtime::buffer_pool::bench_util::{
324 AcquireError, BufferPoolExt, PooledBufMut, Reservation, ReserveError, ReserveFuture,
325 };
326}
327
328/// HTTP related components and utils
329pub(crate) mod http;
330
331/// Transfer types
332pub(crate) mod transfer;
333
334/// Work scheduler
335pub(crate) mod scheduler;
336
337/// Execution runtime
338pub(crate) mod runtime;
339
340/// Telemetry target constants
341pub(crate) mod telemetry;
342
343/// Transfer progress and I/O metrics, and client memory metrics.
344pub mod metrics;
345
346/// Body-read retry loop layered over the latency deadline guard
347pub(crate) mod retry;
348
349pub use self::client::Client;
350pub use self::config::Config;
351use self::config::ConfigLoader;
352pub use self::runtime::{ShutdownCompletion, ShutdownError, ShutdownErrorKind};
353pub use self::transfer::SchedulingCtl;
354
355/// Create a [`ConfigLoader`] that loads configuration from the environment.
356///
357/// This is the usual way to configure a [`Client`]; see
358/// [`ConfigLoader`] for what [`load`](ConfigLoader::load) resolves.
359pub fn from_env() -> ConfigLoader {
360 ConfigLoader::default()
361}
362
363/// Compiles the package README's examples as doctests.
364#[cfg(doctest)]
365#[doc = include_str!("../README.md")]
366struct ReadmeDoctests;
367
368#[cfg(test)]
369mod tests;