use aws_runtime::sdk_feature::AwsSdkFeature;
use aws_runtime::user_agent::{AdditionalMetadata, ApiMetadata, AwsUserAgent, FrameworkMetadata};
use aws_sdk_s3::config::{AppName, Intercept, IntoShared};
use aws_types::os_shim_internal::Env;
use std::sync::Arc;
use crate::config::{ConcurrencyMode, MemoryConfig, PartSize};
use crate::runtime::platform::MachineProfile;
use crate::Config;
const RUST_TM_VERSION: &str = concat!("rust-tm#", env!("CARGO_PKG_VERSION"));
const INTERCEPTOR_NAME: &str = "S3TransferManager";
const OPERATION_INTERCEPTOR_NAME: &str = "S3TransferManagerOperation";
pub(crate) fn client_features(config: &Config, profile: &MachineProfile) -> Vec<AwsSdkFeature> {
let mut features = Vec::new();
if matches!(config.part_size(), PartSize::Target(_)) {
features.push(AwsSdkFeature::S3CustomPartSize);
}
let custom_concurrency = match config.concurrency() {
ConcurrencyMode::Auto => false,
ConcurrencyMode::TargetThroughput(_) | ConcurrencyMode::Explicit(_) => true,
};
if custom_concurrency {
features.push(AwsSdkFeature::S3CustomThroughput);
}
if matches!(config.memory(), MemoryConfig::Explicit(_)) {
features.push(AwsSdkFeature::S3CustomMemoryLimit);
}
if profile.instance_type.is_some() {
features.push(AwsSdkFeature::S3OnEc2);
}
features
}
pub(crate) fn install(builder: &mut aws_sdk_s3::config::Builder, features: Vec<AwsSdkFeature>) {
builder.push_interceptor(S3TransferManagerInterceptor { features }.into_shared());
}
#[derive(Clone, Debug, Default)]
pub(crate) struct OperationFeatures(Arc<[AwsSdkFeature]>);
impl OperationFeatures {
pub(crate) fn new(features: impl IntoIterator<Item = AwsSdkFeature>) -> Self {
Self(features.into_iter().collect())
}
pub(crate) fn apply(
&self,
builder: aws_sdk_s3::config::Builder,
) -> aws_sdk_s3::config::Builder {
if self.0.is_empty() {
builder
} else {
builder.interceptor(self.clone())
}
}
pub(crate) fn apply_to_client(&self, client: &aws_sdk_s3::Client) -> aws_sdk_s3::Client {
aws_sdk_s3::Client::from_conf(self.apply(client.config().to_builder()).build())
}
}
impl Intercept for OperationFeatures {
fn name(&self) -> &'static str {
OPERATION_INTERCEPTOR_NAME
}
fn read_before_execution(
&self,
_ctx: &aws_sdk_s3::config::interceptors::BeforeSerializationInterceptorContextRef<'_>,
cfg: &mut aws_sdk_s3::config::ConfigBag,
) -> Result<(), aws_sdk_s3::error::BoxError> {
for feature in self.0.iter() {
cfg.interceptor_state().store_append(feature.clone());
}
Ok(())
}
}
pub(crate) fn install_on_client(
client: aws_sdk_s3::Client,
features: Vec<AwsSdkFeature>,
) -> aws_sdk_s3::Client {
if client
.config()
.interceptors()
.any(|interceptor| interceptor.name() == INTERCEPTOR_NAME)
{
return client;
}
let mut builder = client.config().to_builder();
install(&mut builder, features);
aws_sdk_s3::Client::from_conf(builder.build())
}
#[derive(Debug)]
struct S3TransferManagerInterceptor {
features: Vec<AwsSdkFeature>,
}
impl Intercept for S3TransferManagerInterceptor {
fn name(&self) -> &'static str {
INTERCEPTOR_NAME
}
fn read_before_execution(
&self,
_ctx: &aws_sdk_s3::config::interceptors::BeforeSerializationInterceptorContextRef<'_>,
cfg: &mut aws_sdk_s3::config::ConfigBag,
) -> Result<(), aws_sdk_s3::error::BoxError> {
cfg.interceptor_state()
.store_append(AwsSdkFeature::S3Transfer);
for feature in &self.features {
cfg.interceptor_state().store_append(feature.clone());
}
let existing = cfg.load::<AwsUserAgent>().cloned();
let base = existing.or_else(|| {
cfg.load::<ApiMetadata>().cloned().map(|api_metadata| {
let mut ua = AwsUserAgent::new_from_environment(Env::real(), api_metadata);
if let Some(app_name) = cfg.load::<AppName>().cloned() {
ua.set_app_name(app_name);
}
for framework_metadata in framework_metadata(cfg) {
ua.add_framework_metadata(framework_metadata);
}
ua
})
});
let Some(mut ua) = base else { return Ok(()) };
if let Ok(metadata) = AdditionalMetadata::new(RUST_TM_VERSION) {
ua = ua.with_additional_metadata(metadata);
}
cfg.interceptor_state().store_put(ua);
Ok(())
}
}
const MAX_FRAMEWORK_METADATA: usize = 10;
fn framework_metadata(cfg: &aws_sdk_s3::config::ConfigBag) -> Vec<FrameworkMetadata> {
let appended: Vec<&FrameworkMetadata> = cfg.load::<FrameworkMetadata>().collect();
let mut kept: Vec<FrameworkMetadata> = Vec::new();
for metadata in appended.into_iter().rev() {
if kept.len() == MAX_FRAMEWORK_METADATA {
break;
}
if !kept.contains(metadata) {
kept.push(metadata.clone());
}
}
kept
}
#[cfg(test)]
mod tests {
use crate::config::RuntimeConfig;
use crate::config::{ConcurrencyMode, MemoryConfig, PartSize};
use crate::runtime::platform::MachineProfile;
use crate::types::ByteSize;
use aws_config::{BehaviorVersion, Region};
use aws_runtime::user_agent::{AwsUserAgent, BusinessMetric, FrameworkMetadata};
use aws_sdk_s3::config::interceptors::BeforeTransmitInterceptorContextRef;
use aws_sdk_s3::config::{
AppName, Credentials, Intercept, IntoShared, SharedCredentialsProvider,
};
use aws_sdk_s3::config::{ConfigBag, RuntimeComponents};
use aws_sdk_s3::error::BoxError;
use aws_sdk_s3::operation::get_object::GetObjectOutput;
use aws_sdk_s3::operation::list_objects_v2::ListObjectsV2Output;
use aws_sdk_s3::operation::put_object::PutObjectOutput;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::Object;
use aws_smithy_http_client::test_util::{capture_request, CaptureRequestReceiver};
use aws_smithy_mocks::{mock, mock_client, RuleMode};
use aws_smithy_runtime_api::client::orchestrator::Metadata;
use std::borrow::Cow;
use std::sync::{Arc, Mutex};
fn test_credentials() -> SharedCredentialsProvider {
SharedCredentialsProvider::new(Credentials::new(
"ANOTREAL",
"notrealrnrELgWzOk3IfjzDKtFBhDby",
None,
None,
"test",
))
}
async fn captured_user_agent(
runtime_config: RuntimeConfig,
tweak: impl FnOnce(aws_sdk_s3::config::Builder) -> aws_sdk_s3::config::Builder,
) -> String {
let (http_client, captured) = capture_request(None);
let builder = aws_sdk_s3::config::Builder::default()
.http_client(http_client)
.region(Region::from_static("us-west-2"))
.credentials_provider(test_credentials())
.behavior_version(BehaviorVersion::latest());
let s3_config = crate::config::S3ClientConfig::from_sdk_builder(tweak(builder));
captured_user_agent_from(runtime_config, s3_config, captured).await
}
async fn captured_user_agent_from(
runtime_config: RuntimeConfig,
s3_config: crate::config::S3ClientConfig,
captured: CaptureRequestReceiver,
) -> String {
let tm = crate::Client::new(
crate::Config::builder()
.runtime_config(runtime_config)
.s3_config(s3_config)
.build()
.expect("valid config"),
);
let mut handle = tm.download().bucket("foo").key("bar").initiate().unwrap();
let _ = handle.body_mut().next().await;
captured
.expect_request()
.headers()
.get("x-amz-user-agent")
.expect("user agent header is set")
.to_string()
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn preserves_customer_app_name() {
let ua = captured_user_agent(RuntimeConfig::multi_thread_tokio(), |b| {
b.app_name(AppName::new("my-app").unwrap())
})
.await;
assert!(ua.contains("app/my-app"), "app id missing from {ua:?}");
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn emits_transfer_manager_version() {
let ua = captured_user_agent(RuntimeConfig::multi_thread_tokio(), |b| b).await;
let expected = concat!("md/rust-tm#", env!("CARGO_PKG_VERSION"));
assert!(ua.contains(expected), "{expected:?} missing from {ua:?}");
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn emits_our_marker_once() {
let ua = captured_user_agent(RuntimeConfig::default(), |b| b).await;
assert_eq!(ua.matches("md/rust-tm#").count(), 1, "{ua:?}");
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn sdk_config_framework_metadata_joins_our_attribution() {
let (http_client, captured) = capture_request(None);
let sdk_config = aws_types::SdkConfig::builder()
.http_client(http_client)
.region(Region::from_static("us-west-2"))
.credentials_provider(test_credentials())
.behavior_version(BehaviorVersion::latest())
.framework_metadata(framework("first-framework", "1.0"))
.framework_metadata(framework("second-framework", "2.0"))
.build();
let ua = captured_user_agent_from(
RuntimeConfig::multi_thread_tokio(),
crate::config::S3ClientConfig::from(&sdk_config),
captured,
)
.await;
let ours = concat!("md/rust-tm#", env!("CARGO_PKG_VERSION"));
assert!(
ua.contains(ours),
"our attribution was displaced by the framework's: {ua:?}"
);
let first = ua.find("lib/first-framework/1.0");
let second = ua.find("lib/second-framework/2.0");
assert!(
matches!((first, second), (Some(first), Some(second)) if first < second),
"framework metadata missing or out of order in {ua:?}"
);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn repeated_framework_metadata_is_carried_once() {
let ua = captured_user_agent(RuntimeConfig::multi_thread_tokio(), |b| {
b.framework_metadata(framework("some-framework", "1.3"))
.framework_metadata(framework("some-framework", "1.3"))
})
.await;
assert_eq!(ua.matches("lib/some-framework/1.3").count(), 1, "{ua:?}");
}
fn framework(name: &'static str, version: &'static str) -> FrameworkMetadata {
FrameworkMetadata::new(name, Some(Cow::Borrowed(version)))
.expect("valid framework metadata")
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn preserves_user_agent_already_in_config_bag() {
let ua = captured_user_agent(RuntimeConfig::multi_thread_tokio(), |b| {
b.with_test_defaults()
})
.await;
assert!(
ua.contains("api/test-service/0.123"),
"seeded user agent was replaced: {ua:?}"
);
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn emits_s3_transfer_business_metric() {
let ua = captured_user_agent(RuntimeConfig::multi_thread_tokio(), |b| b).await;
let metrics = ua
.split_whitespace()
.find_map(|section| section.strip_prefix("m/"))
.expect("user agent carries a metrics section");
assert!(
metrics.split(',').any(|id| id == "G"),
"S3Transfer metric missing from {metrics:?} in {ua:?}"
);
}
fn provided_client(
http_client: impl aws_sdk_s3::config::HttpClient + 'static,
) -> aws_sdk_s3::Client {
aws_sdk_s3::Client::from_conf(
aws_sdk_s3::config::Builder::default()
.http_client(http_client)
.region(Region::from_static("us-west-2"))
.credentials_provider(test_credentials())
.behavior_version(BehaviorVersion::latest())
.build(),
)
}
async fn download_through(client: aws_sdk_s3::Client) {
let tm = crate::Client::new(
crate::Config::builder()
.runtime_config(RuntimeConfig::multi_thread_tokio())
.part_size(PartSize::Target(ByteSize::new(8 << 20)))
.sdk_client(client)
.build()
.expect("valid config"),
);
let mut handle = tm.download().bucket("foo").key("bar").initiate().unwrap();
let _ = handle.body_mut().next().await;
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn provided_client_gets_attribution() {
let (http_client, captured) = capture_request(None);
download_through(provided_client(http_client)).await;
let ua = captured
.expect_request()
.headers()
.get("x-amz-user-agent")
.expect("user agent header is set")
.to_string();
let ours = concat!("md/rust-tm#", env!("CARGO_PKG_VERSION"));
assert!(ua.contains(ours), "{ours:?} missing from {ua:?}");
let metrics = ua
.split_whitespace()
.find_map(|section| section.strip_prefix("m/"))
.expect("user agent carries a metrics section");
for metric in [BusinessMetric::S3Transfer, BusinessMetric::S3CustomPartSize] {
let id = id_of(metric);
assert!(
metrics.split(',').any(|m| m == id),
"{id} missing from {metrics:?} in {ua:?}"
);
}
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn rebuilding_a_provided_client_keeps_its_configuration() {
let (http_client, captured) = capture_request(None);
let client = aws_sdk_s3::Client::from_conf(
aws_sdk_s3::config::Builder::default()
.http_client(http_client)
.region(Region::from_static("eu-central-1"))
.credentials_provider(test_credentials())
.behavior_version(BehaviorVersion::latest())
.app_name(AppName::new("callers-app").unwrap())
.endpoint_url("https://s3.example.internal")
.force_path_style(true)
.interceptor(MarkHeader)
.build(),
);
download_through(client).await;
let request = captured.expect_request();
assert_eq!(request.method(), "HEAD");
assert_eq!(
request.uri(),
"https://s3.example.internal/foo/bar",
"endpoint or path style lost in the rebuild"
);
let authorization = request
.headers()
.get("authorization")
.expect("request is signed");
assert!(
authorization.contains("ANOTREAL/") && authorization.contains("eu-central-1/s3/"),
"credentials or region lost in the rebuild: {authorization:?}"
);
assert_eq!(
request.headers().get("x-callers-interceptor"),
Some("ran"),
"the caller's interceptor was lost in the rebuild"
);
let ua = request
.headers()
.get("x-amz-user-agent")
.expect("user agent header is set")
.to_string();
assert!(ua.contains("app/callers-app"), "app name lost: {ua:?}");
let ours = concat!("md/rust-tm#", env!("CARGO_PKG_VERSION"));
assert!(ua.contains(ours), "{ours:?} missing from {ua:?}");
}
#[derive(Debug)]
struct MarkHeader;
impl Intercept for MarkHeader {
fn name(&self) -> &'static str {
"MarkHeader"
}
fn modify_before_signing(
&self,
ctx: &mut aws_sdk_s3::config::interceptors::BeforeTransmitInterceptorContextMut<'_>,
_components: &aws_sdk_s3::config::RuntimeComponents,
_cfg: &mut aws_sdk_s3::config::ConfigBag,
) -> Result<(), aws_sdk_s3::error::BoxError> {
ctx.request_mut()
.headers_mut()
.insert("x-callers-interceptor", "ran");
Ok(())
}
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn an_already_instrumented_client_is_not_attributed_twice() {
let (http_client, captured) = capture_request(None);
let first = crate::Client::new(
crate::Config::builder()
.runtime_config(RuntimeConfig::multi_thread_tokio())
.sdk_client(provided_client(http_client))
.build()
.expect("valid config"),
);
download_through(first.handle.s3_client.clone()).await;
let ua = captured
.expect_request()
.headers()
.get("x-amz-user-agent")
.expect("user agent header is set")
.to_string();
assert_eq!(ua.matches("md/rust-tm#").count(), 1, "{ua:?}");
}
#[derive(Debug)]
struct OverrideUserAgent;
impl Intercept for OverrideUserAgent {
fn name(&self) -> &'static str {
"OverrideUserAgent"
}
fn modify_before_signing(
&self,
ctx: &mut aws_sdk_s3::config::interceptors::BeforeTransmitInterceptorContextMut<'_>,
_components: &aws_sdk_s3::config::RuntimeComponents,
_cfg: &mut aws_sdk_s3::config::ConfigBag,
) -> Result<(), aws_sdk_s3::error::BoxError> {
let headers = ctx.request_mut().headers_mut();
headers.insert("user-agent", OWNED_USER_AGENT);
headers.insert("x-amz-user-agent", OWNED_USER_AGENT);
Ok(())
}
}
const OWNED_USER_AGENT: &str = "aws-cli/2.36.23 md/command#s3.cp";
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn a_caller_can_own_both_headers() {
let ua = captured_user_agent(RuntimeConfig::multi_thread_tokio(), |mut b| {
b.push_interceptor(OverrideUserAgent.into_shared());
b
})
.await;
assert_eq!(ua, OWNED_USER_AGENT);
}
fn id_of(metric: BusinessMetric) -> String {
let header = AwsUserAgent::for_tests()
.with_business_metric(metric)
.aws_ua_header();
metric_ids(&header).remove(0)
}
fn metric_ids(ua: &str) -> Vec<String> {
ua.split_whitespace()
.find_map(|section| section.strip_prefix("m/"))
.map(|metrics| metrics.split(',').map(str::to_owned).collect())
.unwrap_or_default()
}
#[derive(Debug)]
struct Sent {
operation: String,
attribution: Vec<BusinessMetric>,
}
#[derive(Debug, Clone, Default)]
struct MetricsRecorder(Arc<Mutex<Vec<Sent>>>);
impl MetricsRecorder {
#[track_caller]
fn take(&self) -> Vec<Sent> {
let sent = std::mem::take(&mut *self.0.lock().unwrap());
assert!(!sent.is_empty(), "no request was sent");
sent
}
}
impl Intercept for MetricsRecorder {
fn name(&self) -> &'static str {
"MetricsRecorder"
}
fn read_before_transmit(
&self,
ctx: &BeforeTransmitInterceptorContextRef<'_>,
_components: &RuntimeComponents,
cfg: &mut ConfigBag,
) -> Result<(), BoxError> {
let operation = cfg.load::<Metadata>().expect("operation metadata").name();
let ua = ctx
.request()
.headers()
.get("x-amz-user-agent")
.unwrap_or_default();
self.0.lock().unwrap().push(Sent {
operation: operation.to_owned(),
attribution: attribution(&metric_ids(ua)),
});
Ok(())
}
}
fn attribution(ids: &[String]) -> Vec<BusinessMetric> {
use BusinessMetric::*;
[
S3CustomPartSize,
S3CustomThroughput,
S3CustomMemoryLimit,
S3OnEc2,
S3FilePath,
S3TransferUploadDirectory,
S3TransferDownloadDirectory,
]
.into_iter()
.filter(|metric| ids.contains(&id_of(metric.clone())))
.collect()
}
fn off_ec2() -> crate::config::Builder {
crate::Config::builder().machine_profile(Some(MachineProfile {
instance_type: None,
vcpus: 4,
ram_bytes: None,
}))
}
fn tm_over(builder: crate::config::Builder, recorder: &MetricsRecorder) -> crate::Client {
let list = mock!(aws_sdk_s3::Client::list_objects_v2).then_output(|| {
ListObjectsV2Output::builder()
.contents(Object::builder().key("a.txt").size(1).build())
.build()
});
let get = mock!(aws_sdk_s3::Client::get_object).then_output(|| {
GetObjectOutput::builder()
.content_length(1)
.content_range("bytes 0-0/1")
.e_tag("test-etag")
.body(ByteStream::from_static(b"x"))
.build()
});
let put = mock!(aws_sdk_s3::Client::put_object)
.then_output(|| PutObjectOutput::builder().build());
let head = crate::operation::download::test_util::fixtures::head_rule(1);
let s3 = mock_client!(
aws_sdk_s3,
RuleMode::MatchAny,
&[list, head, get, put],
|c| c.interceptor(recorder.clone())
);
crate::Client::new(
builder
.runtime_config(RuntimeConfig::multi_thread_tokio())
.sdk_client(s3)
.build()
.expect("valid config"),
)
}
async fn download(tm: &crate::Client) {
let mut handle = tm.download().bucket("foo").key("a.txt").initiate().unwrap();
while handle.body_mut().next().await.is_some() {}
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn configuration_metrics_follow_the_configuration() {
let pool = crate::runtime::buffer_pool::BufferPool::from_capacity(
crate::config::MemoryBudgetConfig::Limit(ByteSize::new(256 * 1024 * 1024)),
None,
Default::default(),
)
.unwrap();
let ec2 = MachineProfile {
instance_type: Some("m6i.large".to_owned()),
vcpus: 2,
ram_bytes: None,
};
let cases = vec![
("defaults", off_ec2(), vec![]),
(
"part size",
off_ec2().part_size(PartSize::Target(ByteSize::new(8 << 20))),
vec![BusinessMetric::S3CustomPartSize],
),
(
"concurrency",
off_ec2().concurrency(ConcurrencyMode::Explicit(4)),
vec![BusinessMetric::S3CustomThroughput],
),
(
"throughput target",
off_ec2().concurrency(ConcurrencyMode::TargetThroughput(
crate::config::TargetThroughput::gbps(25.0).unwrap(),
)),
vec![BusinessMetric::S3CustomThroughput],
),
(
"memory",
off_ec2().memory(MemoryConfig::Explicit(pool)),
vec![BusinessMetric::S3CustomMemoryLimit],
),
(
"ec2",
off_ec2().machine_profile(Some(ec2)),
vec![BusinessMetric::S3OnEc2],
),
];
for (name, builder, expected) in cases {
let recorder = MetricsRecorder::default();
download(&tm_over(builder, &recorder)).await;
assert_eq!(expected, recorder.take()[0].attribution, "{name}");
}
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn upload_objects_emits_upload_directory_and_file_path() {
let recorder = MetricsRecorder::default();
let tm = tm_over(off_ec2(), &recorder);
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("a.txt"), b"x").unwrap();
tm.upload_objects()
.bucket("foo")
.source(dir.path())
.initiate()
.unwrap()
.join()
.await
.unwrap();
for sent in recorder.take() {
assert_eq!(
vec![
BusinessMetric::S3FilePath,
BusinessMetric::S3TransferUploadDirectory
],
sent.attribution,
"{}",
sent.operation
);
}
let body = crate::io::InputStream::from(bytes::Bytes::from_static(b"x"));
tm.upload()
.bucket("foo")
.key("k")
.body(body)
.initiate()
.unwrap()
.join()
.await
.unwrap();
for sent in recorder.take() {
assert_eq!(Vec::<BusinessMetric>::new(), sent.attribution);
}
}
#[cfg_attr(miri, ignore)]
#[tokio::test]
async fn download_objects_emits_download_directory_and_file_path_on_objects() {
let recorder = MetricsRecorder::default();
let tm = tm_over(off_ec2(), &recorder);
let dir = tempfile::tempdir().unwrap();
tm.download_objects()
.bucket("foo")
.destination(dir.path())
.initiate()
.unwrap()
.join()
.await
.unwrap();
let sent = recorder.take();
let (listings, objects): (Vec<_>, Vec<_>) = sent
.iter()
.partition(|sent| sent.operation == "ListObjectsV2");
assert!(!listings.is_empty() && !objects.is_empty(), "{sent:?}");
for sent in listings {
assert_eq!(
vec![BusinessMetric::S3TransferDownloadDirectory],
sent.attribution
);
}
for sent in objects {
assert_eq!(
vec![
BusinessMetric::S3FilePath,
BusinessMetric::S3TransferDownloadDirectory
],
sent.attribution
);
}
download(&tm).await;
for sent in recorder.take() {
assert_eq!(Vec::<BusinessMetric>::new(), sent.attribution);
}
}
}