use crate::config::environment_names::logging::otlp as env_otlp;
use crate::metrics::MetricsRegistry;
use opentelemetry_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest;
use opentelemetry_proto::tonic::collector::metrics::v1::metrics_service_client::MetricsServiceClient;
use opentelemetry_proto::tonic::common::v1::{AnyValue, InstrumentationScope, KeyValue, any_value};
use opentelemetry_proto::tonic::metrics::v1::{
AggregationTemporality, DataPointFlags, Gauge, Histogram, HistogramDataPoint, Metric,
NumberDataPoint, ResourceMetrics, ScopeMetrics, Sum, Summary, SummaryDataPoint, metric,
number_data_point, summary_data_point::ValueAtQuantile,
};
use opentelemetry_proto::tonic::resource::v1::Resource;
use prometheus::proto::{MetricFamily, MetricType};
use std::collections::HashMap;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use tokio_util::sync::CancellationToken;
const SCOPE_NAME: &str = "dynamo.runtime.metrics";
pub fn to_resource_metrics(
collected: &crate::metrics::CollectedFamilies,
resource_attrs: &[KeyValue],
start_time: SystemTime,
) -> ResourceMetrics {
let families = &collected.families;
let now = unix_nanos(SystemTime::now());
let start = unix_nanos(start_time);
let created: HashMap<(String, Vec<(String, String)>), u64> = families
.iter()
.filter(|f| f.name().ends_with("_created"))
.flat_map(|f| {
let parent = f.name().trim_end_matches("_created").to_string();
f.get_metric().iter().map(move |m| {
(
(parent.clone(), label_key(m)),
(m.get_gauge().value() * 1_000_000_000.0) as u64,
)
})
})
.collect();
let metrics = families
.iter()
.filter(|family| !family.name().ends_with("_created"))
.filter_map(|family| {
to_metric(
family,
start,
now,
&created,
&collected.units,
&collected.prom_types,
)
})
.collect();
ResourceMetrics {
resource: Some(Resource {
attributes: resource_attrs.to_vec(),
dropped_attributes_count: 0,
entity_refs: Vec::new(),
}),
scope_metrics: vec![ScopeMetrics {
scope: Some(InstrumentationScope {
name: SCOPE_NAME.to_string(),
version: env!("CARGO_PKG_VERSION").to_string(),
attributes: Vec::new(),
dropped_attributes_count: 0,
}),
metrics,
schema_url: String::new(),
}],
schema_url: String::new(),
}
}
fn label_key(metric: &prometheus::proto::Metric) -> Vec<(String, String)> {
let mut labels: Vec<(String, String)> = metric
.get_label()
.iter()
.map(|l| (l.name().to_string(), l.value().to_string()))
.collect();
labels.sort();
labels
}
type CreatedTimes = HashMap<(String, Vec<(String, String)>), u64>;
fn ucum(word: &str) -> &str {
match word {
"days" => "d",
"hours" => "h",
"minutes" => "min",
"seconds" => "s",
"milliseconds" => "ms",
"microseconds" => "us",
"nanoseconds" => "ns",
"bytes" => "By",
"kibibytes" => "KiBy",
"mebibytes" => "MiBy",
"gibibytes" => "GiBy",
"tebibytes" => "TiBy",
"kilobytes" => "kBy",
"megabytes" => "MBy",
"gigabytes" => "GBy",
"terabytes" => "TBy",
"meters" => "m",
"volts" => "V",
"amperes" => "A",
"joules" => "J",
"watts" => "W",
"grams" => "g",
"celsius" => "Cel",
"hertz" => "Hz",
"percent" => "%",
other => other,
}
}
fn unit_for(family: &MetricFamily, units: &HashMap<String, String>) -> String {
units
.get(family.name())
.map(|word| ucum(word).to_string())
.unwrap_or_default()
}
fn type_metadata(prom_type: &str) -> KeyValue {
KeyValue {
key: "prometheus.type".to_string(),
value: Some(AnyValue {
value: Some(any_value::Value::StringValue(prom_type.to_string())),
}),
}
}
fn prometheus_type_name(family: &MetricFamily) -> &'static str {
match family.get_field_type() {
MetricType::COUNTER => "counter",
MetricType::GAUGE => "gauge",
MetricType::HISTOGRAM => "histogram",
MetricType::SUMMARY => "summary",
MetricType::UNTYPED => "unknown",
}
}
fn to_metric(
family: &MetricFamily,
start: u64,
now: u64,
created: &CreatedTimes,
units: &HashMap<String, String>,
prom_types: &HashMap<String, String>,
) -> Option<Metric> {
let prom_type = prom_types
.get(family.name())
.map(String::as_str)
.unwrap_or_else(|| prometheus_type_name(family));
if prom_type == "info" || prom_type == "stateset" {
return Some(Metric {
name: family.name().to_string(),
description: family.help().to_string(),
unit: unit_for(family, units),
metadata: vec![type_metadata(prom_type)],
data: Some(metric::Data::Sum(Sum {
data_points: family
.get_metric()
.iter()
.map(|m| number_point(m, m.get_gauge().value(), 0, now))
.collect(),
aggregation_temporality: AggregationTemporality::Cumulative as i32,
is_monotonic: false,
})),
});
}
let data = match family.get_field_type() {
MetricType::COUNTER => metric::Data::Sum(Sum {
data_points: family
.get_metric()
.iter()
.map(|m| {
let start = start_for(family, m, start, created);
number_point(m, m.get_counter().value(), start, now)
})
.collect(),
aggregation_temporality: AggregationTemporality::Cumulative as i32,
is_monotonic: true,
}),
MetricType::GAUGE => metric::Data::Gauge(Gauge {
data_points: family
.get_metric()
.iter()
.map(|m| number_point(m, m.get_gauge().value(), 0, now))
.collect(),
}),
MetricType::HISTOGRAM => metric::Data::Histogram(Histogram {
data_points: family
.get_metric()
.iter()
.map(|m| histogram_point(m, start_for(family, m, start, created), now))
.collect(),
aggregation_temporality: AggregationTemporality::Cumulative as i32,
}),
MetricType::SUMMARY => metric::Data::Summary(Summary {
data_points: family
.get_metric()
.iter()
.map(|m| summary_point(m, start_for(family, m, start, created), now))
.collect(),
}),
MetricType::UNTYPED => metric::Data::Gauge(Gauge {
data_points: family
.get_metric()
.iter()
.map(|m| number_point(m, m.untyped.value(), 0, now))
.collect(),
}),
};
Some(Metric {
name: family.name().to_string(),
description: family.help().to_string(),
unit: unit_for(family, units),
metadata: vec![type_metadata(prom_type)],
data: Some(data),
})
}
fn observed_at(metric: &prometheus::proto::Metric, now: u64) -> u64 {
match metric.timestamp_ms() {
ms if ms <= 0 => now,
ms => (ms as u64).saturating_mul(1_000_000),
}
}
fn start_for(
family: &MetricFamily,
metric: &prometheus::proto::Metric,
fallback: u64,
created: &CreatedTimes,
) -> u64 {
if created.is_empty() {
return fallback;
}
created
.get(&(family.name().to_string(), label_key(metric)))
.copied()
.unwrap_or(fallback)
}
fn number_point(
metric: &prometheus::proto::Metric,
value: f64,
start: u64,
now: u64,
) -> NumberDataPoint {
let (flags, value) = if value.is_nan() {
(DataPointFlags::NoRecordedValueMask as u32, None)
} else {
(0, Some(number_data_point::Value::AsDouble(value)))
};
NumberDataPoint {
attributes: attributes(metric),
start_time_unix_nano: start,
time_unix_nano: observed_at(metric, now),
exemplars: Vec::new(),
flags,
value,
}
}
fn histogram_point(metric: &prometheus::proto::Metric, start: u64, now: u64) -> HistogramDataPoint {
let h = metric.get_histogram();
let mut bounds = Vec::new();
let mut counts = Vec::new();
let mut previous = 0u64;
for bucket in h.get_bucket() {
let cumulative = bucket.cumulative_count();
counts.push(cumulative.saturating_sub(previous));
previous = cumulative;
if bucket.upper_bound().is_finite() {
bounds.push(bucket.upper_bound());
}
}
if counts.len() == bounds.len() {
counts.push(h.sample_count().saturating_sub(previous));
}
HistogramDataPoint {
attributes: attributes(metric),
start_time_unix_nano: start,
time_unix_nano: observed_at(metric, now),
count: h.sample_count(),
sum: (!h.sample_sum().is_nan()).then(|| h.sample_sum()),
bucket_counts: counts,
explicit_bounds: bounds,
exemplars: Vec::new(),
flags: 0,
min: None,
max: None,
}
}
fn summary_point(metric: &prometheus::proto::Metric, start: u64, now: u64) -> SummaryDataPoint {
let s = metric.get_summary();
SummaryDataPoint {
attributes: attributes(metric),
start_time_unix_nano: start,
time_unix_nano: observed_at(metric, now),
count: s.sample_count(),
sum: if s.sample_sum().is_nan() {
0.0
} else {
s.sample_sum()
},
quantile_values: s
.get_quantile()
.iter()
.map(|q| ValueAtQuantile {
quantile: q.quantile(),
value: q.value(),
})
.collect(),
flags: 0,
}
}
fn attributes(metric: &prometheus::proto::Metric) -> Vec<KeyValue> {
metric
.get_label()
.iter()
.map(|label| KeyValue {
key: label.name().to_string(),
value: Some(AnyValue {
value: Some(any_value::Value::StringValue(label.value().to_string())),
}),
})
.collect()
}
fn unix_nanos(time: SystemTime) -> u64 {
time.duration_since(UNIX_EPOCH)
.map(|d| d.as_nanos() as u64)
.unwrap_or_default()
}
pub struct ExportConfig {
pub endpoint: String,
pub interval: Duration,
pub service_name: String,
pub headers: Vec<(String, String)>,
pub resource_attributes: Vec<(String, String)>,
}
pub(crate) fn parse_key_value_list(raw: &str) -> Vec<(String, String)> {
raw.split(',')
.filter_map(|pair| pair.split_once('='))
.map(|(k, v)| (k.trim().to_string(), v.trim().to_string()))
.filter(|(k, _)| !k.is_empty())
.collect()
}
impl ExportConfig {
pub fn from_env() -> anyhow::Result<Option<Self>> {
let Ok(exporter) = std::env::var(env_otlp::OTEL_METRICS_EXPORTER) else {
return Ok(None);
};
if !exporter.trim().eq_ignore_ascii_case("otlp") {
return Ok(None);
}
let metrics_protocol = std::env::var(env_otlp::OTEL_EXPORTER_OTLP_METRICS_PROTOCOL).ok();
let generic_protocol = std::env::var(env_otlp::OTEL_EXPORTER_OTLP_PROTOCOL).ok();
let configured = [
(
metrics_protocol.as_deref(),
env_otlp::OTEL_EXPORTER_OTLP_METRICS_PROTOCOL,
),
(
generic_protocol.as_deref(),
env_otlp::OTEL_EXPORTER_OTLP_PROTOCOL,
),
]
.into_iter()
.find_map(|(value, source)| {
value
.map(str::trim)
.filter(|value| !value.is_empty())
.map(|value| (value, source))
});
if let Some((value, source)) = configured
&& !value.eq_ignore_ascii_case("grpc")
{
anyhow::bail!(
"{source}={value} is not supported for metrics; only grpc is implemented. \
Set {}=grpc to override.",
env_otlp::OTEL_EXPORTER_OTLP_METRICS_PROTOCOL
);
}
let protocol = crate::logging::OtlpProtocol::Grpc;
let headers = std::env::var(env_otlp::OTEL_EXPORTER_OTLP_METRICS_HEADERS)
.or_else(|_| std::env::var(env_otlp::OTEL_EXPORTER_OTLP_HEADERS))
.map(|raw| parse_key_value_list(&raw))
.unwrap_or_default();
let resource_attributes = std::env::var(env_otlp::OTEL_RESOURCE_ATTRIBUTES)
.map(|raw| parse_key_value_list(&raw))
.unwrap_or_default();
Ok(Some(Self {
headers,
resource_attributes,
endpoint: crate::logging::resolve_otlp_endpoint(
protocol,
std::env::var(env_otlp::OTEL_EXPORTER_OTLP_METRICS_ENDPOINT).ok(),
std::env::var(env_otlp::OTEL_EXPORTER_OTLP_ENDPOINT).ok(),
"/v1/metrics",
),
interval: export_interval(),
service_name: crate::logging::get_service_name(),
}))
}
}
fn export_interval() -> Duration {
const DEFAULT_MS: u64 = 60_000;
let millis = std::env::var(env_otlp::OTEL_METRIC_EXPORT_INTERVAL)
.ok()
.and_then(|value| value.trim().parse::<u64>().ok())
.filter(|millis| *millis > 0)
.unwrap_or(DEFAULT_MS);
Duration::from_millis(millis)
}
fn attributes_for(config: &ExportConfig) -> Vec<KeyValue> {
let mut attrs: Vec<KeyValue> = config
.resource_attributes
.iter()
.filter(|(key, _)| key != "service.name")
.map(|(key, value)| KeyValue {
key: key.clone(),
value: Some(AnyValue {
value: Some(any_value::Value::StringValue(value.clone())),
}),
})
.collect();
attrs.push(KeyValue {
key: "service.name".to_string(),
value: Some(AnyValue {
value: Some(any_value::Value::StringValue(config.service_name.clone())),
}),
});
attrs
}
pub async fn run(registry: MetricsRegistry, config: ExportConfig, cancel: CancellationToken) {
let attrs = attributes_for(&config);
let start_time = SystemTime::now();
const MAX_RPC_DEADLINE: Duration = Duration::from_secs(30);
let rpc_deadline = config.interval.min(MAX_RPC_DEADLINE);
let mut ticker = tokio::time::interval(config.interval);
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
let mut client: Option<MetricsServiceClient<tonic::transport::Channel>> = None;
loop {
tokio::select! {
_ = cancel.cancelled() => break,
_ = ticker.tick() => {}
}
if client.is_none() {
let connecting = async {
tonic::transport::Endpoint::from_shared(config.endpoint.clone())?
.connect_timeout(rpc_deadline)
.timeout(rpc_deadline)
.connect()
.await
.map_err(anyhow::Error::from)
};
let connected = tokio::select! {
_ = cancel.cancelled() => break,
result = connecting => result,
};
match connected {
Ok(channel) => client = Some(MetricsServiceClient::new(channel)),
Err(error) => {
tracing::warn!(%error, endpoint = %config.endpoint, "OTLP metrics connect failed");
continue;
}
}
}
let collector = registry.clone();
let collected =
tokio::task::spawn_blocking(move || collector.metric_families_combined()).await;
let families = match collected {
Ok(Ok(families)) => families,
Ok(Err(error)) => {
tracing::warn!(%error, "OTLP metrics collection failed");
continue;
}
Err(error) => {
tracing::warn!(%error, "OTLP metrics collection task failed");
continue;
}
};
let request = ExportMetricsServiceRequest {
resource_metrics: vec![to_resource_metrics(&families, &attrs, start_time)],
};
let mut request = tonic::Request::new(request);
for (key, value) in &config.headers {
match (
tonic::metadata::MetadataKey::from_bytes(key.as_bytes()),
tonic::metadata::MetadataValue::try_from(value),
) {
(Ok(key), Ok(value)) => {
request.metadata_mut().insert(key, value);
}
_ => {
tracing::warn!(header = %key, "skipping unusable OTLP header");
}
}
}
if let Some(connected) = client.as_mut() {
let exported = tokio::select! {
_ = cancel.cancelled() => break,
result = connected.export(request) => result,
};
match exported {
Ok(response) => {
if let Some(partial) = response.into_inner().partial_success
&& partial.rejected_data_points > 0
{
tracing::warn!(
rejected_data_points = partial.rejected_data_points,
error_message = %partial.error_message,
endpoint = %config.endpoint,
"OTLP collector rejected part of the export"
);
}
}
Err(error) => {
tracing::warn!(%error, endpoint = %config.endpoint, "OTLP metrics export failed");
client = None;
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::metrics::prom_typed::{TypedFamily, build_families};
fn collected_from(
built: Vec<crate::metrics::prom_typed::BuiltFamily>,
) -> crate::metrics::CollectedFamilies {
let mut units = HashMap::new();
let mut prom_types = HashMap::new();
let mut families = Vec::new();
for b in built {
if !b.unit.is_empty() {
units.insert(b.family.name().to_string(), b.unit);
}
prom_types.insert(b.family.name().to_string(), b.prom_type);
families.push(b.family);
}
crate::metrics::CollectedFamilies {
families,
units,
prom_types,
}
}
fn export(typed_json: &str) -> Vec<Metric> {
let typed: Vec<TypedFamily> = serde_json::from_str(typed_json).expect("typed");
let collected = collected_from(build_families(typed));
let rm = to_resource_metrics(&collected, &[], UNIX_EPOCH);
rm.scope_metrics.into_iter().next().expect("scope").metrics
}
#[test]
fn histogram_buckets_are_de_cumulated() {
let metrics = export(
r#"[{"name":"d_seconds","help":"","type":"histogram","samples":[
{"name":"d_seconds_bucket","labels":{"le":"0.1"},"value":"2"},
{"name":"d_seconds_bucket","labels":{"le":"0.5"},"value":"5"},
{"name":"d_seconds_bucket","labels":{"le":"+Inf"},"value":"9"},
{"name":"d_seconds_sum","labels":{},"value":"1.5"},
{"name":"d_seconds_count","labels":{},"value":"9"}]}]"#,
);
let Some(metric::Data::Histogram(h)) = &metrics[0].data else {
panic!("expected histogram, got {:?}", metrics[0].data);
};
let point = &h.data_points[0];
assert_eq!(point.explicit_bounds, vec![0.1, 0.5]);
assert_eq!(point.bucket_counts, vec![2, 3, 4]);
assert_eq!(point.bucket_counts.len(), point.explicit_bounds.len() + 1);
assert_eq!(point.count, 9);
assert_eq!(point.sum, Some(1.5));
assert_eq!(
h.aggregation_temporality,
AggregationTemporality::Cumulative as i32
);
}
#[test]
fn counter_maps_to_monotonic_cumulative_sum_with_help() {
let metrics = export(
r#"[{"name":"d_requests","help":"Total requests","type":"counter","samples":[
{"name":"d_requests_total","labels":{"model":"a"},"value":"17"}]}]"#,
);
assert_eq!(
metrics[0].name, "d_requests",
"spec: the Prometheus name is used as-is and SHOULD NOT be altered"
);
assert_eq!(metrics[0].description, "Total requests");
let Some(metric::Data::Sum(sum)) = &metrics[0].data else {
panic!("expected sum, got {:?}", metrics[0].data);
};
assert!(sum.is_monotonic);
assert_eq!(
sum.aggregation_temporality,
AggregationTemporality::Cumulative as i32
);
assert_eq!(
sum.data_points[0].value,
Some(number_data_point::Value::AsDouble(17.0))
);
assert_eq!(sum.data_points[0].attributes[0].key, "model");
}
#[test]
fn config_is_opt_in_and_rejects_unsupported_protocol() {
temp_env::with_vars([(env_otlp::OTEL_METRICS_EXPORTER, None::<&str>)], || {
assert!(ExportConfig::from_env().expect("disabled").is_none())
});
temp_env::with_vars(
[
(env_otlp::OTEL_METRICS_EXPORTER, Some("otlp")),
(
env_otlp::OTEL_EXPORTER_OTLP_METRICS_ENDPOINT,
Some("http://collector:4317"),
),
(env_otlp::OTEL_METRIC_EXPORT_INTERVAL, None),
],
|| {
let config = ExportConfig::from_env()
.expect("enabled")
.expect("some config");
assert_eq!(config.endpoint, "http://collector:4317");
assert_eq!(config.interval, Duration::from_millis(60_000));
},
);
temp_env::with_vars(
[
(env_otlp::OTEL_METRICS_EXPORTER, Some("otlp")),
(
env_otlp::OTEL_EXPORTER_OTLP_METRICS_PROTOCOL,
Some("http/protobuf"),
),
],
|| assert!(ExportConfig::from_env().is_err()),
);
temp_env::with_vars(
[
(env_otlp::OTEL_METRICS_EXPORTER, Some("otlp")),
(
env_otlp::OTEL_EXPORTER_OTLP_METRICS_PROTOCOL,
Some("htp/protbuf"),
),
],
|| assert!(ExportConfig::from_env().is_err()),
);
temp_env::with_vars(
[
(env_otlp::OTEL_METRICS_EXPORTER, Some("otlp")),
(env_otlp::OTEL_EXPORTER_OTLP_METRICS_PROTOCOL, None),
(env_otlp::OTEL_EXPORTER_OTLP_PROTOCOL, Some("http/protobuf")),
],
|| assert!(ExportConfig::from_env().is_err()),
);
}
#[test]
fn native_dynamo_families_survive_the_mapping() {
let registry = MetricsRegistry::new();
let counter =
prometheus::IntCounter::new("dynamo_requests_total", "Requests").expect("counter");
counter.inc_by(7);
registry
.get_prometheus_registry()
.register(Box::new(counter))
.expect("register counter");
let histogram = prometheus::Histogram::with_opts(
prometheus::HistogramOpts::new("dynamo_latency_seconds", "Latency")
.buckets(vec![0.1, 1.0]),
)
.expect("histogram");
histogram.observe(0.5);
registry
.get_prometheus_registry()
.register(Box::new(histogram))
.expect("register histogram");
let collected = registry.metric_families_combined().expect("combined");
let metrics = to_resource_metrics(&collected, &[], UNIX_EPOCH)
.scope_metrics
.into_iter()
.next()
.expect("scope")
.metrics;
let by_name = |n: &str| metrics.iter().find(|m| m.name == n).cloned();
let Some(metric::Data::Sum(sum)) = by_name("dynamo_requests_total").expect("counter").data
else {
panic!("counter should map to a Sum");
};
assert!(sum.is_monotonic);
assert_eq!(
sum.data_points[0].value,
Some(number_data_point::Value::AsDouble(7.0))
);
let Some(metric::Data::Histogram(h)) =
by_name("dynamo_latency_seconds").expect("histogram").data
else {
panic!("histogram should map to a Histogram");
};
assert_eq!(h.data_points[0].count, 1);
assert_eq!(h.data_points[0].sum, Some(0.5));
}
#[test]
fn untyped_family_maps_to_a_gauge() {
let mut family = MetricFamily::new();
family.set_name("legacy_metric".to_string());
family.set_field_type(MetricType::UNTYPED);
let mut metric = prometheus::proto::Metric::new();
let mut untyped = prometheus::proto::Untyped::new();
untyped.set_value(3.5);
metric.untyped = Some(untyped).into();
family.mut_metric().push(metric);
let collected = crate::metrics::CollectedFamilies {
families: vec![family],
units: HashMap::new(),
prom_types: HashMap::new(),
};
let metrics = to_resource_metrics(&collected, &[], UNIX_EPOCH)
.scope_metrics
.into_iter()
.next()
.expect("scope")
.metrics;
assert_eq!(metrics.len(), 1, "UNTYPED family was dropped");
let Some(metric::Data::Gauge(g)) = &metrics[0].data else {
panic!("expected gauge, got {:?}", metrics[0].data);
};
assert_eq!(
g.data_points[0].value,
Some(number_data_point::Value::AsDouble(3.5))
);
}
#[cfg(feature = "integration")]
#[tokio::test]
async fn exporter_is_waited_for_and_releases_on_phase_one() {
use crate::distributed::distributed_test_utils::create_test_drt_async;
use tokio::net::TcpListener;
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
let port = listener.local_addr().expect("addr").port();
temp_env::async_with_vars(
[
(env_otlp::OTEL_METRICS_EXPORTER, Some("otlp")),
(
env_otlp::OTEL_EXPORTER_OTLP_METRICS_ENDPOINT,
Some(format!("http://127.0.0.1:{port}").as_str()),
),
(env_otlp::OTEL_METRIC_EXPORT_INTERVAL, Some("600000")),
],
async {
let drt = create_test_drt_async().await;
let tracker = drt.runtime().graceful_shutdown_tracker();
assert!(
tracker.get_count() > 0,
"exporter did not register for graceful shutdown"
);
drt.runtime().shutdown();
let released = tokio::time::timeout(Duration::from_secs(10), async {
while tracker.get_count() > 0 {
tokio::time::sleep(Duration::from_millis(50)).await;
}
})
.await;
assert!(
released.is_ok(),
"exporter never released its shutdown guard; Phase 2 would block until timeout"
);
},
)
.await;
}
#[test]
fn headers_and_resource_attributes_are_read_from_the_environment() {
temp_env::with_vars(
[
(env_otlp::OTEL_METRICS_EXPORTER, Some("otlp")),
(env_otlp::OTEL_EXPORTER_OTLP_ENDPOINT, Some("http://c:4317")),
(env_otlp::OTEL_METRIC_EXPORT_INTERVAL, None),
(
env_otlp::OTEL_EXPORTER_OTLP_HEADERS,
Some("authorization=Bearer abc==,x-key=v"),
),
(
env_otlp::OTEL_RESOURCE_ATTRIBUTES,
Some("deployment.environment=prod,service.name=ignored"),
),
],
|| {
let config = ExportConfig::from_env().expect("enabled").expect("some");
assert_eq!(
config.headers,
vec![
("authorization".to_string(), "Bearer abc==".to_string()),
("x-key".to_string(), "v".to_string()),
]
);
assert_eq!(
config.resource_attributes,
vec![
("deployment.environment".to_string(), "prod".to_string()),
("service.name".to_string(), "ignored".to_string()),
]
);
let empty = crate::metrics::CollectedFamilies {
families: Vec::new(),
units: HashMap::new(),
prom_types: HashMap::new(),
};
let rm = to_resource_metrics(&empty, &attributes_for(&config), UNIX_EPOCH);
let keys: Vec<&str> = rm
.resource
.as_ref()
.expect("resource")
.attributes
.iter()
.map(|kv| kv.key.as_str())
.collect();
assert_eq!(keys, vec!["deployment.environment", "service.name"]);
},
);
}
#[test]
fn metrics_headers_replace_the_generic_ones() {
temp_env::with_vars(
[
(env_otlp::OTEL_METRICS_EXPORTER, Some("otlp")),
(env_otlp::OTEL_EXPORTER_OTLP_ENDPOINT, Some("http://c:4317")),
(env_otlp::OTEL_METRIC_EXPORT_INTERVAL, None),
(env_otlp::OTEL_EXPORTER_OTLP_HEADERS, Some("generic=1")),
(
env_otlp::OTEL_EXPORTER_OTLP_METRICS_HEADERS,
Some("specific=2"),
),
],
|| {
let config = ExportConfig::from_env().expect("enabled").expect("some");
assert_eq!(
config.headers,
vec![("specific".to_string(), "2".to_string())]
);
},
);
}
#[cfg(feature = "integration")]
#[tokio::test]
async fn export_starts_without_the_system_status_server() {
use crate::distributed::distributed_test_utils::create_test_drt_async;
use tokio::net::TcpListener;
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
let port = listener.local_addr().expect("addr").port();
let connected = temp_env::async_with_vars(
[
(env_otlp::OTEL_METRICS_EXPORTER, Some("otlp")),
(
env_otlp::OTEL_EXPORTER_OTLP_METRICS_ENDPOINT,
Some(format!("http://127.0.0.1:{port}").as_str()),
),
(env_otlp::OTEL_METRIC_EXPORT_INTERVAL, Some("100")),
],
async {
let _drt = create_test_drt_async().await;
tokio::time::timeout(Duration::from_secs(10), listener.accept())
.await
.is_ok()
},
)
.await;
assert!(
connected,
"exporter never dialled the collector with the system status server disabled"
);
}
#[test]
fn histogram_without_a_count_is_dropped() {
let metrics = export(
r#"[{"name":"d_ok","help":"ok","type":"histogram","samples":[
{"name":"d_ok_bucket","labels":{"le":"1"},"value":"2"},
{"name":"d_ok_sum","labels":{},"value":"1.0"},
{"name":"d_ok_count","labels":{},"value":"2"}]},
{"name":"d_missing","help":"missing","type":"histogram","samples":[
{"name":"d_missing_bucket","labels":{"le":"1"},"value":"2"},
{"name":"d_missing_sum","labels":{},"value":"1.0"}]}]"#,
);
let names: Vec<&str> = metrics.iter().map(|m| m.name.as_str()).collect();
assert!(names.contains(&"d_ok"), "got {names:?}");
assert!(
!names.contains(&"d_missing"),
"a histogram with no _count must be dropped: {names:?}"
);
let zero = export(
r#"[{"name":"d_zero","help":"z","type":"histogram","samples":[
{"name":"d_zero_bucket","labels":{"le":"1"},"value":"0"},
{"name":"d_zero_count","labels":{},"value":"0"}]}]"#,
);
assert_eq!(zero.len(), 1, "a zero count must still be exported");
}
#[test]
fn unit_metadata_is_carried_and_translated() {
let collected = collected_from(build_families(
serde_json::from_str(
r#"[{"name":"d_latency_seconds","help":"L","type":"gauge","unit":"seconds",
"samples":[{"name":"d_latency_seconds","labels":{},"value":"0.5"}]},
{"name":"d_widgets","help":"W","type":"gauge","unit":"widgets",
"samples":[{"name":"d_widgets","labels":{},"value":"2"}]},
{"name":"d_plain","help":"P","type":"gauge",
"samples":[{"name":"d_plain","labels":{},"value":"1"}]}]"#,
)
.expect("fixture"),
));
let metrics = to_resource_metrics(&collected, &[], UNIX_EPOCH)
.scope_metrics
.into_iter()
.next()
.expect("scope")
.metrics;
let unit_of = |n: &str| {
metrics
.iter()
.find(|m| m.name == n)
.unwrap_or_else(|| panic!("{n} missing"))
.unit
.clone()
};
assert_eq!(unit_of("d_latency_seconds"), "s", "translated to UCUM");
assert_eq!(unit_of("d_widgets"), "widgets");
assert_eq!(unit_of("d_plain"), "", "no UNIT metadata, no unit");
assert!(metrics.iter().any(|m| m.name == "d_latency_seconds"));
}
#[test]
fn metric_names_are_not_rewritten() {
let metrics = export(
r#"[{"name":"d_latency_seconds","help":"Latency","type":"gauge","samples":[
{"name":"d_latency_seconds","labels":{},"value":"0.5"}]},
{"name":"d_request_bytes","help":"Bytes","type":"counter","samples":[
{"name":"d_request_bytes_total","labels":{},"value":"9"}]}]"#,
);
let names: Vec<&str> = metrics.iter().map(|m| m.name.as_str()).collect();
assert!(names.contains(&"d_latency_seconds"), "got {names:?}");
assert!(names.contains(&"d_request_bytes"), "got {names:?}");
let latency = metrics
.iter()
.find(|m| m.name == "d_latency_seconds")
.unwrap();
assert_eq!(latency.metadata[0].key, "prometheus.type");
}
#[test]
fn nan_becomes_no_recorded_value() {
let metrics = export(
r#"[{"name":"d_level","help":"Level","type":"gauge","samples":[
{"name":"d_level","labels":{},"value":"NaN"}]},
{"name":"d_latency","help":"Latency","type":"histogram","samples":[
{"name":"d_latency_bucket","labels":{"le":"1"},"value":"0"},
{"name":"d_latency_sum","labels":{},"value":"NaN"},
{"name":"d_latency_count","labels":{},"value":"0"}]}]"#,
);
let Some(metric::Data::Gauge(g)) = &metrics
.iter()
.find(|m| m.name == "d_level")
.expect("gauge")
.data
else {
panic!("expected gauge");
};
assert_eq!(
g.data_points[0].flags,
DataPointFlags::NoRecordedValueMask as u32
);
assert!(
g.data_points[0].value.is_none(),
"the value must be unset, not a NaN"
);
let Some(metric::Data::Histogram(h)) = &metrics
.iter()
.find(|m| m.name == "d_latency")
.expect("histogram")
.data
else {
panic!("expected histogram");
};
assert!(
h.data_points[0].sum.is_none(),
"an unrecorded histogram sum must be unset rather than NaN"
);
}
#[test]
fn created_becomes_the_start_time_and_gauges_carry_none() {
let metrics = export(
r#"[{"name":"d_requests","help":"Requests","type":"counter","samples":[
{"name":"d_requests_total","labels":{"model":"a"},"value":"17"},
{"name":"d_requests_created","labels":{"model":"a"},"value":"1700000000"}]},
{"name":"d_queue","help":"Queue","type":"gauge","samples":[
{"name":"d_queue","labels":{},"value":"3"}]}]"#,
);
assert!(
!metrics.iter().any(|m| m.name.ends_with("_created")),
"_created must not be exported as a metric: {:?}",
metrics.iter().map(|m| &m.name).collect::<Vec<_>>()
);
let counter = metrics
.iter()
.find(|m| m.name == "d_requests")
.expect("counter");
let Some(metric::Data::Sum(sum)) = &counter.data else {
panic!("expected sum");
};
assert_eq!(
sum.data_points[0].start_time_unix_nano, 1_700_000_000_000_000_000,
"the counter must start at its _created time"
);
let gauge = metrics.iter().find(|m| m.name == "d_queue").expect("gauge");
let Some(metric::Data::Gauge(g)) = &gauge.data else {
panic!("expected gauge");
};
assert_eq!(
g.data_points[0].start_time_unix_nano, 0,
"spec: gauges carry no start time"
);
}
#[test]
fn sample_timestamps_survive_instead_of_the_export_instant() {
let metrics = export(
r#"[{"name":"engine_queue","help":"Queue","type":"gauge","samples":[
{"name":"engine_queue","labels":{},"value":"4","timestamp":1700000000.0}]},
{"name":"engine_fresh","help":"Fresh","type":"gauge","samples":[
{"name":"engine_fresh","labels":{},"value":"1"}]}]"#,
);
let point = |name: &str| {
let m = metrics.iter().find(|m| m.name == name).expect("metric");
let Some(metric::Data::Gauge(g)) = &m.data else {
panic!("expected gauge");
};
g.data_points[0].time_unix_nano
};
assert_eq!(
point("engine_queue"),
1_700_000_000_000_000_000,
"a carried timestamp must be reported as the observation time"
);
assert_ne!(
point("engine_fresh"),
0,
"a sample without one still gets the export instant"
);
}
#[test]
fn info_family_keeps_its_rendered_name() {
let metrics = export(
r#"[{"name":"example_build","help":"Build info","type":"info","samples":[
{"name":"example_build_info","labels":{"version":"1.2.3"},"value":"1"}]}]"#,
);
assert_eq!(metrics[0].name, "example_build_info");
let Some(metric::Data::Sum(sum)) = &metrics[0].data else {
panic!("expected a Sum, got {:?}", metrics[0].data);
};
assert!(!sum.is_monotonic, "info is a non-monotonic sum");
assert_eq!(sum.data_points[0].attributes[0].key, "version");
assert_eq!(metrics[0].metadata[0].key, "prometheus.type");
let Some(any_value::Value::StringValue(t)) =
&metrics[0].metadata[0].value.as_ref().unwrap().value
else {
panic!("expected a string");
};
assert_eq!(t, "info", "the original type word, not the proto collapse");
}
#[test]
fn summary_quantiles_survive() {
let metrics = export(
r#"[{"name":"d_pause_seconds","help":"","type":"summary","samples":[
{"name":"d_pause_seconds","labels":{"quantile":"0.99"},"value":"0.2"},
{"name":"d_pause_seconds_sum","labels":{},"value":"12.5"},
{"name":"d_pause_seconds_count","labels":{},"value":"300"}]}]"#,
);
let Some(metric::Data::Summary(s)) = &metrics[0].data else {
panic!("expected summary, got {:?}", metrics[0].data);
};
let point = &s.data_points[0];
assert_eq!(point.count, 300);
assert_eq!(point.sum, 12.5);
assert_eq!(point.quantile_values[0].quantile, 0.99);
assert_eq!(point.quantile_values[0].value, 0.2);
}
}