use std::{
fmt,
future::Future,
pin::Pin,
sync::{
Arc,
atomic::{AtomicU64, Ordering},
},
task::{Context, Poll},
time::Duration,
};
use axum::{extract::MatchedPath, http::Request, response::Response};
use tower::{Layer, Service};
use crate::{
clock::{SharedClock, system_clock},
observability::{
MetricLabels, MetricRecorder, MetricRegistry, MetricsError, metric_labels, metric_names,
},
};
mod response;
pub use response::{MetricsResponse, OPENMETRICS_CONTENT_TYPE, PROMETHEUS_CONTENT_TYPE};
pub const HTTP_METRICS_UNMATCHED_ROUTE: &str = "unmatched";
fn route_label<B>(request: &Request<B>) -> &str {
request
.extensions()
.get::<MatchedPath>()
.map(MatchedPath::as_str)
.unwrap_or(HTTP_METRICS_UNMATCHED_ROUTE)
}
fn method_label(method: &str) -> &'static str {
match method {
"GET" => "GET",
"HEAD" => "HEAD",
"POST" => "POST",
"PUT" => "PUT",
"DELETE" => "DELETE",
"CONNECT" => "CONNECT",
"OPTIONS" => "OPTIONS",
"TRACE" => "TRACE",
"PATCH" => "PATCH",
"QUERY" => "QUERY",
_ => "_OTHER",
}
}
#[derive(Clone)]
pub struct HttpMetricsLayer {
recorder: Arc<dyn MetricRecorder>,
clock: SharedClock,
failed_recordings: Arc<AtomicU64>,
}
impl HttpMetricsLayer {
#[must_use]
pub fn new(recorder: Arc<dyn MetricRecorder>) -> Self {
Self {
recorder,
clock: system_clock(),
failed_recordings: Arc::new(AtomicU64::new(0)),
}
}
#[must_use]
pub fn in_memory() -> (Self, Arc<MetricRegistry>) {
let registry = Arc::new(MetricRegistry::new());
(Self::new(registry.clone()), registry)
}
#[must_use]
pub fn with_clock(mut self, clock: SharedClock) -> Self {
self.clock = clock;
self
}
#[must_use]
pub fn recorder(&self) -> &Arc<dyn MetricRecorder> {
&self.recorder
}
#[must_use]
pub fn failed_recordings(&self) -> u64 {
self.failed_recordings.load(Ordering::Relaxed)
}
#[must_use]
pub fn clock(&self) -> &SharedClock {
&self.clock
}
}
impl fmt::Debug for HttpMetricsLayer {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("HttpMetricsLayer")
.finish_non_exhaustive()
}
}
impl<S> Layer<S> for HttpMetricsLayer {
type Service = HttpMetricsService<S>;
fn layer(&self, inner: S) -> Self::Service {
HttpMetricsService {
inner,
recorder: self.recorder.clone(),
clock: self.clock.clone(),
failed_recordings: self.failed_recordings.clone(),
}
}
}
#[derive(Clone)]
pub struct HttpMetricsService<S> {
inner: S,
recorder: Arc<dyn MetricRecorder>,
clock: SharedClock,
failed_recordings: Arc<AtomicU64>,
}
impl<S> fmt::Debug for HttpMetricsService<S>
where
S: fmt::Debug,
{
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("HttpMetricsService")
.field("inner", &self.inner)
.finish_non_exhaustive()
}
}
impl<S, ReqBody, ResBody> Service<Request<ReqBody>> for HttpMetricsService<S>
where
S: Service<Request<ReqBody>, Response = Response<ResBody>>,
S::Future: Send + 'static,
S::Error: Send + 'static,
{
type Response = Response<ResBody>;
type Error = S::Error;
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
fn poll_ready(&mut self, context: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
self.inner.poll_ready(context)
}
fn call(&mut self, request: Request<ReqBody>) -> Self::Future {
let start = self.clock.now();
let recorder = self.recorder.clone();
let clock = self.clock.clone();
let failed_recordings = self.failed_recordings.clone();
let method = method_label(request.method().as_str());
let route = route_label(&request).to_owned();
let future = self.inner.call(request);
Box::pin(async move {
let result = future.await;
let recording = match &result {
Ok(response) => record_http_metrics(
recorder.as_ref(),
method,
&route,
response.status().as_u16(),
clock.now().saturating_duration_since(start),
),
Err(_) => MetricLabels::new([
(metric_labels::METHOD, method),
(metric_labels::ROUTE, route.as_str()),
(metric_labels::ERROR, "service"),
])
.and_then(|labels| {
recorder.increment_counter(metric_names::HTTP_SERVER_ERRORS_TOTAL, 1, labels)
}),
};
if let Err(error) = recording {
let previous = failed_recordings
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |value| {
Some(value.saturating_add(1))
})
.unwrap_or_else(|value| value);
let failures = previous.saturating_add(1);
if failures.is_power_of_two() {
tracing::warn!(failures, error = %error, "HTTP metric recording failed");
}
}
result
})
}
}
pub fn record_http_metrics(
recorder: &dyn MetricRecorder,
method: &str,
route: &str,
status: u16,
elapsed: Duration,
) -> Result<(), MetricsError> {
let status = status.to_string();
let labels = MetricLabels::new([
(metric_labels::METHOD, method_label(method)),
(metric_labels::ROUTE, route),
(metric_labels::STATUS, status.as_str()),
])?;
recorder.increment_counter(metric_names::HTTP_SERVER_REQUESTS_TOTAL, 1, labels.clone())?;
recorder.observe_histogram(
metric_names::HTTP_SERVER_REQUEST_DURATION_SECONDS,
elapsed.as_secs_f64(),
labels,
)
}
#[cfg(test)]
mod tests {
use axum::{
Router,
body::{Body, to_bytes},
http::{StatusCode, header::CONTENT_TYPE},
response::IntoResponse,
routing::get,
};
use tower::ServiceExt;
use super::*;
use crate::observability::MetricsSnapshot;
#[tokio::test]
async fn metrics_compose_with_request_and_response_body_wrappers() {
use http_body_util::{BodyExt, Full};
use tower_http::timeout::{RequestBodyTimeoutLayer, ResponseBodyTimeoutLayer, TimeoutBody};
let (layer, registry) = HttpMetricsLayer::in_memory();
let inner = tower::service_fn(|request: Request<TimeoutBody<Body>>| async move {
let body = request.into_body().collect().await.unwrap().to_bytes();
Ok::<_, std::convert::Infallible>(Response::new(Full::new(body)))
});
let service = RequestBodyTimeoutLayer::new(Duration::from_secs(1))
.layer(layer.layer(ResponseBodyTimeoutLayer::new(Duration::from_secs(1)).layer(inner)));
let response = service
.oneshot(Request::new(Body::from("payload")))
.await
.unwrap();
assert_eq!(
response.into_body().collect().await.unwrap().to_bytes(),
"payload"
);
assert_eq!(registry.snapshot().unwrap().samples().len(), 2);
assert_eq!(layer.failed_recordings(), 0);
}
#[tokio::test]
async fn recording_failures_are_shared_and_do_not_change_responses() {
let registry = Arc::new(MetricRegistry::with_limits(
crate::observability::MetricRegistryLimits::new(1, 1).unwrap(),
));
let layer = HttpMetricsLayer::new(registry.clone());
let app = Router::new()
.route("/", get(|| async { "ok" }))
.layer(layer.clone());
for _ in 0..3 {
let response = app
.clone()
.oneshot(Request::new(Body::empty()))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(to_bytes(response.into_body(), 16).await.unwrap(), "ok");
}
assert_eq!(layer.failed_recordings(), 3);
assert_eq!(registry.snapshot().unwrap().samples()[0].value(), 3.0);
let service = layer
.clone()
.layer(tower::service_fn(|_: Request<()>| async {
Err::<Response<()>, _>("service failed")
}));
assert_eq!(
service.oneshot(Request::new(())).await.unwrap_err(),
"service failed"
);
assert_eq!(layer.failed_recordings(), 4);
}
#[tokio::test]
async fn oversized_routes_are_rejected_without_losing_the_route_label() {
let route = format!("/{}", "x".repeat(300));
let (layer, registry) = HttpMetricsLayer::in_memory();
let app = Router::new()
.route(&route, get(|| async {}))
.layer(layer.clone());
let response = app
.oneshot(Request::get(&route).body(Body::empty()).unwrap())
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(layer.failed_recordings(), 1);
assert!(registry.snapshot().unwrap().samples().is_empty());
let larger = MetricRegistry::with_limits(
crate::observability::MetricRegistryLimits::default()
.with_max_label_value_length(1024)
.unwrap(),
);
record_http_metrics(&larger, "GET", &route, 200, Duration::ZERO).unwrap();
for sample in larger.snapshot().unwrap().samples() {
assert!(
sample
.labels()
.iter()
.any(|(key, value)| key == "route" && value == &route)
);
}
assert_eq!(
record_http_metrics(registry.as_ref(), "GET", "bad\nroute", 200, Duration::ZERO),
Err(MetricsError::InvalidLabelValue)
);
assert!(registry.snapshot().unwrap().samples().is_empty());
}
#[tokio::test]
async fn metrics_accept_non_clone_services_and_call_the_ready_instance() {
struct ReadyService(bool);
impl Service<Request<Body>> for ReadyService {
type Response = Response;
type Error = std::convert::Infallible;
type Future = std::future::Ready<Result<Response, Self::Error>>;
fn poll_ready(&mut self, _: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
self.0 = true;
Poll::Ready(Ok(()))
}
fn call(&mut self, _: Request<Body>) -> Self::Future {
assert!(
std::mem::take(&mut self.0),
"call must use the ready service"
);
std::future::ready(Ok("ok".into_response()))
}
}
let (layer, registry) = HttpMetricsLayer::in_memory();
let mut service = layer.layer(ReadyService(false));
for _ in 0..2 {
service
.ready()
.await
.unwrap()
.call(Request::new(Body::empty()))
.await
.unwrap();
}
let snapshot = registry.snapshot().unwrap();
let requests = snapshot
.samples()
.iter()
.find(|sample| sample.name() == metric_names::HTTP_SERVER_REQUESTS_TOTAL)
.unwrap();
assert_eq!(requests.value(), 2.0);
}
#[tokio::test]
async fn unknown_methods_share_one_label_and_leave_room_for_normal_requests() {
let registry = Arc::new(MetricRegistry::with_limits(
crate::observability::MetricRegistryLimits::new(10, 2).unwrap(),
));
let app = Router::new()
.route("/health", get(|| async { "ok" }))
.layer(HttpMetricsLayer::new(registry.clone()));
for method in ["CUSTOM_A", "CUSTOM_B", "GET"] {
app.clone()
.oneshot(
Request::builder()
.uri("/health")
.method(method)
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
}
let snapshot = registry.snapshot().unwrap();
let requests: Vec<_> = snapshot
.samples()
.iter()
.filter(|sample| sample.name() == metric_names::HTTP_SERVER_REQUESTS_TOTAL)
.collect();
assert_eq!(requests.len(), 2);
for (method, count) in [("_OTHER", 2.0), ("GET", 1.0)] {
assert!(requests.iter().any(|sample| sample.value() == count
&& sample.labels().contains(&("method".into(), method.into()))));
}
}
#[tokio::test]
async fn metrics_layer_records_requests() {
let (layer, registry) = HttpMetricsLayer::in_memory();
let response = Router::new()
.route("/health", get(|| async { "ok" }))
.layer(layer)
.oneshot(
Request::get("/health")
.body(Body::empty())
.expect("request should build"),
)
.await
.expect("request should complete");
assert_eq!(response.status(), StatusCode::OK);
let snapshot = registry.snapshot().expect("snapshot should build");
let sample = snapshot
.samples()
.iter()
.find(|sample| sample.name() == metric_names::HTTP_SERVER_REQUESTS_TOTAL)
.expect("request counter should be recorded");
assert_eq!(
sample
.labels()
.iter()
.find(|(name, _)| name == metric_labels::ROUTE)
.map(|(_, value)| value.as_str()),
Some("/health")
);
}
#[tokio::test]
async fn metrics_layer_uses_matched_route_instead_of_raw_identifier() {
let (layer, registry) = HttpMetricsLayer::in_memory();
let response = Router::new()
.route("/users/{id}", get(|| async { "ok" }))
.layer(layer)
.oneshot(
Request::get("/users/123")
.body(Body::empty())
.expect("request should build"),
)
.await
.expect("request should complete");
assert_eq!(response.status(), StatusCode::OK);
let snapshot = registry.snapshot().expect("snapshot should build");
let sample = snapshot
.samples()
.iter()
.find(|sample| sample.name() == metric_names::HTTP_SERVER_REQUESTS_TOTAL)
.expect("request counter should be recorded");
assert_eq!(
sample
.labels()
.iter()
.find(|(name, _)| name == metric_labels::ROUTE)
.map(|(_, value)| value.as_str()),
Some("/users/{id}")
);
}
#[test]
fn route_label_strategy_never_uses_raw_path_by_default() {
let request = Request::get("/users/123")
.body(Body::empty())
.expect("request should build");
assert_eq!(route_label(&request), HTTP_METRICS_UNMATCHED_ROUTE);
}
#[tokio::test]
async fn metrics_response_uses_openmetrics_content_type() {
let response = MetricsResponse(MetricsSnapshot::new(vec![
crate::observability::MetricSample::counter("jobs_total", 1)
.expect("metric sample should build"),
]))
.into_response();
assert_eq!(
response
.headers()
.get(CONTENT_TYPE)
.and_then(|value| value.to_str().ok()),
Some(OPENMETRICS_CONTENT_TYPE)
);
let body = to_bytes(response.into_body(), usize::MAX)
.await
.expect("body should read");
assert!(String::from_utf8_lossy(&body).contains("jobs_total"));
}
}