use std::{
fmt,
pin::Pin,
sync::Arc,
task::{Context, Poll},
time::Instant,
};
use axum::{http::StatusCode, response::Response};
use http_body::{Body, Frame, SizeHint};
use pin_project_lite::pin_project;
use crate::{
clock::SharedClock,
observability::{
MetricLabels, MetricRecorder, RecordingFailures, metric_labels, metric_names,
metric_outcomes,
},
};
pin_project! {
pub struct HttpMetricsBody<B> {
#[pin]
inner: B,
observation: Option<BodyObservation>,
}
}
impl<B: fmt::Debug> fmt::Debug for HttpMetricsBody<B> {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("HttpMetricsBody")
.field("inner", &self.inner)
.field("observing", &self.observation.is_some())
.finish()
}
}
impl<B: Body> Body for HttpMetricsBody<B> {
type Data = B::Data;
type Error = B::Error;
fn poll_frame(
self: Pin<&mut Self>,
context: &mut Context<'_>,
) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
let mut this = self.project();
let frame = this.inner.as_mut().poll_frame(context);
let outcome = match &frame {
Poll::Ready(Some(Err(_))) => Some(metric_outcomes::ERROR),
Poll::Ready(None) => Some(metric_outcomes::COMPLETED),
Poll::Ready(Some(Ok(frame))) if frame.is_trailers() || this.inner.is_end_stream() => {
Some(metric_outcomes::COMPLETED)
}
_ => None,
};
if let Some(outcome) = outcome {
finish(this.observation, outcome);
}
frame
}
fn is_end_stream(&self) -> bool {
self.inner.is_end_stream()
}
fn size_hint(&self) -> SizeHint {
self.inner.size_hint()
}
}
pub(super) fn observe_response<B: Body>(
response: Response<B>,
recorder: Arc<dyn MetricRecorder>,
clock: SharedClock,
failures: RecordingFailures,
method: &'static str,
route: String,
enabled: bool,
) -> Response<HttpMetricsBody<B>> {
let status = response.status();
let mut observation = enabled.then(|| BodyObservation {
recorder,
started_at: clock.now(),
clock,
failures,
method,
route,
status,
outcome: metric_outcomes::INCOMPLETE,
});
if method == "HEAD"
|| method == "CONNECT" && status.is_success()
|| status.is_informational()
|| matches!(
status,
StatusCode::NO_CONTENT | StatusCode::RESET_CONTENT | StatusCode::NOT_MODIFIED
)
|| response.body().is_end_stream()
{
finish(&mut observation, metric_outcomes::COMPLETED);
}
response.map(|inner| HttpMetricsBody { inner, observation })
}
fn finish(observation: &mut Option<BodyObservation>, outcome: &'static str) {
if let Some(mut observation) = observation.take() {
observation.outcome = outcome;
}
}
struct BodyObservation {
recorder: Arc<dyn MetricRecorder>,
clock: SharedClock,
failures: RecordingFailures,
started_at: Instant,
method: &'static str,
route: String,
status: StatusCode,
outcome: &'static str,
}
impl Drop for BodyObservation {
fn drop(&mut self) {
let elapsed = self.clock.now().saturating_duration_since(self.started_at);
let status = self.status.as_u16().to_string();
let recording = MetricLabels::new([
(metric_labels::METHOD, self.method),
(metric_labels::ROUTE, self.route.as_str()),
(metric_labels::STATUS, status.as_str()),
(metric_labels::OUTCOME, self.outcome),
])
.and_then(|labels| {
self.recorder.increment_counter(
metric_names::HTTP_SERVER_RESPONSE_BODIES_TOTAL,
1,
labels.clone(),
)?;
self.recorder.observe_histogram(
metric_names::HTTP_SERVER_RESPONSE_BODY_DURATION_SECONDS,
elapsed.as_secs_f64(),
labels,
)
});
self.failures.record(recording, "http_response_body");
}
}
#[cfg(test)]
mod tests {
use std::{
collections::VecDeque,
convert::Infallible,
marker::PhantomPinned,
sync::atomic::{AtomicUsize, Ordering},
time::Duration,
};
use axum::{
body::{Body as AxumBody, Bytes},
http::{HeaderMap, Request},
};
use http_body_util::{BodyExt, Full};
use tokio::sync::oneshot;
use tower::{Layer, ServiceExt};
use super::*;
use crate::{
clock::ManualClock,
http::HttpMetricsLayer,
observability::{MetricRegistry, MetricRegistryLimits},
};
pin_project! {
struct Frames {
frames: VecDeque<Result<Frame<Bytes>, std::io::Error>>,
polls: Arc<AtomicUsize>,
#[pin]
pinned: PhantomPinned,
}
}
impl Body for Frames {
type Data = Bytes;
type Error = std::io::Error;
fn poll_frame(
self: Pin<&mut Self>,
_: &mut Context<'_>,
) -> Poll<Option<Result<Frame<Bytes>, Self::Error>>> {
let this = self.project();
this.polls.fetch_add(1, Ordering::Relaxed);
Poll::Ready(this.frames.pop_front())
}
fn size_hint(&self) -> SizeHint {
SizeHint::with_exact(
self.frames
.iter()
.filter_map(|frame| frame.as_ref().ok()?.data_ref())
.map(|bytes| bytes.len() as u64)
.sum(),
)
}
}
fn frames(polls: Arc<AtomicUsize>, failure: bool) -> Frames {
let mut trailers = HeaderMap::new();
trailers.insert("x-finished", "yes".parse().unwrap());
Frames {
frames: [
Ok(Frame::data(Bytes::from_static(b"hello"))),
if failure {
Err(std::io::Error::other("original body error"))
} else {
Ok(Frame::trailers(trailers))
},
]
.into(),
polls,
pinned: PhantomPinned,
}
}
struct PendingBody(Option<oneshot::Sender<()>>);
impl Body for PendingBody {
type Data = Bytes;
type Error = Infallible;
fn poll_frame(
mut self: Pin<&mut Self>,
_: &mut Context<'_>,
) -> Poll<Option<Result<Frame<Bytes>, Infallible>>> {
if let Some(started) = self.0.take() {
started.send(()).unwrap();
}
Poll::Pending
}
}
fn assert_body_metrics(registry: &MetricRegistry, outcome: &str, seconds: f64) {
let snapshot = registry.snapshot().unwrap();
let body_samples: Vec<_> = snapshot
.samples()
.iter()
.filter(|sample| {
[
metric_names::HTTP_SERVER_RESPONSE_BODIES_TOTAL,
metric_names::HTTP_SERVER_RESPONSE_BODY_DURATION_SECONDS,
]
.contains(&sample.name())
})
.collect();
assert_eq!(body_samples.len(), 2, "{snapshot:?}");
for sample in body_samples {
assert!(
sample
.labels()
.contains(&(metric_labels::OUTCOME.into(), outcome.into()))
);
assert_eq!(
sample.value(),
if sample.name() == metric_names::HTTP_SERVER_RESPONSE_BODIES_TOTAL {
1.0
} else {
seconds
}
);
}
}
#[tokio::test]
async fn pinned_frames_are_lazy_and_preserve_trailers_and_body_errors() {
for failure in [false, true] {
let clock = ManualClock::new();
let (layer, registry) = HttpMetricsLayer::in_memory();
let layer = layer
.with_clock(clock.shared())
.with_response_body_metrics(true);
let polls = Arc::new(AtomicUsize::new(0));
let inner_clock = clock.clone();
let inner_polls = polls.clone();
let service = layer.layer(tower::service_fn(move |_: Request<()>| {
inner_clock.advance(Duration::from_secs(2)).unwrap();
let body = frames(inner_polls.clone(), failure);
async move { Ok::<_, Infallible>(Response::new(body)) }
}));
let response = service.oneshot(Request::new(())).await.unwrap();
assert_eq!(polls.load(Ordering::Relaxed), 0);
assert_eq!(response.body().size_hint().exact(), Some(5));
assert!(!response.body().is_end_stream());
assert_eq!(registry.snapshot().unwrap().samples().len(), 2);
let request_duration = registry
.snapshot()
.unwrap()
.samples()
.iter()
.find(|sample| sample.name() == metric_names::HTTP_SERVER_REQUEST_DURATION_SECONDS)
.unwrap()
.value();
assert_eq!(request_duration, 2.0);
let mut body = Box::pin(response.into_body());
clock.advance(Duration::from_secs(3)).unwrap();
assert_eq!(
body.frame().await.unwrap().unwrap().into_data().unwrap(),
"hello"
);
assert_eq!(registry.snapshot().unwrap().samples().len(), 2);
clock.advance(Duration::from_secs(4)).unwrap();
let frame = body.frame().await.unwrap();
if failure {
assert_eq!(frame.unwrap_err().to_string(), "original body error");
} else {
assert_eq!(frame.unwrap().into_trailers().unwrap()["x-finished"], "yes");
}
assert!(body.frame().await.is_none());
drop(body);
assert_body_metrics(
®istry,
if failure {
metric_outcomes::ERROR
} else {
metric_outcomes::COMPLETED
},
7.0,
);
assert_eq!(layer.failed_recordings(), 0);
}
}
#[tokio::test]
async fn dropped_bodies_are_incomplete_without_claiming_a_disconnect() {
for read_first in [false, true] {
let clock = ManualClock::new();
let (layer, registry) = HttpMetricsLayer::in_memory();
let service = layer
.with_clock(clock.shared())
.with_response_body_metrics(true)
.layer(tower::service_fn(|_: Request<()>| async {
Ok::<_, Infallible>(Response::new(frames(Arc::new(AtomicUsize::new(0)), false)))
}));
let mut body = Box::pin(service.oneshot(Request::new(())).await.unwrap().into_body());
if read_first {
body.frame().await.unwrap().unwrap();
}
clock.advance(Duration::from_secs(5)).unwrap();
drop(body);
assert_body_metrics(®istry, metric_outcomes::INCOMPLETE, 5.0);
}
}
#[tokio::test(start_paused = true)]
async fn pending_body_timeout_and_task_cancellation_have_distinct_outcomes() {
use tower_http::timeout::{ResponseBodyTimeoutLayer, TimeoutError};
for cancel in [false, true] {
let clock = ManualClock::new();
let (layer, registry) = HttpMetricsLayer::in_memory();
let (started, ready) = oneshot::channel();
let inner = tower::service_fn(|request: Request<PendingBody>| async {
Ok::<_, Infallible>(Response::new(request.into_body()))
});
let service = layer
.with_clock(clock.shared())
.with_response_body_metrics(true)
.layer(ResponseBodyTimeoutLayer::new(Duration::from_secs(2)).layer(inner));
let response = service
.oneshot(Request::new(PendingBody(Some(started))))
.await
.unwrap();
let consumer = tokio::spawn(response.into_body().collect());
ready.await.unwrap();
assert_eq!(registry.snapshot().unwrap().samples().len(), 2);
clock.advance(Duration::from_secs(2)).unwrap();
if cancel {
consumer.abort();
assert!(consumer.await.unwrap_err().is_cancelled());
assert_body_metrics(®istry, metric_outcomes::INCOMPLETE, 2.0);
} else {
tokio::time::advance(Duration::from_secs(2)).await;
let error = consumer.await.unwrap().unwrap_err();
assert!(error.is::<TimeoutError>());
assert_body_metrics(®istry, metric_outcomes::ERROR, 2.0);
}
}
}
#[tokio::test]
async fn unknown_length_body_completes_at_eof() {
let clock = ManualClock::new();
let (layer, registry) = HttpMetricsLayer::in_memory();
let service = layer
.with_clock(clock.shared())
.with_response_body_metrics(true)
.layer(tower::service_fn(|_: Request<()>| async {
let mut body = frames(Arc::new(AtomicUsize::new(0)), false);
body.frames.pop_back(); Ok::<_, Infallible>(Response::new(body))
}));
let mut body = Box::pin(service.oneshot(Request::new(())).await.unwrap().into_body());
body.frame().await.unwrap().unwrap();
assert_eq!(registry.snapshot().unwrap().samples().len(), 2);
clock.advance(Duration::from_secs(3)).unwrap();
assert!(body.frame().await.is_none());
drop(body);
assert_body_metrics(®istry, metric_outcomes::COMPLETED, 3.0);
}
#[tokio::test]
async fn empty_and_bodyless_responses_complete_without_polling() {
for (method, status, empty) in [
("GET", StatusCode::OK, true),
("HEAD", StatusCode::OK, false),
("GET", StatusCode::SWITCHING_PROTOCOLS, false),
("CONNECT", StatusCode::OK, false),
("GET", StatusCode::NO_CONTENT, false),
("GET", StatusCode::RESET_CONTENT, false),
("GET", StatusCode::NOT_MODIFIED, false),
] {
let (layer, registry) = HttpMetricsLayer::in_memory();
let clock = ManualClock::new();
let service = layer
.with_clock(clock.shared())
.with_response_body_metrics(true)
.layer(tower::service_fn(move |_: Request<()>| async move {
let mut response = Response::new(Full::new(if empty {
Bytes::new()
} else {
Bytes::from_static(b"not consumed")
}));
*response.status_mut() = status;
Ok::<_, Infallible>(response)
}));
let response = service
.oneshot(Request::builder().method(method).body(()).unwrap())
.await
.unwrap();
clock.advance(Duration::from_secs(5)).unwrap();
drop(response);
assert_body_metrics(®istry, metric_outcomes::COMPLETED, 0.0);
}
}
#[tokio::test]
async fn final_data_frame_completes_once_and_recording_errors_preserve_the_body() {
for exhausted in [false, true] {
let registry = Arc::new(if exhausted {
MetricRegistry::with_limits(MetricRegistryLimits::new(2, 2).unwrap())
} else {
MetricRegistry::new()
});
let clock = ManualClock::new();
let layer = HttpMetricsLayer::new(registry.clone())
.with_clock(clock.shared())
.with_response_body_metrics(true);
let service = layer.layer(tower::service_fn(|_: Request<()>| async {
Ok::<_, Infallible>(Response::new(Full::new(Bytes::from_static(b"payload"))))
}));
let mut body = service.oneshot(Request::new(())).await.unwrap().into_body();
clock.advance(Duration::from_secs(2)).unwrap();
assert_eq!(
body.frame().await.unwrap().unwrap().into_data().unwrap(),
"payload"
);
assert!(body.is_end_stream());
drop(body); if exhausted {
assert_eq!(layer.failed_recordings(), 1);
assert_eq!(registry.snapshot().unwrap().samples().len(), 2);
} else {
assert_body_metrics(®istry, metric_outcomes::COMPLETED, 2.0);
}
}
}
#[tokio::test]
async fn configured_layer_integrates_with_the_middleware_stack() {
use crate::http::{HttpMiddlewareConfig, HttpMiddlewareStack};
let (layer, registry) = HttpMetricsLayer::in_memory();
let app = HttpMiddlewareStack::builder(HttpMiddlewareConfig::standard())
.with_http_metrics_layer(layer.with_response_body_metrics(true))
.build()
.unwrap()
.apply(
axum::Router::new().route("/items/{id}", axum::routing::get(|| async { "body" })),
);
let response = app
.oneshot(Request::get("/items/123").body(AxumBody::empty()).unwrap())
.await
.unwrap();
assert_eq!(
response.into_body().collect().await.unwrap().to_bytes(),
"body"
);
let snapshot = registry.snapshot().unwrap();
assert_eq!(snapshot.samples().len(), 4);
for sample in snapshot.samples() {
assert!(
sample
.labels()
.contains(&(metric_labels::ROUTE.into(), "/items/{id}".into()))
);
}
}
}