#![cfg_attr(
not(test),
deny(
clippy::unwrap_used,
clippy::expect_used,
clippy::panic,
clippy::unreachable,
clippy::todo,
clippy::unimplemented,
clippy::indexing_slicing,
clippy::string_slice,
clippy::arithmetic_side_effects,
)
)]
use std::future::Future;
use std::pin::Pin;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::task::{Context, Poll};
use axum::body::Body;
use axum::http::{HeaderValue, Request, Response, header::RETRY_AFTER};
use axum::response::IntoResponse;
use pin_project_lite::pin_project;
use tower::{Layer, Service};
use crate::admission::{AdaptiveLimiter, Criticality, InboundDeadline, PartitionShares, Sample};
use crate::middleware::MetricsCollector;
use crate::middleware::maintenance::{health_prefix_matches, prefix_with_trailing_slash};
const RETRY_AFTER_SECS: &str = "1";
#[derive(Clone, Copy, Debug)]
pub struct LoadShedExempt;
#[derive(Clone, Copy, Debug)]
pub struct LoadShedEnvelope;
#[derive(Clone, Debug, Default)]
pub struct EnvelopeAdmission(Arc<AtomicBool>);
impl EnvelopeAdmission {
pub fn skip_sample(&self) {
self.0.store(true, Ordering::Release);
}
fn is_skipped(&self) -> bool {
self.0.load(Ordering::Acquire)
}
}
#[derive(Clone)]
pub struct LoadShedLayer {
limit: LimitSource,
shares: PartitionShares,
in_flight: Arc<AtomicUsize>,
classified: Arc<AtomicUsize>,
metrics: MetricsCollector,
paths: Arc<ExemptPaths>,
cors: Option<Arc<crate::config::CorsConfig>>,
}
#[derive(Clone)]
enum LimitSource {
Static(usize),
Adaptive(Arc<AdaptiveLimiter>),
}
#[derive(Clone)]
struct ExemptPaths {
health_prefix: String,
health_prefix_slash: String,
probe_paths: Vec<String>,
}
impl LoadShedLayer {
#[must_use]
pub fn new(limit: usize, metrics: MetricsCollector) -> Self {
Self::with_source(LimitSource::Static(limit), metrics)
}
#[must_use]
pub fn adaptive(limiter: Arc<AdaptiveLimiter>, metrics: MetricsCollector) -> Self {
metrics.set_admission_limit(limiter.limit());
Self::with_source(LimitSource::Adaptive(limiter), metrics)
}
fn with_source(limit: LimitSource, metrics: MetricsCollector) -> Self {
if let LimitSource::Static(limit) = limit {
metrics.set_admission_limit(limit);
}
Self {
limit,
shares: PartitionShares::default(),
in_flight: Arc::new(AtomicUsize::new(0)),
classified: Arc::new(AtomicUsize::new(0)),
metrics,
paths: Arc::new(ExemptPaths {
health_prefix: String::new(),
health_prefix_slash: String::new(),
probe_paths: Vec::new(),
}),
cors: None,
}
}
#[must_use]
pub const fn with_partitions(mut self, shares: PartitionShares) -> Self {
self.shares = shares;
self
}
fn current_limit(&self) -> usize {
match &self.limit {
LimitSource::Static(limit) => *limit,
LimitSource::Adaptive(limiter) => limiter.limit(),
}
}
#[must_use]
pub fn with_health_prefix(mut self, prefix: impl Into<String>) -> Self {
let paths = Arc::make_mut(&mut self.paths);
paths.health_prefix = prefix.into();
paths.health_prefix_slash = prefix_with_trailing_slash(&paths.health_prefix);
self
}
#[must_use]
pub fn with_probe_paths(mut self, paths: Vec<String>) -> Self {
Arc::make_mut(&mut self.paths).probe_paths = paths;
self
}
#[must_use]
pub fn with_cors(mut self, cors: Option<Arc<crate::config::CorsConfig>>) -> Self {
self.cors = cors;
self
}
}
impl<S> Layer<S> for LoadShedLayer {
type Service = LoadShedService<S>;
fn layer(&self, inner: S) -> Self::Service {
LoadShedService {
inner,
layer: self.clone(),
}
}
}
#[derive(Clone)]
pub struct LoadShedService<S> {
inner: S,
layer: LoadShedLayer,
}
impl<S> LoadShedService<S> {
fn is_exempt<B>(&self, req: &Request<B>) -> bool {
let path = req.uri().path();
let prefix_matched = health_prefix_matches(
path,
&self.layer.paths.health_prefix,
&self.layer.paths.health_prefix_slash,
);
prefix_matched
|| self
.layer
.paths
.probe_paths
.iter()
.any(|probe| probe == path)
}
fn shed<B, F>(&self, req: &Request<B>, criticality: Criticality) -> LoadShedFuture<F> {
self.layer.metrics.record_request_shed_for(criticality);
let cors_origin = self
.layer
.cors
.as_ref()
.and_then(|_| req.headers().get(http::header::ORIGIN).cloned());
LoadShedFuture::ShortCircuit {
response: Some(build_shed_response(
self.layer.cors.as_deref(),
cors_origin.as_ref(),
)),
}
}
}
impl<S, ReqBody> Service<Request<ReqBody>> for LoadShedService<S>
where
S: Service<Request<ReqBody>, Response = Response<Body>>,
{
type Response = Response<Body>;
type Error = S::Error;
type Future = LoadShedFuture<S::Future>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
self.inner.poll_ready(cx)
}
fn call(&mut self, mut req: Request<ReqBody>) -> Self::Future {
let limit = self.layer.current_limit();
if limit == 0 || self.is_exempt(&req) {
return LoadShedFuture::Forward {
inner: self.inner.call(req),
guard: None,
};
}
if req.extensions().get::<LoadShedExempt>().is_some() {
let Some(&criticality) = req.extensions().get::<Criticality>() else {
return LoadShedFuture::Forward {
inner: self.inner.call(req),
guard: None,
};
};
let threshold = self.layer.shares.threshold(criticality, limit);
if claim(&self.layer.classified, threshold).is_none() {
if let Some(envelope) = req.extensions().get::<EnvelopeAdmission>() {
envelope.skip_sample();
}
return self.shed(&req, criticality);
}
return LoadShedFuture::Forward {
inner: self.inner.call(req),
guard: Some(InFlightGuard {
in_flight: None,
classified: Some(Arc::clone(&self.layer.classified)),
sampler: None,
envelope: None,
}),
};
}
let criticality = req
.extensions()
.get::<Criticality>()
.copied()
.unwrap_or_default();
let threshold = self.layer.shares.threshold(criticality, limit);
let in_flight = &self.layer.in_flight;
let Some(current) = claim(in_flight, threshold) else {
return self.shed(&req, criticality);
};
let envelope = req
.extensions()
.get::<LoadShedEnvelope>()
.is_some()
.then(EnvelopeAdmission::default);
if let Some(envelope) = &envelope {
req.extensions_mut().insert(envelope.clone());
}
let classified = envelope.is_none().then(|| {
self.layer.classified.fetch_add(1, Ordering::AcqRel);
Arc::clone(&self.layer.classified)
});
let deadline = req.extensions().get::<InboundDeadline>().copied();
let sampler = match (&self.layer.limit, deadline) {
(LimitSource::Static(_), _) | (_, Some(InboundDeadline::Off)) => None,
(LimitSource::Adaptive(limiter), deadline) => Some(Sampler {
limiter: Arc::clone(limiter),
metrics: self.layer.metrics.clone(),
start: tokio::time::Instant::now(),
in_flight: current.saturating_add(1),
deadline: match deadline {
Some(InboundDeadline::At(at)) => Some(at),
_ => None,
},
}),
};
LoadShedFuture::Forward {
inner: self.inner.call(req),
guard: Some(InFlightGuard {
in_flight: Some(Arc::clone(in_flight)),
classified,
sampler,
envelope,
}),
}
}
}
fn claim(counter: &AtomicUsize, threshold: usize) -> Option<usize> {
let mut current = counter.load(Ordering::Acquire);
loop {
if current >= threshold {
return None;
}
match counter.compare_exchange_weak(
current,
current.saturating_add(1),
Ordering::AcqRel,
Ordering::Acquire,
) {
Ok(_) => return Some(current),
Err(observed) => current = observed,
}
}
}
struct Sampler {
limiter: Arc<AdaptiveLimiter>,
metrics: MetricsCollector,
start: tokio::time::Instant,
in_flight: usize,
deadline: Option<tokio::time::Instant>,
}
impl Sampler {
fn record(self, dropped: bool) {
let now = tokio::time::Instant::now();
let sample = Sample {
rtt: now.saturating_duration_since(self.start),
in_flight: self.in_flight,
dropped,
at: self.limiter.elapsed(now),
};
let metrics = &self.metrics;
let _ = self
.limiter
.record_with(sample, |limit| metrics.set_admission_limit(limit));
}
}
struct InFlightGuard {
in_flight: Option<Arc<AtomicUsize>>,
classified: Option<Arc<AtomicUsize>>,
sampler: Option<Sampler>,
envelope: Option<EnvelopeAdmission>,
}
impl InFlightGuard {
fn complete(&mut self, status: Option<axum::http::StatusCode>) {
let Some(sampler) = self.sampler.take() else {
return;
};
if self
.envelope
.as_ref()
.is_some_and(EnvelopeAdmission::is_skipped)
{
return;
}
match status {
Some(s) if s.is_client_error() || s == axum::http::StatusCode::SERVICE_UNAVAILABLE => {}
Some(s) => sampler.record(s == axum::http::StatusCode::GATEWAY_TIMEOUT),
None => sampler.record(true),
}
}
}
impl Drop for InFlightGuard {
fn drop(&mut self) {
if let Some(sampler) = self.sampler.take()
&& !self
.envelope
.as_ref()
.is_some_and(EnvelopeAdmission::is_skipped)
&& sampler
.deadline
.is_some_and(|d| tokio::time::Instant::now() >= d)
{
sampler.record(true);
}
if let Some(classified) = &self.classified {
classified.fetch_sub(1, Ordering::AcqRel);
}
if let Some(in_flight) = &self.in_flight {
in_flight.fetch_sub(1, Ordering::AcqRel);
}
}
}
fn build_shed_response(
cors: Option<&crate::config::CorsConfig>,
origin: Option<&HeaderValue>,
) -> Response<Body> {
let mut response = crate::error::AutumnError::service_unavailable_msg(
"Too many concurrent requests; try again shortly.",
)
.into_response();
response
.headers_mut()
.insert(RETRY_AFTER, HeaderValue::from_static(RETRY_AFTER_SECS));
if let Some(cors) = cors {
crate::router::mirror_cors_headers(cors, origin, &mut response);
}
response
}
pin_project! {
#[project = LoadShedFutureProj]
pub enum LoadShedFuture<F> {
ShortCircuit { response: Option<Response<Body>> },
Forward {
#[pin]
inner: F,
guard: Option<InFlightGuard>,
},
}
}
impl<F, E> Future for LoadShedFuture<F>
where
F: Future<Output = Result<Response<Body>, E>>,
{
type Output = Result<Response<Body>, E>;
#[allow(
clippy::expect_used,
reason = "unreachable: future not polled after Ready"
)]
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
match self.project() {
LoadShedFutureProj::ShortCircuit { response } => Poll::Ready(Ok(response
.take()
.expect("LoadShedFuture polled after completion"))),
LoadShedFutureProj::Forward { inner, guard } => {
let output = std::task::ready!(inner.poll(cx));
if let Some(guard) = guard.as_mut() {
guard.complete(output.as_ref().ok().map(Response::status));
}
guard.take();
Poll::Ready(output)
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use axum::Router;
use axum::routing::get;
use std::sync::atomic::AtomicUsize as StdAtomicUsize;
use std::time::Duration;
use tokio::sync::Notify;
use tower::ServiceExt;
fn make_app(layer: LoadShedLayer) -> Router {
Router::new()
.route("/", get(|| async { "ok" }))
.route("/work", get(|| async { "ok" }))
.route("/actuator/health", get(|| async { "healthy" }))
.route("/live", get(|| async { "live" }))
.layer(layer)
}
async fn status(app: Router, uri: &str) -> axum::http::StatusCode {
app.oneshot(Request::builder().uri(uri).body(Body::empty()).unwrap())
.await
.unwrap()
.status()
}
#[tokio::test]
async fn below_ceiling_passes_through() {
let layer = LoadShedLayer::new(10, MetricsCollector::new());
let app = make_app(layer);
assert_eq!(status(app, "/").await, axum::http::StatusCode::OK);
}
#[tokio::test]
async fn disabled_zero_limit_never_sheds() {
let layer = LoadShedLayer::new(0, MetricsCollector::new());
let app = make_app(layer);
for _ in 0..5 {
assert_eq!(status(app.clone(), "/").await, axum::http::StatusCode::OK);
}
}
async fn blocking_handler(gate: Arc<Notify>, entered: Arc<StdAtomicUsize>) -> &'static str {
entered.fetch_add(1, Ordering::SeqCst);
gate.notified().await;
"released"
}
fn make_blocking_app(
layer: LoadShedLayer,
gate: Arc<Notify>,
entered: Arc<StdAtomicUsize>,
) -> Router {
Router::new()
.route(
"/block",
get(move || blocking_handler(gate.clone(), entered.clone())),
)
.route("/", get(|| async { "root" }))
.route("/work", get(|| async { "work" }))
.route("/actuator/health", get(|| async { "healthy" }))
.route("/live", get(|| async { "live" }))
.layer(layer)
}
async fn wait_for_entered(entered: &Arc<StdAtomicUsize>, expected: usize) {
tokio::time::timeout(Duration::from_secs(5), async {
while entered.load(Ordering::SeqCst) < expected {
tokio::time::sleep(Duration::from_millis(5)).await;
}
})
.await
.expect("handlers did not reach the expected in-flight count in time");
}
#[tokio::test]
async fn at_ceiling_sheds_with_503_and_retry_after() {
let metrics = MetricsCollector::new();
let layer = LoadShedLayer::new(2, metrics.clone());
let gate = Arc::new(Notify::new());
let entered = Arc::new(StdAtomicUsize::new(0));
let app = make_blocking_app(layer, gate.clone(), entered.clone());
let mut handles = Vec::new();
for _ in 0..2 {
let app = app.clone();
handles.push(tokio::spawn(async move {
app.oneshot(
Request::builder()
.uri("/block")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap()
.status()
}));
}
wait_for_entered(&entered, 2).await;
let resp = app
.clone()
.oneshot(
Request::builder()
.uri("/block")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), axum::http::StatusCode::SERVICE_UNAVAILABLE);
assert!(
resp.headers().contains_key(RETRY_AFTER),
"shed response must carry Retry-After"
);
assert_eq!(metrics.snapshot().http.requests_shed_total, 1);
gate.notify_waiters();
for handle in handles {
assert_eq!(handle.await.unwrap(), axum::http::StatusCode::OK);
}
}
#[tokio::test]
async fn released_slots_are_reusable() {
let metrics = MetricsCollector::new();
let layer = LoadShedLayer::new(1, metrics.clone());
let gate = Arc::new(Notify::new());
let entered = Arc::new(StdAtomicUsize::new(0));
let app = make_blocking_app(layer.clone(), gate.clone(), entered.clone());
let held = {
let app = app.clone();
tokio::spawn(async move {
app.oneshot(
Request::builder()
.uri("/block")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap()
.status()
})
};
wait_for_entered(&entered, 1).await;
assert_eq!(layer.in_flight.load(Ordering::Acquire), 1);
gate.notify_waiters();
assert_eq!(held.await.unwrap(), axum::http::StatusCode::OK);
tokio::time::timeout(Duration::from_secs(5), async {
while layer.in_flight.load(Ordering::Acquire) != 0 {
tokio::time::sleep(Duration::from_millis(5)).await;
}
})
.await
.expect("in-flight counter should return to 0 after completion");
assert_eq!(
status(app, "/actuator/health").await,
axum::http::StatusCode::OK
);
}
#[tokio::test]
async fn cancelled_request_still_releases_its_slot() {
let metrics = MetricsCollector::new();
let layer = LoadShedLayer::new(1, metrics);
let gate = Arc::new(Notify::new());
let entered = Arc::new(StdAtomicUsize::new(0));
let app = make_blocking_app(layer.clone(), gate.clone(), entered.clone());
let fut = app.clone().oneshot(
Request::builder()
.uri("/block")
.body(Body::empty())
.unwrap(),
);
let mut fut = Box::pin(fut);
let () = futures::future::poll_fn(|cx| {
let _ = Pin::new(&mut fut).poll(cx);
Poll::Ready(())
})
.await;
wait_for_entered(&entered, 1).await;
assert_eq!(layer.in_flight.load(Ordering::Acquire), 1);
drop(fut);
assert_eq!(layer.in_flight.load(Ordering::Acquire), 0);
}
#[tokio::test]
async fn guard_is_released_as_soon_as_inner_future_resolves() {
let metrics = MetricsCollector::new();
let layer = LoadShedLayer::new(1, metrics);
let app = make_app(layer.clone());
let mut fut =
Box::pin(app.oneshot(Request::builder().uri("/").body(Body::empty()).unwrap()));
let resp = std::future::poll_fn(|cx| Pin::new(&mut fut).poll(cx)).await;
assert_eq!(resp.unwrap().status(), axum::http::StatusCode::OK);
assert_eq!(
layer.in_flight.load(Ordering::Acquire),
0,
"slot must be released on Poll::Ready, not deferred until drop"
);
}
fn request(uri: &str, criticality: Option<crate::admission::Criticality>) -> Request<Body> {
let mut req = Request::builder().uri(uri).body(Body::empty()).unwrap();
if let Some(c) = criticality {
req.extensions_mut().insert(c);
}
req
}
#[tokio::test]
async fn sheddable_is_shed_before_critical() {
use crate::admission::Criticality;
let metrics = MetricsCollector::new();
let layer = LoadShedLayer::new(2, metrics.clone());
let gate = Arc::new(Notify::new());
let entered = Arc::new(StdAtomicUsize::new(0));
let app = make_blocking_app(layer, gate.clone(), entered.clone());
let held = {
let app = app.clone();
tokio::spawn(
async move { app.oneshot(request("/block", None)).await.unwrap().status() },
)
};
wait_for_entered(&entered, 1).await;
let shed = app
.clone()
.oneshot(request("/work", Some(Criticality::Sheddable)))
.await
.unwrap();
assert_eq!(shed.status(), axum::http::StatusCode::SERVICE_UNAVAILABLE);
let crit = app
.clone()
.oneshot(request("/work", Some(Criticality::Critical)))
.await
.unwrap();
assert_eq!(crit.status(), axum::http::StatusCode::OK);
let snap = metrics.snapshot().http;
assert_eq!(snap.requests_shed_total, 1);
assert_eq!(snap.admission.shed_sheddable, 1);
assert_eq!(snap.admission.shed_critical, 0);
gate.notify_waiters();
assert_eq!(held.await.unwrap(), axum::http::StatusCode::OK);
}
#[tokio::test]
async fn custom_shares_reserve_headroom_for_critical() {
use crate::admission::{Criticality, PartitionShares};
let layer = LoadShedLayer::new(2, MetricsCollector::new())
.with_partitions(PartitionShares::new(0.5, 0.0).unwrap());
let gate = Arc::new(Notify::new());
let entered = Arc::new(StdAtomicUsize::new(0));
let app = make_blocking_app(layer, gate.clone(), entered.clone());
let held = {
let app = app.clone();
tokio::spawn(
async move { app.oneshot(request("/block", None)).await.unwrap().status() },
)
};
wait_for_entered(&entered, 1).await;
assert_eq!(
app.clone()
.oneshot(request("/work", None))
.await
.unwrap()
.status(),
axum::http::StatusCode::SERVICE_UNAVAILABLE,
"default may fill only half the limit"
);
assert_eq!(
app.clone()
.oneshot(request("/work", Some(Criticality::Critical)))
.await
.unwrap()
.status(),
axum::http::StatusCode::OK
);
gate.notify_waiters();
assert_eq!(held.await.unwrap(), axum::http::StatusCode::OK);
}
fn aimd_limiter(initial: usize) -> Arc<crate::admission::AdaptiveLimiter> {
use crate::admission::{AdaptiveLimiter, Aimd, LimitAlgorithm, LimitBounds};
AdaptiveLimiter::new(LimitAlgorithm::Aimd(Aimd::new(
LimitBounds::new(1, 100, initial).unwrap(),
Duration::from_secs(1),
)))
}
#[tokio::test]
async fn adaptive_limit_grows_on_fast_full_use() {
let metrics = MetricsCollector::new();
let limiter = aimd_limiter(2);
let app = make_app(LoadShedLayer::adaptive(
Arc::clone(&limiter),
metrics.clone(),
));
assert_eq!(metrics.snapshot().http.admission.limit, 2);
assert_eq!(
status(app.clone(), "/work").await,
axum::http::StatusCode::OK
);
assert_eq!(limiter.limit(), 3);
assert_eq!(metrics.snapshot().http.admission.limit, 3);
}
#[tokio::test]
async fn adaptive_limit_backs_off_on_gateway_timeout() {
let limiter = aimd_limiter(50);
let app = Router::new()
.route(
"/slow-upstream",
get(|| async { axum::http::StatusCode::GATEWAY_TIMEOUT }),
)
.layer(LoadShedLayer::adaptive(
Arc::clone(&limiter),
MetricsCollector::new(),
));
assert_eq!(
status(app, "/slow-upstream").await,
axum::http::StatusCode::GATEWAY_TIMEOUT
);
assert_eq!(limiter.limit(), 45, "a 504 is a drop: x0.9");
}
#[tokio::test]
async fn adaptive_limit_sheds_at_the_current_limit() {
let limiter = aimd_limiter(1);
let layer = LoadShedLayer::adaptive(Arc::clone(&limiter), MetricsCollector::new());
let gate = Arc::new(Notify::new());
let entered = Arc::new(StdAtomicUsize::new(0));
let app = make_blocking_app(layer, gate.clone(), entered.clone());
let held = {
let app = app.clone();
tokio::spawn(
async move { app.oneshot(request("/block", None)).await.unwrap().status() },
)
};
wait_for_entered(&entered, 1).await;
assert_eq!(
status(app.clone(), "/work").await,
axum::http::StatusCode::SERVICE_UNAVAILABLE
);
assert_eq!(limiter.limit(), 1, "a shed request is not a sample");
gate.notify_waiters();
assert_eq!(held.await.unwrap(), axum::http::StatusCode::OK);
}
#[tokio::test]
async fn adaptive_exempt_paths_are_not_sampled() {
let limiter = aimd_limiter(2);
let app = make_app(
LoadShedLayer::adaptive(Arc::clone(&limiter), MetricsCollector::new())
.with_health_prefix("/actuator"),
);
assert_eq!(
status(app, "/actuator/health").await,
axum::http::StatusCode::OK
);
assert_eq!(limiter.limit(), 2);
}
fn hang_app(layer: LoadShedLayer) -> Router {
Router::new()
.route(
"/hang",
get(|| async {
tokio::time::sleep(Duration::from_secs(3600)).await;
"never"
}),
)
.route(
"/missing",
get(|| async { axum::http::StatusCode::NOT_FOUND }),
)
.route(
"/maintenance",
get(|| async { axum::http::StatusCode::SERVICE_UNAVAILABLE }),
)
.layer(layer)
}
#[tokio::test(start_paused = true)]
async fn adaptive_client_cancel_records_nothing() {
let limiter = aimd_limiter(50);
let layer = LoadShedLayer::adaptive(Arc::clone(&limiter), MetricsCollector::new());
let app = hang_app(layer.clone());
let res =
tokio::time::timeout(Duration::from_secs(5), app.oneshot(request("/hang", None))).await;
assert!(res.is_err());
assert_eq!(limiter.limit(), 50);
assert_eq!(layer.in_flight.load(Ordering::Acquire), 0);
}
#[tokio::test]
async fn adaptive_client_errors_and_503_are_not_sampled() {
let limiter = aimd_limiter(1);
let app = hang_app(LoadShedLayer::adaptive(
Arc::clone(&limiter),
MetricsCollector::new(),
));
for uri in ["/missing", "/maintenance"] {
app.clone().oneshot(request(uri, None)).await.unwrap();
}
assert_eq!(
limiter.limit(),
1,
"a fast 404 or 503 must not grow the limit"
);
}
#[tokio::test]
async fn adaptive_timeout_off_routes_are_not_sampled() {
let limiter = aimd_limiter(1);
let app = make_app(LoadShedLayer::adaptive(
Arc::clone(&limiter),
MetricsCollector::new(),
));
let mut req = request("/work", None);
req.extensions_mut().insert(InboundDeadline::Off);
assert_eq!(
app.oneshot(req).await.unwrap().status(),
axum::http::StatusCode::OK
);
assert_eq!(limiter.limit(), 1);
}
#[tokio::test(start_paused = true)]
async fn adaptive_deadline_cancel_is_a_drop() {
let limiter = aimd_limiter(50);
let layer = LoadShedLayer::adaptive(Arc::clone(&limiter), MetricsCollector::new());
let app = hang_app(layer.clone());
let mut req = request("/hang", None);
req.extensions_mut().insert(InboundDeadline::At(
tokio::time::Instant::now() + Duration::from_secs(5),
));
let res = tokio::time::timeout(Duration::from_secs(5), app.oneshot(req)).await;
assert!(res.is_err(), "the timeout cancels the request");
assert_eq!(limiter.limit(), 45);
assert_eq!(layer.in_flight.load(Ordering::Acquire), 0);
}
#[tokio::test]
async fn load_shed_exempt_marker_bypasses_the_ceiling() {
let metrics = MetricsCollector::new();
let layer = LoadShedLayer::new(1, metrics);
let gate = Arc::new(Notify::new());
let entered = Arc::new(StdAtomicUsize::new(0));
let app = make_blocking_app(layer, gate.clone(), entered.clone());
let held = {
let app = app.clone();
tokio::spawn(async move {
app.oneshot(
Request::builder()
.uri("/block")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap()
.status()
})
};
wait_for_entered(&entered, 1).await;
assert_eq!(
status(app.clone(), "/block").await,
axum::http::StatusCode::SERVICE_UNAVAILABLE,
"sanity: the ceiling is actually saturated"
);
let mut exempt_req = Request::builder()
.uri("/block")
.body(Body::empty())
.unwrap();
exempt_req.extensions_mut().insert(LoadShedExempt);
let exempt_fut = {
let app = app.clone();
tokio::spawn(async move { app.oneshot(exempt_req).await.unwrap().status() })
};
wait_for_entered(&entered, 2).await;
gate.notify_waiters();
assert_eq!(exempt_fut.await.unwrap(), axum::http::StatusCode::OK);
assert_eq!(held.await.unwrap(), axum::http::StatusCode::OK);
}
#[tokio::test]
async fn exempt_replay_is_rechecked_against_its_class() {
use crate::admission::Criticality;
let layer = LoadShedLayer::new(2, MetricsCollector::new());
let gate = Arc::new(Notify::new());
let entered = Arc::new(StdAtomicUsize::new(0));
let app = make_blocking_app(layer, gate.clone(), entered.clone());
let mut held = Vec::new();
for _ in 0..2 {
let app = app.clone();
held.push(tokio::spawn(async move {
let mut req = request("/block", Some(Criticality::Critical));
req.extensions_mut().insert(LoadShedEnvelope);
app.oneshot(req).await.unwrap().status()
}));
}
wait_for_entered(&entered, 2).await;
let replay = |c, uri| {
let mut req = request(uri, Some(c));
req.extensions_mut().insert(LoadShedExempt);
req
};
let first = {
let app = app.clone();
tokio::spawn(async move {
app.oneshot(replay(Criticality::Sheddable, "/block"))
.await
.unwrap()
.status()
})
};
wait_for_entered(&entered, 3).await;
assert_eq!(
app.clone()
.oneshot(replay(Criticality::Sheddable, "/work"))
.await
.unwrap()
.status(),
axum::http::StatusCode::SERVICE_UNAVAILABLE,
"a sheddable tool over its share is shed"
);
assert_eq!(
app.clone()
.oneshot(replay(Criticality::Critical, "/work"))
.await
.unwrap()
.status(),
axum::http::StatusCode::OK,
"a critical tool below the limit is not shed"
);
gate.notify_waiters();
assert_eq!(first.await.unwrap(), axum::http::StatusCode::OK);
for h in held {
assert_eq!(h.await.unwrap(), axum::http::StatusCode::OK);
}
}
#[tokio::test]
async fn replay_burst_sheds_only_the_excess() {
use crate::admission::{Criticality, PartitionShares};
let layer = LoadShedLayer::new(10, MetricsCollector::new())
.with_partitions(PartitionShares::new(1.0, 0.5).unwrap());
let gate = Arc::new(Notify::new());
let entered = Arc::new(StdAtomicUsize::new(0));
let counters = (Arc::clone(&layer.in_flight), Arc::clone(&layer.classified));
let app = make_blocking_app(layer, gate.clone(), entered.clone());
let mut envelopes = Vec::new();
for _ in 0..6 {
let app = app.clone();
envelopes.push(tokio::spawn(async move {
let mut req = request("/block", Some(Criticality::Critical));
req.extensions_mut().insert(LoadShedEnvelope);
app.oneshot(req).await.unwrap().status()
}));
}
wait_for_entered(&entered, 6).await;
let mut replays = Vec::new();
for _ in 0..6 {
let app = app.clone();
replays.push(tokio::spawn(async move {
let mut req = request("/block", Some(Criticality::Sheddable));
req.extensions_mut().insert(LoadShedExempt);
app.oneshot(req).await.unwrap().status()
}));
}
wait_for_entered(&entered, 11).await;
gate.notify_waiters();
let mut statuses = Vec::new();
for h in replays {
statuses.push(h.await.unwrap());
}
assert_eq!(
statuses
.iter()
.filter(|s| **s == axum::http::StatusCode::SERVICE_UNAVAILABLE)
.count(),
1
);
for h in envelopes {
assert_eq!(h.await.unwrap(), axum::http::StatusCode::OK);
}
assert_eq!(counters.0.load(Ordering::Acquire), 0, "total count leaks");
assert_eq!(counters.1.load(Ordering::Acquire), 0, "class count leaks");
}
#[tokio::test]
async fn replay_shed_gives_no_limiter_sample() {
use crate::admission::{Criticality, PartitionShares};
use std::convert::Infallible;
async fn run_envelope(replay_class: Criticality) -> usize {
let limiter = aimd_limiter(1);
let layer = LoadShedLayer::adaptive(Arc::clone(&limiter), MetricsCollector::new())
.with_partitions(PartitionShares::new(1.0, 0.0).unwrap());
let replay_svc =
layer
.clone()
.layer(tower::service_fn(|_req: Request<Body>| async move {
Ok::<_, Infallible>(Response::new(Body::empty()))
}));
let envelope_svc = layer.layer(tower::service_fn(move |req: Request<Body>| {
let mut replay_svc = replay_svc.clone();
async move {
let handle = req
.extensions()
.get::<EnvelopeAdmission>()
.cloned()
.expect("the layer gives the envelope a handle");
let mut replay = request("/work", Some(replay_class));
replay.extensions_mut().insert(LoadShedExempt);
replay.extensions_mut().insert(handle);
let _ = replay_svc.call(replay).await.unwrap();
Ok::<_, Infallible>(Response::new(Body::empty()))
}
}));
let mut req = request("/mcp", Some(Criticality::Critical));
req.extensions_mut().insert(LoadShedEnvelope);
let resp = envelope_svc.oneshot(req).await.unwrap();
assert_eq!(resp.status(), axum::http::StatusCode::OK);
limiter.limit()
}
assert_eq!(
run_envelope(Criticality::Critical).await,
2,
"control: a fast envelope grows the limit"
);
assert_eq!(
run_envelope(Criticality::Sheddable).await,
1,
"a shed replay is not a sample"
);
}
#[tokio::test]
async fn empty_health_prefix_exempts_root_path_like_maintenance_does() {
let metrics = MetricsCollector::new();
let layer = LoadShedLayer::new(1, metrics)
.with_health_prefix("")
.with_probe_paths(vec![]);
let gate = Arc::new(Notify::new());
let entered = Arc::new(StdAtomicUsize::new(0));
let app = make_blocking_app(layer, gate.clone(), entered.clone());
let held = {
let app = app.clone();
tokio::spawn(async move {
app.oneshot(
Request::builder()
.uri("/block")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap()
.status()
})
};
wait_for_entered(&entered, 1).await;
assert_eq!(status(app.clone(), "/").await, axum::http::StatusCode::OK);
gate.notify_waiters();
assert_eq!(held.await.unwrap(), axum::http::StatusCode::OK);
}
#[tokio::test]
async fn shed_503_carries_cors_headers_when_configured() {
let metrics = MetricsCollector::new();
let cors = crate::config::CorsConfig {
allowed_origins: vec!["http://other.example".to_owned()],
..Default::default()
};
let layer = LoadShedLayer::new(1, metrics).with_cors(Some(Arc::new(cors)));
let gate = Arc::new(Notify::new());
let entered = Arc::new(StdAtomicUsize::new(0));
let app = make_blocking_app(layer, gate.clone(), entered.clone());
let held = {
let app = app.clone();
tokio::spawn(async move {
app.oneshot(
Request::builder()
.uri("/block")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap()
.status()
})
};
wait_for_entered(&entered, 1).await;
let resp = app
.clone()
.oneshot(
Request::builder()
.uri("/block")
.header(http::header::ORIGIN, "http://other.example")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(resp.status(), axum::http::StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(
resp.headers()
.get(http::header::ACCESS_CONTROL_ALLOW_ORIGIN)
.and_then(|v| v.to_str().ok()),
Some("http://other.example"),
"shed 503 must mirror the matching CORS origin"
);
gate.notify_waiters();
assert_eq!(held.await.unwrap(), axum::http::StatusCode::OK);
}
#[tokio::test]
async fn actuator_health_bypasses_ceiling() {
let metrics = MetricsCollector::new();
let layer = LoadShedLayer::new(1, metrics)
.with_health_prefix("/actuator")
.with_probe_paths(vec!["/live".to_owned()]);
let gate = Arc::new(Notify::new());
let entered = Arc::new(StdAtomicUsize::new(0));
let app = make_blocking_app(layer, gate.clone(), entered.clone());
let held = {
let app = app.clone();
tokio::spawn(async move {
app.oneshot(
Request::builder()
.uri("/block")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap()
.status()
})
};
wait_for_entered(&entered, 1).await;
assert_eq!(
status(app.clone(), "/actuator/health").await,
axum::http::StatusCode::OK
);
assert_eq!(
status(app.clone(), "/live").await,
axum::http::StatusCode::OK
);
gate.notify_waiters();
assert_eq!(held.await.unwrap(), axum::http::StatusCode::OK);
}
}