use std::{
convert::Infallible,
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
};
use async_trait::async_trait;
use axum::{
Router,
body::{Body, to_bytes},
http::{
HeaderMap, HeaderName, HeaderValue, Request, StatusCode,
header::{AUTHORIZATION, CACHE_CONTROL, CONTENT_TYPE, COOKIE, SET_COOKIE},
},
middleware::{self, Next},
response::{IntoResponse, Response},
routing::post,
};
use tower::ServiceExt;
use super::{
DEFAULT_IDEMPOTENCY_MAX_BODY_BYTES, DEFAULT_IDEMPOTENCY_PROCESSING_TTL,
DEFAULT_IDEMPOTENCY_TTL, IDEMPOTENCY_KEY_HEADER_NAME, IdempotencyConfig,
IdempotencyConfigError, IdempotencyKey, IdempotencyLayer, IdempotencyLease,
IdempotencyRejection, IdempotencyRequestHash, IdempotencyReservation, IdempotencyRuntime,
IdempotencyScope, IdempotencyStore, IdempotencyStoreError, MemoryIdempotencyStore,
StoredIdempotencyResponse, idempotency_key_from_headers, is_valid_idempotency_key,
run_idempotent_service,
};
use crate::observability::{MetricRegistry, metric_names};
use crate::result::{ClassifyError, ErrorKind};
#[test]
fn poisoned_memory_store_stays_failed_and_is_not_retryable() {
let store = MemoryIdempotencyStore::new();
let state = store.state.clone();
assert!(
std::thread::spawn(move || {
let _guard = state.lock().unwrap();
panic!("poison the store state");
})
.join()
.is_err()
);
for _ in 0..2 {
let error = store.record_count().unwrap_err();
assert!(matches!(error, IdempotencyStoreError::LockPoisoned));
assert_eq!(error.error_kind(), ErrorKind::Internal);
assert!(!error.error_descriptor().retryable());
}
assert!(
IdempotencyStoreError::CapacityExceeded
.error_descriptor()
.retryable()
);
assert!(
IdempotencyStoreError::Store(std::io::Error::other("unavailable").into())
.error_descriptor()
.retryable()
);
}
fn with_request_context(mut request: Request<Body>, request_id: &str) -> Request<Body> {
request.extensions_mut().insert(
crate::http::RequestContext::new(crate::http::RequestId::new(request_id).unwrap())
.with_traceparent(
crate::http::TraceParent::new(
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
)
.unwrap(),
)
.with_session_id("private-session"),
);
request
}
async fn assert_correlated_problem(response: Response, code: &str, request_id: &str) {
assert_eq!(response.headers()[CACHE_CONTROL], "no-store");
assert_eq!(response.headers()[CONTENT_TYPE], "application/problem+json");
let bytes = to_bytes(response.into_body(), 4096).await.unwrap();
let problem: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
assert_eq!(problem["code"], code);
assert_eq!(problem["request_id"], request_id);
assert_eq!(problem["trace_id"], "4bf92f3577b34da6a3ce929d0e0e4736");
assert!(!String::from_utf8_lossy(&bytes).contains("private-session"));
}
fn assert_store_error_metric(metrics: &MetricRegistry, stage: &str, error_code: &str, count: f64) {
let snapshot = metrics.snapshot().unwrap();
let errors: Vec<_> = snapshot
.samples()
.iter()
.filter(|sample| sample.name() == metric_names::IDEMPOTENCY_STORE_ERRORS_TOTAL)
.collect();
assert_eq!(errors.len(), 1);
assert_eq!(errors[0].value(), count);
assert_eq!(
errors[0].labels(),
crate::observability::MetricLabels::new([("stage", stage), ("error_code", error_code),])
.unwrap()
.as_slice()
);
}
fn idempotency_config(
processing_ttl: std::time::Duration,
max_response_body_bytes: usize,
) -> IdempotencyConfig {
IdempotencyConfig::new(
DEFAULT_IDEMPOTENCY_TTL,
processing_ttl,
DEFAULT_IDEMPOTENCY_MAX_BODY_BYTES,
max_response_body_bytes,
)
.unwrap()
}
fn in_memory_runtime() -> IdempotencyRuntime {
IdempotencyRuntime::in_memory(IdempotencyConfig::default()).unwrap()
}
#[tokio::test]
async fn idempotency_keeps_bodyless_statuses_without_an_invented_content_length() {
use axum::http::{
Method,
header::{CONTENT_LENGTH, ETAG},
};
use tower::Layer;
for (status, original_length) in [
(StatusCode::NOT_MODIFIED, None),
(StatusCode::NOT_MODIFIED, Some("14")),
(StatusCode::NO_CONTENT, None),
(StatusCode::OK, None),
(StatusCode::OK, Some("14")),
] {
for method in [Method::GET, Method::HEAD] {
if status == StatusCode::OK && method != Method::HEAD {
continue;
}
for outside_router in [false, true] {
let executions = Arc::new(AtomicUsize::new(0));
let calls = executions.clone();
let app = Router::new().route(
"/resource",
axum::routing::get(move || {
let calls = calls.clone();
async move {
calls.fetch_add(1, Ordering::SeqCst);
let mut response =
Response::builder().status(status).header(ETAG, "\"v1\"");
if let Some(length) = original_length {
response = response.header(CONTENT_LENGTH, length);
}
let body = if status == StatusCode::NO_CONTENT {
Body::empty()
} else {
Body::from_stream(Body::empty().into_data_stream())
};
response.body(body).unwrap()
}
}),
);
let layer = IdempotencyLayer::new(in_memory_runtime());
let app = if outside_router {
Router::new().route_service("/resource", layer.layer(app))
} else {
app.layer(layer)
};
for replayed in [false, true] {
let response = app
.clone()
.oneshot(
Request::builder()
.method(method.clone())
.uri("/resource")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "bodyless")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), status);
assert_eq!(
response.headers().contains_key("idempotency-replayed"),
replayed
);
assert_eq!(response.headers()[ETAG], "\"v1\"");
let expected_length = if replayed { None } else { original_length };
assert_eq!(
response
.headers()
.get(CONTENT_LENGTH)
.map(|value| value.to_str().unwrap()),
expected_length,
"{method} {status}, outside_router={outside_router}, replayed={replayed}"
);
assert!(
to_bytes(response.into_body(), 4096)
.await
.unwrap()
.is_empty()
);
}
assert_eq!(executions.load(Ordering::SeqCst), 1);
}
}
}
}
#[tokio::test]
async fn idempotency_head_replay_does_not_replace_representation_length_with_zero() {
use axum::http::header::CONTENT_LENGTH;
use tower::Layer;
for outside_router in [false, true] {
let executions = Arc::new(AtomicUsize::new(0));
let calls = executions.clone();
let app = Router::new().route(
"/resource",
axum::routing::get(move || {
let calls = calls.clone();
async move {
calls.fetch_add(1, Ordering::SeqCst);
"representation"
}
}),
);
let layer = IdempotencyLayer::new(in_memory_runtime());
let app = if outside_router {
Router::new().route_service("/resource", layer.layer(app))
} else {
app.layer(layer)
};
for replayed in [false, true] {
let response = app
.clone()
.oneshot(
Request::head("/resource")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "head")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response.headers().contains_key("idempotency-replayed"),
replayed
);
if let Some(length) = response.headers().get(CONTENT_LENGTH) {
assert_eq!(
length, "14",
"outside_router={outside_router}, replayed={replayed}"
);
}
assert!(
to_bytes(response.into_body(), 4096)
.await
.unwrap()
.is_empty()
);
}
assert_eq!(executions.load(Ordering::SeqCst), 1);
}
}
#[tokio::test]
async fn idempotency_preserves_ordinary_empty_and_nonempty_response_lengths() {
use axum::http::header::CONTENT_LENGTH;
for body in ["", "representation"] {
let app = Router::new()
.route("/resource", axum::routing::get(move || async move { body }))
.layer(IdempotencyLayer::new(in_memory_runtime()));
for replayed in [false, true] {
let response = app
.clone()
.oneshot(
Request::get("/resource")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "ordinary")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(response.headers()[CONTENT_LENGTH], body.len().to_string());
assert_eq!(
response.headers().contains_key("idempotency-replayed"),
replayed
);
assert_eq!(to_bytes(response.into_body(), 4096).await.unwrap(), body);
}
}
}
fn body_with_trailers(value: &'static str, trailers: HeaderMap) -> Body {
use http_body_util::{BodyExt, Full};
Body::new(
Full::new(axum::body::Bytes::from_static(value.as_bytes()))
.with_trailers(std::future::ready(Some(Ok::<_, Infallible>(trailers)))),
)
}
#[tokio::test]
async fn request_trailers_are_rejected_before_reserving_or_executing() {
let store = Arc::new(MemoryIdempotencyStore::new());
let runtime = IdempotencyRuntime::new(store.clone(), IdempotencyConfig::default());
let mut trailers = HeaderMap::new();
trailers.insert("content-digest", "sha-256=:digest:".parse().unwrap());
let response = run_idempotent_service(
runtime,
Request::post("/jobs")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "trailers")
.body(body_with_trailers("payload", trailers))
.unwrap(),
|_| async { panic!("a request with unsupported trailers must not execute") },
)
.await
.map(|result: Result<Response, Infallible>| result.unwrap())
.unwrap_err()
.into_response();
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
let body = to_bytes(response.into_body(), 4096).await.unwrap();
let problem: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(problem["code"], "idempotency_request_trailers_unsupported");
assert_eq!(store.record_count().unwrap(), 0);
}
#[tokio::test]
async fn response_trailers_terminalize_once_instead_of_losing_the_result() {
let runtime = in_memory_runtime();
let executions = AtomicUsize::new(0);
for replayed in [false, true] {
let response = run_idempotent_service(
runtime.clone(),
Request::post("/jobs")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "trailers")
.body(Body::empty())
.unwrap(),
|_| async {
executions.fetch_add(1, Ordering::SeqCst);
let mut trailers = HeaderMap::new();
trailers.insert("grpc-status", "13".parse().unwrap());
Ok::<_, Infallible>(Response::new(body_with_trailers("payload", trailers)))
},
)
.await
.unwrap()
.unwrap();
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(
response.headers().contains_key("idempotency-replayed"),
replayed
);
let body = to_bytes(response.into_body(), 4096).await.unwrap();
assert_eq!(body, super::model::IDEMPOTENCY_TERMINAL_FAILURE_BODY);
}
assert_eq!(executions.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn empty_trailers_do_not_prevent_idempotent_execution_or_replay() {
let runtime = in_memory_runtime();
let executions = AtomicUsize::new(0);
for replayed in [false, true] {
let response = run_idempotent_service(
runtime.clone(),
Request::post("/jobs")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "empty-trailers")
.body(body_with_trailers("payload", HeaderMap::new()))
.unwrap(),
|_| async {
executions.fetch_add(1, Ordering::SeqCst);
Ok::<_, Infallible>(Response::new(body_with_trailers("ok", HeaderMap::new())))
},
)
.await
.unwrap()
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
response.headers().contains_key("idempotency-replayed"),
replayed
);
assert_eq!(to_bytes(response.into_body(), 4096).await.unwrap(), "ok");
}
assert_eq!(executions.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn fingerprinted_headers_distinguish_missing_empty_repeated_and_opaque_values() {
for (name, config) in [
(CONTENT_TYPE, IdempotencyConfig::default()),
(
HeaderName::from_static("x-version"),
IdempotencyConfig::default()
.with_fingerprint_headers([HeaderName::from_static("x-version")]),
),
] {
let executions = Arc::new(AtomicUsize::new(0));
let calls = executions.clone();
let app = Router::new()
.route(
"/jobs",
post(move || {
let calls = calls.clone();
async move {
calls.fetch_add(1, Ordering::SeqCst);
StatusCode::CREATED
}
}),
)
.layer(IdempotencyLayer::new(
IdempotencyRuntime::in_memory(config).unwrap(),
));
let cases: &[(&str, &[&[u8]], StatusCode, bool)] = &[
("missing", &[], StatusCode::CREATED, false),
("missing", &[], StatusCode::CREATED, true),
("missing", &[b""], StatusCode::CONFLICT, false),
("repeated", &[b"a", b"bc"], StatusCode::CREATED, false),
("repeated", &[b"a", b"bc"], StatusCode::CREATED, true),
("repeated", &[b"a"], StatusCode::CONFLICT, false),
("repeated", &[b"a", b"changed"], StatusCode::CONFLICT, false),
("repeated", &[b"ab", b"c"], StatusCode::CONFLICT, false),
("repeated", &[b"a,bc"], StatusCode::CONFLICT, false),
("repeated", &[b"bc", b"a"], StatusCode::CONFLICT, false),
("repeated", &[b" a", b"bc"], StatusCode::CONFLICT, false),
("opaque", &[b"opaque-\xff"], StatusCode::CREATED, false),
("opaque", &[b"opaque-\xff"], StatusCode::CREATED, true),
("opaque", &[b"opaque-\xfe"], StatusCode::CONFLICT, false),
];
for (attempt, &(key, values, expected_status, replayed)) in cases.iter().enumerate() {
let mut request = Request::post("/jobs")
.header(IDEMPOTENCY_KEY_HEADER_NAME, key)
.header("x-request-id", attempt.to_string())
.body(Body::from("same body"))
.unwrap();
for value in values {
request
.headers_mut()
.append(&name, HeaderValue::from_bytes(value).unwrap());
}
let response = app.clone().oneshot(request).await.unwrap();
assert_eq!(
response.status(),
expected_status,
"{name}, attempt {attempt}"
);
assert_eq!(
response
.headers()
.contains_key(super::IDEMPOTENCY_REPLAYED_HEADER_NAME),
replayed,
"attempt {attempt}",
);
}
assert_eq!(executions.load(Ordering::SeqCst), 3);
}
}
#[tokio::test]
async fn fingerprint_header_names_are_canonical_and_policy_changes_conflict() {
let store = Arc::new(MemoryIdempotencyStore::new());
let config = IdempotencyConfig::default().with_fingerprint_headers([
HeaderName::from_static("if-match"),
HeaderName::from_static("prefer"),
]);
let equivalent = IdempotencyConfig::default().with_fingerprint_headers([
HeaderName::from_bytes(b"Prefer").unwrap(),
HeaderName::from_static("if-match"),
HeaderName::from_static("prefer"),
]);
assert_eq!(config, equivalent);
let executions = AtomicUsize::new(0);
for (config, expected) in [
(config, StatusCode::CREATED),
(equivalent, StatusCode::CREATED),
(IdempotencyConfig::default(), StatusCode::CONFLICT),
(
IdempotencyConfig::default()
.with_fingerprint_headers([HeaderName::from_static("if-match")]),
StatusCode::CONFLICT,
),
] {
let result = run_idempotent_service(
IdempotencyRuntime::new(store.clone(), config),
Request::post("/jobs")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "same")
.body(Body::empty())
.unwrap(),
|_| async {
executions.fetch_add(1, Ordering::SeqCst);
Ok::<_, Infallible>(StatusCode::CREATED.into_response())
},
)
.await;
let response = match result {
Ok(response) => response.unwrap(),
Err(rejection) => rejection.into_response(),
};
assert_eq!(response.status(), expected);
}
assert_eq!(executions.load(Ordering::SeqCst), 1);
}
#[test]
fn configuration_rejects_expiry_and_response_capacity_overflow_before_execution() {
use std::time::Duration;
for invalid_ttl in [Duration::ZERO, Duration::MAX] {
assert_eq!(
IdempotencyConfig::new(invalid_ttl, DEFAULT_IDEMPOTENCY_PROCESSING_TTL, 1024, 1024),
Err(IdempotencyConfigError::InvalidTtl)
);
assert_eq!(
IdempotencyConfig::new(DEFAULT_IDEMPOTENCY_TTL, invalid_ttl, 1024, 1024),
Err(IdempotencyConfigError::InvalidProcessingTtl)
);
}
let max_body = usize::MAX - super::DEFAULT_MAX_IDEMPOTENCY_STORED_HEADER_BYTES;
for invalid_limit in [0, max_body + 1, usize::MAX] {
assert_eq!(
IdempotencyConfig::new(
DEFAULT_IDEMPOTENCY_TTL,
DEFAULT_IDEMPOTENCY_PROCESSING_TTL,
1024,
invalid_limit,
),
Err(IdempotencyConfigError::InvalidResponseBodyLimit)
);
}
assert!(
IdempotencyConfig::new(
DEFAULT_IDEMPOTENCY_TTL,
DEFAULT_IDEMPOTENCY_PROCESSING_TTL,
1024,
max_body,
)
.is_ok()
);
}
#[tokio::test]
async fn nested_routers_share_keys_without_replaying_different_request_targets() {
use axum::extract::OriginalUri;
let executions = Arc::new(AtomicUsize::new(0));
let calls = executions.clone();
let nested = Router::new()
.route(
"/jobs",
post(move |OriginalUri(uri): OriginalUri| {
let calls = calls.clone();
async move {
calls.fetch_add(1, Ordering::SeqCst);
uri.to_string()
}
}),
)
.layer(IdempotencyLayer::new(in_memory_runtime()));
let app = Router::new().nest("/a", nested.clone()).nest("/b", nested);
for (uri, key, status, replayed, calls) in [
("/a/jobs?queue=1", "same", StatusCode::OK, false, 1),
("/a/jobs?queue=1", "same", StatusCode::OK, true, 1),
("/b/jobs?queue=1", "same", StatusCode::CONFLICT, false, 1),
("/a/jobs?queue=2", "same", StatusCode::CONFLICT, false, 1),
("/b/jobs?queue=1", "fresh", StatusCode::OK, false, 2),
] {
let response = app
.clone()
.oneshot(
Request::post(uri)
.header(IDEMPOTENCY_KEY_HEADER_NAME, key)
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), status, "{uri}");
assert_eq!(
response
.headers()
.contains_key(super::IDEMPOTENCY_REPLAYED_HEADER_NAME),
replayed,
);
let body = to_bytes(response.into_body(), 1024).await.unwrap();
if status == StatusCode::OK {
assert_eq!(body, uri);
}
assert_eq!(executions.load(Ordering::SeqCst), calls);
}
}
struct InterruptedBody;
impl http_body::Body for InterruptedBody {
type Data = axum::body::Bytes;
type Error = std::io::Error;
fn poll_frame(
self: std::pin::Pin<&mut Self>,
_context: &mut std::task::Context<'_>,
) -> std::task::Poll<Option<Result<http_body::Frame<Self::Data>, Self::Error>>> {
std::task::Poll::Ready(Some(Err(std::io::Error::new(
std::io::ErrorKind::ConnectionReset,
"private transport error",
))))
}
}
#[tokio::test]
async fn unrepresentable_headers_commit_a_terminal_failure_without_reexecution() {
for value in [
HeaderValue::from_str(&"x".repeat(super::DEFAULT_MAX_IDEMPOTENCY_STORED_HEADER_BYTES))
.unwrap(),
HeaderValue::from_bytes(b"opaque-\xff").unwrap(),
] {
let runtime = in_memory_runtime();
let executions = AtomicUsize::new(0);
let mut first_body = None;
for attempt in 0..2 {
let response = run_idempotent_service(
runtime.clone(),
Request::post("/")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "uncapturable-header")
.body(Body::empty())
.unwrap(),
|_| async {
executions.fetch_add(1, Ordering::SeqCst);
let mut response = (StatusCode::CREATED, "created").into_response();
response.headers_mut().insert("location", value.clone());
Ok::<_, Infallible>(response)
},
)
.await
.unwrap()
.unwrap();
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(
response
.headers()
.contains_key(super::IDEMPOTENCY_REPLAYED_HEADER_NAME),
attempt == 1
);
let body = to_bytes(response.into_body(), 4096).await.unwrap();
if let Some(first) = &first_body {
assert_eq!(&body, first);
} else {
first_body = Some(body);
}
}
assert_eq!(executions.load(Ordering::SeqCst), 1);
}
}
#[tokio::test]
async fn metric_capacity_failures_are_counted_without_affecting_execution_or_replay() {
use crate::observability::{MetricLabels, MetricRegistry, MetricRegistryLimits};
let metrics = Arc::new(MetricRegistry::with_limits(
MetricRegistryLimits::new(1, 1).unwrap(),
));
metrics
.increment_counter("occupied", 1, MetricLabels::default())
.unwrap();
let runtime = in_memory_runtime().with_metrics(Some(metrics));
let executions = AtomicUsize::new(0);
for _ in 0..3 {
let response = run_idempotent_service(
runtime.clone(),
Request::post("/")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "metrics-full")
.body(Body::empty())
.unwrap(),
|_| async {
executions.fetch_add(1, Ordering::SeqCst);
Ok::<_, Infallible>("ok".into_response())
},
)
.await
.unwrap()
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
}
assert_eq!(runtime.failed_recordings(), 3);
assert_eq!(executions.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn reservation_errors_are_separate_from_successful_reservation_decisions() {
let metrics = Arc::new(MetricRegistry::new());
let store = MemoryIdempotencyStore::with_clock_and_max_records(crate::clock::system_clock(), 1)
.unwrap();
let runtime = IdempotencyRuntime::new(Arc::new(store), IdempotencyConfig::default())
.with_metrics(Some(metrics.clone()));
let executions = AtomicUsize::new(0);
for key in ["first", "second"] {
let result = run_idempotent_service(
runtime.clone(),
Request::post("/")
.header(IDEMPOTENCY_KEY_HEADER_NAME, key)
.body(Body::empty())
.unwrap(),
|_| async {
executions.fetch_add(1, Ordering::SeqCst);
Ok::<_, Infallible>("ok".into_response())
},
)
.await;
if key == "first" {
assert!(result.is_ok());
} else {
assert!(matches!(
result,
Err(IdempotencyRejection::StoreUnavailable)
));
}
}
assert_eq!(executions.load(Ordering::SeqCst), 1);
assert_store_error_metric(
&metrics,
"reserve",
"idempotency_store_capacity_exceeded",
1.0,
);
let snapshot = metrics.snapshot().unwrap();
assert_eq!(snapshot.samples().len(), 2);
assert_eq!(
snapshot
.samples()
.iter()
.find(|sample| sample.name() == metric_names::IDEMPOTENCY_REQUESTS_TOTAL)
.unwrap()
.value(),
1.0
);
}
#[tokio::test]
async fn every_foreground_commit_failure_is_counted_once() {
let metrics = Arc::new(MetricRegistry::new());
for mode in ["normal", "service_error", "capture_error"] {
let store = Arc::new(CommitFailingStore::default());
let runtime = IdempotencyRuntime::new(
store.clone(),
idempotency_config(
DEFAULT_IDEMPOTENCY_PROCESSING_TTL,
if mode == "capture_error" { 1 } else { 1024 },
),
)
.with_metrics(Some(metrics.clone()));
let result = run_idempotent_service(
runtime,
Request::post("/")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "private-key")
.body(Body::empty())
.unwrap(),
|_| async {
if mode == "service_error" {
Err("handler error")
} else {
Ok("response".into_response())
}
},
)
.await;
match mode {
"normal" => assert!(matches!(
result,
Err(IdempotencyRejection::StoreUnavailable)
)),
"service_error" => assert!(matches!(result, Ok(Err("handler error")))),
_ => assert_eq!(
result.unwrap().unwrap().status(),
StatusCode::INTERNAL_SERVER_ERROR
),
}
assert_eq!(store.commit_attempts.load(Ordering::SeqCst), 1);
assert_eq!(store.abort_attempts.load(Ordering::SeqCst), 0);
assert_eq!(store.inner.record_count().unwrap(), 1);
}
assert_store_error_metric(
&metrics,
"commit",
"idempotency_store_capacity_exceeded",
3.0,
);
assert_eq!(metrics.snapshot().unwrap().samples().len(), 2);
}
#[tokio::test]
async fn request_body_read_failures_are_distinct_from_size_limits() {
for oversized in [false, true] {
let body = || {
if oversized {
Body::from("too large")
} else {
Body::new(InterruptedBody)
}
};
let expected_status = if oversized {
StatusCode::PAYLOAD_TOO_LARGE
} else {
StatusCode::BAD_REQUEST
};
let suffix = if oversized {
"too_large"
} else {
"read_failed"
};
let store = Arc::new(MemoryIdempotencyStore::new());
let config = IdempotencyConfig::new(
DEFAULT_IDEMPOTENCY_TTL,
DEFAULT_IDEMPOTENCY_PROCESSING_TTL,
4,
1024,
)
.unwrap();
let runtime = IdempotencyRuntime::new(store.clone(), config);
let request = Request::post("/")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "body-test")
.body(body())
.unwrap();
let calls = AtomicUsize::new(0);
let rejection = run_idempotent_service(runtime, request, |_| async {
calls.fetch_add(1, Ordering::SeqCst);
Ok::<_, Infallible>("ok".into_response())
})
.await
.unwrap_err();
assert_eq!(rejection.error_kind(), ErrorKind::Validation);
assert_eq!(
rejection.error_code().as_str(),
format!("idempotency_request_body_{suffix}")
);
let response = rejection.into_response();
assert_eq!(response.status(), expected_status);
let bytes = to_bytes(response.into_body(), 4096).await.unwrap();
let problem: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
assert_eq!(
problem["code"],
format!("idempotency_request_body_{suffix}")
);
assert!(!String::from_utf8_lossy(&bytes).contains("private transport error"));
assert_eq!(calls.load(Ordering::SeqCst), 0);
assert_eq!(store.record_count().unwrap(), 0);
}
}
#[derive(Default)]
struct CommitFailingStore {
inner: MemoryIdempotencyStore,
commit_attempts: AtomicUsize,
abort_attempts: AtomicUsize,
commit_finished: tokio::sync::Notify,
}
#[async_trait]
impl IdempotencyStore for CommitFailingStore {
async fn reserve(
&self,
key: IdempotencyKey,
request_hash: IdempotencyRequestHash,
processing_ttl: std::time::Duration,
response_capacity_bytes: usize,
) -> Result<IdempotencyReservation, IdempotencyStoreError> {
self.inner
.reserve(key, request_hash, processing_ttl, response_capacity_bytes)
.await
}
async fn commit(
&self,
_key: &IdempotencyKey,
_lease: super::IdempotencyLease,
_response: StoredIdempotencyResponse,
_retention_ttl: std::time::Duration,
) -> Result<(), IdempotencyStoreError> {
self.commit_attempts.fetch_add(1, Ordering::SeqCst);
self.commit_finished.notify_one();
Err(IdempotencyStoreError::CapacityExceeded)
}
async fn abort(
&self,
key: &IdempotencyKey,
lease: super::IdempotencyLease,
) -> Result<(), IdempotencyStoreError> {
self.abort_attempts.fetch_add(1, Ordering::SeqCst);
self.inner.abort(key, lease).await
}
}
#[test]
fn validates_idempotency_keys() {
assert!(is_valid_idempotency_key("op_018f.test:1"));
assert!(!is_valid_idempotency_key(""));
assert!(!is_valid_idempotency_key("bad/key"));
}
#[test]
fn key_and_fingerprint_constructors_preserve_exact_identifiers() {
for value in [
" key",
"key ",
"\tkey\t",
"\r\nkey\r\n",
"\u{00a0}key",
"key\u{2003}",
] {
assert!(!is_valid_idempotency_key(value));
assert!(IdempotencyKey::new(value).is_err());
assert!(IdempotencyKey::parse(value).is_none());
assert!(IdempotencyRequestHash::new(value).is_err());
}
let value = "Operation_01.Hash:ABC";
assert!(is_valid_idempotency_key(value));
assert_eq!(IdempotencyKey::new(value).unwrap().as_str(), value);
assert_eq!(IdempotencyRequestHash::new(value).unwrap().as_str(), value);
}
#[test]
fn idempotency_debug_output_redacts_keys_hashes_and_response_payloads() {
let key_value = "private-idempotency-key";
let hash_value = "private-request-hash";
let header_value = "private-response-header";
let body_value = "private-response-body";
let key = IdempotencyKey::new(key_value).expect("key should validate");
let hash = IdempotencyRequestHash::new(hash_value).expect("hash should validate");
let response = StoredIdempotencyResponse::new(StatusCode::OK, body_value)
.try_with_header("content-type", header_value)
.unwrap();
let lease = IdempotencyLease::new();
let lease_value = lease.0.to_string();
let output = format!("{key} {key:?} {hash:?} {response:?} {lease:?}");
assert!(output.contains("[REDACTED]"));
assert!(output.contains("body_bytes"));
for secret in [key_value, hash_value, header_value, body_value] {
assert!(!output.contains(secret));
}
assert!(!output.contains(&lease_value));
}
#[test]
fn idempotency_errors_have_stable_classification() {
let invalid_key = IdempotencyKey::new("bad/key")
.expect_err("key should be invalid")
.error_descriptor();
let store = IdempotencyStoreError::InconsistentRecord.error_descriptor();
let execution_result_unavailable =
IdempotencyRejection::ExecutionResultUnavailable.error_descriptor();
let unverified_scope = IdempotencyRejection::UnverifiedAuthenticationScope.error_descriptor();
assert_eq!(invalid_key.kind(), ErrorKind::Validation);
assert_eq!(
invalid_key.code().as_str(),
"idempotency_key_invalid_characters"
);
assert_eq!(store.kind(), ErrorKind::Internal);
assert_eq!(store.code().as_str(), "idempotency_record_inconsistent");
assert_eq!(execution_result_unavailable.kind(), ErrorKind::Internal);
assert_eq!(
execution_result_unavailable.code().as_str(),
"idempotency_execution_result_unavailable"
);
assert_eq!(unverified_scope.kind(), ErrorKind::Authorization);
assert_eq!(
unverified_scope.code().as_str(),
"idempotency_authentication_scope_required"
);
}
#[test]
fn extracts_key_from_headers() {
let mut headers = HeaderMap::new();
headers.insert(
IDEMPOTENCY_KEY_HEADER_NAME,
HeaderValue::from_static("request-1"),
);
assert_eq!(
idempotency_key_from_headers(&headers)
.as_ref()
.map(IdempotencyKey::as_str),
Some("request-1")
);
}
#[test]
fn header_extraction_accepts_only_surrounding_http_whitespace() {
let mut headers = HeaderMap::new();
for value in [
"request-1",
" request-1 ",
"\trequest-1\t",
" \trequest-1\t ",
] {
headers.insert(
IDEMPOTENCY_KEY_HEADER_NAME,
HeaderValue::from_str(value).unwrap(),
);
assert_eq!(
idempotency_key_from_headers(&headers).unwrap().as_str(),
"request-1"
);
}
for value in [" \t ", "request 1", "request\t1", "\u{00a0}request-1"] {
headers.insert(
IDEMPOTENCY_KEY_HEADER_NAME,
HeaderValue::from_str(value).unwrap(),
);
assert!(idempotency_key_from_headers(&headers).is_none());
}
}
#[tokio::test]
async fn repeated_idempotency_keys_are_rejected_before_execution() {
for second in ["request-1", "request-2"] {
let request = Request::post("/")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.header(IDEMPOTENCY_KEY_HEADER_NAME, second)
.body(Body::empty())
.unwrap();
assert!(idempotency_key_from_headers(request.headers()).is_none());
let store = Arc::new(MemoryIdempotencyStore::new());
let result: Result<Result<Response, Infallible>, _> = run_idempotent_service(
IdempotencyRuntime::new(store.clone(), IdempotencyConfig::default()),
request,
|_| async { panic!("ambiguous keys must not execute the handler") },
)
.await;
let rejection = result.unwrap_err();
assert!(matches!(rejection, IdempotencyRejection::InvalidKey));
let response = rejection.into_response();
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
let body = to_bytes(response.into_body(), 4096).await.unwrap();
let problem: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(problem["code"], "invalid_idempotency_key");
assert_eq!(store.record_count().unwrap(), 0);
}
}
#[tokio::test]
async fn memory_store_reserves_conflicts_and_replays() {
let clock = crate::clock::ManualClock::new();
let store = MemoryIdempotencyStore::with_clock(clock.shared());
let key = IdempotencyKey::new("request-1").unwrap();
let hash = IdempotencyRequestHash::new("hash-1").unwrap();
let lease = match store
.reserve(
key.clone(),
hash.clone(),
std::time::Duration::from_secs(60),
64,
)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
assert_eq!(
store
.reserve(
key.clone(),
IdempotencyRequestHash::new("hash-2").unwrap(),
std::time::Duration::from_secs(60),
64,
)
.await
.unwrap(),
IdempotencyReservation::Conflict
);
let stored = StoredIdempotencyResponse::new(StatusCode::OK, b"ok".to_vec());
store
.commit(&key, lease, stored, std::time::Duration::from_secs(60))
.await
.unwrap();
let IdempotencyReservation::Replay(replay) = store
.reserve(key, hash, std::time::Duration::from_secs(60), 64)
.await
.unwrap()
else {
panic!("completed response must be replayed");
};
assert_eq!(replay.body(), b"ok");
clock.advance(std::time::Duration::from_secs(60)).unwrap();
assert_eq!(store.prune_expired().unwrap(), 1);
assert_eq!(store.stored_response_bytes().unwrap(), 0);
let response = super::response_from_stored(replay, &axum::http::Method::POST).unwrap();
assert_eq!(to_bytes(response.into_body(), 64).await.unwrap(), "ok");
}
#[tokio::test]
async fn abort_releases_capacity_without_touching_a_newer_lease() {
let store = MemoryIdempotencyStore::with_clock_and_max_records(crate::clock::system_clock(), 1)
.unwrap();
let key = IdempotencyKey::new("request-1").unwrap();
let hash = IdempotencyRequestHash::new("hash-1").unwrap();
let first_lease = match store
.reserve(
key.clone(),
hash.clone(),
std::time::Duration::from_secs(60),
64,
)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
store.abort(&key, first_lease).await.unwrap();
assert_eq!(store.record_count().unwrap(), 0);
assert_eq!(store.reserved_response_bytes().unwrap(), 0);
let second_lease = match store
.reserve(key.clone(), hash, std::time::Duration::from_secs(60), 64)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
assert_ne!(first_lease, second_lease);
assert!(matches!(
store
.commit(
&key,
first_lease,
StoredIdempotencyResponse::new(StatusCode::OK, b"stale".to_vec()),
std::time::Duration::from_secs(60),
)
.await,
Err(IdempotencyStoreError::InconsistentRecord)
));
assert_eq!(store.record_count().unwrap(), 1);
assert_eq!(store.stored_response_bytes().unwrap(), 0);
}
#[tokio::test]
async fn cancelled_execution_is_terminalized_and_never_reexecuted() {
let store = Arc::new(MemoryIdempotencyStore::new());
let runtime = IdempotencyRuntime::new(
store.clone(),
idempotency_config(
std::time::Duration::from_secs(60),
DEFAULT_IDEMPOTENCY_MAX_BODY_BYTES,
),
);
let request = Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "cancelled-request")
.body(Body::empty())
.unwrap();
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let executions = Arc::new(AtomicUsize::new(0));
let executions_for_task = executions.clone();
let runtime_for_task = runtime.clone();
let task = tokio::spawn(async move {
super::run_idempotent_service(runtime_for_task, request, |_request| async move {
executions_for_task.fetch_add(1, Ordering::SeqCst);
let _ = started_tx.send(());
std::future::pending::<Result<axum::response::Response, std::convert::Infallible>>()
.await
})
.await
});
started_rx
.await
.expect("request should acquire a reservation before blocking");
assert_eq!(store.record_count().unwrap(), 1);
assert!(store.reserved_response_bytes().unwrap() > 0);
task.abort();
assert!(
task.await
.expect_err("request should be cancelled")
.is_cancelled()
);
for _ in 0..128 {
let terminalized = store
.state
.lock()
.unwrap()
.records
.values()
.any(|record| matches!(record.state, super::memory::EntryState::Completed(_)));
if terminalized {
break;
}
tokio::task::yield_now().await;
}
assert_eq!(store.record_count().unwrap(), 1);
assert_eq!(store.reserved_response_bytes().unwrap(), 0);
assert!(store.stored_response_bytes().unwrap() > 0);
let executions_for_retry = executions.clone();
let response = super::run_idempotent_service(
runtime,
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "cancelled-request")
.body(Body::empty())
.unwrap(),
move |_request| async move {
executions_for_retry.fetch_add(1, Ordering::SeqCst);
Ok::<_, std::convert::Infallible>(
(StatusCode::OK, "unexpected execution").into_response(),
)
},
)
.await
.unwrap()
.unwrap();
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(
response
.headers()
.get(super::IDEMPOTENCY_REPLAYED_HEADER_NAME)
.and_then(|value| value.to_str().ok()),
Some("true")
);
assert_eq!(executions.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn cancelled_commit_errors_and_metric_write_failures_are_counted_once() {
use crate::observability::{MetricLabels, MetricRegistryLimits};
for full_metrics in [false, true] {
let metrics = Arc::new(if full_metrics {
MetricRegistry::with_limits(MetricRegistryLimits::new(1, 1).unwrap())
} else {
MetricRegistry::new()
});
if full_metrics {
metrics
.increment_counter("occupied", 1, MetricLabels::default())
.unwrap();
}
let store = Arc::new(CommitFailingStore::default());
let runtime = IdempotencyRuntime::new(store.clone(), IdempotencyConfig::default())
.with_metrics(Some(metrics.clone()));
let task_runtime = runtime.clone();
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let task = tokio::spawn(async move {
run_idempotent_service(
task_runtime,
Request::post("/")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "cancelled")
.body(Body::empty())
.unwrap(),
|_| async {
started_tx.send(()).unwrap();
std::future::pending::<Result<Response, Infallible>>().await
},
)
.await
});
started_rx.await.unwrap();
task.abort();
assert!(task.await.unwrap_err().is_cancelled());
store.commit_finished.notified().await;
assert_eq!(store.commit_attempts.load(Ordering::SeqCst), 1);
assert_eq!(store.abort_attempts.load(Ordering::SeqCst), 0);
assert!(store.inner.reserved_response_bytes().unwrap() > 0);
assert_eq!(
runtime.failed_recordings(),
if full_metrics { 2 } else { 0 }
);
if full_metrics {
assert_eq!(metrics.snapshot().unwrap().samples().len(), 1);
} else {
assert_store_error_metric(
&metrics,
"cancel_commit",
"idempotency_store_capacity_exceeded",
1.0,
);
}
}
}
#[tokio::test(start_paused = true)]
async fn cancelled_execution_bounds_stalled_commit_and_preserves_processing_lease() {
struct HangingCommitStore {
inner: MemoryIdempotencyStore,
commit_started: tokio::sync::Notify,
commit_dropped: tokio::sync::Notify,
}
#[async_trait]
impl IdempotencyStore for HangingCommitStore {
async fn reserve(
&self,
key: IdempotencyKey,
request_hash: IdempotencyRequestHash,
processing_ttl: std::time::Duration,
response_capacity_bytes: usize,
) -> Result<IdempotencyReservation, IdempotencyStoreError> {
self.inner
.reserve(key, request_hash, processing_ttl, response_capacity_bytes)
.await
}
async fn commit(
&self,
_: &IdempotencyKey,
_: IdempotencyLease,
_: StoredIdempotencyResponse,
_: std::time::Duration,
) -> Result<(), IdempotencyStoreError> {
struct CommitGuard<'a>(&'a tokio::sync::Notify);
impl Drop for CommitGuard<'_> {
fn drop(&mut self) {
self.0.notify_one();
}
}
let _guard = CommitGuard(&self.commit_dropped);
self.commit_started.notify_one();
std::future::pending().await
}
async fn abort(
&self,
_: &IdempotencyKey,
_: IdempotencyLease,
) -> Result<(), IdempotencyStoreError> {
panic!("cancelled execution must not release its processing reservation")
}
}
let store = Arc::new(HangingCommitStore {
inner: MemoryIdempotencyStore::default(),
commit_started: tokio::sync::Notify::new(),
commit_dropped: tokio::sync::Notify::new(),
});
let metrics = Arc::new(MetricRegistry::new());
let runtime = IdempotencyRuntime::new(store.clone(), IdempotencyConfig::default())
.with_metrics(Some(metrics.clone()));
let request = || {
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "cancelled-stalled-commit")
.body(Body::empty())
.unwrap()
};
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let task_runtime = runtime.clone();
let task = tokio::spawn(async move {
run_idempotent_service(task_runtime, request(), |_| async {
started_tx.send(()).unwrap();
std::future::pending::<Result<Response, Infallible>>().await
})
.await
});
started_rx.await.unwrap();
task.abort();
assert!(task.await.unwrap_err().is_cancelled());
store.commit_started.notified().await;
tokio::time::timeout(
std::time::Duration::from_secs(10),
store.commit_dropped.notified(),
)
.await
.expect("background commit must be cancelled within a bounded time");
assert_store_error_metric(&metrics, "cancel_commit", "idempotency_store_timeout", 1.0);
assert_eq!(store.inner.record_count().unwrap(), 1);
assert!(store.inner.reserved_response_bytes().unwrap() > 0);
assert_eq!(store.inner.stored_response_bytes().unwrap(), 0);
assert!(matches!(
run_idempotent_service::<_, _, Infallible>(runtime, request(), |_| async {
panic!("the protected operation must not execute again");
})
.await,
Err(IdempotencyRejection::Processing)
));
}
#[tokio::test]
async fn inner_service_error_is_terminalized_before_the_error_is_propagated() {
let store = Arc::new(MemoryIdempotencyStore::new());
let runtime = IdempotencyRuntime::new(store.clone(), IdempotencyConfig::default());
let executions = Arc::new(AtomicUsize::new(0));
let executions_for_first = executions.clone();
let first = super::run_idempotent_service(
runtime.clone(),
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "service-error")
.body(Body::empty())
.unwrap(),
move |_request| async move {
executions_for_first.fetch_add(1, Ordering::SeqCst);
Err::<axum::response::Response, _>("downstream failure")
},
)
.await
.expect("idempotency orchestration should succeed");
assert!(matches!(first, Err("downstream failure")));
assert_eq!(store.record_count().unwrap(), 1);
assert_eq!(store.reserved_response_bytes().unwrap(), 0);
assert!(store.stored_response_bytes().unwrap() > 0);
let executions_for_retry = executions.clone();
let replay = super::run_idempotent_service(
runtime,
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "service-error")
.body(Body::empty())
.unwrap(),
move |_request| async move {
executions_for_retry.fetch_add(1, Ordering::SeqCst);
Ok::<_, &'static str>((StatusCode::OK, "unexpected execution").into_response())
},
)
.await
.unwrap()
.unwrap();
assert_eq!(replay.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(
replay
.headers()
.get(super::IDEMPOTENCY_REPLAYED_HEADER_NAME)
.and_then(|value| value.to_str().ok()),
Some("true")
);
assert_eq!(executions.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn expired_processing_lease_cannot_commit_into_a_new_reservation() {
let clock = crate::clock::ManualClock::new();
let store = MemoryIdempotencyStore::with_clock_and_max_records(clock.shared(), 1).unwrap();
let key = IdempotencyKey::new("request-expiry").unwrap();
let hash = IdempotencyRequestHash::new("hash-expiry").unwrap();
let processing_ttl = std::time::Duration::from_secs(5);
let first_lease = match store
.reserve(key.clone(), hash.clone(), processing_ttl, 64)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
clock
.advance(processing_ttl + std::time::Duration::from_secs(1))
.unwrap();
let second_lease = match store
.reserve(key.clone(), hash, processing_ttl, 64)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
assert_ne!(first_lease, second_lease);
assert!(matches!(
store
.commit(
&key,
first_lease,
StoredIdempotencyResponse::new(StatusCode::OK, b"stale".to_vec()),
std::time::Duration::from_secs(60),
)
.await,
Err(IdempotencyStoreError::InconsistentRecord)
));
store
.commit(
&key,
second_lease,
StoredIdempotencyResponse::new(StatusCode::OK, b"current".to_vec()),
std::time::Duration::from_secs(60),
)
.await
.unwrap();
assert_eq!(store.record_count().unwrap(), 1);
}
#[tokio::test]
async fn completed_retention_ttl_starts_when_the_response_is_committed() {
let clock = crate::clock::ManualClock::new();
let store = MemoryIdempotencyStore::with_clock_and_max_records(clock.shared(), 1).unwrap();
let key = IdempotencyKey::new("request-retention").unwrap();
let hash = IdempotencyRequestHash::new("hash-retention").unwrap();
let processing_ttl = std::time::Duration::from_secs(10);
let retention_ttl = std::time::Duration::from_secs(20);
let lease = match store
.reserve(key.clone(), hash.clone(), processing_ttl, 64)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
clock.advance(std::time::Duration::from_secs(9)).unwrap();
store
.commit(
&key,
lease,
StoredIdempotencyResponse::new(StatusCode::OK, b"completed".to_vec()),
retention_ttl,
)
.await
.unwrap();
clock.advance(std::time::Duration::from_secs(19)).unwrap();
assert!(matches!(
store
.reserve(key.clone(), hash.clone(), processing_ttl, 64)
.await
.unwrap(),
IdempotencyReservation::Replay(_)
));
clock.advance(std::time::Duration::from_secs(2)).unwrap();
assert!(matches!(
store.reserve(key, hash, processing_ttl, 64).await.unwrap(),
IdempotencyReservation::Reserved(_)
));
}
#[tokio::test]
async fn expiry_releases_only_expired_response_and_reservation_bytes() {
use std::time::Duration;
let clock = crate::clock::ManualClock::new();
let store = MemoryIdempotencyStore::with_clock_and_limits(clock.shared(), 4, 128).unwrap();
for index in 0..4 {
let key = IdempotencyKey::new(format!("request-{index}")).unwrap();
let ttl = Duration::from_secs(if index < 2 { 5 } else { 10 });
let reservation = store
.reserve(
key.clone(),
IdempotencyRequestHash::new("hash").unwrap(),
ttl,
16,
)
.await
.unwrap();
let IdempotencyReservation::Reserved(lease) = reservation else {
panic!("new key must be reserved");
};
if index % 2 == 0 {
let mut response = StoredIdempotencyResponse::new(StatusCode::OK, b"ok".to_vec());
response.headers.push(("x-test".into(), "yes".into()));
store.commit(&key, lease, response, ttl).await.unwrap();
}
}
assert_eq!(store.stored_response_bytes().unwrap(), 22);
assert_eq!(store.reserved_response_bytes().unwrap(), 32);
assert_eq!(store.prune_expired().unwrap(), 0);
clock.advance(Duration::from_secs(5)).unwrap();
assert_eq!(store.prune_expired().unwrap(), 2);
assert_eq!(store.record_count().unwrap(), 2);
assert_eq!(store.stored_response_bytes().unwrap(), 11);
assert_eq!(store.reserved_response_bytes().unwrap(), 16);
assert_eq!(store.prune_expired().unwrap(), 0);
clock.advance(Duration::from_secs(5)).unwrap();
assert_eq!(store.prune_expired().unwrap(), 2);
assert_eq!(store.stored_response_bytes().unwrap(), 0);
assert_eq!(store.reserved_response_bytes().unwrap(), 0);
}
#[tokio::test]
async fn memory_store_bounds_unexpired_records() {
let store = MemoryIdempotencyStore::with_clock_and_max_records(crate::clock::system_clock(), 2)
.unwrap();
let ttl = std::time::Duration::from_secs(60);
for index in 0..2 {
store
.reserve(
IdempotencyKey::new(format!("request-{index}")).unwrap(),
IdempotencyRequestHash::new(format!("hash-{index}")).unwrap(),
ttl,
64,
)
.await
.unwrap();
}
let result = store
.reserve(
IdempotencyKey::new("request-overflow").unwrap(),
IdempotencyRequestHash::new("hash-overflow").unwrap(),
ttl,
64,
)
.await;
assert!(matches!(
result,
Err(IdempotencyStoreError::CapacityExceeded)
));
assert_eq!(store.record_count().unwrap(), 2);
}
#[tokio::test]
async fn memory_store_bounds_total_stored_response_bytes() {
let store =
MemoryIdempotencyStore::with_clock_and_limits(crate::clock::system_clock(), 2, 7).unwrap();
let key = IdempotencyKey::new("request-sized").unwrap();
let lease = match store
.reserve(
key.clone(),
IdempotencyRequestHash::new("hash-sized").unwrap(),
std::time::Duration::from_secs(60),
7,
)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
let result = store
.commit(
&key,
lease,
StoredIdempotencyResponse::new(StatusCode::OK, vec![0; 8]),
std::time::Duration::from_secs(60),
)
.await;
assert!(matches!(
result,
Err(IdempotencyStoreError::CapacityExceeded)
));
assert_eq!(store.stored_response_bytes().unwrap(), 0);
assert_eq!(store.reserved_response_bytes().unwrap(), 7);
let response = StoredIdempotencyResponse::new(StatusCode::OK, b"ok".to_vec());
let ttl = std::time::Duration::from_secs(60);
store
.commit(&key, lease, response.clone(), ttl)
.await
.unwrap();
assert_eq!(store.reserved_response_bytes().unwrap(), 0);
assert_eq!(store.stored_response_bytes().unwrap(), 2);
assert!(matches!(
store.commit(&key, lease, response.clone(), ttl).await,
Err(IdempotencyStoreError::InconsistentRecord)
));
store.abort(&key, lease).await.unwrap();
assert_eq!(store.stored_response_bytes().unwrap(), 2);
assert_eq!(
store
.reserve(
key,
IdempotencyRequestHash::new("hash-sized").unwrap(),
ttl,
7
)
.await
.unwrap(),
IdempotencyReservation::Replay(response)
);
}
#[test]
fn in_memory_store_reserves_worst_case_response_capacity_before_execution() {
assert!(matches!(
MemoryIdempotencyStore::for_response_body_limit(0),
Err(IdempotencyStoreError::InvalidCapacity)
));
let store =
MemoryIdempotencyStore::for_response_body_limit(super::DEFAULT_IDEMPOTENCY_MAX_BODY_BYTES)
.unwrap();
let per_record = super::DEFAULT_IDEMPOTENCY_MAX_BODY_BYTES
+ super::DEFAULT_MAX_IDEMPOTENCY_STORED_HEADER_BYTES;
assert_eq!(store.max_records(), super::DEFAULT_MAX_IDEMPOTENCY_RECORDS);
assert_eq!(store.max_response_bytes_per_record, per_record);
assert!(per_record <= store.max_stored_response_bytes());
}
#[tokio::test]
async fn completed_small_responses_release_unused_reserved_capacity() {
let store = MemoryIdempotencyStore::with_clock_and_limits_and_reservation(
crate::clock::system_clock(),
4,
10,
6,
)
.unwrap();
let ttl = std::time::Duration::from_secs(60);
for index in 0..4 {
let key = IdempotencyKey::new(format!("request-{index}")).unwrap();
let lease = match store
.reserve(
key.clone(),
IdempotencyRequestHash::new(format!("hash-{index}")).unwrap(),
ttl,
6,
)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
store
.commit(
&key,
lease,
StoredIdempotencyResponse::new(StatusCode::OK, vec![0]),
ttl,
)
.await
.unwrap();
}
assert_eq!(store.record_count().unwrap(), 4);
assert_eq!(store.stored_response_bytes().unwrap(), 4);
assert_eq!(store.reserved_response_bytes().unwrap(), 0);
}
#[test]
fn stored_response_preserves_valid_duplicate_headers() {
let stored = StoredIdempotencyResponse::new(axum::http::StatusCode::OK, b"ok".to_vec())
.try_with_header(CONTENT_TYPE.as_str(), "application/json")
.unwrap()
.try_with_header("warning", "199 first")
.unwrap()
.try_with_header("warning", "299 second")
.unwrap();
assert_eq!(
stored.headers(),
&[
(
CONTENT_TYPE.as_str().to_owned(),
"application/json".to_owned()
),
("warning".to_owned(), "199 first".to_owned()),
("warning".to_owned(), "299 second".to_owned()),
]
);
}
#[tokio::test]
async fn stored_response_filters_connection_nominated_headers_and_replays_duplicates() {
let mut response = Response::new(Body::from("ok"));
response.headers_mut().insert(
"connection",
HeaderValue::from_static("x-hop, keep-alive, Content-Type"),
);
response
.headers_mut()
.insert(CONTENT_TYPE, HeaderValue::from_static("text/plain"));
response
.headers_mut()
.insert("x-hop", HeaderValue::from_static("private"));
response
.headers_mut()
.insert("keep-alive", HeaderValue::from_static("timeout=5"));
response.headers_mut().insert(
super::IDEMPOTENCY_REPLAYED_HEADER_NAME,
HeaderValue::from_static("false"),
);
response
.headers_mut()
.append("warning", HeaderValue::from_static("199 first"));
response
.headers_mut()
.append("warning", HeaderValue::from_static("299 second"));
let (parts, body) = response.into_parts();
let body = to_bytes(body, usize::MAX).await.unwrap().to_vec();
let stored = super::stored_response_from_parts(&parts, body).unwrap();
assert!(stored.headers().iter().all(|(name, _)| {
name != "connection"
&& name != "content-type"
&& name != "x-hop"
&& name != "keep-alive"
&& name != super::IDEMPOTENCY_REPLAYED_HEADER_NAME
}));
let replay = super::response_from_stored(stored, &axum::http::Method::POST).unwrap();
let warnings = replay
.headers()
.get_all("warning")
.iter()
.collect::<Vec<_>>();
assert_eq!(warnings.len(), 2);
assert_eq!(warnings[0], "199 first");
assert_eq!(warnings[1], "299 second");
let replay_markers = replay
.headers()
.get_all(super::IDEMPOTENCY_REPLAYED_HEADER_NAME)
.iter()
.collect::<Vec<_>>();
assert_eq!(replay_markers.len(), 1);
assert_eq!(replay_markers[0], "true");
}
#[tokio::test]
async fn insufficient_store_capacity_rejects_before_execution() {
let store = Arc::new(CommitFailingStore {
inner: MemoryIdempotencyStore::for_response_body_limit(64).unwrap(),
..Default::default()
});
let runtime = IdempotencyRuntime::new(
store.clone(),
idempotency_config(DEFAULT_IDEMPOTENCY_PROCESSING_TTL, 1024),
);
let executions = AtomicUsize::new(0);
let result = run_idempotent_service(
runtime,
Request::post("/")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "insufficient-capacity")
.body(Body::empty())
.unwrap(),
|_| async {
executions.fetch_add(1, Ordering::SeqCst);
Ok::<_, Infallible>(Response::new(Body::empty()))
},
)
.await;
assert!(matches!(
result,
Err(IdempotencyRejection::StoreUnavailable)
));
assert_eq!(executions.load(Ordering::SeqCst), 0);
assert_eq!(store.inner.record_count().unwrap(), 0);
assert_eq!(store.inner.reserved_response_bytes().unwrap(), 0);
assert_eq!(store.commit_attempts.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn in_memory_runtime_reserves_raised_response_limit_before_execution() {
let response_bytes = super::DEFAULT_IDEMPOTENCY_MAX_BODY_BYTES
+ super::DEFAULT_MAX_IDEMPOTENCY_STORED_HEADER_BYTES
+ 1;
let runtime = IdempotencyRuntime::in_memory(idempotency_config(
DEFAULT_IDEMPOTENCY_PROCESSING_TTL,
response_bytes,
))
.unwrap();
let executions = Arc::new(AtomicUsize::new(0));
let executions_for_service = executions.clone();
let result = run_idempotent_service(
runtime,
Request::post("/")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "raised-response-limit")
.body(Body::empty())
.unwrap(),
move |_| async move {
executions_for_service.fetch_add(1, Ordering::SeqCst);
Ok::<_, Infallible>(Response::new(Body::from(vec![0_u8; response_bytes])))
},
)
.await
.expect("configured capacity should be reserved")
.expect("service is infallible");
assert_eq!(executions.load(Ordering::SeqCst), 1);
assert_eq!(
to_bytes(result.into_body(), usize::MAX)
.await
.unwrap()
.len(),
response_bytes
);
}
#[test]
fn in_memory_runtime_rejects_response_limit_exceeding_store_capacity() {
let runtime = IdempotencyRuntime::in_memory(idempotency_config(
DEFAULT_IDEMPOTENCY_PROCESSING_TTL,
super::DEFAULT_MAX_IDEMPOTENCY_STORED_RESPONSE_BYTES,
));
assert!(matches!(
runtime,
Err(IdempotencyStoreError::InvalidCapacity)
));
}
#[tokio::test]
async fn middleware_replays_matching_requests() {
async fn create_order() -> Response {
let mut response = (StatusCode::CREATED, "created").into_response();
response
.headers_mut()
.insert(SET_COOKIE, HeaderValue::from_static("session=fresh"));
response
}
let runtime = in_memory_runtime();
let app = Router::new()
.route("/orders", post(create_order))
.layer(IdempotencyLayer::new(runtime));
let response = app
.clone()
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.body(axum::body::Body::from(r#"{"sku":"A"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CREATED);
assert_eq!(
response.headers().get(SET_COOKIE),
Some(&HeaderValue::from_static("session=fresh"))
);
let response = app
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.body(axum::body::Body::from(r#"{"sku":"A"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CREATED);
assert_eq!(
response
.headers()
.get(super::IDEMPOTENCY_REPLAYED_HEADER_NAME),
Some(&HeaderValue::from_static("true"))
);
assert!(response.headers().get(SET_COOKIE).is_none());
}
#[tokio::test]
async fn replay_preserves_the_original_problem_context() {
let app = Router::new()
.route(
"/orders",
post(|context: crate::http::RequestContext| async move {
crate::http::ApiProblem::bad_request(
crate::http::ProblemCode::INVALID_QUERY,
"invalid order",
)
.with_request_context(&context)
}),
)
.layer(IdempotencyLayer::new(in_memory_runtime()));
let mut first_body = None;
for request_id in ["original-request", "replay-request"] {
let response = app
.clone()
.oneshot(with_request_context(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.body(Body::empty())
.unwrap(),
request_id,
))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
assert_eq!(response.headers()[CACHE_CONTROL], "no-store");
assert_eq!(
response
.headers()
.contains_key(super::IDEMPOTENCY_REPLAYED_HEADER_NAME),
first_body.is_some()
);
let bytes = to_bytes(response.into_body(), 4096).await.unwrap();
if let Some(first_body) = &first_body {
assert_eq!(&bytes, first_body);
} else {
let problem: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
assert_eq!(problem["request_id"], "original-request");
assert_eq!(problem["trace_id"], "4bf92f3577b34da6a3ce929d0e0e4736");
first_body = Some(bytes);
}
}
}
#[tokio::test]
async fn middleware_rejections_include_the_installed_request_context() {
for (runtime, key, status, code) in [
(
in_memory_runtime(),
"invalid/key",
StatusCode::BAD_REQUEST,
"invalid_idempotency_key",
),
(
IdempotencyRuntime::new(
Arc::new(CommitFailingStore::default()),
IdempotencyConfig::default(),
),
"valid-key",
StatusCode::SERVICE_UNAVAILABLE,
"idempotency_store_unavailable",
),
] {
let app = Router::new()
.route("/orders", post(|| async { "created" }))
.layer(IdempotencyLayer::new(runtime));
let response = app
.oneshot(with_request_context(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, key)
.header("x-request-id", "untrusted-request")
.body(Body::empty())
.unwrap(),
"verified-request",
))
.await
.unwrap();
assert_eq!(response.status(), status);
assert_correlated_problem(response, code, "verified-request").await;
}
}
#[tokio::test]
async fn required_key_extractor_preserves_request_context() {
let app = Router::new().route(
"/orders",
post(|_: super::RequiredIdempotencyKey| async { "created" }),
);
let response = app
.oneshot(with_request_context(
Request::post("/orders").body(Body::empty()).unwrap(),
"missing-key-request",
))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
assert_correlated_problem(response, "idempotency_key_required", "missing-key-request").await;
}
#[tokio::test]
async fn middleware_rejects_same_key_with_different_body() {
let runtime = in_memory_runtime();
let app = Router::new()
.route("/orders", post(|| async { "created" }))
.layer(IdempotencyLayer::new(runtime));
app.clone()
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.body(axum::body::Body::from(r#"{"sku":"A"}"#))
.unwrap(),
)
.await
.unwrap();
let response = app
.oneshot(with_request_context(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.body(axum::body::Body::from(r#"{"sku":"B"}"#))
.unwrap(),
"conflicting-request",
))
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CONFLICT);
assert_correlated_problem(response, "idempotency_key_conflict", "conflicting-request").await;
}
#[tokio::test]
async fn middleware_rejects_unverified_authentication_context() {
let runtime = in_memory_runtime();
let app = Router::new()
.route("/orders", post(|| async { "must not execute" }))
.layer(IdempotencyLayer::new(runtime));
let response = app
.clone()
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.header(AUTHORIZATION, "Bearer unverified")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::FORBIDDEN);
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
let problem: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(problem["code"], "idempotency_authentication_scope_required");
let response = app
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-2")
.header(COOKIE, "session=unverified")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::FORBIDDEN);
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
let problem: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(problem["code"], "idempotency_authentication_scope_required");
}
#[tokio::test]
async fn middleware_requires_scope_for_principals_without_credential_headers() {
use crate::{
auth::Principal,
http::{AuthenticationChallenge, CurrentPrincipal, RequestContext, VerifiedAuthentication},
};
let calls = Arc::new(AtomicUsize::new(0));
let app = Router::new()
.route(
"/orders",
post({
let calls = calls.clone();
move |principal: CurrentPrincipal| {
calls.fetch_add(1, Ordering::SeqCst);
async move { principal.id().to_owned() }
}
}),
)
.layer(IdempotencyLayer::new(in_memory_runtime()))
.with_state(AuthenticationChallenge::try_new("Bearer").unwrap());
for scoped in [false, true] {
for identity in ["user_1", "user_2"] {
let mut request = Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "shared-client-key")
.body(Body::empty())
.unwrap();
let principal = Principal::user(identity).unwrap();
if scoped {
VerifiedAuthentication::new(principal, IdempotencyScope::new(identity))
.install(&mut request);
} else {
let mut context = RequestContext::from_headers(request.headers());
context.set_principal(principal);
request.extensions_mut().insert(context);
}
let response = app.clone().oneshot(request).await.unwrap();
assert_eq!(
response.status(),
if scoped {
StatusCode::OK
} else {
StatusCode::FORBIDDEN
}
);
let body = to_bytes(response.into_body(), 1024).await.unwrap();
if scoped {
assert_eq!(body, identity);
} else {
let problem: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(problem["code"], "idempotency_authentication_scope_required");
assert_eq!(calls.load(Ordering::SeqCst), 0);
}
}
}
assert_eq!(calls.load(Ordering::SeqCst), 2);
}
#[tokio::test]
async fn middleware_namespaces_records_by_verified_authentication_context() {
async fn install_verified_scope(mut request: Request<Body>, next: Next) -> Response {
let scope = IdempotencyScope::namespaced(
"test.authenticated-user",
request
.headers()
.get(AUTHORIZATION)
.expect("test authorization should be present")
.as_bytes(),
);
request.extensions_mut().insert(scope);
next.run(request).await
}
let runtime = in_memory_runtime();
let app = Router::new()
.route(
"/orders",
post(|headers: HeaderMap| async move {
headers
.get(AUTHORIZATION)
.and_then(|value| value.to_str().ok())
.unwrap_or("anonymous")
.to_owned()
}),
)
.layer(IdempotencyLayer::new(runtime))
.layer(middleware::from_fn(install_verified_scope));
let response = app
.clone()
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "shared-client-key")
.header(AUTHORIZATION, "Bearer user-one")
.body(Body::from(r#"{"sku":"A"}"#))
.unwrap(),
)
.await
.unwrap();
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
assert_eq!(&body[..], b"Bearer user-one");
let response = app
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "shared-client-key")
.header(AUTHORIZATION, "Bearer user-two")
.body(Body::from(r#"{"sku":"A"}"#))
.unwrap(),
)
.await
.unwrap();
assert!(
response
.headers()
.get(super::IDEMPOTENCY_REPLAYED_HEADER_NAME)
.is_none()
);
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
assert_eq!(&body[..], b"Bearer user-two");
}
#[tokio::test]
async fn oversized_response_is_terminalized_and_replayed_without_reexecution() {
let store = Arc::new(MemoryIdempotencyStore::for_response_body_limit(1).unwrap());
let runtime = IdempotencyRuntime::new(
store.clone(),
idempotency_config(DEFAULT_IDEMPOTENCY_PROCESSING_TTL, 1),
);
let executions = Arc::new(AtomicUsize::new(0));
let executions_for_handler = executions.clone();
let app = Router::new()
.route(
"/orders",
post(move || {
let executions = executions_for_handler.clone();
async move {
executions.fetch_add(1, Ordering::SeqCst);
"too large"
}
}),
)
.layer(IdempotencyLayer::new(runtime));
let first = app
.clone()
.oneshot(with_request_context(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.body(Body::empty())
.unwrap(),
"first-request",
))
.await
.unwrap();
assert_eq!(first.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(first.headers()[CACHE_CONTROL], "no-store");
assert_eq!(first.headers()[CONTENT_TYPE], "application/problem+json");
assert!(
first
.headers()
.get(super::IDEMPOTENCY_REPLAYED_HEADER_NAME)
.is_none()
);
let first_body = to_bytes(first.into_body(), usize::MAX).await.unwrap();
let replay = app
.oneshot(with_request_context(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.body(Body::empty())
.unwrap(),
"replay-request",
))
.await
.unwrap();
assert_eq!(replay.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(replay.headers()[CACHE_CONTROL], "no-store");
assert_eq!(replay.headers()[CONTENT_TYPE], "application/problem+json");
assert_eq!(
replay
.headers()
.get(super::IDEMPOTENCY_REPLAYED_HEADER_NAME)
.and_then(|value| value.to_str().ok()),
Some("true")
);
let replay_body = to_bytes(replay.into_body(), usize::MAX).await.unwrap();
assert_eq!(replay_body, first_body);
assert_eq!(replay_body, super::model::IDEMPOTENCY_TERMINAL_FAILURE_BODY);
assert_eq!(executions.load(Ordering::SeqCst), 1);
assert_eq!(store.record_count().unwrap(), 1);
assert_eq!(store.reserved_response_bytes().unwrap(), 0);
let stored_size = {
let state = store.state.lock().unwrap();
let response = state
.records
.values()
.next()
.and_then(|record| match &record.state {
super::memory::EntryState::Completed(response) => Some(response),
super::memory::EntryState::Processing { .. } => None,
})
.expect("terminal response should be stored");
super::stored_response_size(response)
};
assert_eq!(store.stored_response_bytes().unwrap(), stored_size);
}
#[tokio::test]
async fn failed_terminal_commit_keeps_the_processing_lease_and_never_aborts() {
let store = Arc::new(CommitFailingStore::default());
let runtime = IdempotencyRuntime::new(
store.clone(),
idempotency_config(DEFAULT_IDEMPOTENCY_PROCESSING_TTL, 1),
);
let executions = Arc::new(AtomicUsize::new(0));
let executions_for_handler = executions.clone();
let app = Router::new()
.route(
"/orders",
post(move || {
let executions = executions_for_handler.clone();
async move {
executions.fetch_add(1, Ordering::SeqCst);
"too large"
}
}),
)
.layer(IdempotencyLayer::new(runtime));
let first = app
.clone()
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "commit-failure")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(first.status(), StatusCode::INTERNAL_SERVER_ERROR);
let retry = app
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "commit-failure")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(retry.status(), StatusCode::CONFLICT);
assert_eq!(executions.load(Ordering::SeqCst), 1);
assert_eq!(store.commit_attempts.load(Ordering::SeqCst), 1);
assert_eq!(store.abort_attempts.load(Ordering::SeqCst), 0);
assert_eq!(store.inner.record_count().unwrap(), 1);
assert_eq!(store.inner.stored_response_bytes().unwrap(), 0);
assert!(store.inner.reserved_response_bytes().unwrap() > 0);
}
#[tokio::test]
async fn layer_allows_requests_without_key_by_default() {
let app = Router::new()
.route("/orders", post(|| async { "created" }))
.layer(IdempotencyLayer::new(in_memory_runtime()));
let response = app
.oneshot(
Request::post("/orders")
.body(axum::body::Body::from(r#"{"sku":"A"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
}