use std::{
convert::Infallible,
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
};
use async_trait::async_trait;
use axum::{
Router,
body::{Body, to_bytes},
http::{
HeaderMap, HeaderValue, Request, StatusCode,
header::{AUTHORIZATION, CONTENT_TYPE, COOKIE, SET_COOKIE},
},
middleware::{self, Next},
response::{IntoResponse, Response},
routing::post,
};
use tower::ServiceExt;
use super::{
DEFAULT_IDEMPOTENCY_MAX_BODY_BYTES, DEFAULT_IDEMPOTENCY_PROCESSING_TTL,
DEFAULT_IDEMPOTENCY_TTL, IDEMPOTENCY_KEY_HEADER_NAME, IdempotencyConfig,
IdempotencyConfigError, IdempotencyKey, IdempotencyLayer, IdempotencyLease,
IdempotencyRejection, IdempotencyRequestHash, IdempotencyReservation, IdempotencyRuntime,
IdempotencyScope, IdempotencyStore, IdempotencyStoreError, MemoryIdempotencyStore,
StoredIdempotencyResponse, idempotency_key_from_headers, is_valid_idempotency_key,
run_idempotent_service,
};
use crate::result::{ClassifyError, ErrorKind};
fn idempotency_config(
processing_ttl: std::time::Duration,
max_response_body_bytes: usize,
) -> IdempotencyConfig {
IdempotencyConfig::new(
DEFAULT_IDEMPOTENCY_TTL,
processing_ttl,
DEFAULT_IDEMPOTENCY_MAX_BODY_BYTES,
max_response_body_bytes,
)
.unwrap()
}
fn in_memory_runtime() -> IdempotencyRuntime {
IdempotencyRuntime::in_memory(IdempotencyConfig::default()).unwrap()
}
#[test]
fn configuration_rejects_expiry_and_response_capacity_overflow_before_execution() {
use std::time::Duration;
for invalid_ttl in [Duration::ZERO, Duration::MAX] {
assert_eq!(
IdempotencyConfig::new(invalid_ttl, DEFAULT_IDEMPOTENCY_PROCESSING_TTL, 1024, 1024),
Err(IdempotencyConfigError::InvalidTtl)
);
assert_eq!(
IdempotencyConfig::new(DEFAULT_IDEMPOTENCY_TTL, invalid_ttl, 1024, 1024),
Err(IdempotencyConfigError::InvalidProcessingTtl)
);
}
let max_body = usize::MAX - super::DEFAULT_MAX_IDEMPOTENCY_STORED_HEADER_BYTES;
for invalid_limit in [0, max_body + 1, usize::MAX] {
assert_eq!(
IdempotencyConfig::new(
DEFAULT_IDEMPOTENCY_TTL,
DEFAULT_IDEMPOTENCY_PROCESSING_TTL,
1024,
invalid_limit,
),
Err(IdempotencyConfigError::InvalidResponseBodyLimit)
);
}
assert!(
IdempotencyConfig::new(
DEFAULT_IDEMPOTENCY_TTL,
DEFAULT_IDEMPOTENCY_PROCESSING_TTL,
1024,
max_body,
)
.is_ok()
);
}
#[tokio::test]
async fn nested_routers_share_keys_without_replaying_different_request_targets() {
use axum::extract::OriginalUri;
let executions = Arc::new(AtomicUsize::new(0));
let calls = executions.clone();
let nested = Router::new()
.route(
"/jobs",
post(move |OriginalUri(uri): OriginalUri| {
let calls = calls.clone();
async move {
calls.fetch_add(1, Ordering::SeqCst);
uri.to_string()
}
}),
)
.layer(IdempotencyLayer::new(in_memory_runtime()));
let app = Router::new().nest("/a", nested.clone()).nest("/b", nested);
for (uri, key, status, replayed, calls) in [
("/a/jobs?queue=1", "same", StatusCode::OK, false, 1),
("/a/jobs?queue=1", "same", StatusCode::OK, true, 1),
("/b/jobs?queue=1", "same", StatusCode::CONFLICT, false, 1),
("/a/jobs?queue=2", "same", StatusCode::CONFLICT, false, 1),
("/b/jobs?queue=1", "fresh", StatusCode::OK, false, 2),
] {
let response = app
.clone()
.oneshot(
Request::post(uri)
.header(IDEMPOTENCY_KEY_HEADER_NAME, key)
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), status, "{uri}");
assert_eq!(
response
.headers()
.contains_key(super::IDEMPOTENCY_REPLAYED_HEADER_NAME),
replayed,
);
let body = to_bytes(response.into_body(), 1024).await.unwrap();
if status == StatusCode::OK {
assert_eq!(body, uri);
}
assert_eq!(executions.load(Ordering::SeqCst), calls);
}
}
struct InterruptedBody;
impl http_body::Body for InterruptedBody {
type Data = axum::body::Bytes;
type Error = std::io::Error;
fn poll_frame(
self: std::pin::Pin<&mut Self>,
_context: &mut std::task::Context<'_>,
) -> std::task::Poll<Option<Result<http_body::Frame<Self::Data>, Self::Error>>> {
std::task::Poll::Ready(Some(Err(std::io::Error::new(
std::io::ErrorKind::ConnectionReset,
"private transport error",
))))
}
}
#[tokio::test]
async fn unrepresentable_headers_commit_a_terminal_failure_without_reexecution() {
for value in [
HeaderValue::from_str(&"x".repeat(super::DEFAULT_MAX_IDEMPOTENCY_STORED_HEADER_BYTES))
.unwrap(),
HeaderValue::from_bytes(b"opaque-\xff").unwrap(),
] {
let runtime = in_memory_runtime();
let executions = AtomicUsize::new(0);
let mut first_body = None;
for attempt in 0..2 {
let response = run_idempotent_service(
runtime.clone(),
Request::post("/")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "uncapturable-header")
.body(Body::empty())
.unwrap(),
|_| async {
executions.fetch_add(1, Ordering::SeqCst);
let mut response = (StatusCode::CREATED, "created").into_response();
response.headers_mut().insert("location", value.clone());
Ok::<_, Infallible>(response)
},
)
.await
.unwrap()
.unwrap();
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(
response
.headers()
.contains_key(super::IDEMPOTENCY_REPLAYED_HEADER_NAME),
attempt == 1
);
let body = to_bytes(response.into_body(), 4096).await.unwrap();
if let Some(first) = &first_body {
assert_eq!(&body, first);
} else {
first_body = Some(body);
}
}
assert_eq!(executions.load(Ordering::SeqCst), 1);
}
}
#[tokio::test]
async fn metric_capacity_failures_are_counted_without_affecting_execution_or_replay() {
use crate::observability::{MetricLabels, MetricRegistry, MetricRegistryLimits};
let metrics = Arc::new(MetricRegistry::with_limits(
MetricRegistryLimits::new(1, 1).unwrap(),
));
metrics
.increment_counter("occupied", 1, MetricLabels::default())
.unwrap();
let runtime = in_memory_runtime().with_metrics(Some(metrics));
let executions = AtomicUsize::new(0);
for _ in 0..3 {
let response = run_idempotent_service(
runtime.clone(),
Request::post("/")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "metrics-full")
.body(Body::empty())
.unwrap(),
|_| async {
executions.fetch_add(1, Ordering::SeqCst);
Ok::<_, Infallible>("ok".into_response())
},
)
.await
.unwrap()
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
}
assert_eq!(runtime.failed_recordings(), 3);
assert_eq!(executions.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn request_body_read_failures_are_distinct_from_size_limits() {
for oversized in [false, true] {
let body = || {
if oversized {
Body::from("too large")
} else {
Body::new(InterruptedBody)
}
};
let expected_status = if oversized {
StatusCode::PAYLOAD_TOO_LARGE
} else {
StatusCode::BAD_REQUEST
};
let suffix = if oversized {
"too_large"
} else {
"read_failed"
};
let store = Arc::new(MemoryIdempotencyStore::new());
let config = IdempotencyConfig::new(
DEFAULT_IDEMPOTENCY_TTL,
DEFAULT_IDEMPOTENCY_PROCESSING_TTL,
4,
1024,
)
.unwrap();
let runtime = IdempotencyRuntime::new(store.clone(), config);
let request = Request::post("/")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "body-test")
.body(body())
.unwrap();
let calls = AtomicUsize::new(0);
let rejection = run_idempotent_service(runtime, request, |_| async {
calls.fetch_add(1, Ordering::SeqCst);
Ok::<_, Infallible>("ok".into_response())
})
.await
.unwrap_err();
assert_eq!(rejection.error_kind(), ErrorKind::Validation);
assert_eq!(
rejection.error_code().as_str(),
format!("idempotency_request_body_{suffix}")
);
let response = rejection.into_response();
assert_eq!(response.status(), expected_status);
let bytes = to_bytes(response.into_body(), 4096).await.unwrap();
let problem: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
assert_eq!(
problem["code"],
format!("idempotency_request_body_{suffix}")
);
assert!(!String::from_utf8_lossy(&bytes).contains("private transport error"));
assert_eq!(calls.load(Ordering::SeqCst), 0);
assert_eq!(store.record_count().unwrap(), 0);
}
}
#[derive(Default)]
struct CommitFailingStore {
inner: MemoryIdempotencyStore,
commit_attempts: AtomicUsize,
abort_attempts: AtomicUsize,
}
#[async_trait]
impl IdempotencyStore for CommitFailingStore {
async fn reserve(
&self,
key: IdempotencyKey,
request_hash: IdempotencyRequestHash,
processing_ttl: std::time::Duration,
response_capacity_bytes: usize,
) -> Result<IdempotencyReservation, IdempotencyStoreError> {
self.inner
.reserve(key, request_hash, processing_ttl, response_capacity_bytes)
.await
}
async fn commit(
&self,
_key: &IdempotencyKey,
_lease: super::IdempotencyLease,
_response: StoredIdempotencyResponse,
_retention_ttl: std::time::Duration,
) -> Result<(), IdempotencyStoreError> {
self.commit_attempts.fetch_add(1, Ordering::SeqCst);
Err(IdempotencyStoreError::CapacityExceeded)
}
async fn abort(
&self,
key: &IdempotencyKey,
lease: super::IdempotencyLease,
) -> Result<(), IdempotencyStoreError> {
self.abort_attempts.fetch_add(1, Ordering::SeqCst);
self.inner.abort(key, lease).await
}
}
#[test]
fn validates_idempotency_keys() {
assert!(is_valid_idempotency_key("op_018f.test:1"));
assert!(!is_valid_idempotency_key(""));
assert!(!is_valid_idempotency_key("bad/key"));
}
#[test]
fn idempotency_debug_output_redacts_keys_hashes_and_response_payloads() {
let key_value = "private-idempotency-key";
let hash_value = "private-request-hash";
let header_value = "private-response-header";
let body_value = "private-response-body";
let key = IdempotencyKey::new(key_value).expect("key should validate");
let hash = IdempotencyRequestHash::new(hash_value).expect("hash should validate");
let response = StoredIdempotencyResponse::new(StatusCode::OK, body_value)
.try_with_header("content-type", header_value)
.unwrap();
let lease = IdempotencyLease::new();
let lease_value = lease.0.to_string();
let output = format!("{key} {key:?} {hash:?} {response:?} {lease:?}");
assert!(output.contains("[REDACTED]"));
assert!(output.contains("body_bytes"));
for secret in [key_value, hash_value, header_value, body_value] {
assert!(!output.contains(secret));
}
assert!(!output.contains(&lease_value));
}
#[test]
fn idempotency_errors_have_stable_classification() {
let invalid_key = IdempotencyKey::new("bad/key")
.expect_err("key should be invalid")
.error_descriptor();
let store = IdempotencyStoreError::InconsistentRecord.error_descriptor();
let execution_result_unavailable =
IdempotencyRejection::ExecutionResultUnavailable.error_descriptor();
let unverified_scope = IdempotencyRejection::UnverifiedAuthenticationScope.error_descriptor();
assert_eq!(invalid_key.kind(), ErrorKind::Validation);
assert_eq!(
invalid_key.code().as_str(),
"idempotency_key_invalid_characters"
);
assert_eq!(store.kind(), ErrorKind::Internal);
assert_eq!(store.code().as_str(), "idempotency_record_inconsistent");
assert_eq!(execution_result_unavailable.kind(), ErrorKind::Internal);
assert_eq!(
execution_result_unavailable.code().as_str(),
"idempotency_execution_result_unavailable"
);
assert_eq!(unverified_scope.kind(), ErrorKind::Authentication);
assert_eq!(
unverified_scope.code().as_str(),
"idempotency_authentication_scope_required"
);
}
#[test]
fn extracts_key_from_headers() {
let mut headers = HeaderMap::new();
headers.insert(
IDEMPOTENCY_KEY_HEADER_NAME,
HeaderValue::from_static("request-1"),
);
assert_eq!(
idempotency_key_from_headers(&headers)
.as_ref()
.map(IdempotencyKey::as_str),
Some("request-1")
);
}
#[tokio::test]
async fn repeated_idempotency_keys_are_rejected_before_execution() {
for second in ["request-1", "request-2"] {
let request = Request::post("/")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.header(IDEMPOTENCY_KEY_HEADER_NAME, second)
.body(Body::empty())
.unwrap();
assert!(idempotency_key_from_headers(request.headers()).is_none());
let store = Arc::new(MemoryIdempotencyStore::new());
let result: Result<Result<Response, Infallible>, _> = run_idempotent_service(
IdempotencyRuntime::new(store.clone(), IdempotencyConfig::default()),
request,
|_| async { panic!("ambiguous keys must not execute the handler") },
)
.await;
let rejection = result.unwrap_err();
assert!(matches!(rejection, IdempotencyRejection::InvalidKey));
let response = rejection.into_response();
assert_eq!(response.status(), StatusCode::BAD_REQUEST);
let body = to_bytes(response.into_body(), 4096).await.unwrap();
let problem: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(problem["code"], "invalid_idempotency_key");
assert_eq!(store.record_count().unwrap(), 0);
}
}
#[tokio::test]
async fn memory_store_reserves_conflicts_and_replays() {
let store = MemoryIdempotencyStore::new();
let key = IdempotencyKey::new("request-1").unwrap();
let hash = IdempotencyRequestHash::new("hash-1").unwrap();
let lease = match store
.reserve(
key.clone(),
hash.clone(),
std::time::Duration::from_secs(60),
64,
)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
assert_eq!(
store
.reserve(
key.clone(),
IdempotencyRequestHash::new("hash-2").unwrap(),
std::time::Duration::from_secs(60),
64,
)
.await
.unwrap(),
IdempotencyReservation::Conflict
);
store
.commit(
&key,
lease,
StoredIdempotencyResponse::new(axum::http::StatusCode::OK, b"ok".to_vec()),
std::time::Duration::from_secs(60),
)
.await
.unwrap();
assert!(matches!(
store
.reserve(key, hash, std::time::Duration::from_secs(60), 64)
.await
.unwrap(),
IdempotencyReservation::Replay(_)
));
}
#[tokio::test]
async fn abort_releases_capacity_without_touching_a_newer_lease() {
let store = MemoryIdempotencyStore::with_clock_and_max_records(crate::clock::system_clock(), 1)
.unwrap();
let key = IdempotencyKey::new("request-1").unwrap();
let hash = IdempotencyRequestHash::new("hash-1").unwrap();
let first_lease = match store
.reserve(
key.clone(),
hash.clone(),
std::time::Duration::from_secs(60),
64,
)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
store.abort(&key, first_lease).await.unwrap();
assert_eq!(store.record_count().unwrap(), 0);
assert_eq!(store.reserved_response_bytes().unwrap(), 0);
let second_lease = match store
.reserve(key.clone(), hash, std::time::Duration::from_secs(60), 64)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
assert_ne!(first_lease, second_lease);
assert!(matches!(
store
.commit(
&key,
first_lease,
StoredIdempotencyResponse::new(StatusCode::OK, b"stale".to_vec()),
std::time::Duration::from_secs(60),
)
.await,
Err(IdempotencyStoreError::InconsistentRecord)
));
assert_eq!(store.record_count().unwrap(), 1);
assert_eq!(store.stored_response_bytes().unwrap(), 0);
}
#[tokio::test]
async fn cancelled_execution_is_terminalized_and_never_reexecuted() {
let store = Arc::new(MemoryIdempotencyStore::new());
let runtime = IdempotencyRuntime::new(
store.clone(),
idempotency_config(
std::time::Duration::from_secs(60),
DEFAULT_IDEMPOTENCY_MAX_BODY_BYTES,
),
);
let request = Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "cancelled-request")
.body(Body::empty())
.unwrap();
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let executions = Arc::new(AtomicUsize::new(0));
let executions_for_task = executions.clone();
let runtime_for_task = runtime.clone();
let task = tokio::spawn(async move {
super::run_idempotent_service(runtime_for_task, request, |_request| async move {
executions_for_task.fetch_add(1, Ordering::SeqCst);
let _ = started_tx.send(());
std::future::pending::<Result<axum::response::Response, std::convert::Infallible>>()
.await
})
.await
});
started_rx
.await
.expect("request should acquire a reservation before blocking");
assert_eq!(store.record_count().unwrap(), 1);
assert!(store.reserved_response_bytes().unwrap() > 0);
task.abort();
assert!(
task.await
.expect_err("request should be cancelled")
.is_cancelled()
);
for _ in 0..128 {
let terminalized = store
.state
.lock()
.unwrap()
.records
.values()
.any(|record| matches!(record.state, super::memory::EntryState::Completed(_)));
if terminalized {
break;
}
tokio::task::yield_now().await;
}
assert_eq!(store.record_count().unwrap(), 1);
assert_eq!(store.reserved_response_bytes().unwrap(), 0);
assert!(store.stored_response_bytes().unwrap() > 0);
let executions_for_retry = executions.clone();
let response = super::run_idempotent_service(
runtime,
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "cancelled-request")
.body(Body::empty())
.unwrap(),
move |_request| async move {
executions_for_retry.fetch_add(1, Ordering::SeqCst);
Ok::<_, std::convert::Infallible>(
(StatusCode::OK, "unexpected execution").into_response(),
)
},
)
.await
.unwrap()
.unwrap();
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(
response
.headers()
.get(super::IDEMPOTENCY_REPLAYED_HEADER_NAME)
.and_then(|value| value.to_str().ok()),
Some("true")
);
assert_eq!(executions.load(Ordering::SeqCst), 1);
}
#[tokio::test(start_paused = true)]
async fn cancelled_execution_bounds_stalled_commit_and_preserves_processing_lease() {
struct HangingCommitStore {
inner: MemoryIdempotencyStore,
commit_started: tokio::sync::Notify,
commit_dropped: tokio::sync::Notify,
}
#[async_trait]
impl IdempotencyStore for HangingCommitStore {
async fn reserve(
&self,
key: IdempotencyKey,
request_hash: IdempotencyRequestHash,
processing_ttl: std::time::Duration,
response_capacity_bytes: usize,
) -> Result<IdempotencyReservation, IdempotencyStoreError> {
self.inner
.reserve(key, request_hash, processing_ttl, response_capacity_bytes)
.await
}
async fn commit(
&self,
_: &IdempotencyKey,
_: IdempotencyLease,
_: StoredIdempotencyResponse,
_: std::time::Duration,
) -> Result<(), IdempotencyStoreError> {
struct CommitGuard<'a>(&'a tokio::sync::Notify);
impl Drop for CommitGuard<'_> {
fn drop(&mut self) {
self.0.notify_one();
}
}
let _guard = CommitGuard(&self.commit_dropped);
self.commit_started.notify_one();
std::future::pending().await
}
async fn abort(
&self,
_: &IdempotencyKey,
_: IdempotencyLease,
) -> Result<(), IdempotencyStoreError> {
panic!("cancelled execution must not release its processing reservation")
}
}
let store = Arc::new(HangingCommitStore {
inner: MemoryIdempotencyStore::default(),
commit_started: tokio::sync::Notify::new(),
commit_dropped: tokio::sync::Notify::new(),
});
let runtime = IdempotencyRuntime::new(store.clone(), IdempotencyConfig::default());
let request = || {
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "cancelled-stalled-commit")
.body(Body::empty())
.unwrap()
};
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let task_runtime = runtime.clone();
let task = tokio::spawn(async move {
run_idempotent_service(task_runtime, request(), |_| async {
started_tx.send(()).unwrap();
std::future::pending::<Result<Response, Infallible>>().await
})
.await
});
started_rx.await.unwrap();
task.abort();
assert!(task.await.unwrap_err().is_cancelled());
store.commit_started.notified().await;
tokio::time::timeout(
std::time::Duration::from_secs(10),
store.commit_dropped.notified(),
)
.await
.expect("background commit must be cancelled within a bounded time");
assert_eq!(store.inner.record_count().unwrap(), 1);
assert!(store.inner.reserved_response_bytes().unwrap() > 0);
assert_eq!(store.inner.stored_response_bytes().unwrap(), 0);
assert!(matches!(
run_idempotent_service::<_, _, Infallible>(runtime, request(), |_| async {
panic!("the protected operation must not execute again");
})
.await,
Err(IdempotencyRejection::Processing)
));
}
#[tokio::test]
async fn inner_service_error_is_terminalized_before_the_error_is_propagated() {
let store = Arc::new(MemoryIdempotencyStore::new());
let runtime = IdempotencyRuntime::new(store.clone(), IdempotencyConfig::default());
let executions = Arc::new(AtomicUsize::new(0));
let executions_for_first = executions.clone();
let first = super::run_idempotent_service(
runtime.clone(),
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "service-error")
.body(Body::empty())
.unwrap(),
move |_request| async move {
executions_for_first.fetch_add(1, Ordering::SeqCst);
Err::<axum::response::Response, _>("downstream failure")
},
)
.await
.expect("idempotency orchestration should succeed");
assert!(matches!(first, Err("downstream failure")));
assert_eq!(store.record_count().unwrap(), 1);
assert_eq!(store.reserved_response_bytes().unwrap(), 0);
assert!(store.stored_response_bytes().unwrap() > 0);
let executions_for_retry = executions.clone();
let replay = super::run_idempotent_service(
runtime,
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "service-error")
.body(Body::empty())
.unwrap(),
move |_request| async move {
executions_for_retry.fetch_add(1, Ordering::SeqCst);
Ok::<_, &'static str>((StatusCode::OK, "unexpected execution").into_response())
},
)
.await
.unwrap()
.unwrap();
assert_eq!(replay.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(
replay
.headers()
.get(super::IDEMPOTENCY_REPLAYED_HEADER_NAME)
.and_then(|value| value.to_str().ok()),
Some("true")
);
assert_eq!(executions.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn expired_processing_lease_cannot_commit_into_a_new_reservation() {
let clock = crate::clock::ManualClock::new();
let store = MemoryIdempotencyStore::with_clock_and_max_records(clock.shared(), 1).unwrap();
let key = IdempotencyKey::new("request-expiry").unwrap();
let hash = IdempotencyRequestHash::new("hash-expiry").unwrap();
let processing_ttl = std::time::Duration::from_secs(5);
let first_lease = match store
.reserve(key.clone(), hash.clone(), processing_ttl, 64)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
clock
.advance(processing_ttl + std::time::Duration::from_secs(1))
.unwrap();
let second_lease = match store
.reserve(key.clone(), hash, processing_ttl, 64)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
assert_ne!(first_lease, second_lease);
assert!(matches!(
store
.commit(
&key,
first_lease,
StoredIdempotencyResponse::new(StatusCode::OK, b"stale".to_vec()),
std::time::Duration::from_secs(60),
)
.await,
Err(IdempotencyStoreError::InconsistentRecord)
));
store
.commit(
&key,
second_lease,
StoredIdempotencyResponse::new(StatusCode::OK, b"current".to_vec()),
std::time::Duration::from_secs(60),
)
.await
.unwrap();
assert_eq!(store.record_count().unwrap(), 1);
}
#[tokio::test]
async fn completed_retention_ttl_starts_when_the_response_is_committed() {
let clock = crate::clock::ManualClock::new();
let store = MemoryIdempotencyStore::with_clock_and_max_records(clock.shared(), 1).unwrap();
let key = IdempotencyKey::new("request-retention").unwrap();
let hash = IdempotencyRequestHash::new("hash-retention").unwrap();
let processing_ttl = std::time::Duration::from_secs(10);
let retention_ttl = std::time::Duration::from_secs(20);
let lease = match store
.reserve(key.clone(), hash.clone(), processing_ttl, 64)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
clock.advance(std::time::Duration::from_secs(9)).unwrap();
store
.commit(
&key,
lease,
StoredIdempotencyResponse::new(StatusCode::OK, b"completed".to_vec()),
retention_ttl,
)
.await
.unwrap();
clock.advance(std::time::Duration::from_secs(19)).unwrap();
assert!(matches!(
store
.reserve(key.clone(), hash.clone(), processing_ttl, 64)
.await
.unwrap(),
IdempotencyReservation::Replay(_)
));
clock.advance(std::time::Duration::from_secs(2)).unwrap();
assert!(matches!(
store.reserve(key, hash, processing_ttl, 64).await.unwrap(),
IdempotencyReservation::Reserved(_)
));
}
#[tokio::test]
async fn expiry_releases_only_expired_response_and_reservation_bytes() {
use std::time::Duration;
let clock = crate::clock::ManualClock::new();
let store = MemoryIdempotencyStore::with_clock_and_limits(clock.shared(), 4, 128).unwrap();
for index in 0..4 {
let key = IdempotencyKey::new(format!("request-{index}")).unwrap();
let ttl = Duration::from_secs(if index < 2 { 5 } else { 10 });
let reservation = store
.reserve(
key.clone(),
IdempotencyRequestHash::new("hash").unwrap(),
ttl,
16,
)
.await
.unwrap();
let IdempotencyReservation::Reserved(lease) = reservation else {
panic!("new key must be reserved");
};
if index % 2 == 0 {
let mut response = StoredIdempotencyResponse::new(StatusCode::OK, b"ok".to_vec());
response.headers.push(("x-test".into(), "yes".into()));
store.commit(&key, lease, response, ttl).await.unwrap();
}
}
assert_eq!(store.stored_response_bytes().unwrap(), 22);
assert_eq!(store.reserved_response_bytes().unwrap(), 32);
assert_eq!(store.prune_expired().unwrap(), 0);
clock.advance(Duration::from_secs(5)).unwrap();
assert_eq!(store.prune_expired().unwrap(), 2);
assert_eq!(store.record_count().unwrap(), 2);
assert_eq!(store.stored_response_bytes().unwrap(), 11);
assert_eq!(store.reserved_response_bytes().unwrap(), 16);
assert_eq!(store.prune_expired().unwrap(), 0);
clock.advance(Duration::from_secs(5)).unwrap();
assert_eq!(store.prune_expired().unwrap(), 2);
assert_eq!(store.stored_response_bytes().unwrap(), 0);
assert_eq!(store.reserved_response_bytes().unwrap(), 0);
}
#[tokio::test]
async fn memory_store_bounds_unexpired_records() {
let store = MemoryIdempotencyStore::with_clock_and_max_records(crate::clock::system_clock(), 2)
.unwrap();
let ttl = std::time::Duration::from_secs(60);
for index in 0..2 {
store
.reserve(
IdempotencyKey::new(format!("request-{index}")).unwrap(),
IdempotencyRequestHash::new(format!("hash-{index}")).unwrap(),
ttl,
64,
)
.await
.unwrap();
}
let result = store
.reserve(
IdempotencyKey::new("request-overflow").unwrap(),
IdempotencyRequestHash::new("hash-overflow").unwrap(),
ttl,
64,
)
.await;
assert!(matches!(
result,
Err(IdempotencyStoreError::CapacityExceeded)
));
assert_eq!(store.record_count().unwrap(), 2);
}
#[tokio::test]
async fn memory_store_bounds_total_stored_response_bytes() {
let store =
MemoryIdempotencyStore::with_clock_and_limits(crate::clock::system_clock(), 2, 7).unwrap();
let key = IdempotencyKey::new("request-sized").unwrap();
let lease = match store
.reserve(
key.clone(),
IdempotencyRequestHash::new("hash-sized").unwrap(),
std::time::Duration::from_secs(60),
7,
)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
let result = store
.commit(
&key,
lease,
StoredIdempotencyResponse::new(StatusCode::OK, vec![0; 8]),
std::time::Duration::from_secs(60),
)
.await;
assert!(matches!(
result,
Err(IdempotencyStoreError::CapacityExceeded)
));
assert_eq!(store.stored_response_bytes().unwrap(), 0);
assert_eq!(store.reserved_response_bytes().unwrap(), 7);
let response = StoredIdempotencyResponse::new(StatusCode::OK, b"ok".to_vec());
let ttl = std::time::Duration::from_secs(60);
store
.commit(&key, lease, response.clone(), ttl)
.await
.unwrap();
assert_eq!(store.reserved_response_bytes().unwrap(), 0);
assert_eq!(store.stored_response_bytes().unwrap(), 2);
assert!(matches!(
store.commit(&key, lease, response.clone(), ttl).await,
Err(IdempotencyStoreError::InconsistentRecord)
));
store.abort(&key, lease).await.unwrap();
assert_eq!(store.stored_response_bytes().unwrap(), 2);
assert_eq!(
store
.reserve(
key,
IdempotencyRequestHash::new("hash-sized").unwrap(),
ttl,
7
)
.await
.unwrap(),
IdempotencyReservation::Replay(response)
);
}
#[test]
fn in_memory_store_reserves_worst_case_response_capacity_before_execution() {
assert!(matches!(
MemoryIdempotencyStore::for_response_body_limit(0),
Err(IdempotencyStoreError::InvalidCapacity)
));
let store =
MemoryIdempotencyStore::for_response_body_limit(super::DEFAULT_IDEMPOTENCY_MAX_BODY_BYTES)
.unwrap();
let per_record = super::DEFAULT_IDEMPOTENCY_MAX_BODY_BYTES
+ super::DEFAULT_MAX_IDEMPOTENCY_STORED_HEADER_BYTES;
assert_eq!(store.max_records(), super::DEFAULT_MAX_IDEMPOTENCY_RECORDS);
assert_eq!(store.max_response_bytes_per_record, per_record);
assert!(per_record <= store.max_stored_response_bytes());
}
#[tokio::test]
async fn completed_small_responses_release_unused_reserved_capacity() {
let store = MemoryIdempotencyStore::with_clock_and_limits_and_reservation(
crate::clock::system_clock(),
4,
10,
6,
)
.unwrap();
let ttl = std::time::Duration::from_secs(60);
for index in 0..4 {
let key = IdempotencyKey::new(format!("request-{index}")).unwrap();
let lease = match store
.reserve(
key.clone(),
IdempotencyRequestHash::new(format!("hash-{index}")).unwrap(),
ttl,
6,
)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
store
.commit(
&key,
lease,
StoredIdempotencyResponse::new(StatusCode::OK, vec![0]),
ttl,
)
.await
.unwrap();
}
assert_eq!(store.record_count().unwrap(), 4);
assert_eq!(store.stored_response_bytes().unwrap(), 4);
assert_eq!(store.reserved_response_bytes().unwrap(), 0);
}
#[test]
fn stored_response_preserves_valid_duplicate_headers() {
let stored = StoredIdempotencyResponse::new(axum::http::StatusCode::OK, b"ok".to_vec())
.try_with_header(CONTENT_TYPE.as_str(), "application/json")
.unwrap()
.try_with_header("warning", "199 first")
.unwrap()
.try_with_header("warning", "299 second")
.unwrap();
assert_eq!(
stored.headers(),
&[
(
CONTENT_TYPE.as_str().to_owned(),
"application/json".to_owned()
),
("warning".to_owned(), "199 first".to_owned()),
("warning".to_owned(), "299 second".to_owned()),
]
);
}
#[tokio::test]
async fn stored_response_filters_connection_nominated_headers_and_replays_duplicates() {
let mut response = Response::new(Body::from("ok"));
response.headers_mut().insert(
"connection",
HeaderValue::from_static("x-hop, keep-alive, Content-Type"),
);
response
.headers_mut()
.insert(CONTENT_TYPE, HeaderValue::from_static("text/plain"));
response
.headers_mut()
.insert("x-hop", HeaderValue::from_static("private"));
response
.headers_mut()
.insert("keep-alive", HeaderValue::from_static("timeout=5"));
response.headers_mut().insert(
super::IDEMPOTENCY_REPLAYED_HEADER_NAME,
HeaderValue::from_static("false"),
);
response
.headers_mut()
.append("warning", HeaderValue::from_static("199 first"));
response
.headers_mut()
.append("warning", HeaderValue::from_static("299 second"));
let (parts, body) = response.into_parts();
let body = to_bytes(body, usize::MAX).await.unwrap().to_vec();
let stored = super::stored_response_from_parts(&parts, body).unwrap();
assert!(stored.headers().iter().all(|(name, _)| {
name != "connection"
&& name != "content-type"
&& name != "x-hop"
&& name != "keep-alive"
&& name != super::IDEMPOTENCY_REPLAYED_HEADER_NAME
}));
let replay = super::response_from_stored(stored).unwrap();
let warnings = replay
.headers()
.get_all("warning")
.iter()
.collect::<Vec<_>>();
assert_eq!(warnings.len(), 2);
assert_eq!(warnings[0], "199 first");
assert_eq!(warnings[1], "299 second");
let replay_markers = replay
.headers()
.get_all(super::IDEMPOTENCY_REPLAYED_HEADER_NAME)
.iter()
.collect::<Vec<_>>();
assert_eq!(replay_markers.len(), 1);
assert_eq!(replay_markers[0], "true");
}
#[tokio::test]
async fn insufficient_store_capacity_rejects_before_execution() {
let store = Arc::new(CommitFailingStore {
inner: MemoryIdempotencyStore::for_response_body_limit(64).unwrap(),
..Default::default()
});
let runtime = IdempotencyRuntime::new(
store.clone(),
idempotency_config(DEFAULT_IDEMPOTENCY_PROCESSING_TTL, 1024),
);
let executions = AtomicUsize::new(0);
let result = run_idempotent_service(
runtime,
Request::post("/")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "insufficient-capacity")
.body(Body::empty())
.unwrap(),
|_| async {
executions.fetch_add(1, Ordering::SeqCst);
Ok::<_, Infallible>(Response::new(Body::empty()))
},
)
.await;
assert!(matches!(
result,
Err(IdempotencyRejection::StoreUnavailable)
));
assert_eq!(executions.load(Ordering::SeqCst), 0);
assert_eq!(store.inner.record_count().unwrap(), 0);
assert_eq!(store.inner.reserved_response_bytes().unwrap(), 0);
assert_eq!(store.commit_attempts.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn in_memory_runtime_reserves_raised_response_limit_before_execution() {
let response_bytes = super::DEFAULT_IDEMPOTENCY_MAX_BODY_BYTES
+ super::DEFAULT_MAX_IDEMPOTENCY_STORED_HEADER_BYTES
+ 1;
let runtime = IdempotencyRuntime::in_memory(idempotency_config(
DEFAULT_IDEMPOTENCY_PROCESSING_TTL,
response_bytes,
))
.unwrap();
let executions = Arc::new(AtomicUsize::new(0));
let executions_for_service = executions.clone();
let result = run_idempotent_service(
runtime,
Request::post("/")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "raised-response-limit")
.body(Body::empty())
.unwrap(),
move |_| async move {
executions_for_service.fetch_add(1, Ordering::SeqCst);
Ok::<_, Infallible>(Response::new(Body::from(vec![0_u8; response_bytes])))
},
)
.await
.expect("configured capacity should be reserved")
.expect("service is infallible");
assert_eq!(executions.load(Ordering::SeqCst), 1);
assert_eq!(
to_bytes(result.into_body(), usize::MAX)
.await
.unwrap()
.len(),
response_bytes
);
}
#[test]
fn in_memory_runtime_rejects_response_limit_exceeding_store_capacity() {
let runtime = IdempotencyRuntime::in_memory(idempotency_config(
DEFAULT_IDEMPOTENCY_PROCESSING_TTL,
super::DEFAULT_MAX_IDEMPOTENCY_STORED_RESPONSE_BYTES,
));
assert!(matches!(
runtime,
Err(IdempotencyStoreError::InvalidCapacity)
));
}
#[tokio::test]
async fn middleware_replays_matching_requests() {
async fn create_order() -> Response {
let mut response = (StatusCode::CREATED, "created").into_response();
response
.headers_mut()
.insert(SET_COOKIE, HeaderValue::from_static("session=fresh"));
response
}
let runtime = in_memory_runtime();
let app = Router::new()
.route("/orders", post(create_order))
.layer(IdempotencyLayer::new(runtime));
let response = app
.clone()
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.body(axum::body::Body::from(r#"{"sku":"A"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CREATED);
assert_eq!(
response.headers().get(SET_COOKIE),
Some(&HeaderValue::from_static("session=fresh"))
);
let response = app
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.body(axum::body::Body::from(r#"{"sku":"A"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CREATED);
assert_eq!(
response
.headers()
.get(super::IDEMPOTENCY_REPLAYED_HEADER_NAME),
Some(&HeaderValue::from_static("true"))
);
assert!(response.headers().get(SET_COOKIE).is_none());
}
#[tokio::test]
async fn middleware_rejects_same_key_with_different_body() {
let runtime = in_memory_runtime();
let app = Router::new()
.route("/orders", post(|| async { "created" }))
.layer(IdempotencyLayer::new(runtime));
app.clone()
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.body(axum::body::Body::from(r#"{"sku":"A"}"#))
.unwrap(),
)
.await
.unwrap();
let response = app
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.body(axum::body::Body::from(r#"{"sku":"B"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CONFLICT);
}
#[tokio::test]
async fn middleware_rejects_unverified_authentication_context() {
let runtime = in_memory_runtime();
let app = Router::new()
.route("/orders", post(|| async { "must not execute" }))
.layer(IdempotencyLayer::new(runtime));
let response = app
.clone()
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.header(AUTHORIZATION, "Bearer unverified")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
let problem: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(problem["code"], "idempotency_authentication_scope_required");
let response = app
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-2")
.header(COOKIE, "session=unverified")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
let problem: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(problem["code"], "idempotency_authentication_scope_required");
}
#[tokio::test]
async fn middleware_namespaces_records_by_verified_authentication_context() {
async fn install_verified_scope(mut request: Request<Body>, next: Next) -> Response {
let scope = IdempotencyScope::namespaced(
"test.authenticated-user",
request
.headers()
.get(AUTHORIZATION)
.expect("test authorization should be present")
.as_bytes(),
);
request.extensions_mut().insert(scope);
next.run(request).await
}
let runtime = in_memory_runtime();
let app = Router::new()
.route(
"/orders",
post(|headers: HeaderMap| async move {
headers
.get(AUTHORIZATION)
.and_then(|value| value.to_str().ok())
.unwrap_or("anonymous")
.to_owned()
}),
)
.layer(IdempotencyLayer::new(runtime))
.layer(middleware::from_fn(install_verified_scope));
let response = app
.clone()
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "shared-client-key")
.header(AUTHORIZATION, "Bearer user-one")
.body(Body::from(r#"{"sku":"A"}"#))
.unwrap(),
)
.await
.unwrap();
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
assert_eq!(&body[..], b"Bearer user-one");
let response = app
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "shared-client-key")
.header(AUTHORIZATION, "Bearer user-two")
.body(Body::from(r#"{"sku":"A"}"#))
.unwrap(),
)
.await
.unwrap();
assert!(
response
.headers()
.get(super::IDEMPOTENCY_REPLAYED_HEADER_NAME)
.is_none()
);
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
assert_eq!(&body[..], b"Bearer user-two");
}
#[tokio::test]
async fn oversized_response_is_terminalized_and_replayed_without_reexecution() {
let store = Arc::new(MemoryIdempotencyStore::new());
let runtime = IdempotencyRuntime::new(
store.clone(),
idempotency_config(DEFAULT_IDEMPOTENCY_PROCESSING_TTL, 1),
);
let executions = Arc::new(AtomicUsize::new(0));
let executions_for_handler = executions.clone();
let app = Router::new()
.route(
"/orders",
post(move || {
let executions = executions_for_handler.clone();
async move {
executions.fetch_add(1, Ordering::SeqCst);
"too large"
}
}),
)
.layer(IdempotencyLayer::new(runtime));
let first = app
.clone()
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(first.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert!(
first
.headers()
.get(super::IDEMPOTENCY_REPLAYED_HEADER_NAME)
.is_none()
);
let first_body = to_bytes(first.into_body(), usize::MAX).await.unwrap();
let replay = app
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(replay.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(
replay
.headers()
.get(super::IDEMPOTENCY_REPLAYED_HEADER_NAME)
.and_then(|value| value.to_str().ok()),
Some("true")
);
let replay_body = to_bytes(replay.into_body(), usize::MAX).await.unwrap();
assert_eq!(replay_body, first_body);
assert_eq!(executions.load(Ordering::SeqCst), 1);
assert_eq!(store.record_count().unwrap(), 1);
assert_eq!(store.reserved_response_bytes().unwrap(), 0);
let stored_size = {
let state = store.state.lock().unwrap();
let response = state
.records
.values()
.next()
.and_then(|record| match &record.state {
super::memory::EntryState::Completed(response) => Some(response),
super::memory::EntryState::Processing { .. } => None,
})
.expect("terminal response should be stored");
super::stored_response_size(response)
};
assert_eq!(store.stored_response_bytes().unwrap(), stored_size);
}
#[tokio::test]
async fn failed_terminal_commit_keeps_the_processing_lease_and_never_aborts() {
let store = Arc::new(CommitFailingStore::default());
let runtime = IdempotencyRuntime::new(
store.clone(),
idempotency_config(DEFAULT_IDEMPOTENCY_PROCESSING_TTL, 1),
);
let executions = Arc::new(AtomicUsize::new(0));
let executions_for_handler = executions.clone();
let app = Router::new()
.route(
"/orders",
post(move || {
let executions = executions_for_handler.clone();
async move {
executions.fetch_add(1, Ordering::SeqCst);
"too large"
}
}),
)
.layer(IdempotencyLayer::new(runtime));
let first = app
.clone()
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "commit-failure")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(first.status(), StatusCode::INTERNAL_SERVER_ERROR);
let retry = app
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "commit-failure")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(retry.status(), StatusCode::CONFLICT);
assert_eq!(executions.load(Ordering::SeqCst), 1);
assert_eq!(store.commit_attempts.load(Ordering::SeqCst), 1);
assert_eq!(store.abort_attempts.load(Ordering::SeqCst), 0);
assert_eq!(store.inner.record_count().unwrap(), 1);
assert_eq!(store.inner.stored_response_bytes().unwrap(), 0);
assert!(store.inner.reserved_response_bytes().unwrap() > 0);
}
#[tokio::test]
async fn layer_allows_requests_without_key_by_default() {
let app = Router::new()
.route("/orders", post(|| async { "created" }))
.layer(IdempotencyLayer::new(in_memory_runtime()));
let response = app
.oneshot(
Request::post("/orders")
.body(axum::body::Body::from(r#"{"sku":"A"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
}