use std::{
convert::Infallible,
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
};
use async_trait::async_trait;
use axum::{
Router,
body::{Body, to_bytes},
http::{
HeaderMap, HeaderValue, Request, StatusCode,
header::{AUTHORIZATION, CONTENT_TYPE, COOKIE, SET_COOKIE},
},
middleware::{self, Next},
response::{IntoResponse, Response},
routing::post,
};
use tower::ServiceExt;
use super::{
DEFAULT_IDEMPOTENCY_MAX_BODY_BYTES, DEFAULT_IDEMPOTENCY_PROCESSING_TTL,
DEFAULT_IDEMPOTENCY_TTL, IDEMPOTENCY_KEY_HEADER_NAME, IdempotencyConfig, IdempotencyKey,
IdempotencyLayer, IdempotencyLease, IdempotencyRejection, IdempotencyRequestHash,
IdempotencyReservation, IdempotencyRuntime, IdempotencyScope, IdempotencyStore,
IdempotencyStoreError, MemoryIdempotencyStore, StoredIdempotencyResponse,
idempotency_key_from_headers, idempotency_middleware, is_valid_idempotency_key,
run_idempotent_service,
};
use crate::result::{ClassifyError, ErrorKind};
fn idempotency_config(
processing_ttl: std::time::Duration,
max_response_body_bytes: usize,
) -> IdempotencyConfig {
IdempotencyConfig::new(
DEFAULT_IDEMPOTENCY_TTL,
processing_ttl,
DEFAULT_IDEMPOTENCY_MAX_BODY_BYTES,
max_response_body_bytes,
)
.unwrap()
}
fn in_memory_runtime() -> IdempotencyRuntime {
IdempotencyRuntime::in_memory(IdempotencyConfig::default()).unwrap()
}
#[derive(Default)]
struct CommitFailingStore {
inner: MemoryIdempotencyStore,
commit_attempts: AtomicUsize,
abort_attempts: AtomicUsize,
}
#[async_trait]
impl IdempotencyStore for CommitFailingStore {
async fn reserve(
&self,
key: IdempotencyKey,
request_hash: IdempotencyRequestHash,
processing_ttl: std::time::Duration,
) -> Result<IdempotencyReservation, IdempotencyStoreError> {
self.inner.reserve(key, request_hash, processing_ttl).await
}
async fn commit(
&self,
_key: &IdempotencyKey,
_lease: super::IdempotencyLease,
_response: StoredIdempotencyResponse,
_retention_ttl: std::time::Duration,
) -> Result<(), IdempotencyStoreError> {
self.commit_attempts.fetch_add(1, Ordering::SeqCst);
Err(IdempotencyStoreError::CapacityExceeded)
}
async fn abort(
&self,
key: &IdempotencyKey,
lease: super::IdempotencyLease,
) -> Result<(), IdempotencyStoreError> {
self.abort_attempts.fetch_add(1, Ordering::SeqCst);
self.inner.abort(key, lease).await
}
}
#[test]
fn validates_idempotency_keys() {
assert!(is_valid_idempotency_key("op_018f.test:1"));
assert!(!is_valid_idempotency_key(""));
assert!(!is_valid_idempotency_key("bad/key"));
}
#[test]
fn idempotency_debug_output_redacts_keys_hashes_and_response_payloads() {
let key_value = "private-idempotency-key";
let hash_value = "private-request-hash";
let header_value = "private-response-header";
let body_value = "private-response-body";
let key = IdempotencyKey::new(key_value).expect("key should validate");
let hash = IdempotencyRequestHash::new(hash_value).expect("hash should validate");
let response = StoredIdempotencyResponse::new(StatusCode::OK, body_value)
.with_header("content-type", header_value);
let lease = IdempotencyLease::new();
let lease_value = lease.0.to_string();
let output = format!("{key} {key:?} {hash:?} {response:?} {lease:?}");
assert!(output.contains("[REDACTED]"));
assert!(output.contains("body_bytes"));
for secret in [key_value, hash_value, header_value, body_value] {
assert!(!output.contains(secret));
}
assert!(!output.contains(&lease_value));
}
#[test]
fn idempotency_errors_have_stable_classification() {
let invalid_key = IdempotencyKey::new("bad/key")
.expect_err("key should be invalid")
.error_descriptor();
let store = IdempotencyStoreError::InconsistentRecord.error_descriptor();
let execution_result_unavailable =
IdempotencyRejection::ExecutionResultUnavailable.error_descriptor();
let unverified_scope = IdempotencyRejection::UnverifiedAuthenticationScope.error_descriptor();
assert_eq!(invalid_key.kind(), ErrorKind::Validation);
assert_eq!(
invalid_key.code().as_str(),
"idempotency_key_invalid_characters"
);
assert_eq!(store.kind(), ErrorKind::Internal);
assert_eq!(store.code().as_str(), "idempotency_record_inconsistent");
assert_eq!(execution_result_unavailable.kind(), ErrorKind::Internal);
assert_eq!(
execution_result_unavailable.code().as_str(),
"idempotency_execution_result_unavailable"
);
assert_eq!(unverified_scope.kind(), ErrorKind::Authentication);
assert_eq!(
unverified_scope.code().as_str(),
"idempotency_authentication_scope_required"
);
}
#[test]
fn extracts_key_from_headers() {
let mut headers = HeaderMap::new();
headers.insert(
IDEMPOTENCY_KEY_HEADER_NAME,
HeaderValue::from_static("request-1"),
);
assert_eq!(
idempotency_key_from_headers(&headers)
.as_ref()
.map(IdempotencyKey::as_str),
Some("request-1")
);
}
#[tokio::test]
async fn memory_store_reserves_conflicts_and_replays() {
let store = MemoryIdempotencyStore::new();
let key = IdempotencyKey::new("request-1").unwrap();
let hash = IdempotencyRequestHash::new("hash-1").unwrap();
let lease = match store
.reserve(
key.clone(),
hash.clone(),
std::time::Duration::from_secs(60),
)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
assert_eq!(
store
.reserve(
key.clone(),
IdempotencyRequestHash::new("hash-2").unwrap(),
std::time::Duration::from_secs(60)
)
.await
.unwrap(),
IdempotencyReservation::Conflict
);
store
.commit(
&key,
lease,
StoredIdempotencyResponse::new(axum::http::StatusCode::OK, b"ok".to_vec()),
std::time::Duration::from_secs(60),
)
.await
.unwrap();
assert!(matches!(
store
.reserve(key, hash, std::time::Duration::from_secs(60))
.await
.unwrap(),
IdempotencyReservation::Replay(_)
));
}
#[tokio::test]
async fn memory_store_reports_inconsistent_completed_records() {
let store = MemoryIdempotencyStore::new();
let key = IdempotencyKey::new("request-1").unwrap();
let hash = IdempotencyRequestHash::new("hash-1").unwrap();
store
.reserve(
key.clone(),
hash.clone(),
std::time::Duration::from_secs(60),
)
.await
.unwrap();
{
let mut state = store.state.lock().unwrap();
let record = state.records.get_mut(&key).unwrap();
record.state = super::IdempotencyState::Completed;
record.response = None;
}
let result = store
.reserve(key, hash, std::time::Duration::from_secs(60))
.await;
assert!(matches!(
result,
Err(super::IdempotencyStoreError::InconsistentRecord)
));
}
#[tokio::test]
async fn abort_releases_capacity_without_touching_a_newer_lease() {
let store = MemoryIdempotencyStore::with_clock_and_max_records(crate::clock::system_clock(), 1)
.unwrap();
let key = IdempotencyKey::new("request-1").unwrap();
let hash = IdempotencyRequestHash::new("hash-1").unwrap();
let first_lease = match store
.reserve(
key.clone(),
hash.clone(),
std::time::Duration::from_secs(60),
)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
store.abort(&key, first_lease).await.unwrap();
assert_eq!(store.record_count().unwrap(), 0);
assert_eq!(store.reserved_response_bytes().unwrap(), 0);
let second_lease = match store
.reserve(key.clone(), hash, std::time::Duration::from_secs(60))
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
assert_ne!(first_lease, second_lease);
assert!(matches!(
store
.commit(
&key,
first_lease,
StoredIdempotencyResponse::new(StatusCode::OK, b"stale".to_vec()),
std::time::Duration::from_secs(60),
)
.await,
Err(IdempotencyStoreError::InconsistentRecord)
));
assert_eq!(store.record_count().unwrap(), 1);
assert_eq!(store.stored_response_bytes().unwrap(), 0);
}
#[tokio::test]
async fn cancelled_execution_is_terminalized_and_never_reexecuted() {
let store = Arc::new(MemoryIdempotencyStore::new());
let runtime = IdempotencyRuntime::new(
store.clone(),
idempotency_config(
std::time::Duration::from_secs(60),
DEFAULT_IDEMPOTENCY_MAX_BODY_BYTES,
),
);
let request = Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "cancelled-request")
.body(Body::empty())
.unwrap();
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let executions = Arc::new(AtomicUsize::new(0));
let executions_for_task = executions.clone();
let runtime_for_task = runtime.clone();
let task = tokio::spawn(async move {
super::run_idempotent_service(runtime_for_task, request, |_request| async move {
executions_for_task.fetch_add(1, Ordering::SeqCst);
let _ = started_tx.send(());
std::future::pending::<Result<axum::response::Response, std::convert::Infallible>>()
.await
})
.await
});
started_rx
.await
.expect("request should acquire a reservation before blocking");
assert_eq!(store.record_count().unwrap(), 1);
assert!(store.reserved_response_bytes().unwrap() > 0);
task.abort();
assert!(
task.await
.expect_err("request should be cancelled")
.is_cancelled()
);
for _ in 0..128 {
let terminalized = store.state.lock().unwrap().records.values().any(|record| {
record.state == super::IdempotencyState::Completed && record.response.is_some()
});
if terminalized {
break;
}
tokio::task::yield_now().await;
}
assert_eq!(store.record_count().unwrap(), 1);
assert_eq!(store.reserved_response_bytes().unwrap(), 0);
assert!(store.stored_response_bytes().unwrap() > 0);
let executions_for_retry = executions.clone();
let response = super::run_idempotent_service(
runtime,
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "cancelled-request")
.body(Body::empty())
.unwrap(),
move |_request| async move {
executions_for_retry.fetch_add(1, Ordering::SeqCst);
Ok::<_, std::convert::Infallible>(
(StatusCode::OK, "unexpected execution").into_response(),
)
},
)
.await
.unwrap()
.unwrap();
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(
response
.headers()
.get(super::IDEMPOTENCY_REPLAYED_HEADER_NAME)
.and_then(|value| value.to_str().ok()),
Some("true")
);
assert_eq!(executions.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn inner_service_error_is_terminalized_before_the_error_is_propagated() {
let store = Arc::new(MemoryIdempotencyStore::new());
let runtime = IdempotencyRuntime::new(store.clone(), IdempotencyConfig::default());
let executions = Arc::new(AtomicUsize::new(0));
let executions_for_first = executions.clone();
let first = super::run_idempotent_service(
runtime.clone(),
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "service-error")
.body(Body::empty())
.unwrap(),
move |_request| async move {
executions_for_first.fetch_add(1, Ordering::SeqCst);
Err::<axum::response::Response, _>("downstream failure")
},
)
.await
.expect("idempotency orchestration should succeed");
assert!(matches!(first, Err("downstream failure")));
assert_eq!(store.record_count().unwrap(), 1);
assert_eq!(store.reserved_response_bytes().unwrap(), 0);
assert!(store.stored_response_bytes().unwrap() > 0);
let executions_for_retry = executions.clone();
let replay = super::run_idempotent_service(
runtime,
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "service-error")
.body(Body::empty())
.unwrap(),
move |_request| async move {
executions_for_retry.fetch_add(1, Ordering::SeqCst);
Ok::<_, &'static str>((StatusCode::OK, "unexpected execution").into_response())
},
)
.await
.unwrap()
.unwrap();
assert_eq!(replay.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(
replay
.headers()
.get(super::IDEMPOTENCY_REPLAYED_HEADER_NAME)
.and_then(|value| value.to_str().ok()),
Some("true")
);
assert_eq!(executions.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn expired_processing_lease_cannot_commit_into_a_new_reservation() {
let clock = crate::clock::ManualClock::new();
let store = MemoryIdempotencyStore::with_clock_and_max_records(clock.shared(), 1).unwrap();
let key = IdempotencyKey::new("request-expiry").unwrap();
let hash = IdempotencyRequestHash::new("hash-expiry").unwrap();
let processing_ttl = std::time::Duration::from_secs(5);
let first_lease = match store
.reserve(key.clone(), hash.clone(), processing_ttl)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
clock.advance(processing_ttl + std::time::Duration::from_secs(1));
let second_lease = match store
.reserve(key.clone(), hash, processing_ttl)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
assert_ne!(first_lease, second_lease);
assert!(matches!(
store
.commit(
&key,
first_lease,
StoredIdempotencyResponse::new(StatusCode::OK, b"stale".to_vec()),
std::time::Duration::from_secs(60),
)
.await,
Err(IdempotencyStoreError::InconsistentRecord)
));
store
.commit(
&key,
second_lease,
StoredIdempotencyResponse::new(StatusCode::OK, b"current".to_vec()),
std::time::Duration::from_secs(60),
)
.await
.unwrap();
assert_eq!(store.record_count().unwrap(), 1);
}
#[tokio::test]
async fn completed_retention_ttl_starts_when_the_response_is_committed() {
let clock = crate::clock::ManualClock::new();
let store = MemoryIdempotencyStore::with_clock_and_max_records(clock.shared(), 1).unwrap();
let key = IdempotencyKey::new("request-retention").unwrap();
let hash = IdempotencyRequestHash::new("hash-retention").unwrap();
let processing_ttl = std::time::Duration::from_secs(10);
let retention_ttl = std::time::Duration::from_secs(20);
let lease = match store
.reserve(key.clone(), hash.clone(), processing_ttl)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
clock.advance(std::time::Duration::from_secs(9));
store
.commit(
&key,
lease,
StoredIdempotencyResponse::new(StatusCode::OK, b"completed".to_vec()),
retention_ttl,
)
.await
.unwrap();
clock.advance(std::time::Duration::from_secs(19));
assert!(matches!(
store
.reserve(key.clone(), hash.clone(), processing_ttl)
.await
.unwrap(),
IdempotencyReservation::Replay(_)
));
clock.advance(std::time::Duration::from_secs(2));
assert!(matches!(
store.reserve(key, hash, processing_ttl).await.unwrap(),
IdempotencyReservation::Reserved(_)
));
}
#[tokio::test]
async fn memory_store_bounds_unexpired_records() {
let store = MemoryIdempotencyStore::with_clock_and_max_records(crate::clock::system_clock(), 2)
.unwrap();
let ttl = std::time::Duration::from_secs(60);
for index in 0..2 {
store
.reserve(
IdempotencyKey::new(format!("request-{index}")).unwrap(),
IdempotencyRequestHash::new(format!("hash-{index}")).unwrap(),
ttl,
)
.await
.unwrap();
}
let result = store
.reserve(
IdempotencyKey::new("request-overflow").unwrap(),
IdempotencyRequestHash::new("hash-overflow").unwrap(),
ttl,
)
.await;
assert!(matches!(
result,
Err(IdempotencyStoreError::CapacityExceeded)
));
assert_eq!(store.record_count().unwrap(), 2);
}
#[tokio::test]
async fn memory_store_bounds_total_stored_response_bytes() {
let store =
MemoryIdempotencyStore::with_clock_and_limits(crate::clock::system_clock(), 2, 7).unwrap();
let key = IdempotencyKey::new("request-sized").unwrap();
let lease = match store
.reserve(
key.clone(),
IdempotencyRequestHash::new("hash-sized").unwrap(),
std::time::Duration::from_secs(60),
)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
let result = store
.commit(
&key,
lease,
StoredIdempotencyResponse::new(StatusCode::OK, vec![0; 8]),
std::time::Duration::from_secs(60),
)
.await;
assert!(matches!(
result,
Err(IdempotencyStoreError::CapacityExceeded)
));
assert_eq!(store.stored_response_bytes().unwrap(), 0);
}
#[test]
fn in_memory_store_reserves_worst_case_response_capacity_before_execution() {
assert!(matches!(
MemoryIdempotencyStore::for_response_body_limit(0),
Err(IdempotencyStoreError::InvalidCapacity)
));
let store =
MemoryIdempotencyStore::for_response_body_limit(super::DEFAULT_IDEMPOTENCY_MAX_BODY_BYTES)
.unwrap();
let per_record = super::DEFAULT_IDEMPOTENCY_MAX_BODY_BYTES
+ super::DEFAULT_MAX_IDEMPOTENCY_STORED_HEADER_BYTES;
assert_eq!(store.max_records(), super::DEFAULT_MAX_IDEMPOTENCY_RECORDS);
assert_eq!(store.max_response_bytes_per_record, per_record);
assert!(per_record <= store.max_stored_response_bytes());
}
#[tokio::test]
async fn completed_small_responses_release_unused_reserved_capacity() {
let store = MemoryIdempotencyStore::with_clock_and_limits_and_reservation(
crate::clock::system_clock(),
4,
10,
6,
)
.unwrap();
let ttl = std::time::Duration::from_secs(60);
for index in 0..4 {
let key = IdempotencyKey::new(format!("request-{index}")).unwrap();
let lease = match store
.reserve(
key.clone(),
IdempotencyRequestHash::new(format!("hash-{index}")).unwrap(),
ttl,
)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
store
.commit(
&key,
lease,
StoredIdempotencyResponse::new(StatusCode::OK, vec![0]),
ttl,
)
.await
.unwrap();
}
assert_eq!(store.record_count().unwrap(), 4);
assert_eq!(store.stored_response_bytes().unwrap(), 4);
assert_eq!(store.reserved_response_bytes().unwrap(), 0);
}
#[test]
fn stored_response_filters_unstable_headers_and_preserves_duplicates() {
let stored = StoredIdempotencyResponse::new(axum::http::StatusCode::OK, b"ok".to_vec())
.with_header(CONTENT_TYPE.as_str(), "application/json")
.with_header("warning", "199 first")
.with_header("warning", "299 second")
.with_header(SET_COOKIE.as_str(), "session=unsafe")
.with_header("bad header", "ignored");
assert_eq!(
stored.headers(),
&[
(
CONTENT_TYPE.as_str().to_owned(),
"application/json".to_owned()
),
("warning".to_owned(), "199 first".to_owned()),
("warning".to_owned(), "299 second".to_owned()),
]
);
}
#[tokio::test]
async fn stored_response_filters_connection_nominated_headers_and_replays_duplicates() {
let mut response = Response::new(Body::from("ok"));
response
.headers_mut()
.insert("connection", HeaderValue::from_static("x-hop, keep-alive"));
response
.headers_mut()
.insert("x-hop", HeaderValue::from_static("private"));
response
.headers_mut()
.insert("keep-alive", HeaderValue::from_static("timeout=5"));
response.headers_mut().insert(
super::IDEMPOTENCY_REPLAYED_HEADER_NAME,
HeaderValue::from_static("false"),
);
response
.headers_mut()
.append("warning", HeaderValue::from_static("199 first"));
response
.headers_mut()
.append("warning", HeaderValue::from_static("299 second"));
let (parts, body) = response.into_parts();
let body = to_bytes(body, usize::MAX).await.unwrap().to_vec();
let stored = super::stored_response_from_parts(&parts, body);
assert!(stored.headers().iter().all(|(name, _)| {
name != "connection"
&& name != "x-hop"
&& name != "keep-alive"
&& name != super::IDEMPOTENCY_REPLAYED_HEADER_NAME
}));
let replay = super::response_from_stored(stored).unwrap();
let warnings = replay
.headers()
.get_all("warning")
.iter()
.collect::<Vec<_>>();
assert_eq!(warnings.len(), 2);
assert_eq!(warnings[0], "199 first");
assert_eq!(warnings[1], "299 second");
let replay_markers = replay
.headers()
.get_all(super::IDEMPOTENCY_REPLAYED_HEADER_NAME)
.iter()
.collect::<Vec<_>>();
assert_eq!(replay_markers.len(), 1);
assert_eq!(replay_markers[0], "true");
}
#[tokio::test]
async fn in_memory_runtime_reserves_raised_response_limit_before_execution() {
let response_bytes = super::DEFAULT_IDEMPOTENCY_MAX_BODY_BYTES
+ super::DEFAULT_MAX_IDEMPOTENCY_STORED_HEADER_BYTES
+ 1;
let runtime = IdempotencyRuntime::in_memory(idempotency_config(
DEFAULT_IDEMPOTENCY_PROCESSING_TTL,
response_bytes,
))
.unwrap();
let executions = Arc::new(AtomicUsize::new(0));
let executions_for_service = executions.clone();
let result = run_idempotent_service(
runtime,
Request::post("/")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "raised-response-limit")
.body(Body::empty())
.unwrap(),
move |_| async move {
executions_for_service.fetch_add(1, Ordering::SeqCst);
Ok::<_, Infallible>(Response::new(Body::from(vec![0_u8; response_bytes])))
},
)
.await
.expect("configured capacity should be reserved")
.expect("service is infallible");
assert_eq!(executions.load(Ordering::SeqCst), 1);
assert_eq!(
to_bytes(result.into_body(), usize::MAX)
.await
.unwrap()
.len(),
response_bytes
);
}
#[tokio::test]
async fn impossible_in_memory_response_limit_fails_before_execution() {
let runtime = IdempotencyRuntime::in_memory(idempotency_config(
DEFAULT_IDEMPOTENCY_PROCESSING_TTL,
usize::MAX,
));
let executions = Arc::new(AtomicUsize::new(0));
assert!(matches!(
runtime,
Err(IdempotencyStoreError::InvalidCapacity)
));
assert_eq!(executions.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn middleware_replays_matching_requests() {
async fn create_order() -> Response {
let mut response = (StatusCode::CREATED, "created").into_response();
response
.headers_mut()
.insert(SET_COOKIE, HeaderValue::from_static("session=fresh"));
response
}
let runtime = in_memory_runtime();
let app = Router::new()
.route("/orders", post(create_order))
.layer(middleware::from_fn(idempotency_middleware))
.layer(axum::Extension(runtime));
let response = app
.clone()
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.body(axum::body::Body::from(r#"{"sku":"A"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CREATED);
assert_eq!(
response.headers().get(SET_COOKIE),
Some(&HeaderValue::from_static("session=fresh"))
);
let response = app
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.body(axum::body::Body::from(r#"{"sku":"A"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CREATED);
assert_eq!(
response
.headers()
.get(super::IDEMPOTENCY_REPLAYED_HEADER_NAME),
Some(&HeaderValue::from_static("true"))
);
assert!(response.headers().get(SET_COOKIE).is_none());
}
#[tokio::test]
async fn middleware_rejects_same_key_with_different_body() {
let runtime = in_memory_runtime();
let app = Router::new()
.route("/orders", post(|| async { "created" }))
.layer(middleware::from_fn(idempotency_middleware))
.layer(axum::Extension(runtime));
app.clone()
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.body(axum::body::Body::from(r#"{"sku":"A"}"#))
.unwrap(),
)
.await
.unwrap();
let response = app
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.body(axum::body::Body::from(r#"{"sku":"B"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CONFLICT);
}
#[tokio::test]
async fn middleware_rejects_unverified_authentication_context() {
let runtime = in_memory_runtime();
let app = Router::new()
.route("/orders", post(|| async { "must not execute" }))
.layer(middleware::from_fn(idempotency_middleware))
.layer(axum::Extension(runtime));
let response = app
.clone()
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.header(AUTHORIZATION, "Bearer unverified")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
let problem: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(problem["code"], "idempotency_authentication_scope_required");
let response = app
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-2")
.header(COOKIE, "session=unverified")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
let problem: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(problem["code"], "idempotency_authentication_scope_required");
}
#[tokio::test]
async fn middleware_namespaces_records_by_verified_authentication_context() {
async fn install_verified_scope(mut request: Request<Body>, next: Next) -> Response {
let scope = IdempotencyScope::namespaced(
"test.authenticated-user",
request
.headers()
.get(AUTHORIZATION)
.expect("test authorization should be present")
.as_bytes(),
);
request.extensions_mut().insert(scope);
next.run(request).await
}
let runtime = in_memory_runtime();
let app = Router::new()
.route(
"/orders",
post(|headers: HeaderMap| async move {
headers
.get(AUTHORIZATION)
.and_then(|value| value.to_str().ok())
.unwrap_or("anonymous")
.to_owned()
}),
)
.layer(middleware::from_fn(idempotency_middleware))
.layer(middleware::from_fn(install_verified_scope))
.layer(axum::Extension(runtime));
let response = app
.clone()
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "shared-client-key")
.header(AUTHORIZATION, "Bearer user-one")
.body(Body::from(r#"{"sku":"A"}"#))
.unwrap(),
)
.await
.unwrap();
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
assert_eq!(&body[..], b"Bearer user-one");
let response = app
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "shared-client-key")
.header(AUTHORIZATION, "Bearer user-two")
.body(Body::from(r#"{"sku":"A"}"#))
.unwrap(),
)
.await
.unwrap();
assert!(
response
.headers()
.get(super::IDEMPOTENCY_REPLAYED_HEADER_NAME)
.is_none()
);
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
assert_eq!(&body[..], b"Bearer user-two");
}
#[tokio::test]
async fn oversized_response_is_terminalized_and_replayed_without_reexecution() {
let store = Arc::new(MemoryIdempotencyStore::new());
let runtime = IdempotencyRuntime::new(
store.clone(),
idempotency_config(DEFAULT_IDEMPOTENCY_PROCESSING_TTL, 1),
);
let executions = Arc::new(AtomicUsize::new(0));
let executions_for_handler = executions.clone();
let app = Router::new()
.route(
"/orders",
post(move || {
let executions = executions_for_handler.clone();
async move {
executions.fetch_add(1, Ordering::SeqCst);
"too large"
}
}),
)
.layer(middleware::from_fn(idempotency_middleware))
.layer(axum::Extension(runtime));
let first = app
.clone()
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(first.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert!(
first
.headers()
.get(super::IDEMPOTENCY_REPLAYED_HEADER_NAME)
.is_none()
);
let first_body = to_bytes(first.into_body(), usize::MAX).await.unwrap();
let replay = app
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(replay.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(
replay
.headers()
.get(super::IDEMPOTENCY_REPLAYED_HEADER_NAME)
.and_then(|value| value.to_str().ok()),
Some("true")
);
let replay_body = to_bytes(replay.into_body(), usize::MAX).await.unwrap();
assert_eq!(replay_body, first_body);
assert_eq!(executions.load(Ordering::SeqCst), 1);
assert_eq!(store.record_count().unwrap(), 1);
assert_eq!(store.reserved_response_bytes().unwrap(), 0);
let stored_size = {
let state = store.state.lock().unwrap();
let response = state
.records
.values()
.next()
.and_then(|record| record.response.as_ref())
.expect("terminal response should be stored");
super::stored_response_size(response)
};
assert_eq!(store.stored_response_bytes().unwrap(), stored_size);
}
#[tokio::test]
async fn failed_terminal_commit_keeps_the_processing_lease_and_never_aborts() {
let store = Arc::new(CommitFailingStore::default());
let runtime = IdempotencyRuntime::new(
store.clone(),
idempotency_config(DEFAULT_IDEMPOTENCY_PROCESSING_TTL, 1),
);
let executions = Arc::new(AtomicUsize::new(0));
let executions_for_handler = executions.clone();
let app = Router::new()
.route(
"/orders",
post(move || {
let executions = executions_for_handler.clone();
async move {
executions.fetch_add(1, Ordering::SeqCst);
"too large"
}
}),
)
.layer(middleware::from_fn(idempotency_middleware))
.layer(axum::Extension(runtime));
let first = app
.clone()
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "commit-failure")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(first.status(), StatusCode::INTERNAL_SERVER_ERROR);
let retry = app
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "commit-failure")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(retry.status(), StatusCode::CONFLICT);
assert_eq!(executions.load(Ordering::SeqCst), 1);
assert_eq!(store.commit_attempts.load(Ordering::SeqCst), 1);
assert_eq!(store.abort_attempts.load(Ordering::SeqCst), 0);
assert_eq!(store.inner.record_count().unwrap(), 1);
assert_eq!(store.inner.stored_response_bytes().unwrap(), 0);
assert!(store.inner.reserved_response_bytes().unwrap() > 0);
}
#[tokio::test]
async fn layer_allows_requests_without_key_by_default() {
let app = Router::new()
.route("/orders", post(|| async { "created" }))
.layer(IdempotencyLayer::new(in_memory_runtime()));
let response = app
.oneshot(
Request::post("/orders")
.body(axum::body::Body::from(r#"{"sku":"A"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
}