use std::{
borrow::Cow,
collections::{HashMap, HashSet},
fmt,
future::Future,
pin::Pin,
sync::Arc,
sync::Mutex,
task::{Context, Poll},
time::{Duration, Instant},
};
use async_trait::async_trait;
use axum::{
body::{Body, to_bytes},
extract::FromRequestParts,
http::{
HeaderMap, HeaderName, HeaderValue, Request, StatusCode,
header::{AUTHORIZATION, CONNECTION, CONTENT_TYPE, COOKIE},
request::Parts,
response,
},
middleware::Next,
response::{IntoResponse, Response},
};
use sha2::{Digest, Sha256};
use thiserror::Error;
use tower::{Layer, Service};
use uuid::Uuid;
use crate::{
clock::{SharedClock, system_clock},
http::{ApiProblem, HttpRequestPolicy, ProblemCode},
observability::{MetricLabels, MetricRecorder, metric_labels, metric_names, metric_outcomes},
result::{ClassifyError, ErrorKind},
};
pub const IDEMPOTENCY_KEY_HEADER_NAME: &str = "idempotency-key";
pub const IDEMPOTENCY_REPLAYED_HEADER_NAME: &str = "idempotency-replayed";
pub const MAX_IDEMPOTENCY_KEY_LENGTH: usize = 128;
pub const DEFAULT_IDEMPOTENCY_TTL: Duration = Duration::from_secs(60 * 60 * 24);
pub const DEFAULT_IDEMPOTENCY_PROCESSING_TTL: Duration = Duration::from_secs(5 * 60);
pub const DEFAULT_IDEMPOTENCY_MAX_BODY_BYTES: usize = 1024 * 1024;
pub const DEFAULT_MAX_IDEMPOTENCY_RECORDS: usize = 1_024;
pub const DEFAULT_MAX_IDEMPOTENCY_STORED_RESPONSE_BYTES: usize = 64 * 1024 * 1024;
pub const DEFAULT_MAX_IDEMPOTENCY_STORED_HEADER_BYTES: usize = 16 * 1024;
const IDEMPOTENCY_TERMINAL_FAILURE_MEDIA_TYPE: &str = "application/problem+json";
const IDEMPOTENCY_TERMINAL_FAILURE_BODY: &[u8] = br#"{"type":"about:blank","title":"Internal Server Error","status":500,"code":"internal_error","detail":"internal server error","details":{"code":"idempotency_execution_result_unavailable"}}"#;
#[derive(Clone, PartialEq, Eq)]
pub struct IdempotencyScope([u8; 32]);
impl IdempotencyScope {
#[must_use]
pub fn new(value: impl AsRef<[u8]>) -> Self {
Self(Sha256::digest(value.as_ref()).into())
}
#[must_use]
pub fn namespaced(namespace: impl AsRef<[u8]>, value: impl AsRef<[u8]>) -> Self {
let mut hasher = Sha256::new();
hasher.update(b"footing-idempotency-scope-v1\0");
hash_length_prefixed(&mut hasher, namespace.as_ref());
hash_length_prefixed(&mut hasher, value.as_ref());
Self(hasher.finalize().into())
}
}
impl fmt::Debug for IdempotencyScope {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("IdempotencyScope([REDACTED])")
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum IdempotencyRequestPolicy {
Disabled,
HeaderOptional,
Required,
}
#[derive(Clone, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub struct IdempotencyKey(String);
impl IdempotencyKey {
pub fn new(value: impl Into<String>) -> Result<Self, IdempotencyKeyError> {
let value = value.into().trim().to_owned();
validate_idempotency_key(&value)?;
Ok(Self(value))
}
#[must_use]
pub fn parse(value: &str) -> Option<Self> {
Self::new(value).ok()
}
#[must_use]
pub fn as_str(&self) -> &str {
&self.0
}
}
impl fmt::Debug for IdempotencyKey {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("[REDACTED]")
}
}
impl fmt::Display for IdempotencyKey {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("[REDACTED]")
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Error)]
#[non_exhaustive]
pub enum IdempotencyKeyError {
#[error("idempotency key must not be empty")]
Empty,
#[error("idempotency key must be less than or equal to {max} bytes")]
TooLong {
#[doc = "The configured upper bound."]
max: usize,
},
#[error("idempotency key contains unsupported characters")]
InvalidCharacters,
}
impl ClassifyError for IdempotencyKeyError {
fn error_kind(&self) -> ErrorKind {
ErrorKind::Validation
}
fn error_code(&self) -> Cow<'static, str> {
Cow::Borrowed(match self {
Self::Empty => "idempotency_key_empty",
Self::TooLong { .. } => "idempotency_key_too_long",
Self::InvalidCharacters => "idempotency_key_invalid_characters",
})
}
}
#[derive(Clone, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub struct IdempotencyRequestHash(String);
impl IdempotencyRequestHash {
pub fn new(value: impl Into<String>) -> Result<Self, IdempotencyKeyError> {
let value = value.into().trim().to_owned();
validate_idempotency_key(&value)?;
Ok(Self(value))
}
#[must_use]
pub fn as_str(&self) -> &str {
&self.0
}
}
impl fmt::Debug for IdempotencyRequestHash {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("[REDACTED]")
}
}
#[derive(Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct StoredIdempotencyResponse {
status: StatusCode,
headers: Vec<(String, String)>,
body: Vec<u8>,
}
impl fmt::Debug for StoredIdempotencyResponse {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("StoredIdempotencyResponse")
.field("status", &self.status)
.field("header_count", &self.headers.len())
.field("body_bytes", &self.body.len())
.finish()
}
}
impl StoredIdempotencyResponse {
#[must_use]
pub fn new(status: StatusCode, body: impl Into<Vec<u8>>) -> Self {
Self {
status,
headers: Vec::new(),
body: body.into(),
}
}
#[must_use]
pub fn with_header(mut self, name: impl Into<String>, value: impl Into<String>) -> Self {
let name = name.into();
let value = value.into();
let Ok(name) = HeaderName::try_from(name.as_str()) else {
return self;
};
let Ok(value) = HeaderValue::try_from(value.as_str()) else {
return self;
};
let Ok(value) = value.to_str() else {
return self;
};
if !is_storable_response_header(&name) {
return self;
}
self.headers
.push((name.as_str().to_owned(), value.to_owned()));
self
}
#[must_use]
pub const fn status(&self) -> StatusCode {
self.status
}
#[must_use]
pub fn body(&self) -> &[u8] {
&self.body
}
#[must_use]
pub fn headers(&self) -> &[(String, String)] {
&self.headers
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum IdempotencyReservation {
Reserved(IdempotencyLease),
Processing,
Replay(StoredIdempotencyResponse),
Conflict,
}
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
#[non_exhaustive]
pub struct IdempotencyLease(Uuid);
impl fmt::Debug for IdempotencyLease {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter.write_str("IdempotencyLease([REDACTED])")
}
}
impl IdempotencyLease {
#[must_use]
pub fn new() -> Self {
Self(Uuid::now_v7())
}
}
impl Default for IdempotencyLease {
fn default() -> Self {
Self::new()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum IdempotencyState {
Processing,
Completed,
}
#[derive(Debug, Error)]
#[non_exhaustive]
pub enum IdempotencyStoreError {
#[error("idempotency store capacity must be greater than zero")]
InvalidCapacity,
#[error("idempotency store capacity is exhausted")]
CapacityExceeded,
#[error("idempotency record expiry is out of range")]
InvalidExpiry,
#[error("idempotency store lock is poisoned")]
LockPoisoned,
#[error("idempotency store record is inconsistent")]
InconsistentRecord,
}
impl ClassifyError for IdempotencyStoreError {
fn error_kind(&self) -> ErrorKind {
match self {
Self::InvalidCapacity | Self::InvalidExpiry => ErrorKind::Validation,
Self::CapacityExceeded | Self::LockPoisoned => ErrorKind::Unavailable,
Self::InconsistentRecord => ErrorKind::Internal,
}
}
fn error_code(&self) -> Cow<'static, str> {
Cow::Borrowed(match self {
Self::InvalidCapacity => "idempotency_store_invalid_capacity",
Self::CapacityExceeded => "idempotency_store_capacity_exceeded",
Self::InvalidExpiry => "idempotency_record_invalid_expiry",
Self::LockPoisoned => "idempotency_store_unavailable",
Self::InconsistentRecord => "idempotency_record_inconsistent",
})
}
}
#[async_trait]
pub trait IdempotencyStore: Send + Sync {
async fn reserve(
&self,
key: IdempotencyKey,
request_hash: IdempotencyRequestHash,
processing_ttl: Duration,
) -> Result<IdempotencyReservation, IdempotencyStoreError>;
async fn reserve_with_capacity(
&self,
key: IdempotencyKey,
request_hash: IdempotencyRequestHash,
processing_ttl: Duration,
_response_capacity_bytes: usize,
) -> Result<IdempotencyReservation, IdempotencyStoreError> {
self.reserve(key, request_hash, processing_ttl).await
}
async fn commit(
&self,
key: &IdempotencyKey,
lease: IdempotencyLease,
response: StoredIdempotencyResponse,
retention_ttl: Duration,
) -> Result<(), IdempotencyStoreError>;
async fn abort(
&self,
key: &IdempotencyKey,
lease: IdempotencyLease,
) -> Result<(), IdempotencyStoreError>;
}
#[derive(Clone)]
#[non_exhaustive]
pub struct IdempotencyRuntime {
store: Arc<dyn IdempotencyStore>,
ttl: Duration,
processing_ttl: Duration,
max_request_body_bytes: usize,
max_response_body_bytes: usize,
metrics: Option<Arc<dyn MetricRecorder>>,
}
impl fmt::Debug for IdempotencyRuntime {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("IdempotencyRuntime")
.field("ttl", &self.ttl)
.field("processing_ttl", &self.processing_ttl)
.field("max_request_body_bytes", &self.max_request_body_bytes)
.field("max_response_body_bytes", &self.max_response_body_bytes)
.finish_non_exhaustive()
}
}
impl IdempotencyRuntime {
#[must_use]
pub fn new(store: Arc<dyn IdempotencyStore>) -> Self {
Self {
store,
ttl: DEFAULT_IDEMPOTENCY_TTL,
processing_ttl: DEFAULT_IDEMPOTENCY_PROCESSING_TTL,
max_request_body_bytes: DEFAULT_IDEMPOTENCY_MAX_BODY_BYTES,
max_response_body_bytes: DEFAULT_IDEMPOTENCY_MAX_BODY_BYTES,
metrics: None,
}
}
#[must_use]
pub fn in_memory() -> Self {
Self::new(Arc::new(MemoryIdempotencyStore::new()))
}
#[must_use]
pub fn with_ttl(mut self, ttl: Duration) -> Self {
self.ttl = ttl;
self
}
#[must_use]
pub fn with_processing_ttl(mut self, processing_ttl: Duration) -> Self {
self.processing_ttl = processing_ttl;
self
}
#[must_use]
pub fn with_max_request_body_bytes(mut self, max_body_bytes: usize) -> Self {
self.max_request_body_bytes = max_body_bytes;
self
}
#[must_use]
pub fn with_max_response_body_bytes(mut self, max_body_bytes: usize) -> Self {
self.max_response_body_bytes = max_body_bytes;
self
}
#[must_use]
pub fn with_metrics(mut self, metrics: Option<Arc<dyn MetricRecorder>>) -> Self {
self.metrics = metrics;
self
}
#[must_use]
pub fn store(&self) -> &Arc<dyn IdempotencyStore> {
&self.store
}
#[must_use]
pub const fn ttl(&self) -> Duration {
self.ttl
}
#[must_use]
pub const fn processing_ttl(&self) -> Duration {
self.processing_ttl
}
#[must_use]
pub const fn max_request_body_bytes(&self) -> usize {
self.max_request_body_bytes
}
#[must_use]
pub const fn max_response_body_bytes(&self) -> usize {
self.max_response_body_bytes
}
#[must_use]
pub fn metrics(&self) -> Option<&Arc<dyn MetricRecorder>> {
self.metrics.as_ref()
}
}
#[derive(Debug, Error)]
#[non_exhaustive]
pub enum IdempotencyRejection {
#[error("idempotency key is invalid")]
InvalidKey,
#[error("idempotency request is already processing")]
Processing,
#[error("idempotency key was reused with a different request")]
Conflict,
#[error("idempotency request body is too large")]
RequestBodyTooLarge,
#[error("idempotency requires a verified authentication scope")]
UnverifiedAuthenticationScope,
#[error("idempotency execution result is unavailable")]
ExecutionResultUnavailable,
#[error("idempotency store is unavailable")]
StoreUnavailable,
}
impl ClassifyError for IdempotencyRejection {
fn error_kind(&self) -> ErrorKind {
match self {
Self::InvalidKey | Self::RequestBodyTooLarge => ErrorKind::Validation,
Self::UnverifiedAuthenticationScope => ErrorKind::Authentication,
Self::Processing | Self::Conflict => ErrorKind::Conflict,
Self::ExecutionResultUnavailable => ErrorKind::Internal,
Self::StoreUnavailable => ErrorKind::Unavailable,
}
}
fn error_code(&self) -> Cow<'static, str> {
Cow::Borrowed(match self {
Self::InvalidKey => "invalid_idempotency_key",
Self::Processing => "idempotency_request_processing",
Self::Conflict => "idempotency_key_conflict",
Self::RequestBodyTooLarge => "idempotency_request_body_too_large",
Self::UnverifiedAuthenticationScope => "idempotency_authentication_scope_required",
Self::ExecutionResultUnavailable => "idempotency_execution_result_unavailable",
Self::StoreUnavailable => "idempotency_store_unavailable",
})
}
}
impl IntoResponse for IdempotencyRejection {
fn into_response(self) -> Response {
match self {
Self::InvalidKey => ApiProblem::bad_request(
ProblemCode::from_static("invalid_idempotency_key"),
"idempotency key is invalid",
)
.into_response(),
Self::Processing => ApiProblem::conflict(
ProblemCode::from_static("idempotency_request_processing"),
"an idempotency request with this key is already processing",
)
.into_response(),
Self::Conflict => ApiProblem::conflict(
ProblemCode::from_static("idempotency_key_conflict"),
"idempotency key was reused with a different request",
)
.into_response(),
Self::RequestBodyTooLarge => ApiProblem::payload_too_large(
ProblemCode::from_static("idempotency_request_body_too_large"),
"idempotency request body is too large",
)
.into_response(),
Self::UnverifiedAuthenticationScope => ApiProblem::unauthorized(
ProblemCode::from_static("idempotency_authentication_scope_required"),
"authentication must be verified before applying idempotency",
)
.into_response(),
Self::ExecutionResultUnavailable => terminal_failure_response(),
Self::StoreUnavailable => ApiProblem::service_unavailable(
ProblemCode::from_static("idempotency_store_unavailable"),
"idempotency store is unavailable",
)
.into_response(),
}
}
}
#[derive(Debug, Clone)]
struct IdempotencyEntry {
lease: IdempotencyLease,
request_hash: IdempotencyRequestHash,
state: IdempotencyState,
response: Option<StoredIdempotencyResponse>,
reserved_response_bytes: usize,
expires_at: Instant,
}
#[derive(Clone)]
#[non_exhaustive]
pub struct MemoryIdempotencyStore {
state: Arc<Mutex<MemoryIdempotencyState>>,
clock: SharedClock,
max_records: usize,
max_stored_response_bytes: usize,
max_response_bytes_per_record: usize,
}
#[derive(Debug, Default)]
struct MemoryIdempotencyState {
records: HashMap<IdempotencyKey, IdempotencyEntry>,
stored_response_bytes: usize,
reserved_response_bytes: usize,
}
impl MemoryIdempotencyStore {
#[must_use]
pub fn new() -> Self {
Self::with_clock(system_clock())
}
#[must_use]
pub fn with_clock(clock: SharedClock) -> Self {
Self {
state: Arc::new(Mutex::new(MemoryIdempotencyState::default())),
clock,
max_records: DEFAULT_MAX_IDEMPOTENCY_RECORDS,
max_stored_response_bytes: DEFAULT_MAX_IDEMPOTENCY_STORED_RESPONSE_BYTES,
max_response_bytes_per_record: DEFAULT_MAX_IDEMPOTENCY_STORED_RESPONSE_BYTES,
}
}
pub fn for_response_body_limit(
max_response_body_bytes: usize,
) -> Result<Self, IdempotencyStoreError> {
if max_response_body_bytes == 0 {
return Err(IdempotencyStoreError::InvalidCapacity);
}
let per_record_bytes = max_response_body_bytes
.checked_add(DEFAULT_MAX_IDEMPOTENCY_STORED_HEADER_BYTES)
.ok_or(IdempotencyStoreError::InvalidCapacity)?;
if per_record_bytes == 0 || per_record_bytes > DEFAULT_MAX_IDEMPOTENCY_STORED_RESPONSE_BYTES
{
return Err(IdempotencyStoreError::InvalidCapacity);
}
Self::with_clock_and_limits_and_reservation(
system_clock(),
DEFAULT_MAX_IDEMPOTENCY_RECORDS,
DEFAULT_MAX_IDEMPOTENCY_STORED_RESPONSE_BYTES,
per_record_bytes,
)
}
pub fn with_clock_and_max_records(
clock: SharedClock,
max_records: usize,
) -> Result<Self, IdempotencyStoreError> {
if max_records == 0 {
return Err(IdempotencyStoreError::InvalidCapacity);
}
Self::with_clock_and_limits(
clock,
max_records,
DEFAULT_MAX_IDEMPOTENCY_STORED_RESPONSE_BYTES,
)
}
fn with_clock_and_limits(
clock: SharedClock,
max_records: usize,
max_stored_response_bytes: usize,
) -> Result<Self, IdempotencyStoreError> {
Self::with_clock_and_limits_and_reservation(
clock,
max_records,
max_stored_response_bytes,
max_stored_response_bytes,
)
}
fn with_clock_and_limits_and_reservation(
clock: SharedClock,
max_records: usize,
max_stored_response_bytes: usize,
max_response_bytes_per_record: usize,
) -> Result<Self, IdempotencyStoreError> {
if max_records == 0
|| max_stored_response_bytes == 0
|| max_response_bytes_per_record == 0
|| max_response_bytes_per_record > max_stored_response_bytes
{
return Err(IdempotencyStoreError::InvalidCapacity);
}
Ok(Self {
state: Arc::new(Mutex::new(MemoryIdempotencyState::default())),
clock,
max_records,
max_stored_response_bytes,
max_response_bytes_per_record,
})
}
#[must_use]
pub fn clock(&self) -> &SharedClock {
&self.clock
}
#[must_use]
pub const fn max_records(&self) -> usize {
self.max_records
}
#[must_use]
pub const fn max_stored_response_bytes(&self) -> usize {
self.max_stored_response_bytes
}
pub fn record_count(&self) -> Result<usize, IdempotencyStoreError> {
self.state
.lock()
.map(|state| state.records.len())
.map_err(|_| IdempotencyStoreError::LockPoisoned)
}
pub fn stored_response_bytes(&self) -> Result<usize, IdempotencyStoreError> {
self.state
.lock()
.map(|state| state.stored_response_bytes)
.map_err(|_| IdempotencyStoreError::LockPoisoned)
}
pub fn reserved_response_bytes(&self) -> Result<usize, IdempotencyStoreError> {
self.state
.lock()
.map(|state| state.reserved_response_bytes)
.map_err(|_| IdempotencyStoreError::LockPoisoned)
}
pub fn prune_expired(&self) -> Result<usize, IdempotencyStoreError> {
let now = self.clock.now();
let mut state = self
.state
.lock()
.map_err(|_| IdempotencyStoreError::LockPoisoned)?;
Ok(prune_expired_records(&mut state, now))
}
fn reserve_capacity(
&self,
key: IdempotencyKey,
request_hash: IdempotencyRequestHash,
processing_ttl: Duration,
response_capacity_bytes: usize,
) -> Result<IdempotencyReservation, IdempotencyStoreError> {
if response_capacity_bytes == 0
|| response_capacity_bytes > self.max_response_bytes_per_record
{
return Err(IdempotencyStoreError::CapacityExceeded);
}
let now = self.clock.now();
let mut state = self
.state
.lock()
.map_err(|_| IdempotencyStoreError::LockPoisoned)?;
prune_expired_records(&mut state, now);
let expires_at = expiry_after(now, processing_ttl)?;
match state.records.get(&key) {
Some(record) if record.request_hash != request_hash => {
Ok(IdempotencyReservation::Conflict)
}
Some(record) if record.state == IdempotencyState::Processing => {
Ok(IdempotencyReservation::Processing)
}
Some(record) => record
.response
.clone()
.map(IdempotencyReservation::Replay)
.ok_or(IdempotencyStoreError::InconsistentRecord),
None => {
if state.records.len() >= self.max_records {
return Err(IdempotencyStoreError::CapacityExceeded);
}
let allocated_bytes = state
.stored_response_bytes
.checked_add(state.reserved_response_bytes)
.and_then(|bytes| bytes.checked_add(response_capacity_bytes))
.ok_or(IdempotencyStoreError::CapacityExceeded)?;
if allocated_bytes > self.max_stored_response_bytes {
return Err(IdempotencyStoreError::CapacityExceeded);
}
state.reserved_response_bytes = state
.reserved_response_bytes
.checked_add(response_capacity_bytes)
.ok_or(IdempotencyStoreError::CapacityExceeded)?;
let lease = IdempotencyLease::new();
state.records.insert(
key,
IdempotencyEntry {
lease,
request_hash,
state: IdempotencyState::Processing,
response: None,
reserved_response_bytes: response_capacity_bytes,
expires_at,
},
);
Ok(IdempotencyReservation::Reserved(lease))
}
}
}
}
impl Default for MemoryIdempotencyStore {
fn default() -> Self {
Self::new()
}
}
impl fmt::Debug for MemoryIdempotencyStore {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
let state = self.state.lock().ok();
formatter
.debug_struct("MemoryIdempotencyStore")
.field(
"record_count",
&state.as_ref().map(|state| state.records.len()),
)
.field(
"stored_response_bytes",
&state.as_ref().map(|state| state.stored_response_bytes),
)
.field(
"reserved_response_bytes",
&state.as_ref().map(|state| state.reserved_response_bytes),
)
.field("max_records", &self.max_records)
.field("max_stored_response_bytes", &self.max_stored_response_bytes)
.field(
"max_response_bytes_per_record",
&self.max_response_bytes_per_record,
)
.finish_non_exhaustive()
}
}
#[async_trait]
impl IdempotencyStore for MemoryIdempotencyStore {
async fn reserve(
&self,
key: IdempotencyKey,
request_hash: IdempotencyRequestHash,
processing_ttl: Duration,
) -> Result<IdempotencyReservation, IdempotencyStoreError> {
let default_response_capacity = DEFAULT_IDEMPOTENCY_MAX_BODY_BYTES
.checked_add(DEFAULT_MAX_IDEMPOTENCY_STORED_HEADER_BYTES)
.ok_or(IdempotencyStoreError::InvalidCapacity)?
.min(self.max_response_bytes_per_record);
self.reserve_capacity(key, request_hash, processing_ttl, default_response_capacity)
}
async fn reserve_with_capacity(
&self,
key: IdempotencyKey,
request_hash: IdempotencyRequestHash,
processing_ttl: Duration,
response_capacity_bytes: usize,
) -> Result<IdempotencyReservation, IdempotencyStoreError> {
self.reserve_capacity(key, request_hash, processing_ttl, response_capacity_bytes)
}
async fn commit(
&self,
key: &IdempotencyKey,
lease: IdempotencyLease,
response: StoredIdempotencyResponse,
retention_ttl: Duration,
) -> Result<(), IdempotencyStoreError> {
let now = self.clock.now();
let expires_at = expiry_after(now, retention_ttl)?;
let mut state = self
.state
.lock()
.map_err(|_| IdempotencyStoreError::LockPoisoned)?;
prune_expired_records(&mut state, now);
let Some(existing) = state.records.get(key) else {
return Err(IdempotencyStoreError::InconsistentRecord);
};
if existing.lease != lease || existing.state != IdempotencyState::Processing {
return Err(IdempotencyStoreError::InconsistentRecord);
}
let existing_bytes = existing.response.as_ref().map_or(0, stored_response_size);
let reserved_bytes = existing.reserved_response_bytes;
let response_bytes = stored_response_size(&response);
if response_bytes > reserved_bytes {
return Err(IdempotencyStoreError::CapacityExceeded);
}
let next_stored_bytes = state
.stored_response_bytes
.saturating_sub(existing_bytes)
.checked_add(response_bytes)
.ok_or(IdempotencyStoreError::CapacityExceeded)?;
let next_reserved_bytes = state.reserved_response_bytes.saturating_sub(reserved_bytes);
let next_allocated_bytes = next_stored_bytes
.checked_add(next_reserved_bytes)
.ok_or(IdempotencyStoreError::CapacityExceeded)?;
if next_allocated_bytes > self.max_stored_response_bytes {
return Err(IdempotencyStoreError::CapacityExceeded);
}
if let Some(record) = state.records.get_mut(key) {
record.state = IdempotencyState::Completed;
record.response = Some(response);
record.reserved_response_bytes = 0;
record.expires_at = expires_at;
state.stored_response_bytes = next_stored_bytes;
state.reserved_response_bytes = next_reserved_bytes;
}
Ok(())
}
async fn abort(
&self,
key: &IdempotencyKey,
lease: IdempotencyLease,
) -> Result<(), IdempotencyStoreError> {
let mut state = self
.state
.lock()
.map_err(|_| IdempotencyStoreError::LockPoisoned)?;
let should_remove = state.records.get(key).is_some_and(|record| {
record.lease == lease && record.state == IdempotencyState::Processing
});
if !should_remove {
return Ok(());
}
let record = state
.records
.remove(key)
.ok_or(IdempotencyStoreError::InconsistentRecord)?;
state.reserved_response_bytes = state
.reserved_response_bytes
.saturating_sub(record.reserved_response_bytes);
state.stored_response_bytes = state
.stored_response_bytes
.saturating_sub(record.response.as_ref().map_or(0, stored_response_size));
Ok(())
}
}
fn expiry_after(now: Instant, ttl: Duration) -> Result<Instant, IdempotencyStoreError> {
if ttl.is_zero() {
return Err(IdempotencyStoreError::InvalidExpiry);
}
now.checked_add(ttl)
.ok_or(IdempotencyStoreError::InvalidExpiry)
}
fn prune_expired_records(state: &mut MemoryIdempotencyState, now: Instant) -> usize {
let before = state.records.len();
state.records.retain(|_, record| record.expires_at > now);
state.stored_response_bytes = state
.records
.values()
.filter_map(|record| record.response.as_ref())
.map(stored_response_size)
.fold(0usize, usize::saturating_add);
state.reserved_response_bytes = state
.records
.values()
.map(|record| record.reserved_response_bytes)
.fold(0usize, usize::saturating_add);
before - state.records.len()
}
fn stored_response_size(response: &StoredIdempotencyResponse) -> usize {
response.body.len()
+ response
.headers
.iter()
.map(|(name, value)| name.len().saturating_add(value.len()))
.sum::<usize>()
}
pub async fn idempotency_middleware(request: Request<Body>, next: Next) -> Response {
let should_apply = request.headers().contains_key(IDEMPOTENCY_KEY_HEADER_NAME);
let Some(runtime) = request.extensions().get::<IdempotencyRuntime>().cloned() else {
return next.run(request).await;
};
if !should_apply {
return next.run(request).await;
}
match run_idempotent_request(runtime, request, next).await {
Ok(response) => response,
Err(rejection) => rejection.into_response(),
}
}
#[derive(Clone)]
#[non_exhaustive]
pub struct IdempotencyLayer {
runtime: IdempotencyRuntime,
request_policy: HttpRequestPolicy,
}
impl IdempotencyLayer {
#[must_use]
pub fn new(runtime: IdempotencyRuntime) -> Self {
Self {
runtime,
request_policy: HttpRequestPolicy::default(),
}
}
#[must_use]
pub fn with_request_policy(mut self, request_policy: HttpRequestPolicy) -> Self {
self.request_policy = request_policy;
self
}
#[must_use]
pub const fn runtime(&self) -> &IdempotencyRuntime {
&self.runtime
}
#[must_use]
pub const fn request_policy(&self) -> &HttpRequestPolicy {
&self.request_policy
}
}
impl fmt::Debug for IdempotencyLayer {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("IdempotencyLayer")
.field("runtime", &self.runtime)
.finish()
}
}
impl<S> Layer<S> for IdempotencyLayer {
type Service = IdempotencyService<S>;
fn layer(&self, inner: S) -> Self::Service {
IdempotencyService {
inner,
runtime: self.runtime.clone(),
request_policy: self.request_policy.clone(),
}
}
}
#[derive(Clone)]
#[non_exhaustive]
pub struct IdempotencyService<S> {
inner: S,
runtime: IdempotencyRuntime,
request_policy: HttpRequestPolicy,
}
impl<S> fmt::Debug for IdempotencyService<S>
where
S: fmt::Debug,
{
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("IdempotencyService")
.field("inner", &self.inner)
.field("runtime", &self.runtime)
.finish()
}
}
impl<S> Service<Request<Body>> for IdempotencyService<S>
where
S: Service<Request<Body>, Response = Response> + Clone + Send + 'static,
S::Future: Send + 'static,
S::Error: Send + 'static,
{
type Response = Response;
type Error = S::Error;
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
fn poll_ready(&mut self, context: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
self.inner.poll_ready(context)
}
fn call(&mut self, request: Request<Body>) -> Self::Future {
let runtime = self.runtime.clone();
let request_policy = self.request_policy.clone();
let clone = self.inner.clone();
let mut inner = std::mem::replace(&mut self.inner, clone);
Box::pin(async move {
if !should_run_idempotency(&request_policy, &request) {
return inner.call(request).await;
}
let response = match run_idempotent_service(runtime, request, |request| async move {
inner.call(request).await
})
.await
{
Ok(response) => response?,
Err(rejection) => rejection.into_response(),
};
Ok(response)
})
}
}
fn should_run_idempotency<B>(request_policy: &HttpRequestPolicy, request: &Request<B>) -> bool {
match request_policy.policy_for(request).idempotency() {
IdempotencyRequestPolicy::Disabled => false,
IdempotencyRequestPolicy::HeaderOptional => {
request.headers().contains_key(IDEMPOTENCY_KEY_HEADER_NAME)
}
IdempotencyRequestPolicy::Required => true,
}
}
pub async fn run_idempotent_request(
runtime: IdempotencyRuntime,
request: Request<Body>,
next: Next,
) -> Result<Response, IdempotencyRejection> {
run_idempotent_service(runtime, request, |request| async move {
Ok::<_, std::convert::Infallible>(next.run(request).await)
})
.await?
.map_err(|never| match never {})
}
struct IdempotencyReservationGuard {
store: Arc<dyn IdempotencyStore>,
lease: IdempotencyLease,
retention_ttl: Duration,
state: IdempotencyReservationGuardState,
}
enum IdempotencyReservationGuardState {
Reserved(IdempotencyKey),
Executing(IdempotencyKey),
Indeterminate,
Resolved,
}
impl IdempotencyReservationGuard {
fn new(
store: Arc<dyn IdempotencyStore>,
key: IdempotencyKey,
lease: IdempotencyLease,
retention_ttl: Duration,
) -> Self {
Self {
store,
lease,
retention_ttl,
state: IdempotencyReservationGuardState::Reserved(key),
}
}
fn start_execution(&mut self) {
let state = std::mem::replace(
&mut self.state,
IdempotencyReservationGuardState::Indeterminate,
);
self.state = match state {
IdempotencyReservationGuardState::Reserved(key) => {
IdempotencyReservationGuardState::Executing(key)
}
state => state,
};
}
async fn commit(
&mut self,
response: StoredIdempotencyResponse,
) -> Result<(), IdempotencyStoreError> {
let key = match &self.state {
IdempotencyReservationGuardState::Executing(key) => key.clone(),
_ => return Err(IdempotencyStoreError::InconsistentRecord),
};
match self
.store
.commit(&key, self.lease, response, self.retention_ttl)
.await
{
Ok(()) => {
self.state = IdempotencyReservationGuardState::Resolved;
Ok(())
}
Err(error) => {
self.state = IdempotencyReservationGuardState::Indeterminate;
Err(error)
}
}
}
async fn commit_terminal_failure(&mut self) -> Result<(), IdempotencyStoreError> {
self.commit(terminal_failure_stored_response()).await
}
}
impl Drop for IdempotencyReservationGuard {
fn drop(&mut self) {
let state = std::mem::replace(&mut self.state, IdempotencyReservationGuardState::Resolved);
let Ok(runtime) = tokio::runtime::Handle::try_current() else {
return;
};
let store = self.store.clone();
let lease = self.lease;
match state {
IdempotencyReservationGuardState::Reserved(key) => {
drop(runtime.spawn(async move {
if let Err(error) = store.abort(&key, lease).await {
tracing::warn!(
error = %error,
"failed to release an unstarted idempotency reservation"
);
}
}));
}
IdempotencyReservationGuardState::Executing(key) => {
let retention_ttl = self.retention_ttl;
drop(runtime.spawn(async move {
if let Err(error) = store
.commit(
&key,
lease,
terminal_failure_stored_response(),
retention_ttl,
)
.await
{
tracing::warn!(
error = %error,
"failed to terminalize a cancelled idempotent execution"
);
}
}));
}
IdempotencyReservationGuardState::Indeterminate
| IdempotencyReservationGuardState::Resolved => {}
}
}
}
pub async fn run_idempotent_service<F, Fut, E>(
runtime: IdempotencyRuntime,
request: Request<Body>,
call: F,
) -> Result<Result<Response, E>, IdempotencyRejection>
where
F: FnOnce(Request<Body>) -> Fut,
Fut: Future<Output = Result<Response, E>>,
{
let client_key = parse_required_idempotency_key(request.headers())?;
let (parts, body) = request.into_parts();
let store_key = scoped_store_key(&parts, &client_key)?;
let body = to_bytes(body, runtime.max_request_body_bytes)
.await
.map_err(|_| IdempotencyRejection::RequestBodyTooLarge)?;
let request_hash = request_hash_from_parts(&parts, &body);
let request = Request::from_parts(parts, Body::from(body));
let response_capacity_bytes = runtime
.max_response_body_bytes
.checked_add(DEFAULT_MAX_IDEMPOTENCY_STORED_HEADER_BYTES)
.ok_or(IdempotencyRejection::StoreUnavailable)?;
let reservation = runtime
.store
.reserve_with_capacity(
store_key.clone(),
request_hash,
runtime.processing_ttl,
response_capacity_bytes,
)
.await
.map_err(|_| IdempotencyRejection::StoreUnavailable)?;
record_idempotency_metric(&runtime, reservation_metric_outcome(&reservation));
match reservation {
IdempotencyReservation::Reserved(lease) => {
let mut reservation = IdempotencyReservationGuard::new(
runtime.store.clone(),
store_key,
lease,
runtime.ttl,
);
reservation.start_execution();
let response = match call(request).await {
Ok(response) => response,
Err(error) => {
if let Err(commit_error) = reservation.commit_terminal_failure().await {
tracing::warn!(
error = %commit_error,
"failed to terminalize an idempotent service error"
);
}
return Ok(Err(error));
}
};
store_and_return_response(&runtime, &mut reservation, response)
.await
.map(Ok)
}
IdempotencyReservation::Processing => Err(IdempotencyRejection::Processing),
IdempotencyReservation::Replay(stored) => response_from_stored(stored).map(Ok),
IdempotencyReservation::Conflict => Err(IdempotencyRejection::Conflict),
}
}
fn reservation_metric_outcome(reservation: &IdempotencyReservation) -> &'static str {
match reservation {
IdempotencyReservation::Reserved(_) => metric_outcomes::RESERVED,
IdempotencyReservation::Processing => metric_outcomes::PROCESSING,
IdempotencyReservation::Replay(_) => metric_outcomes::REPLAY,
IdempotencyReservation::Conflict => metric_outcomes::CONFLICT,
}
}
fn record_idempotency_metric(runtime: &IdempotencyRuntime, outcome: &'static str) {
if let Some(metrics) = runtime.metrics() {
let _ = metrics.increment_counter(
metric_names::IDEMPOTENCY_REQUESTS_TOTAL,
1,
MetricLabels::from_lossy([(metric_labels::OUTCOME, outcome)]),
);
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct RequiredIdempotencyKey(pub IdempotencyKey);
impl RequiredIdempotencyKey {
#[must_use]
pub fn into_inner(self) -> IdempotencyKey {
self.0
}
}
impl<S> FromRequestParts<S> for RequiredIdempotencyKey
where
S: Send + Sync,
{
type Rejection = ApiProblem;
async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result<Self, Self::Rejection> {
idempotency_key_from_headers(&parts.headers)
.map(Self)
.ok_or_else(|| {
ApiProblem::bad_request(
ProblemCode::from_static("idempotency_key_required"),
"a valid Idempotency-Key header is required",
)
})
}
}
#[must_use]
pub fn idempotency_key_from_headers(headers: &HeaderMap) -> Option<IdempotencyKey> {
headers
.get(IDEMPOTENCY_KEY_HEADER_NAME)
.and_then(|value| value.to_str().ok())
.and_then(IdempotencyKey::parse)
}
pub fn parse_required_idempotency_key(
headers: &HeaderMap,
) -> Result<IdempotencyKey, IdempotencyRejection> {
headers
.get(IDEMPOTENCY_KEY_HEADER_NAME)
.ok_or(IdempotencyRejection::InvalidKey)?
.to_str()
.ok()
.and_then(IdempotencyKey::parse)
.ok_or(IdempotencyRejection::InvalidKey)
}
#[must_use]
pub fn is_valid_idempotency_key(value: &str) -> bool {
validate_idempotency_key(value).is_ok()
}
fn validate_idempotency_key(value: &str) -> Result<(), IdempotencyKeyError> {
if value.is_empty() {
return Err(IdempotencyKeyError::Empty);
}
if value.len() > MAX_IDEMPOTENCY_KEY_LENGTH {
return Err(IdempotencyKeyError::TooLong {
max: MAX_IDEMPOTENCY_KEY_LENGTH,
});
}
if !value
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.' | b':'))
{
return Err(IdempotencyKeyError::InvalidCharacters);
}
Ok(())
}
fn request_hash_from_parts(parts: &Parts, body: &[u8]) -> IdempotencyRequestHash {
let mut hasher = Sha256::new();
hasher.update(parts.method.as_str().as_bytes());
hasher.update(b"\n");
hasher.update(parts.uri.to_string().as_bytes());
hasher.update(b"\n");
if let Some(content_type) = parts.headers.get(CONTENT_TYPE) {
hasher.update(content_type.as_bytes());
}
hasher.update(b"\n");
hasher.update(body);
IdempotencyRequestHash(to_lower_hex(&hasher.finalize()))
}
fn scoped_store_key(
parts: &Parts,
client_key: &IdempotencyKey,
) -> Result<IdempotencyKey, IdempotencyRejection> {
let mut hasher = Sha256::new();
hasher.update(b"footing-idempotency-store-key-v2\0");
if let Some(scope) = parts.extensions.get::<IdempotencyScope>() {
hasher.update(b"explicit\0");
hasher.update(scope.0);
} else if parts.headers.contains_key(AUTHORIZATION) || parts.headers.contains_key(COOKIE) {
return Err(IdempotencyRejection::UnverifiedAuthenticationScope);
} else {
hasher.update(b"anonymous\0");
}
hash_length_prefixed(&mut hasher, client_key.as_str().as_bytes());
Ok(IdempotencyKey(to_lower_hex(&hasher.finalize())))
}
fn hash_length_prefixed(hasher: &mut Sha256, value: &[u8]) {
hasher.update(u64::try_from(value.len()).unwrap_or(u64::MAX).to_be_bytes());
hasher.update(value);
}
async fn store_and_return_response(
runtime: &IdempotencyRuntime,
reservation: &mut IdempotencyReservationGuard,
response: Response,
) -> Result<Response, IdempotencyRejection> {
let (parts, body) = response.into_parts();
let body = match to_bytes(body, runtime.max_response_body_bytes).await {
Ok(body) => body,
Err(error) => {
tracing::warn!(
error = %error,
"failed to capture an idempotent response; committing a terminal failure"
);
if let Err(commit_error) = reservation.commit_terminal_failure().await {
tracing::warn!(
error = %commit_error,
"failed to persist an idempotent terminal failure"
);
}
return Ok(terminal_failure_response());
}
};
let stored = stored_response_from_parts(&parts, body.to_vec());
reservation
.commit(stored)
.await
.map_err(|_| IdempotencyRejection::StoreUnavailable)?;
Ok(Response::from_parts(parts, Body::from(body)))
}
fn terminal_failure_stored_response() -> StoredIdempotencyResponse {
StoredIdempotencyResponse::new(
StatusCode::INTERNAL_SERVER_ERROR,
IDEMPOTENCY_TERMINAL_FAILURE_BODY,
)
.with_header(
CONTENT_TYPE.as_str(),
IDEMPOTENCY_TERMINAL_FAILURE_MEDIA_TYPE,
)
}
fn terminal_failure_response() -> Response {
let mut response = Response::new(Body::from(IDEMPOTENCY_TERMINAL_FAILURE_BODY));
*response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR;
response.headers_mut().insert(
CONTENT_TYPE,
HeaderValue::from_static(IDEMPOTENCY_TERMINAL_FAILURE_MEDIA_TYPE),
);
response
}
fn response_from_stored(
stored: StoredIdempotencyResponse,
) -> Result<Response, IdempotencyRejection> {
let mut builder = response::Builder::new().status(stored.status);
for (name, value) in &stored.headers {
let Ok(name) = HeaderName::try_from(name.as_str()) else {
continue;
};
let Ok(value) = HeaderValue::try_from(value.as_str()) else {
continue;
};
builder = builder.header(name, value);
}
builder = builder.header(IDEMPOTENCY_REPLAYED_HEADER_NAME, "true");
builder
.body(Body::from(stored.body))
.map_err(|_| IdempotencyRejection::StoreUnavailable)
}
fn stored_response_from_parts(parts: &response::Parts, body: Vec<u8>) -> StoredIdempotencyResponse {
let mut stored = StoredIdempotencyResponse::new(parts.status, body);
let mut stored_header_bytes = 0usize;
let connection_headers = connection_nominated_headers(&parts.headers);
if let Some(value) = parts
.headers
.get(CONTENT_TYPE)
.and_then(|value| value.to_str().ok())
{
let header_bytes = CONTENT_TYPE.as_str().len().saturating_add(value.len());
if header_bytes <= DEFAULT_MAX_IDEMPOTENCY_STORED_HEADER_BYTES {
stored_header_bytes = header_bytes;
stored = stored.with_header(CONTENT_TYPE.as_str(), value);
}
}
for (name, value) in &parts.headers {
if name != CONTENT_TYPE
&& is_storable_response_header(name)
&& !connection_headers.contains(name)
&& let Ok(value) = value.to_str()
{
let header_bytes = name.as_str().len().saturating_add(value.len());
let Some(next_header_bytes) = stored_header_bytes.checked_add(header_bytes) else {
continue;
};
if next_header_bytes > DEFAULT_MAX_IDEMPOTENCY_STORED_HEADER_BYTES {
continue;
}
stored = stored.with_header(name.as_str(), value);
stored_header_bytes = next_header_bytes;
}
}
stored
}
fn connection_nominated_headers(headers: &HeaderMap) -> HashSet<HeaderName> {
headers
.get_all(CONNECTION)
.iter()
.filter_map(|value| value.to_str().ok())
.flat_map(|value| value.split(','))
.filter_map(|name| HeaderName::try_from(name.trim()).ok())
.collect()
}
fn is_storable_response_header(name: &HeaderName) -> bool {
if name.as_str() == IDEMPOTENCY_REPLAYED_HEADER_NAME {
return false;
}
!matches!(
name.as_str(),
"connection"
| "content-length"
| "date"
| "keep-alive"
| "proxy-authenticate"
| "proxy-authorization"
| "server"
| "set-cookie"
| "te"
| "trailer"
| "transfer-encoding"
| "upgrade"
)
}
fn to_lower_hex(bytes: &[u8]) -> String {
const HEX: &[u8; 16] = b"0123456789abcdef";
let mut output = String::with_capacity(bytes.len() * 2);
for byte in bytes {
output.push(char::from(HEX[usize::from(byte >> 4)]));
output.push(char::from(HEX[usize::from(byte & 0x0f)]));
}
output
}
#[cfg(test)]
mod tests {
use std::{
convert::Infallible,
sync::{
Arc,
atomic::{AtomicUsize, Ordering},
},
};
use async_trait::async_trait;
use axum::{
Router,
body::{Body, to_bytes},
http::{
HeaderMap, HeaderValue, Request, StatusCode,
header::{AUTHORIZATION, CONTENT_TYPE, COOKIE, SET_COOKIE},
},
middleware::{self, Next},
response::{IntoResponse, Response},
routing::post,
};
use tower::ServiceExt;
use super::{
IDEMPOTENCY_KEY_HEADER_NAME, IdempotencyKey, IdempotencyLayer, IdempotencyLease,
IdempotencyRejection, IdempotencyRequestHash, IdempotencyReservation, IdempotencyRuntime,
IdempotencyScope, IdempotencyStore, IdempotencyStoreError, MemoryIdempotencyStore,
StoredIdempotencyResponse, idempotency_key_from_headers, idempotency_middleware,
is_valid_idempotency_key, run_idempotent_service,
};
use crate::result::{ClassifyError, ErrorKind};
#[derive(Default)]
struct CommitFailingStore {
inner: MemoryIdempotencyStore,
commit_attempts: AtomicUsize,
abort_attempts: AtomicUsize,
}
#[async_trait]
impl IdempotencyStore for CommitFailingStore {
async fn reserve(
&self,
key: IdempotencyKey,
request_hash: IdempotencyRequestHash,
processing_ttl: std::time::Duration,
) -> Result<IdempotencyReservation, IdempotencyStoreError> {
self.inner.reserve(key, request_hash, processing_ttl).await
}
async fn commit(
&self,
_key: &IdempotencyKey,
_lease: super::IdempotencyLease,
_response: StoredIdempotencyResponse,
_retention_ttl: std::time::Duration,
) -> Result<(), IdempotencyStoreError> {
self.commit_attempts.fetch_add(1, Ordering::SeqCst);
Err(IdempotencyStoreError::CapacityExceeded)
}
async fn abort(
&self,
key: &IdempotencyKey,
lease: super::IdempotencyLease,
) -> Result<(), IdempotencyStoreError> {
self.abort_attempts.fetch_add(1, Ordering::SeqCst);
self.inner.abort(key, lease).await
}
}
#[test]
fn validates_idempotency_keys() {
assert!(is_valid_idempotency_key("op_018f.test:1"));
assert!(!is_valid_idempotency_key(""));
assert!(!is_valid_idempotency_key("bad/key"));
}
#[test]
fn idempotency_debug_output_redacts_keys_hashes_and_response_payloads() {
let key_value = "private-idempotency-key";
let hash_value = "private-request-hash";
let header_value = "private-response-header";
let body_value = "private-response-body";
let key = IdempotencyKey::new(key_value).expect("key should validate");
let hash = IdempotencyRequestHash::new(hash_value).expect("hash should validate");
let response = StoredIdempotencyResponse::new(StatusCode::OK, body_value)
.with_header("content-type", header_value);
let lease = IdempotencyLease::new();
let lease_value = lease.0.to_string();
let output = format!("{key} {key:?} {hash:?} {response:?} {lease:?}");
assert!(output.contains("[REDACTED]"));
assert!(output.contains("body_bytes"));
for secret in [key_value, hash_value, header_value, body_value] {
assert!(!output.contains(secret));
}
assert!(!output.contains(&lease_value));
}
#[test]
fn idempotency_errors_have_stable_classification() {
let invalid_key = IdempotencyKey::new("bad/key")
.expect_err("key should be invalid")
.error_descriptor();
let store = IdempotencyStoreError::InconsistentRecord.error_descriptor();
let execution_result_unavailable =
IdempotencyRejection::ExecutionResultUnavailable.error_descriptor();
let unverified_scope =
IdempotencyRejection::UnverifiedAuthenticationScope.error_descriptor();
assert_eq!(invalid_key.kind(), ErrorKind::Validation);
assert_eq!(
invalid_key.code().as_str(),
"idempotency_key_invalid_characters"
);
assert_eq!(store.kind(), ErrorKind::Internal);
assert_eq!(store.code().as_str(), "idempotency_record_inconsistent");
assert_eq!(execution_result_unavailable.kind(), ErrorKind::Internal);
assert_eq!(
execution_result_unavailable.code().as_str(),
"idempotency_execution_result_unavailable"
);
assert_eq!(unverified_scope.kind(), ErrorKind::Authentication);
assert_eq!(
unverified_scope.code().as_str(),
"idempotency_authentication_scope_required"
);
}
#[test]
fn extracts_key_from_headers() {
let mut headers = HeaderMap::new();
headers.insert(
IDEMPOTENCY_KEY_HEADER_NAME,
HeaderValue::from_static("request-1"),
);
assert_eq!(
idempotency_key_from_headers(&headers)
.as_ref()
.map(IdempotencyKey::as_str),
Some("request-1")
);
}
#[tokio::test]
async fn memory_store_reserves_conflicts_and_replays() {
let store = MemoryIdempotencyStore::new();
let key = IdempotencyKey::new("request-1").unwrap();
let hash = IdempotencyRequestHash::new("hash-1").unwrap();
let lease = match store
.reserve(
key.clone(),
hash.clone(),
std::time::Duration::from_secs(60),
)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
assert_eq!(
store
.reserve(
key.clone(),
IdempotencyRequestHash::new("hash-2").unwrap(),
std::time::Duration::from_secs(60)
)
.await
.unwrap(),
IdempotencyReservation::Conflict
);
store
.commit(
&key,
lease,
StoredIdempotencyResponse::new(axum::http::StatusCode::OK, b"ok".to_vec()),
std::time::Duration::from_secs(60),
)
.await
.unwrap();
assert!(matches!(
store
.reserve(key, hash, std::time::Duration::from_secs(60))
.await
.unwrap(),
IdempotencyReservation::Replay(_)
));
}
#[tokio::test]
async fn memory_store_reports_inconsistent_completed_records() {
let store = MemoryIdempotencyStore::new();
let key = IdempotencyKey::new("request-1").unwrap();
let hash = IdempotencyRequestHash::new("hash-1").unwrap();
store
.reserve(
key.clone(),
hash.clone(),
std::time::Duration::from_secs(60),
)
.await
.unwrap();
{
let mut state = store.state.lock().unwrap();
let record = state.records.get_mut(&key).unwrap();
record.state = super::IdempotencyState::Completed;
record.response = None;
}
let result = store
.reserve(key, hash, std::time::Duration::from_secs(60))
.await;
assert!(matches!(
result,
Err(super::IdempotencyStoreError::InconsistentRecord)
));
}
#[tokio::test]
async fn abort_releases_capacity_without_touching_a_newer_lease() {
let store =
MemoryIdempotencyStore::with_clock_and_max_records(crate::clock::system_clock(), 1)
.unwrap();
let key = IdempotencyKey::new("request-1").unwrap();
let hash = IdempotencyRequestHash::new("hash-1").unwrap();
let first_lease = match store
.reserve(
key.clone(),
hash.clone(),
std::time::Duration::from_secs(60),
)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
store.abort(&key, first_lease).await.unwrap();
assert_eq!(store.record_count().unwrap(), 0);
assert_eq!(store.reserved_response_bytes().unwrap(), 0);
let second_lease = match store
.reserve(key.clone(), hash, std::time::Duration::from_secs(60))
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
assert_ne!(first_lease, second_lease);
assert!(matches!(
store
.commit(
&key,
first_lease,
StoredIdempotencyResponse::new(StatusCode::OK, b"stale".to_vec()),
std::time::Duration::from_secs(60),
)
.await,
Err(IdempotencyStoreError::InconsistentRecord)
));
assert_eq!(store.record_count().unwrap(), 1);
assert_eq!(store.stored_response_bytes().unwrap(), 0);
}
#[tokio::test]
async fn cancelled_execution_is_terminalized_and_never_reexecuted() {
let store = Arc::new(MemoryIdempotencyStore::new());
let runtime = IdempotencyRuntime::new(store.clone())
.with_processing_ttl(std::time::Duration::from_secs(60));
let request = Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "cancelled-request")
.body(Body::empty())
.unwrap();
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
let executions = Arc::new(AtomicUsize::new(0));
let executions_for_task = executions.clone();
let runtime_for_task = runtime.clone();
let task = tokio::spawn(async move {
super::run_idempotent_service(runtime_for_task, request, |_request| async move {
executions_for_task.fetch_add(1, Ordering::SeqCst);
let _ = started_tx.send(());
std::future::pending::<Result<axum::response::Response, std::convert::Infallible>>()
.await
})
.await
});
started_rx
.await
.expect("request should acquire a reservation before blocking");
assert_eq!(store.record_count().unwrap(), 1);
assert!(store.reserved_response_bytes().unwrap() > 0);
task.abort();
assert!(
task.await
.expect_err("request should be cancelled")
.is_cancelled()
);
for _ in 0..128 {
let terminalized = store.state.lock().unwrap().records.values().any(|record| {
record.state == super::IdempotencyState::Completed && record.response.is_some()
});
if terminalized {
break;
}
tokio::task::yield_now().await;
}
assert_eq!(store.record_count().unwrap(), 1);
assert_eq!(store.reserved_response_bytes().unwrap(), 0);
assert!(store.stored_response_bytes().unwrap() > 0);
let executions_for_retry = executions.clone();
let response = super::run_idempotent_service(
runtime,
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "cancelled-request")
.body(Body::empty())
.unwrap(),
move |_request| async move {
executions_for_retry.fetch_add(1, Ordering::SeqCst);
Ok::<_, std::convert::Infallible>(
(StatusCode::OK, "unexpected execution").into_response(),
)
},
)
.await
.unwrap()
.unwrap();
assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(
response
.headers()
.get(super::IDEMPOTENCY_REPLAYED_HEADER_NAME)
.and_then(|value| value.to_str().ok()),
Some("true")
);
assert_eq!(executions.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn inner_service_error_is_terminalized_before_the_error_is_propagated() {
let store = Arc::new(MemoryIdempotencyStore::new());
let runtime = IdempotencyRuntime::new(store.clone());
let executions = Arc::new(AtomicUsize::new(0));
let executions_for_first = executions.clone();
let first = super::run_idempotent_service(
runtime.clone(),
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "service-error")
.body(Body::empty())
.unwrap(),
move |_request| async move {
executions_for_first.fetch_add(1, Ordering::SeqCst);
Err::<axum::response::Response, _>("downstream failure")
},
)
.await
.expect("idempotency orchestration should succeed");
assert!(matches!(first, Err("downstream failure")));
assert_eq!(store.record_count().unwrap(), 1);
assert_eq!(store.reserved_response_bytes().unwrap(), 0);
assert!(store.stored_response_bytes().unwrap() > 0);
let executions_for_retry = executions.clone();
let replay = super::run_idempotent_service(
runtime,
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "service-error")
.body(Body::empty())
.unwrap(),
move |_request| async move {
executions_for_retry.fetch_add(1, Ordering::SeqCst);
Ok::<_, &'static str>((StatusCode::OK, "unexpected execution").into_response())
},
)
.await
.unwrap()
.unwrap();
assert_eq!(replay.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(
replay
.headers()
.get(super::IDEMPOTENCY_REPLAYED_HEADER_NAME)
.and_then(|value| value.to_str().ok()),
Some("true")
);
assert_eq!(executions.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn expired_processing_lease_cannot_commit_into_a_new_reservation() {
let clock = crate::clock::ManualClock::new();
let store = MemoryIdempotencyStore::with_clock_and_max_records(clock.shared(), 1).unwrap();
let key = IdempotencyKey::new("request-expiry").unwrap();
let hash = IdempotencyRequestHash::new("hash-expiry").unwrap();
let processing_ttl = std::time::Duration::from_secs(5);
let first_lease = match store
.reserve(key.clone(), hash.clone(), processing_ttl)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
clock.advance(processing_ttl + std::time::Duration::from_secs(1));
let second_lease = match store
.reserve(key.clone(), hash, processing_ttl)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
assert_ne!(first_lease, second_lease);
assert!(matches!(
store
.commit(
&key,
first_lease,
StoredIdempotencyResponse::new(StatusCode::OK, b"stale".to_vec()),
std::time::Duration::from_secs(60),
)
.await,
Err(IdempotencyStoreError::InconsistentRecord)
));
store
.commit(
&key,
second_lease,
StoredIdempotencyResponse::new(StatusCode::OK, b"current".to_vec()),
std::time::Duration::from_secs(60),
)
.await
.unwrap();
assert_eq!(store.record_count().unwrap(), 1);
}
#[tokio::test]
async fn completed_retention_ttl_starts_when_the_response_is_committed() {
let clock = crate::clock::ManualClock::new();
let store = MemoryIdempotencyStore::with_clock_and_max_records(clock.shared(), 1).unwrap();
let key = IdempotencyKey::new("request-retention").unwrap();
let hash = IdempotencyRequestHash::new("hash-retention").unwrap();
let processing_ttl = std::time::Duration::from_secs(10);
let retention_ttl = std::time::Duration::from_secs(20);
let lease = match store
.reserve(key.clone(), hash.clone(), processing_ttl)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
clock.advance(std::time::Duration::from_secs(9));
store
.commit(
&key,
lease,
StoredIdempotencyResponse::new(StatusCode::OK, b"completed".to_vec()),
retention_ttl,
)
.await
.unwrap();
clock.advance(std::time::Duration::from_secs(19));
assert!(matches!(
store
.reserve(key.clone(), hash.clone(), processing_ttl)
.await
.unwrap(),
IdempotencyReservation::Replay(_)
));
clock.advance(std::time::Duration::from_secs(2));
assert!(matches!(
store.reserve(key, hash, processing_ttl).await.unwrap(),
IdempotencyReservation::Reserved(_)
));
}
#[tokio::test]
async fn memory_store_bounds_unexpired_records() {
let store =
MemoryIdempotencyStore::with_clock_and_max_records(crate::clock::system_clock(), 2)
.unwrap();
let ttl = std::time::Duration::from_secs(60);
for index in 0..2 {
store
.reserve(
IdempotencyKey::new(format!("request-{index}")).unwrap(),
IdempotencyRequestHash::new(format!("hash-{index}")).unwrap(),
ttl,
)
.await
.unwrap();
}
let result = store
.reserve(
IdempotencyKey::new("request-overflow").unwrap(),
IdempotencyRequestHash::new("hash-overflow").unwrap(),
ttl,
)
.await;
assert!(matches!(
result,
Err(IdempotencyStoreError::CapacityExceeded)
));
assert_eq!(store.record_count().unwrap(), 2);
}
#[tokio::test]
async fn memory_store_bounds_total_stored_response_bytes() {
let store =
MemoryIdempotencyStore::with_clock_and_limits(crate::clock::system_clock(), 2, 7)
.unwrap();
let key = IdempotencyKey::new("request-sized").unwrap();
let lease = match store
.reserve(
key.clone(),
IdempotencyRequestHash::new("hash-sized").unwrap(),
std::time::Duration::from_secs(60),
)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
let result = store
.commit(
&key,
lease,
StoredIdempotencyResponse::new(StatusCode::OK, vec![0; 8]),
std::time::Duration::from_secs(60),
)
.await;
assert!(matches!(
result,
Err(IdempotencyStoreError::CapacityExceeded)
));
assert_eq!(store.stored_response_bytes().unwrap(), 0);
}
#[test]
fn in_memory_store_reserves_worst_case_response_capacity_before_execution() {
assert!(matches!(
MemoryIdempotencyStore::for_response_body_limit(0),
Err(IdempotencyStoreError::InvalidCapacity)
));
let store = MemoryIdempotencyStore::for_response_body_limit(
super::DEFAULT_IDEMPOTENCY_MAX_BODY_BYTES,
)
.unwrap();
let per_record = super::DEFAULT_IDEMPOTENCY_MAX_BODY_BYTES
+ super::DEFAULT_MAX_IDEMPOTENCY_STORED_HEADER_BYTES;
assert_eq!(store.max_records(), super::DEFAULT_MAX_IDEMPOTENCY_RECORDS);
assert_eq!(store.max_response_bytes_per_record, per_record);
assert!(per_record <= store.max_stored_response_bytes());
}
#[tokio::test]
async fn completed_small_responses_release_unused_reserved_capacity() {
let store = MemoryIdempotencyStore::with_clock_and_limits_and_reservation(
crate::clock::system_clock(),
4,
10,
6,
)
.unwrap();
let ttl = std::time::Duration::from_secs(60);
for index in 0..4 {
let key = IdempotencyKey::new(format!("request-{index}")).unwrap();
let lease = match store
.reserve(
key.clone(),
IdempotencyRequestHash::new(format!("hash-{index}")).unwrap(),
ttl,
)
.await
.unwrap()
{
IdempotencyReservation::Reserved(lease) => lease,
reservation => panic!("unexpected reservation: {reservation:?}"),
};
store
.commit(
&key,
lease,
StoredIdempotencyResponse::new(StatusCode::OK, vec![0]),
ttl,
)
.await
.unwrap();
}
assert_eq!(store.record_count().unwrap(), 4);
assert_eq!(store.stored_response_bytes().unwrap(), 4);
assert_eq!(store.reserved_response_bytes().unwrap(), 0);
}
#[test]
fn stored_response_filters_unstable_headers_and_preserves_duplicates() {
let stored = StoredIdempotencyResponse::new(axum::http::StatusCode::OK, b"ok".to_vec())
.with_header(CONTENT_TYPE.as_str(), "application/json")
.with_header("warning", "199 first")
.with_header("warning", "299 second")
.with_header(SET_COOKIE.as_str(), "session=unsafe")
.with_header("bad header", "ignored");
assert_eq!(
stored.headers(),
&[
(
CONTENT_TYPE.as_str().to_owned(),
"application/json".to_owned()
),
("warning".to_owned(), "199 first".to_owned()),
("warning".to_owned(), "299 second".to_owned()),
]
);
}
#[tokio::test]
async fn stored_response_filters_connection_nominated_headers_and_replays_duplicates() {
let mut response = Response::new(Body::from("ok"));
response
.headers_mut()
.insert("connection", HeaderValue::from_static("x-hop, keep-alive"));
response
.headers_mut()
.insert("x-hop", HeaderValue::from_static("private"));
response
.headers_mut()
.insert("keep-alive", HeaderValue::from_static("timeout=5"));
response.headers_mut().insert(
super::IDEMPOTENCY_REPLAYED_HEADER_NAME,
HeaderValue::from_static("false"),
);
response
.headers_mut()
.append("warning", HeaderValue::from_static("199 first"));
response
.headers_mut()
.append("warning", HeaderValue::from_static("299 second"));
let (parts, body) = response.into_parts();
let body = to_bytes(body, usize::MAX).await.unwrap().to_vec();
let stored = super::stored_response_from_parts(&parts, body);
assert!(stored.headers().iter().all(|(name, _)| {
name != "connection"
&& name != "x-hop"
&& name != "keep-alive"
&& name != super::IDEMPOTENCY_REPLAYED_HEADER_NAME
}));
let replay = super::response_from_stored(stored).unwrap();
let warnings = replay
.headers()
.get_all("warning")
.iter()
.collect::<Vec<_>>();
assert_eq!(warnings.len(), 2);
assert_eq!(warnings[0], "199 first");
assert_eq!(warnings[1], "299 second");
let replay_markers = replay
.headers()
.get_all(super::IDEMPOTENCY_REPLAYED_HEADER_NAME)
.iter()
.collect::<Vec<_>>();
assert_eq!(replay_markers.len(), 1);
assert_eq!(replay_markers[0], "true");
}
#[tokio::test]
async fn in_memory_runtime_reserves_raised_response_limit_before_execution() {
let response_bytes = super::DEFAULT_IDEMPOTENCY_MAX_BODY_BYTES
+ super::DEFAULT_MAX_IDEMPOTENCY_STORED_HEADER_BYTES
+ 1;
let runtime = IdempotencyRuntime::in_memory().with_max_response_body_bytes(response_bytes);
let executions = Arc::new(AtomicUsize::new(0));
let executions_for_service = executions.clone();
let result = run_idempotent_service(
runtime,
Request::post("/")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "raised-response-limit")
.body(Body::empty())
.unwrap(),
move |_| async move {
executions_for_service.fetch_add(1, Ordering::SeqCst);
Ok::<_, Infallible>(Response::new(Body::from(vec![0_u8; response_bytes])))
},
)
.await
.expect("configured capacity should be reserved")
.expect("service is infallible");
assert_eq!(executions.load(Ordering::SeqCst), 1);
assert_eq!(
to_bytes(result.into_body(), usize::MAX)
.await
.unwrap()
.len(),
response_bytes
);
}
#[tokio::test]
async fn impossible_in_memory_response_limit_fails_before_execution() {
let runtime = IdempotencyRuntime::in_memory().with_max_response_body_bytes(usize::MAX);
let executions = Arc::new(AtomicUsize::new(0));
let executions_for_service = executions.clone();
let result = run_idempotent_service(
runtime,
Request::post("/")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "impossible-response-limit")
.body(Body::empty())
.unwrap(),
move |_| async move {
executions_for_service.fetch_add(1, Ordering::SeqCst);
Ok::<_, Infallible>(Response::new(Body::empty()))
},
)
.await;
assert!(matches!(
result,
Err(IdempotencyRejection::StoreUnavailable)
));
assert_eq!(executions.load(Ordering::SeqCst), 0);
}
#[tokio::test]
async fn middleware_replays_matching_requests() {
async fn create_order() -> Response {
let mut response = (StatusCode::CREATED, "created").into_response();
response
.headers_mut()
.insert(SET_COOKIE, HeaderValue::from_static("session=fresh"));
response
}
let runtime = IdempotencyRuntime::in_memory();
let app = Router::new()
.route("/orders", post(create_order))
.layer(middleware::from_fn(idempotency_middleware))
.layer(axum::Extension(runtime));
let response = app
.clone()
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.body(axum::body::Body::from(r#"{"sku":"A"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CREATED);
assert_eq!(
response.headers().get(SET_COOKIE),
Some(&HeaderValue::from_static("session=fresh"))
);
let response = app
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.body(axum::body::Body::from(r#"{"sku":"A"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CREATED);
assert_eq!(
response
.headers()
.get(super::IDEMPOTENCY_REPLAYED_HEADER_NAME)
.and_then(|value| value.to_str().ok()),
Some("true")
);
assert!(
response.headers().get(SET_COOKIE).is_none(),
"Set-Cookie must never be persisted into an idempotency replay"
);
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
assert_eq!(&body[..], b"created");
}
#[tokio::test]
async fn middleware_rejects_same_key_with_different_body() {
let runtime = IdempotencyRuntime::in_memory();
let app = Router::new()
.route("/orders", post(|| async { "created" }))
.layer(middleware::from_fn(idempotency_middleware))
.layer(axum::Extension(runtime));
app.clone()
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.body(axum::body::Body::from(r#"{"sku":"A"}"#))
.unwrap(),
)
.await
.unwrap();
let response = app
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.body(axum::body::Body::from(r#"{"sku":"B"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::CONFLICT);
}
#[tokio::test]
async fn middleware_rejects_unverified_authentication_context() {
let runtime = IdempotencyRuntime::in_memory();
let app = Router::new()
.route("/orders", post(|| async { "must not execute" }))
.layer(middleware::from_fn(idempotency_middleware))
.layer(axum::Extension(runtime));
let response = app
.clone()
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.header(AUTHORIZATION, "Bearer unverified")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
let problem: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(problem["code"], "idempotency_authentication_scope_required");
let response = app
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-2")
.header(COOKIE, "session=unverified")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
let problem: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(problem["code"], "idempotency_authentication_scope_required");
}
#[tokio::test]
async fn middleware_namespaces_records_by_verified_authentication_context() {
async fn install_verified_scope(mut request: Request<Body>, next: Next) -> Response {
let scope = IdempotencyScope::namespaced(
"test.authenticated-user",
request
.headers()
.get(AUTHORIZATION)
.expect("test authorization should be present")
.as_bytes(),
);
request.extensions_mut().insert(scope);
next.run(request).await
}
let runtime = IdempotencyRuntime::in_memory();
let app = Router::new()
.route(
"/orders",
post(|headers: HeaderMap| async move {
headers
.get(AUTHORIZATION)
.and_then(|value| value.to_str().ok())
.unwrap_or("anonymous")
.to_owned()
}),
)
.layer(middleware::from_fn(idempotency_middleware))
.layer(middleware::from_fn(install_verified_scope))
.layer(axum::Extension(runtime));
let response = app
.clone()
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "shared-client-key")
.header(AUTHORIZATION, "Bearer user-one")
.body(Body::from(r#"{"sku":"A"}"#))
.unwrap(),
)
.await
.unwrap();
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
assert_eq!(&body[..], b"Bearer user-one");
let response = app
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "shared-client-key")
.header(AUTHORIZATION, "Bearer user-two")
.body(Body::from(r#"{"sku":"A"}"#))
.unwrap(),
)
.await
.unwrap();
assert!(
response
.headers()
.get(super::IDEMPOTENCY_REPLAYED_HEADER_NAME)
.is_none()
);
let body = to_bytes(response.into_body(), usize::MAX).await.unwrap();
assert_eq!(&body[..], b"Bearer user-two");
}
#[tokio::test]
async fn oversized_response_is_terminalized_and_replayed_without_reexecution() {
let store = Arc::new(MemoryIdempotencyStore::new());
let runtime = IdempotencyRuntime::new(store.clone()).with_max_response_body_bytes(1);
let executions = Arc::new(AtomicUsize::new(0));
let executions_for_handler = executions.clone();
let app = Router::new()
.route(
"/orders",
post(move || {
let executions = executions_for_handler.clone();
async move {
executions.fetch_add(1, Ordering::SeqCst);
"too large"
}
}),
)
.layer(middleware::from_fn(idempotency_middleware))
.layer(axum::Extension(runtime));
let first = app
.clone()
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(first.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert!(
first
.headers()
.get(super::IDEMPOTENCY_REPLAYED_HEADER_NAME)
.is_none()
);
let first_body = to_bytes(first.into_body(), usize::MAX).await.unwrap();
let replay = app
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "request-1")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(replay.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert_eq!(
replay
.headers()
.get(super::IDEMPOTENCY_REPLAYED_HEADER_NAME)
.and_then(|value| value.to_str().ok()),
Some("true")
);
let replay_body = to_bytes(replay.into_body(), usize::MAX).await.unwrap();
assert_eq!(replay_body, first_body);
assert_eq!(executions.load(Ordering::SeqCst), 1);
assert_eq!(store.record_count().unwrap(), 1);
assert_eq!(store.reserved_response_bytes().unwrap(), 0);
let stored_size = {
let state = store.state.lock().unwrap();
let response = state
.records
.values()
.next()
.and_then(|record| record.response.as_ref())
.expect("terminal response should be stored");
super::stored_response_size(response)
};
assert_eq!(store.stored_response_bytes().unwrap(), stored_size);
}
#[tokio::test]
async fn failed_terminal_commit_keeps_the_processing_lease_and_never_aborts() {
let store = Arc::new(CommitFailingStore::default());
let runtime = IdempotencyRuntime::new(store.clone()).with_max_response_body_bytes(1);
let executions = Arc::new(AtomicUsize::new(0));
let executions_for_handler = executions.clone();
let app = Router::new()
.route(
"/orders",
post(move || {
let executions = executions_for_handler.clone();
async move {
executions.fetch_add(1, Ordering::SeqCst);
"too large"
}
}),
)
.layer(middleware::from_fn(idempotency_middleware))
.layer(axum::Extension(runtime));
let first = app
.clone()
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "commit-failure")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(first.status(), StatusCode::INTERNAL_SERVER_ERROR);
let retry = app
.oneshot(
Request::post("/orders")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "commit-failure")
.body(Body::empty())
.unwrap(),
)
.await
.unwrap();
assert_eq!(retry.status(), StatusCode::CONFLICT);
assert_eq!(executions.load(Ordering::SeqCst), 1);
assert_eq!(store.commit_attempts.load(Ordering::SeqCst), 1);
assert_eq!(store.abort_attempts.load(Ordering::SeqCst), 0);
assert_eq!(store.inner.record_count().unwrap(), 1);
assert_eq!(store.inner.stored_response_bytes().unwrap(), 0);
assert!(store.inner.reserved_response_bytes().unwrap() > 0);
}
#[tokio::test]
async fn layer_allows_requests_without_key_by_default() {
let app = Router::new()
.route("/orders", post(|| async { "created" }))
.layer(IdempotencyLayer::new(IdempotencyRuntime::in_memory()));
let response = app
.oneshot(
Request::post("/orders")
.body(axum::body::Body::from(r#"{"sku":"A"}"#))
.unwrap(),
)
.await
.unwrap();
assert_eq!(response.status(), StatusCode::OK);
}
}