use std::{
collections::HashSet,
error::Error as _,
fmt,
future::Future,
pin::Pin,
sync::Arc,
task::{Context, Poll},
time::Duration,
};
use axum::{
body::{Body, Bytes},
extract::{FromRequestParts, OriginalUri},
http::{
HeaderMap, HeaderName, HeaderValue, Request, StatusCode,
header::{AUTHORIZATION, CONNECTION, CONTENT_TYPE, COOKIE},
request::Parts,
response,
},
response::{IntoResponse, Response},
};
use http_body_util::{BodyExt as _, Limited};
use sha2::{Digest, Sha256};
use thiserror::Error;
use tower::{Layer, Service};
use crate::{
http::{ApiProblem, HttpRequestPolicy, ProblemCode, RequestContext, to_lower_hex},
observability::{
MetricLabels, MetricRecorder, RecordingFailures, metric_labels, metric_names,
metric_outcomes,
},
result::{ClassifyError, ErrorKind},
};
use super::{
memory::MemoryIdempotencyStore,
model::{
DEFAULT_MAX_IDEMPOTENCY_STORED_HEADER_BYTES, IDEMPOTENCY_KEY_HEADER_NAME,
IDEMPOTENCY_REPLAYED_HEADER_NAME, IDEMPOTENCY_TERMINAL_FAILURE_BODY,
IDEMPOTENCY_TERMINAL_FAILURE_MEDIA_TYPE, IdempotencyConfig, IdempotencyKey,
IdempotencyLease, IdempotencyRequestHash, IdempotencyRequestPolicy, IdempotencyReservation,
IdempotencyScope, IdempotencyStore, IdempotencyStoreError, StoredIdempotencyResponse,
hash_length_prefixed, is_storable_response_header, validate_idempotency_key,
},
};
const CANCELLED_EXECUTION_COMMIT_TIMEOUT: Duration = Duration::from_secs(5);
#[derive(Clone)]
#[non_exhaustive]
pub struct IdempotencyRuntime {
store: Arc<dyn IdempotencyStore>,
config: IdempotencyConfig,
metrics: Option<Arc<dyn MetricRecorder>>,
recording_failures: RecordingFailures,
}
impl fmt::Debug for IdempotencyRuntime {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("IdempotencyRuntime")
.field("config", &self.config)
.finish_non_exhaustive()
}
}
impl IdempotencyRuntime {
#[must_use]
pub fn new(store: Arc<dyn IdempotencyStore>, config: IdempotencyConfig) -> Self {
Self {
store,
config,
metrics: None,
recording_failures: RecordingFailures::default(),
}
}
pub fn in_memory(config: IdempotencyConfig) -> Result<Self, IdempotencyStoreError> {
let store =
MemoryIdempotencyStore::for_response_body_limit(config.max_response_body_bytes())?;
Ok(Self::new(Arc::new(store), config))
}
#[must_use]
pub fn with_metrics(mut self, metrics: Option<Arc<dyn MetricRecorder>>) -> Self {
self.metrics = metrics;
self
}
#[must_use]
pub fn store(&self) -> &Arc<dyn IdempotencyStore> {
&self.store
}
#[must_use]
pub const fn config(&self) -> &IdempotencyConfig {
&self.config
}
#[must_use]
pub fn metrics(&self) -> Option<&Arc<dyn MetricRecorder>> {
self.metrics.as_ref()
}
#[must_use]
pub fn failed_recordings(&self) -> u64 {
self.recording_failures.count()
}
fn record_store_error(&self, stage: &'static str, error_code: &str) {
if let Some(metrics) = self.metrics() {
self.recording_failures.record(
MetricLabels::new([
(metric_labels::STAGE, stage),
(metric_labels::ERROR_CODE, error_code),
])
.and_then(|labels| {
metrics.increment_counter(
metric_names::IDEMPOTENCY_STORE_ERRORS_TOTAL,
1,
labels,
)
}),
"idempotency",
);
}
}
}
#[derive(Debug, Error)]
#[non_exhaustive]
pub enum IdempotencyRejection {
#[error("idempotency key is invalid")]
InvalidKey,
#[error("idempotency request is already processing")]
Processing,
#[error("idempotency key was reused with a different request")]
Conflict,
#[error("idempotency request body is too large")]
RequestBodyTooLarge,
#[error("idempotency request body could not be read")]
RequestBodyReadFailed,
#[error("idempotency requires a verified authentication scope")]
UnverifiedAuthenticationScope,
#[error("idempotency execution result is unavailable")]
ExecutionResultUnavailable,
#[error("idempotency store is unavailable")]
StoreUnavailable,
#[error("idempotency does not support request trailers")]
RequestTrailersUnsupported,
}
impl ClassifyError for IdempotencyRejection {
fn error_kind(&self) -> ErrorKind {
match self {
Self::InvalidKey
| Self::RequestBodyTooLarge
| Self::RequestBodyReadFailed
| Self::RequestTrailersUnsupported => ErrorKind::Validation,
Self::UnverifiedAuthenticationScope => ErrorKind::Authentication,
Self::Processing | Self::Conflict => ErrorKind::Conflict,
Self::ExecutionResultUnavailable => ErrorKind::Internal,
Self::StoreUnavailable => ErrorKind::Unavailable,
}
}
fn error_code(&self) -> crate::result::ErrorCode {
crate::result::ErrorCode::internal(match self {
Self::InvalidKey => "invalid_idempotency_key",
Self::Processing => "idempotency_request_processing",
Self::Conflict => "idempotency_key_conflict",
Self::RequestBodyTooLarge => "idempotency_request_body_too_large",
Self::RequestBodyReadFailed => "idempotency_request_body_read_failed",
Self::RequestTrailersUnsupported => "idempotency_request_trailers_unsupported",
Self::UnverifiedAuthenticationScope => "idempotency_authentication_scope_required",
Self::ExecutionResultUnavailable => "idempotency_execution_result_unavailable",
Self::StoreUnavailable => "idempotency_store_unavailable",
})
}
}
impl IntoResponse for IdempotencyRejection {
fn into_response(self) -> Response {
match self {
Self::InvalidKey => ApiProblem::bad_request(
ProblemCode::INVALID_IDEMPOTENCY_KEY,
"idempotency key is invalid",
)
.into_response(),
Self::Processing => ApiProblem::conflict(
ProblemCode::IDEMPOTENCY_REQUEST_PROCESSING,
"an idempotency request with this key is already processing",
)
.into_response(),
Self::Conflict => ApiProblem::conflict(
ProblemCode::IDEMPOTENCY_KEY_CONFLICT,
"idempotency key was reused with a different request",
)
.into_response(),
Self::RequestBodyTooLarge => ApiProblem::payload_too_large(
ProblemCode::IDEMPOTENCY_REQUEST_BODY_TOO_LARGE,
"idempotency request body is too large",
)
.into_response(),
Self::RequestBodyReadFailed => ApiProblem::bad_request(
ProblemCode::IDEMPOTENCY_REQUEST_BODY_READ_FAILED,
"request body could not be read",
)
.into_response(),
Self::RequestTrailersUnsupported => ApiProblem::bad_request(
ProblemCode::from_error_code(Self::RequestTrailersUnsupported.error_code()),
"idempotency does not support request trailers",
)
.into_response(),
Self::UnverifiedAuthenticationScope => ApiProblem::unauthorized(
ProblemCode::IDEMPOTENCY_AUTHENTICATION_SCOPE_REQUIRED,
"authentication must be verified before applying idempotency",
)
.into_response(),
Self::ExecutionResultUnavailable => terminal_failure_response(),
Self::StoreUnavailable => ApiProblem::service_unavailable(
ProblemCode::IDEMPOTENCY_STORE_UNAVAILABLE,
"idempotency store is unavailable",
)
.into_response(),
}
}
}
#[derive(Clone)]
#[non_exhaustive]
pub struct IdempotencyLayer {
runtime: IdempotencyRuntime,
request_policy: HttpRequestPolicy,
}
impl IdempotencyLayer {
#[must_use]
pub fn new(runtime: IdempotencyRuntime) -> Self {
Self {
runtime,
request_policy: HttpRequestPolicy::default(),
}
}
#[must_use]
pub fn with_request_policy(mut self, request_policy: HttpRequestPolicy) -> Self {
self.request_policy = request_policy;
self
}
#[must_use]
pub const fn runtime(&self) -> &IdempotencyRuntime {
&self.runtime
}
#[must_use]
pub const fn request_policy(&self) -> &HttpRequestPolicy {
&self.request_policy
}
}
impl fmt::Debug for IdempotencyLayer {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("IdempotencyLayer")
.field("runtime", &self.runtime)
.finish()
}
}
impl<S> Layer<S> for IdempotencyLayer {
type Service = IdempotencyService<S>;
fn layer(&self, inner: S) -> Self::Service {
IdempotencyService {
inner,
runtime: self.runtime.clone(),
request_policy: self.request_policy.clone(),
}
}
}
#[derive(Clone)]
#[non_exhaustive]
pub struct IdempotencyService<S> {
inner: S,
runtime: IdempotencyRuntime,
request_policy: HttpRequestPolicy,
}
impl<S> fmt::Debug for IdempotencyService<S>
where
S: fmt::Debug,
{
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("IdempotencyService")
.field("inner", &self.inner)
.field("runtime", &self.runtime)
.finish()
}
}
impl<S> Service<Request<Body>> for IdempotencyService<S>
where
S: Service<Request<Body>, Response = Response> + Clone + Send + 'static,
S::Future: Send + 'static,
S::Error: Send + 'static,
{
type Response = Response;
type Error = S::Error;
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
fn poll_ready(&mut self, context: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
self.inner.poll_ready(context)
}
fn call(&mut self, mut request: Request<Body>) -> Self::Future {
if !should_run_idempotency(&self.request_policy, &mut request) {
return Box::pin(self.inner.call(request));
}
let runtime = self.runtime.clone();
let clone = self.inner.clone();
let mut inner = std::mem::replace(&mut self.inner, clone);
Box::pin(async move {
let response =
match run_idempotent_service(runtime, request, |request| inner.call(request)).await
{
Ok(response) => response?,
Err(rejection) => rejection.into_response(),
};
Ok(response)
})
}
}
fn should_run_idempotency<B>(request_policy: &HttpRequestPolicy, request: &mut Request<B>) -> bool {
match request_policy.resolve(request).idempotency() {
IdempotencyRequestPolicy::Disabled => false,
IdempotencyRequestPolicy::HeaderOptional => {
request.headers().contains_key(IDEMPOTENCY_KEY_HEADER_NAME)
}
IdempotencyRequestPolicy::Required => true,
}
}
struct IdempotencyReservationGuard {
runtime: IdempotencyRuntime,
lease: IdempotencyLease,
pending_key: Option<IdempotencyKey>,
}
impl IdempotencyReservationGuard {
fn new(runtime: IdempotencyRuntime, key: IdempotencyKey, lease: IdempotencyLease) -> Self {
Self {
runtime,
lease,
pending_key: Some(key),
}
}
async fn commit(
&mut self,
response: StoredIdempotencyResponse,
) -> Result<(), IdempotencyStoreError> {
let key = self
.pending_key
.as_ref()
.ok_or(IdempotencyStoreError::InconsistentRecord)?;
let result = self
.runtime
.store
.commit(key, self.lease, response, self.runtime.config.ttl)
.await;
self.pending_key = None;
if let Err(error) = &result {
self.runtime
.record_store_error("commit", error.error_code().as_str());
}
result
}
async fn commit_terminal_failure(&mut self) -> Result<(), IdempotencyStoreError> {
self.commit(terminal_failure_stored_response()).await
}
}
impl Drop for IdempotencyReservationGuard {
fn drop(&mut self) {
let Some(key) = self.pending_key.take() else {
return;
};
let Ok(handle) = tokio::runtime::Handle::try_current() else {
return;
};
let runtime = self.runtime.clone();
let lease = self.lease;
drop(handle.spawn(async move {
match tokio::time::timeout(
CANCELLED_EXECUTION_COMMIT_TIMEOUT,
runtime.store.commit(
&key,
lease,
terminal_failure_stored_response(),
runtime.config.ttl,
),
)
.await
{
Ok(Ok(())) => {}
Ok(Err(error)) => {
runtime.record_store_error("cancel_commit", error.error_code().as_str());
tracing::warn!(
error_code = %error.error_code(),
"failed to terminalize a cancelled idempotent execution"
);
}
Err(_) => {
runtime.record_store_error("cancel_commit", "idempotency_store_timeout");
tracing::warn!("timed out terminalizing a cancelled idempotent execution")
}
}
}));
}
}
pub async fn run_idempotent_service<F, Fut, E>(
runtime: IdempotencyRuntime,
request: Request<Body>,
call: F,
) -> Result<Result<Response, E>, IdempotencyRejection>
where
F: FnOnce(Request<Body>) -> Fut,
Fut: Future<Output = Result<Response, E>>,
{
let client_key = parse_required_idempotency_key(request.headers())?;
let (parts, body) = request.into_parts();
let store_key = scoped_store_key(&parts, &client_key)?;
let collected = Limited::new(body, runtime.config.max_request_body_bytes)
.collect()
.await
.map_err(axum::Error::new)
.map_err(|error| {
let mut source = error.source();
while let Some(cause) = source {
if cause.is::<http_body_util::LengthLimitError>() {
return IdempotencyRejection::RequestBodyTooLarge;
}
source = cause.source();
}
IdempotencyRejection::RequestBodyReadFailed
})?;
if collected
.trailers()
.is_some_and(|trailers| !trailers.is_empty())
{
return Err(IdempotencyRejection::RequestTrailersUnsupported);
}
let body = collected.to_bytes();
let request_hash = request_hash_from_parts(&parts, &body, runtime.config.fingerprint_headers());
let request = Request::from_parts(parts, Body::from(body));
let response_capacity_bytes =
runtime.config.max_response_body_bytes + DEFAULT_MAX_IDEMPOTENCY_STORED_HEADER_BYTES;
let reservation = runtime
.store
.reserve(
store_key.clone(),
request_hash,
runtime.config.processing_ttl,
response_capacity_bytes,
)
.await
.map_err(|error| {
runtime.record_store_error("reserve", error.error_code().as_str());
IdempotencyRejection::StoreUnavailable
})?;
record_idempotency_metric(&runtime, reservation_metric_outcome(&reservation));
match reservation {
IdempotencyReservation::Reserved(lease) => {
let mut reservation =
IdempotencyReservationGuard::new(runtime.clone(), store_key, lease);
let response = match call(request).await {
Ok(response) => response,
Err(error) => {
if let Err(commit_error) = reservation.commit_terminal_failure().await {
tracing::warn!(
error = %commit_error,
"failed to terminalize an idempotent service error"
);
}
return Ok(Err(error));
}
};
store_and_return_response(&runtime, &mut reservation, response)
.await
.map(Ok)
}
IdempotencyReservation::Processing => Err(IdempotencyRejection::Processing),
IdempotencyReservation::Replay(stored) => response_from_stored(stored).map(Ok),
IdempotencyReservation::Conflict => Err(IdempotencyRejection::Conflict),
}
}
fn reservation_metric_outcome(reservation: &IdempotencyReservation) -> &'static str {
match reservation {
IdempotencyReservation::Reserved(_) => metric_outcomes::RESERVED,
IdempotencyReservation::Processing => metric_outcomes::PROCESSING,
IdempotencyReservation::Replay(_) => metric_outcomes::REPLAY,
IdempotencyReservation::Conflict => metric_outcomes::CONFLICT,
}
}
fn record_idempotency_metric(runtime: &IdempotencyRuntime, outcome: &'static str) {
if let Some(metrics) = runtime.metrics() {
runtime.recording_failures.record(
MetricLabels::new([(metric_labels::OUTCOME, outcome)]).and_then(|labels| {
metrics.increment_counter(metric_names::IDEMPOTENCY_REQUESTS_TOTAL, 1, labels)
}),
"idempotency",
);
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct RequiredIdempotencyKey(pub IdempotencyKey);
impl RequiredIdempotencyKey {
#[must_use]
pub fn into_inner(self) -> IdempotencyKey {
self.0
}
}
impl<S> FromRequestParts<S> for RequiredIdempotencyKey
where
S: Send + Sync,
{
type Rejection = ApiProblem;
async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result<Self, Self::Rejection> {
idempotency_key_from_headers(&parts.headers)
.map(Self)
.ok_or_else(|| {
ApiProblem::bad_request(
ProblemCode::IDEMPOTENCY_KEY_REQUIRED,
"a valid Idempotency-Key header is required",
)
})
}
}
#[must_use]
pub fn idempotency_key_from_headers(headers: &HeaderMap) -> Option<IdempotencyKey> {
let mut values = headers.get_all(IDEMPOTENCY_KEY_HEADER_NAME).iter();
let value = values.next()?;
if values.next().is_some() {
return None;
}
IdempotencyKey::parse(value.to_str().ok()?)
}
pub fn parse_required_idempotency_key(
headers: &HeaderMap,
) -> Result<IdempotencyKey, IdempotencyRejection> {
idempotency_key_from_headers(headers).ok_or(IdempotencyRejection::InvalidKey)
}
#[must_use]
pub fn is_valid_idempotency_key(value: &str) -> bool {
validate_idempotency_key(value).is_ok()
}
fn request_hash_from_parts(
parts: &Parts,
body: &[u8],
fingerprint_headers: &[HeaderName],
) -> IdempotencyRequestHash {
let mut hasher = Sha256::new();
hasher.update(b"footing-idempotency-request-v2\0");
hash_length_prefixed(&mut hasher, parts.method.as_str().as_bytes());
let uri = parts
.extensions
.get::<OriginalUri>()
.map_or(&parts.uri, |original| &original.0);
hash_length_prefixed(&mut hasher, uri.to_string().as_bytes());
hash_length_prefixed(
&mut hasher,
parts
.headers
.get(CONTENT_TYPE)
.map_or(&[], HeaderValue::as_bytes),
);
hash_length_prefixed(&mut hasher, body);
for name in fingerprint_headers {
hash_length_prefixed(&mut hasher, name.as_str().as_bytes());
let values = parts.headers.get_all(name);
hasher.update(
u64::try_from(values.iter().count())
.unwrap_or(u64::MAX)
.to_be_bytes(),
);
for value in values.iter() {
hash_length_prefixed(&mut hasher, value.as_bytes());
}
}
IdempotencyRequestHash(to_lower_hex(&hasher.finalize()))
}
fn scoped_store_key(
parts: &Parts,
client_key: &IdempotencyKey,
) -> Result<IdempotencyKey, IdempotencyRejection> {
let mut hasher = Sha256::new();
hasher.update(b"footing-idempotency-store-key-v2\0");
if let Some(scope) = parts.extensions.get::<IdempotencyScope>() {
hasher.update(b"explicit\0");
hasher.update(scope.0);
} else if parts.headers.contains_key(AUTHORIZATION)
|| parts.headers.contains_key(COOKIE)
|| parts
.extensions
.get::<RequestContext>()
.and_then(RequestContext::principal)
.is_some()
{
return Err(IdempotencyRejection::UnverifiedAuthenticationScope);
} else {
hasher.update(b"anonymous\0");
}
hash_length_prefixed(&mut hasher, client_key.as_str().as_bytes());
Ok(IdempotencyKey(to_lower_hex(&hasher.finalize())))
}
async fn store_and_return_response(
runtime: &IdempotencyRuntime,
reservation: &mut IdempotencyReservationGuard,
response: Response,
) -> Result<Response, IdempotencyRejection> {
let capture = async {
let (parts, body) = response.into_parts();
let collected = Limited::new(body, runtime.config.max_response_body_bytes)
.collect()
.await
.map_err(|_| "response body exceeds the limit or could not be read")?;
if collected
.trailers()
.is_some_and(|trailers| !trailers.is_empty())
{
return Err("response trailers cannot be replayed");
}
let body = collected.to_bytes();
let stored = stored_response_from_parts(&parts, body.to_vec())?;
Ok::<_, &'static str>((Response::from_parts(parts, Body::from(body)), stored))
}
.await;
let (response, stored) = match capture {
Ok(captured) => captured,
Err(error) => {
tracing::warn!(
error = %error,
"failed to capture an idempotent response; committing a terminal failure"
);
if let Err(commit_error) = reservation.commit_terminal_failure().await {
tracing::warn!(
error = %commit_error,
"failed to persist an idempotent terminal failure"
);
}
return Ok(terminal_failure_response());
}
};
reservation
.commit(stored)
.await
.map_err(|_| IdempotencyRejection::StoreUnavailable)?;
Ok(response)
}
fn terminal_failure_stored_response() -> StoredIdempotencyResponse {
StoredIdempotencyResponse {
status: StatusCode::INTERNAL_SERVER_ERROR,
headers: vec![(
CONTENT_TYPE.as_str().to_owned(),
IDEMPOTENCY_TERMINAL_FAILURE_MEDIA_TYPE.to_owned(),
)],
body: Bytes::from_static(IDEMPOTENCY_TERMINAL_FAILURE_BODY),
}
}
fn terminal_failure_response() -> Response {
let mut response = Response::new(Body::from(IDEMPOTENCY_TERMINAL_FAILURE_BODY));
*response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR;
response.headers_mut().insert(
CONTENT_TYPE,
HeaderValue::from_static(IDEMPOTENCY_TERMINAL_FAILURE_MEDIA_TYPE),
);
response
}
pub(super) fn response_from_stored(
stored: StoredIdempotencyResponse,
) -> Result<Response, IdempotencyRejection> {
let mut builder = response::Builder::new().status(stored.status);
for (name, value) in &stored.headers {
builder = builder.header(name.as_str(), value.as_str());
}
builder = builder.header(IDEMPOTENCY_REPLAYED_HEADER_NAME, "true");
builder
.body(Body::from(stored.body))
.map_err(|_| IdempotencyRejection::StoreUnavailable)
}
pub(super) fn stored_response_from_parts(
parts: &response::Parts,
body: Vec<u8>,
) -> Result<StoredIdempotencyResponse, &'static str> {
let mut stored = StoredIdempotencyResponse::new(parts.status, body);
let mut stored_header_bytes = 0usize;
let connection_headers = connection_nominated_headers(&parts.headers);
for (name, value) in &parts.headers {
if is_storable_response_header(name) && !connection_headers.contains(name) {
let value = value.to_str().map_err(|_| "response header is not text")?;
stored_header_bytes = stored_header_bytes
.checked_add(name.as_str().len())
.and_then(|bytes| bytes.checked_add(value.len()))
.filter(|bytes| *bytes <= DEFAULT_MAX_IDEMPOTENCY_STORED_HEADER_BYTES)
.ok_or("response headers exceed the storage limit")?;
stored
.headers
.push((name.as_str().to_owned(), value.to_owned()));
}
}
Ok(stored)
}
fn connection_nominated_headers(headers: &HeaderMap) -> HashSet<HeaderName> {
headers
.get_all(CONNECTION)
.iter()
.filter_map(|value| value.to_str().ok())
.flat_map(|value| value.split(','))
.filter_map(|name| HeaderName::try_from(name.trim()).ok())
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn old_fingerprints_conflict_at_the_same_store_key_until_expiry() {
use std::{
convert::Infallible,
sync::atomic::{AtomicUsize, Ordering},
};
for completed in [false, true] {
let clock = crate::clock::ManualClock::new();
let store = Arc::new(MemoryIdempotencyStore::with_clock(clock.shared()));
let key = IdempotencyKey::new(
"36ee884b2934e32826808ba2112233373faafb3a3b4fcc277d13192bc81ed988",
)
.unwrap();
let old_hash = IdempotencyRequestHash::new(
"109cc3e88b794f9f75bf4983f2f19cbe3ed895f1c694133c410b3b8ddc3f2980",
)
.unwrap();
let IdempotencyReservation::Reserved(lease) = store
.reserve(key.clone(), old_hash, Duration::from_secs(60), 1024)
.await
.unwrap()
else {
panic!("fixture should acquire a reservation");
};
if completed {
store
.commit(
&key,
lease,
StoredIdempotencyResponse::new(StatusCode::CREATED, "old"),
Duration::from_secs(60),
)
.await
.unwrap();
}
let runtime = IdempotencyRuntime::new(store.clone(), IdempotencyConfig::default());
let request = || {
Request::post("/jobs?queue=1")
.header(CONTENT_TYPE, "application/json")
.header(IDEMPOTENCY_KEY_HEADER_NAME, "retry")
.body(Body::from(r#"{"value":1}"#))
.unwrap()
};
let executions = AtomicUsize::new(0);
for expired in [false, true] {
if expired {
clock.advance(Duration::from_secs(61)).unwrap();
}
let result = run_idempotent_service(runtime.clone(), request(), |_| async {
executions.fetch_add(1, Ordering::SeqCst);
Ok::<_, Infallible>(StatusCode::CREATED.into_response())
})
.await;
if expired {
assert_eq!(result.unwrap().unwrap().status(), StatusCode::CREATED);
} else {
assert!(matches!(result, Err(IdempotencyRejection::Conflict)));
assert_eq!(executions.load(Ordering::SeqCst), 0);
assert_eq!(store.record_count().unwrap(), 1);
}
}
assert_eq!(executions.load(Ordering::SeqCst), 1);
}
}
}