use std::{
collections::HashSet,
error::Error as _,
fmt,
future::Future,
pin::Pin,
sync::Arc,
task::{Context, Poll},
time::Duration,
};
use axum::{
body::{Body, to_bytes},
extract::FromRequestParts,
http::{
HeaderMap, HeaderName, HeaderValue, Request, StatusCode,
header::{AUTHORIZATION, CONNECTION, CONTENT_TYPE, COOKIE},
request::Parts,
response,
},
middleware::Next,
response::{IntoResponse, Response},
};
use sha2::{Digest, Sha256};
use thiserror::Error;
use tower::{Layer, Service};
use crate::{
http::{ApiProblem, HttpRequestPolicy, ProblemCode, to_lower_hex},
observability::{MetricLabels, MetricRecorder, metric_labels, metric_names, metric_outcomes},
result::{ClassifyError, ErrorKind},
};
use super::{
memory::MemoryIdempotencyStore,
model::{
DEFAULT_MAX_IDEMPOTENCY_STORED_HEADER_BYTES, IDEMPOTENCY_KEY_HEADER_NAME,
IDEMPOTENCY_REPLAYED_HEADER_NAME, IDEMPOTENCY_TERMINAL_FAILURE_BODY,
IDEMPOTENCY_TERMINAL_FAILURE_MEDIA_TYPE, IdempotencyConfig, IdempotencyKey,
IdempotencyLease, IdempotencyRequestHash, IdempotencyRequestPolicy, IdempotencyReservation,
IdempotencyScope, IdempotencyStore, IdempotencyStoreError, StoredIdempotencyResponse,
hash_length_prefixed, is_storable_response_header, validate_idempotency_key,
},
};
#[derive(Clone)]
#[non_exhaustive]
pub struct IdempotencyRuntime {
store: Arc<dyn IdempotencyStore>,
config: IdempotencyConfig,
metrics: Option<Arc<dyn MetricRecorder>>,
}
impl fmt::Debug for IdempotencyRuntime {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("IdempotencyRuntime")
.field("config", &self.config)
.finish_non_exhaustive()
}
}
impl IdempotencyRuntime {
#[must_use]
pub fn new(store: Arc<dyn IdempotencyStore>, config: IdempotencyConfig) -> Self {
Self {
store,
config,
metrics: None,
}
}
pub fn in_memory(config: IdempotencyConfig) -> Result<Self, IdempotencyStoreError> {
let store =
MemoryIdempotencyStore::for_response_body_limit(config.max_response_body_bytes())?;
Ok(Self::new(Arc::new(store), config))
}
#[must_use]
pub fn with_metrics(mut self, metrics: Option<Arc<dyn MetricRecorder>>) -> Self {
self.metrics = metrics;
self
}
#[must_use]
pub fn store(&self) -> &Arc<dyn IdempotencyStore> {
&self.store
}
#[must_use]
pub const fn config(&self) -> IdempotencyConfig {
self.config
}
#[must_use]
pub fn metrics(&self) -> Option<&Arc<dyn MetricRecorder>> {
self.metrics.as_ref()
}
}
#[derive(Debug, Error)]
#[non_exhaustive]
pub enum IdempotencyRejection {
#[error("idempotency key is invalid")]
InvalidKey,
#[error("idempotency request is already processing")]
Processing,
#[error("idempotency key was reused with a different request")]
Conflict,
#[error("idempotency request body is too large")]
RequestBodyTooLarge,
#[error("idempotency request body could not be read")]
RequestBodyReadFailed,
#[error("idempotency requires a verified authentication scope")]
UnverifiedAuthenticationScope,
#[error("idempotency execution result is unavailable")]
ExecutionResultUnavailable,
#[error("idempotency store is unavailable")]
StoreUnavailable,
}
impl ClassifyError for IdempotencyRejection {
fn error_kind(&self) -> ErrorKind {
match self {
Self::InvalidKey | Self::RequestBodyTooLarge | Self::RequestBodyReadFailed => {
ErrorKind::Validation
}
Self::UnverifiedAuthenticationScope => ErrorKind::Authentication,
Self::Processing | Self::Conflict => ErrorKind::Conflict,
Self::ExecutionResultUnavailable => ErrorKind::Internal,
Self::StoreUnavailable => ErrorKind::Unavailable,
}
}
fn error_code(&self) -> crate::result::ErrorCode {
crate::result::ErrorCode::internal(match self {
Self::InvalidKey => "invalid_idempotency_key",
Self::Processing => "idempotency_request_processing",
Self::Conflict => "idempotency_key_conflict",
Self::RequestBodyTooLarge => "idempotency_request_body_too_large",
Self::RequestBodyReadFailed => "idempotency_request_body_read_failed",
Self::UnverifiedAuthenticationScope => "idempotency_authentication_scope_required",
Self::ExecutionResultUnavailable => "idempotency_execution_result_unavailable",
Self::StoreUnavailable => "idempotency_store_unavailable",
})
}
}
impl IntoResponse for IdempotencyRejection {
fn into_response(self) -> Response {
match self {
Self::InvalidKey => ApiProblem::bad_request(
ProblemCode::INVALID_IDEMPOTENCY_KEY,
"idempotency key is invalid",
)
.into_response(),
Self::Processing => ApiProblem::conflict(
ProblemCode::IDEMPOTENCY_REQUEST_PROCESSING,
"an idempotency request with this key is already processing",
)
.into_response(),
Self::Conflict => ApiProblem::conflict(
ProblemCode::IDEMPOTENCY_KEY_CONFLICT,
"idempotency key was reused with a different request",
)
.into_response(),
Self::RequestBodyTooLarge => ApiProblem::payload_too_large(
ProblemCode::IDEMPOTENCY_REQUEST_BODY_TOO_LARGE,
"idempotency request body is too large",
)
.into_response(),
Self::RequestBodyReadFailed => ApiProblem::bad_request(
ProblemCode::IDEMPOTENCY_REQUEST_BODY_READ_FAILED,
"request body could not be read",
)
.into_response(),
Self::UnverifiedAuthenticationScope => ApiProblem::unauthorized(
ProblemCode::IDEMPOTENCY_AUTHENTICATION_SCOPE_REQUIRED,
"authentication must be verified before applying idempotency",
)
.into_response(),
Self::ExecutionResultUnavailable => terminal_failure_response(),
Self::StoreUnavailable => ApiProblem::service_unavailable(
ProblemCode::IDEMPOTENCY_STORE_UNAVAILABLE,
"idempotency store is unavailable",
)
.into_response(),
}
}
}
pub async fn idempotency_middleware(request: Request<Body>, next: Next) -> Response {
if !request.headers().contains_key(IDEMPOTENCY_KEY_HEADER_NAME) {
return next.run(request).await;
}
let Some(runtime) = request.extensions().get::<IdempotencyRuntime>().cloned() else {
return next.run(request).await;
};
match run_idempotent_request(runtime, request, next).await {
Ok(response) => response,
Err(rejection) => rejection.into_response(),
}
}
#[derive(Clone)]
#[non_exhaustive]
pub struct IdempotencyLayer {
runtime: IdempotencyRuntime,
request_policy: HttpRequestPolicy,
}
impl IdempotencyLayer {
#[must_use]
pub fn new(runtime: IdempotencyRuntime) -> Self {
Self {
runtime,
request_policy: HttpRequestPolicy::default(),
}
}
#[must_use]
pub fn with_request_policy(mut self, request_policy: HttpRequestPolicy) -> Self {
self.request_policy = request_policy;
self
}
#[must_use]
pub const fn runtime(&self) -> &IdempotencyRuntime {
&self.runtime
}
#[must_use]
pub const fn request_policy(&self) -> &HttpRequestPolicy {
&self.request_policy
}
}
impl fmt::Debug for IdempotencyLayer {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("IdempotencyLayer")
.field("runtime", &self.runtime)
.finish()
}
}
impl<S> Layer<S> for IdempotencyLayer {
type Service = IdempotencyService<S>;
fn layer(&self, inner: S) -> Self::Service {
IdempotencyService {
inner,
runtime: self.runtime.clone(),
request_policy: self.request_policy.clone(),
}
}
}
#[derive(Clone)]
#[non_exhaustive]
pub struct IdempotencyService<S> {
inner: S,
runtime: IdempotencyRuntime,
request_policy: HttpRequestPolicy,
}
impl<S> fmt::Debug for IdempotencyService<S>
where
S: fmt::Debug,
{
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("IdempotencyService")
.field("inner", &self.inner)
.field("runtime", &self.runtime)
.finish()
}
}
impl<S> Service<Request<Body>> for IdempotencyService<S>
where
S: Service<Request<Body>, Response = Response> + Clone + Send + 'static,
S::Future: Send + 'static,
S::Error: Send + 'static,
{
type Response = Response;
type Error = S::Error;
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
fn poll_ready(&mut self, context: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
self.inner.poll_ready(context)
}
fn call(&mut self, mut request: Request<Body>) -> Self::Future {
if !should_run_idempotency(&self.request_policy, &mut request) {
return Box::pin(self.inner.call(request));
}
let runtime = self.runtime.clone();
let clone = self.inner.clone();
let mut inner = std::mem::replace(&mut self.inner, clone);
Box::pin(async move {
let response =
match run_idempotent_service(runtime, request, |request| inner.call(request)).await
{
Ok(response) => response?,
Err(rejection) => rejection.into_response(),
};
Ok(response)
})
}
}
fn should_run_idempotency<B>(request_policy: &HttpRequestPolicy, request: &mut Request<B>) -> bool {
match request_policy.resolve(request).idempotency() {
IdempotencyRequestPolicy::Disabled => false,
IdempotencyRequestPolicy::HeaderOptional => {
request.headers().contains_key(IDEMPOTENCY_KEY_HEADER_NAME)
}
IdempotencyRequestPolicy::Required => true,
}
}
pub async fn run_idempotent_request(
runtime: IdempotencyRuntime,
request: Request<Body>,
next: Next,
) -> Result<Response, IdempotencyRejection> {
run_idempotent_service(runtime, request, |request| async move {
Ok::<_, std::convert::Infallible>(next.run(request).await)
})
.await?
.map_err(|never| match never {})
}
struct IdempotencyReservationGuard {
store: Arc<dyn IdempotencyStore>,
lease: IdempotencyLease,
retention_ttl: Duration,
pending_key: Option<IdempotencyKey>,
}
impl IdempotencyReservationGuard {
fn new(
store: Arc<dyn IdempotencyStore>,
key: IdempotencyKey,
lease: IdempotencyLease,
retention_ttl: Duration,
) -> Self {
Self {
store,
lease,
retention_ttl,
pending_key: Some(key),
}
}
async fn commit(
&mut self,
response: StoredIdempotencyResponse,
) -> Result<(), IdempotencyStoreError> {
let key = self
.pending_key
.as_ref()
.ok_or(IdempotencyStoreError::InconsistentRecord)?;
let result = self
.store
.commit(key, self.lease, response, self.retention_ttl)
.await;
self.pending_key = None;
result
}
async fn commit_terminal_failure(&mut self) -> Result<(), IdempotencyStoreError> {
self.commit(terminal_failure_stored_response()).await
}
}
impl Drop for IdempotencyReservationGuard {
fn drop(&mut self) {
let Some(key) = self.pending_key.take() else {
return;
};
let Ok(runtime) = tokio::runtime::Handle::try_current() else {
return;
};
let store = self.store.clone();
let lease = self.lease;
let retention_ttl = self.retention_ttl;
drop(runtime.spawn(async move {
if let Err(error) = store
.commit(
&key,
lease,
terminal_failure_stored_response(),
retention_ttl,
)
.await
{
tracing::warn!(
error = %error,
"failed to terminalize a cancelled idempotent execution"
);
}
}));
}
}
pub async fn run_idempotent_service<F, Fut, E>(
runtime: IdempotencyRuntime,
request: Request<Body>,
call: F,
) -> Result<Result<Response, E>, IdempotencyRejection>
where
F: FnOnce(Request<Body>) -> Fut,
Fut: Future<Output = Result<Response, E>>,
{
let client_key = parse_required_idempotency_key(request.headers())?;
let (parts, body) = request.into_parts();
let store_key = scoped_store_key(&parts, &client_key)?;
let body = to_bytes(body, runtime.config.max_request_body_bytes)
.await
.map_err(|error| {
let mut source = error.source();
while let Some(cause) = source {
if cause.is::<http_body_util::LengthLimitError>() {
return IdempotencyRejection::RequestBodyTooLarge;
}
source = cause.source();
}
IdempotencyRejection::RequestBodyReadFailed
})?;
let request_hash = request_hash_from_parts(&parts, &body);
let request = Request::from_parts(parts, Body::from(body));
let response_capacity_bytes = runtime
.config
.max_response_body_bytes
.checked_add(DEFAULT_MAX_IDEMPOTENCY_STORED_HEADER_BYTES)
.ok_or(IdempotencyRejection::StoreUnavailable)?;
let reservation = runtime
.store
.reserve(
store_key.clone(),
request_hash,
runtime.config.processing_ttl,
response_capacity_bytes,
)
.await
.map_err(|_| IdempotencyRejection::StoreUnavailable)?;
record_idempotency_metric(&runtime, reservation_metric_outcome(&reservation));
match reservation {
IdempotencyReservation::Reserved(lease) => {
let mut reservation = IdempotencyReservationGuard::new(
runtime.store.clone(),
store_key,
lease,
runtime.config.ttl,
);
let response = match call(request).await {
Ok(response) => response,
Err(error) => {
if let Err(commit_error) = reservation.commit_terminal_failure().await {
tracing::warn!(
error = %commit_error,
"failed to terminalize an idempotent service error"
);
}
return Ok(Err(error));
}
};
store_and_return_response(&runtime, &mut reservation, response)
.await
.map(Ok)
}
IdempotencyReservation::Processing => Err(IdempotencyRejection::Processing),
IdempotencyReservation::Replay(stored) => response_from_stored(stored).map(Ok),
IdempotencyReservation::Conflict => Err(IdempotencyRejection::Conflict),
}
}
fn reservation_metric_outcome(reservation: &IdempotencyReservation) -> &'static str {
match reservation {
IdempotencyReservation::Reserved(_) => metric_outcomes::RESERVED,
IdempotencyReservation::Processing => metric_outcomes::PROCESSING,
IdempotencyReservation::Replay(_) => metric_outcomes::REPLAY,
IdempotencyReservation::Conflict => metric_outcomes::CONFLICT,
}
}
fn record_idempotency_metric(runtime: &IdempotencyRuntime, outcome: &'static str) {
if let Some(metrics) = runtime.metrics() {
let _ = metrics.increment_counter(
metric_names::IDEMPOTENCY_REQUESTS_TOTAL,
1,
MetricLabels::from_lossy([(metric_labels::OUTCOME, outcome)]),
);
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub struct RequiredIdempotencyKey(pub IdempotencyKey);
impl RequiredIdempotencyKey {
#[must_use]
pub fn into_inner(self) -> IdempotencyKey {
self.0
}
}
impl<S> FromRequestParts<S> for RequiredIdempotencyKey
where
S: Send + Sync,
{
type Rejection = ApiProblem;
async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result<Self, Self::Rejection> {
idempotency_key_from_headers(&parts.headers)
.map(Self)
.ok_or_else(|| {
ApiProblem::bad_request(
ProblemCode::IDEMPOTENCY_KEY_REQUIRED,
"a valid Idempotency-Key header is required",
)
})
}
}
#[must_use]
pub fn idempotency_key_from_headers(headers: &HeaderMap) -> Option<IdempotencyKey> {
headers
.get(IDEMPOTENCY_KEY_HEADER_NAME)
.and_then(|value| value.to_str().ok())
.and_then(IdempotencyKey::parse)
}
pub fn parse_required_idempotency_key(
headers: &HeaderMap,
) -> Result<IdempotencyKey, IdempotencyRejection> {
headers
.get(IDEMPOTENCY_KEY_HEADER_NAME)
.ok_or(IdempotencyRejection::InvalidKey)?
.to_str()
.ok()
.and_then(IdempotencyKey::parse)
.ok_or(IdempotencyRejection::InvalidKey)
}
#[must_use]
pub fn is_valid_idempotency_key(value: &str) -> bool {
validate_idempotency_key(value).is_ok()
}
fn request_hash_from_parts(parts: &Parts, body: &[u8]) -> IdempotencyRequestHash {
let mut hasher = Sha256::new();
hasher.update(parts.method.as_str().as_bytes());
hasher.update(b"\n");
hasher.update(parts.uri.to_string().as_bytes());
hasher.update(b"\n");
if let Some(content_type) = parts.headers.get(CONTENT_TYPE) {
hasher.update(content_type.as_bytes());
}
hasher.update(b"\n");
hasher.update(body);
IdempotencyRequestHash(to_lower_hex(&hasher.finalize()))
}
fn scoped_store_key(
parts: &Parts,
client_key: &IdempotencyKey,
) -> Result<IdempotencyKey, IdempotencyRejection> {
let mut hasher = Sha256::new();
hasher.update(b"footing-idempotency-store-key-v2\0");
if let Some(scope) = parts.extensions.get::<IdempotencyScope>() {
hasher.update(b"explicit\0");
hasher.update(scope.0);
} else if parts.headers.contains_key(AUTHORIZATION) || parts.headers.contains_key(COOKIE) {
return Err(IdempotencyRejection::UnverifiedAuthenticationScope);
} else {
hasher.update(b"anonymous\0");
}
hash_length_prefixed(&mut hasher, client_key.as_str().as_bytes());
Ok(IdempotencyKey(to_lower_hex(&hasher.finalize())))
}
async fn store_and_return_response(
runtime: &IdempotencyRuntime,
reservation: &mut IdempotencyReservationGuard,
response: Response,
) -> Result<Response, IdempotencyRejection> {
let (parts, body) = response.into_parts();
let body = match to_bytes(body, runtime.config.max_response_body_bytes).await {
Ok(body) => body,
Err(error) => {
tracing::warn!(
error = %error,
"failed to capture an idempotent response; committing a terminal failure"
);
if let Err(commit_error) = reservation.commit_terminal_failure().await {
tracing::warn!(
error = %commit_error,
"failed to persist an idempotent terminal failure"
);
}
return Ok(terminal_failure_response());
}
};
let stored = stored_response_from_parts(&parts, body.to_vec());
reservation
.commit(stored)
.await
.map_err(|_| IdempotencyRejection::StoreUnavailable)?;
Ok(Response::from_parts(parts, Body::from(body)))
}
fn terminal_failure_stored_response() -> StoredIdempotencyResponse {
StoredIdempotencyResponse {
status: StatusCode::INTERNAL_SERVER_ERROR,
headers: vec![(
CONTENT_TYPE.as_str().to_owned(),
IDEMPOTENCY_TERMINAL_FAILURE_MEDIA_TYPE.to_owned(),
)],
body: IDEMPOTENCY_TERMINAL_FAILURE_BODY.to_vec(),
}
}
fn terminal_failure_response() -> Response {
let mut response = Response::new(Body::from(IDEMPOTENCY_TERMINAL_FAILURE_BODY));
*response.status_mut() = StatusCode::INTERNAL_SERVER_ERROR;
response.headers_mut().insert(
CONTENT_TYPE,
HeaderValue::from_static(IDEMPOTENCY_TERMINAL_FAILURE_MEDIA_TYPE),
);
response
}
pub(super) fn response_from_stored(
stored: StoredIdempotencyResponse,
) -> Result<Response, IdempotencyRejection> {
let mut builder = response::Builder::new().status(stored.status);
for (name, value) in &stored.headers {
let Ok(name) = HeaderName::try_from(name.as_str()) else {
continue;
};
let Ok(value) = HeaderValue::try_from(value.as_str()) else {
continue;
};
builder = builder.header(name, value);
}
builder = builder.header(IDEMPOTENCY_REPLAYED_HEADER_NAME, "true");
builder
.body(Body::from(stored.body))
.map_err(|_| IdempotencyRejection::StoreUnavailable)
}
pub(super) fn stored_response_from_parts(
parts: &response::Parts,
body: Vec<u8>,
) -> StoredIdempotencyResponse {
let mut stored = StoredIdempotencyResponse::new(parts.status, body);
let mut stored_header_bytes = 0usize;
let connection_headers = connection_nominated_headers(&parts.headers);
if let Some(value) = parts
.headers
.get(CONTENT_TYPE)
.and_then(|value| value.to_str().ok())
.filter(|_| !connection_headers.contains(&CONTENT_TYPE))
{
let header_bytes = CONTENT_TYPE.as_str().len().saturating_add(value.len());
if header_bytes <= DEFAULT_MAX_IDEMPOTENCY_STORED_HEADER_BYTES {
stored_header_bytes = header_bytes;
stored
.headers
.push((CONTENT_TYPE.as_str().to_owned(), value.to_owned()));
}
}
for (name, value) in &parts.headers {
if name != CONTENT_TYPE
&& is_storable_response_header(name)
&& !connection_headers.contains(name)
&& let Ok(value) = value.to_str()
{
let header_bytes = name.as_str().len().saturating_add(value.len());
let Some(next_header_bytes) = stored_header_bytes.checked_add(header_bytes) else {
continue;
};
if next_header_bytes > DEFAULT_MAX_IDEMPOTENCY_STORED_HEADER_BYTES {
continue;
}
stored
.headers
.push((name.as_str().to_owned(), value.to_owned()));
stored_header_bytes = next_header_bytes;
}
}
stored
}
fn connection_nominated_headers(headers: &HeaderMap) -> HashSet<HeaderName> {
headers
.get_all(CONNECTION)
.iter()
.filter_map(|value| value.to_str().ok())
.flat_map(|value| value.split(','))
.filter_map(|name| HeaderName::try_from(name.trim()).ok())
.collect()
}