use axum::http::StatusCode;
use axum::{
body::Body,
extract::Request,
response::{IntoResponse, Response},
};
use std::future::Future;
use std::time::Duration;
use wordval_cache::Cache;
use wordval_core::api::ApiError;
use crate::middleware::error_response;
use crate::middleware::CurrentUser;
use crate::resources::try_cache;
use crate::tenant::TenantContext;
const CLAIM_TTL_SECS: u64 = 30;
const POLL_RETRIES: u32 = 30;
const POLL_INTERVAL: Duration = Duration::from_millis(100);
const MAX_CACHE_BODY: usize = 1024 * 1024;
const MAX_BUFFER_BODY: usize = 16 * 1024 * 1024;
const MAX_KEY_LEN: usize = 128;
const MAX_FINGERPRINT_BODY: usize = 1024 * 1024;
const FORBIDDEN_REPLAY_HEADERS: &[&str] = &[
"set-cookie",
"set-cookie2",
"authorization",
"proxy-authorization",
"x-request-id",
"content-length",
"transfer-encoding",
"connection",
"keep-alive",
];
#[derive(serde::Serialize, serde::Deserialize)]
struct CachedResponse {
status: u16,
content_type: String,
#[serde(default)]
fingerprint: String,
#[serde(default)]
headers: Vec<(String, String)>,
body: String,
}
impl CachedResponse {
fn encode_body(bytes: &[u8]) -> String {
wordval_utils::Crypto::encode_base64(bytes)
}
fn decode_body(&self) -> Vec<u8> {
wordval_utils::Crypto::decode_base64(&self.body).unwrap_or_default()
}
}
fn build_response(cached: CachedResponse) -> Response<Body> {
let body = cached.decode_body();
let mut builder = Response::builder()
.status(StatusCode::from_u16(cached.status).unwrap_or(StatusCode::OK))
.header("Content-Type", cached.content_type.as_str());
for (name, value) in &cached.headers {
if let (Ok(n), Ok(v)) = (
axum::http::HeaderName::from_bytes(name.as_bytes()),
axum::http::HeaderValue::from_str(value),
) {
builder = builder.header(n, v);
}
}
builder
.body(Body::from(body.clone()))
.unwrap_or_else(|_| Response::new(Body::from(body)))
}
fn extract_idem_key(req: &Request) -> Option<String> {
req.headers()
.get("x-idempotency-key")
.and_then(|v| v.to_str().ok())
.map(|s| s.to_string())
.filter(|s| !s.is_empty() && s.len() <= MAX_KEY_LEN)
}
fn extract_scope(req: &Request) -> Option<String> {
let user = req.extensions().get::<CurrentUser>()?;
let tenant = req
.extensions()
.get::<TenantContext>()
.and_then(|t| t.tenant_id)
.map(|t| format!("t{}", t))
.unwrap_or_else(|| "t-".to_string());
Some(format!(
"u{}:{}:{}:{}",
user.id,
tenant,
req.method().as_str(),
req.uri().path()
))
}
async fn compute_fingerprint(req: Request) -> (Option<String>, Request) {
let (parts, body) = req.into_parts();
match axum::body::to_bytes(body, MAX_BUFFER_BODY).await {
Ok(bytes) => {
let fingerprint = if bytes.len() <= MAX_FINGERPRINT_BODY {
let header = format!(
"{}:{}:{}:",
parts.method.as_str(),
parts.uri.path(),
parts.uri.query().unwrap_or("")
);
Some(wordval_utils::Crypto::sha256_parts(&[
header.as_bytes(),
&bytes,
]))
} else {
None
};
(fingerprint, Request::from_parts(parts, Body::from(bytes)))
}
Err(_) => (None, Request::from_parts(parts, Body::empty())),
}
}
async fn try_fast_path(
cache: &wordval_cache::SharedCache,
resp_key: &str,
fingerprint: &str,
) -> Option<Response> {
match cache.get::<CachedResponse>(resp_key).await {
Ok(Some(cached)) if cached.fingerprint == fingerprint => Some(build_response(cached)),
_ => None,
}
}
async fn fingerprint_conflict(
cache: &wordval_cache::SharedCache,
resp_key: &str,
fingerprint: &str,
) -> bool {
match cache.get::<CachedResponse>(resp_key).await {
Ok(Some(cached)) => cached.fingerprint != fingerprint,
_ => false,
}
}
async fn try_claim_lock(
cache: &wordval_cache::SharedCache,
lock_key: &str,
fingerprint: &str,
) -> Result<bool, String> {
cache
.set_nx(lock_key, &fingerprint.to_string(), CLAIM_TTL_SECS)
.await
.map_err(|e| e.to_string())
}
async fn poll_for_response(cache: &wordval_cache::SharedCache, resp_key: &str) -> Option<Response> {
for _ in 0..POLL_RETRIES {
tokio::time::sleep(POLL_INTERVAL).await;
if let Ok(Some(cached)) = cache.get::<CachedResponse>(resp_key).await {
return Some(build_response(cached));
}
}
None
}
fn conflict_response() -> Response {
error_response(ApiError::conflict("请求正在处理中,请稍后重试"))
}
fn fingerprint_conflict_response() -> Response {
error_response(ApiError::conflict(
"幂等键已被其他请求内容使用,请更换 x-idempotency-key",
))
}
fn anonymous_rejected_response() -> Response {
error_response(ApiError::unauthorized("幂等操作需要登录"))
}
async fn cache_and_return(
cache: &wordval_cache::SharedCache,
resp_key: &str,
lock_key: &str,
fingerprint: &str,
ttl_secs: u64,
parts: axum::http::response::Parts,
body: Body,
) -> Response {
let status = parts.status.as_u16();
let ctype = parts
.headers
.get("Content-Type")
.and_then(|v| v.to_str().ok())
.unwrap_or("application/json; charset=utf-8")
.to_string();
let headers: Vec<(String, String)> = parts
.headers
.iter()
.filter(|(n, _)| !FORBIDDEN_REPLAY_HEADERS.contains(&n.as_str()))
.map(|(n, v)| {
(
n.as_str().to_string(),
v.to_str().unwrap_or_default().to_string(),
)
})
.collect();
match axum::body::to_bytes(body, MAX_BUFFER_BODY).await {
Ok(bytes) => {
if bytes.len() <= MAX_CACHE_BODY {
let cached = CachedResponse {
status,
content_type: ctype,
fingerprint: fingerprint.to_string(),
headers,
body: CachedResponse::encode_body(&bytes),
};
if let Err(e) = cache.set_ex(resp_key, &cached, ttl_secs).await {
let _ = cache.del(lock_key).await;
tracing::error!("幂等响应缓存写入失败: {}", e);
return error_response(ApiError::internal("幂等处理失败,请重试"));
}
}
let _ = cache.del(lock_key).await;
Response::from_parts(parts, Body::from(bytes))
}
Err(_) => {
let _ = cache.del(lock_key).await;
error_response(ApiError::internal("响应体过大,无法完成幂等处理"))
}
}
}
pub async fn idempotent_run<R, F, Fut>(ttl_secs: u64, req: Request, f: F) -> Response
where
R: IntoResponse,
F: FnOnce(Request) -> Fut,
Fut: Future<Output = R> + Send + 'static,
{
let Some(key) = extract_idem_key(&req) else {
return f(req).await.into_response();
};
let Some(scope) = extract_scope(&req) else {
return anonymous_rejected_response();
};
let (fingerprint, req) = compute_fingerprint(req).await;
let Some(fingerprint) = fingerprint else {
return f(req).await.into_response();
};
let Some(cache) = try_cache() else {
return f(req).await.into_response();
};
let path = req.uri().path().to_string();
let resp_key = format!("idem:resp:{}:{}:{}", scope, path, key);
let lock_key = format!("idem:lock:{}:{}:{}", scope, path, key);
if let Some(resp) = try_fast_path(cache, &resp_key, &fingerprint).await {
return resp;
}
if fingerprint_conflict(cache, &resp_key, &fingerprint).await {
return fingerprint_conflict_response();
}
match try_claim_lock(cache, &lock_key, &fingerprint).await {
Ok(true) => {
let resp = f(req).await.into_response();
let (parts, body) = resp.into_parts();
return cache_and_return(
cache,
&resp_key,
&lock_key,
&fingerprint,
ttl_secs,
parts,
body,
)
.await;
}
Ok(false) => {
if fingerprint_conflict(cache, &resp_key, &fingerprint).await {
return fingerprint_conflict_response();
}
if let Some(resp) = poll_for_response(cache, &resp_key).await {
return resp;
}
conflict_response()
}
Err(e) => {
tracing::error!("幂等锁声明失败(缓存故障),拒绝请求: {}", e);
error_response(ApiError::service_unavailable(
"幂等服务暂不可用,请稍后重试",
))
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::middleware::CurrentUser;
use crate::resources::set_cache;
use axum::body::to_bytes;
use axum::http::StatusCode;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Once;
use wordval_cache::{Cache, LocalCache};
static INIT: Once = Once::new();
fn cache() -> &'static wordval_cache::SharedCache {
INIT.call_once(|| {
let _ = set_cache(wordval_cache::SharedCache::Local(LocalCache::new(
100_000,
3600,
)));
});
try_cache().unwrap()
}
fn user(id: i64) -> CurrentUser {
CurrentUser {
id,
username: format!("u{}", id),
roles: vec![],
permissions: vec![],
is_admin: false,
tenant_id: None,
store_id: None,
}
}
fn mk_req(key: Option<&str>, body: &[u8], uid: i64) -> Request {
let mut b = Request::builder()
.method("POST")
.uri("/api/x")
.extension(user(uid));
if let Some(k) = key {
b = b.header("x-idempotency-key", k);
}
b.body(Body::from(body.to_vec())).unwrap()
}
async fn body_str(resp: Response) -> String {
String::from_utf8_lossy(&to_bytes(resp.into_body(), 32 * 1024 * 1024).await.unwrap())
.to_string()
}
#[tokio::test]
async fn no_key_and_long_key_passthrough() {
let c = AtomicUsize::new(0);
let f = |_| {
c.fetch_add(1, Ordering::SeqCst);
async { (StatusCode::OK, "ok") }
};
let r = idempotent_run(60, mk_req(None, b"{}", 1001), f).await;
assert_eq!(r.status(), StatusCode::OK);
let long = "k".repeat(MAX_KEY_LEN + 1);
let _ = idempotent_run(60, mk_req(Some(&long), b"{}", 1001), f).await;
let _ = idempotent_run(60, mk_req(Some(""), b"{}", 1001), f).await;
assert_eq!(c.load(Ordering::SeqCst), 3);
}
#[tokio::test]
async fn anonymous_rejected_401() {
let req = Request::builder()
.method("POST")
.uri("/api/x")
.header("x-idempotency-key", "anon-1")
.body(Body::from("{}"))
.unwrap();
let r = idempotent_run(60, req, |_| async { (StatusCode::OK, "never") }).await;
assert_eq!(r.status(), StatusCode::UNAUTHORIZED);
}
#[tokio::test]
async fn first_exec_then_replay_with_header_filtering() {
let c = AtomicUsize::new(0);
let exec = |_| {
c.fetch_add(1, Ordering::SeqCst);
async {
(
StatusCode::CREATED,
[("x-foo", "bar"), ("set-cookie", "session=1")],
"{\"ok\":1}".to_string(),
)
.into_response()
}
};
let body = b"{\"a\":1}";
let r1 = idempotent_run(3600, mk_req(Some("replay-1"), body, 1002), exec).await;
assert_eq!(r1.status(), StatusCode::CREATED);
assert_eq!(c.load(Ordering::SeqCst), 1);
let r2 = idempotent_run(3600, mk_req(Some("replay-1"), body, 1002), exec).await;
assert_eq!(c.load(Ordering::SeqCst), 1, "重放不应执行业务");
assert_eq!(r2.status(), StatusCode::CREATED);
assert_eq!(
r2.headers().get("x-foo").and_then(|v| v.to_str().ok()),
Some("bar")
);
assert!(r2.headers().get("set-cookie").is_none());
assert!(body_str(r2).await.contains("\"ok\":1"));
}
#[tokio::test]
async fn same_key_different_fingerprint_409() {
let c = AtomicUsize::new(0);
let exec = |_| {
c.fetch_add(1, Ordering::SeqCst);
async { (StatusCode::OK, "done") }
};
let r1 = idempotent_run(3600, mk_req(Some("fp-1"), b"v1", 1003), exec).await;
assert_eq!(r1.status(), StatusCode::OK);
let r2 = idempotent_run(3600, mk_req(Some("fp-1"), b"v2", 1003), exec).await;
assert_eq!(r2.status(), StatusCode::CONFLICT);
assert!(body_str(r2).await.contains("幂等键已被其他请求内容使用"));
assert_eq!(c.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn huge_request_body_passthrough() {
let c = AtomicUsize::new(0);
let big = vec![b'x'; MAX_FINGERPRINT_BODY + 10];
let r = idempotent_run(60, mk_req(Some("huge-req"), &big, 1004), |_| {
c.fetch_add(1, Ordering::SeqCst);
async { (StatusCode::OK, "passed") }
})
.await;
assert_eq!(r.status(), StatusCode::OK);
assert_eq!(c.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn big_response_executes_but_not_cached() {
let c = AtomicUsize::new(0);
let exec = |_| {
c.fetch_add(1, Ordering::SeqCst);
async {
(
StatusCode::OK,
"y".repeat(MAX_CACHE_BODY + 1_024),
)
}
};
let body = b"only-once";
let r1 = idempotent_run(3600, mk_req(Some("big-resp"), body, 1005), exec).await;
assert_eq!(r1.status(), StatusCode::OK);
let r2 = idempotent_run(3600, mk_req(Some("big-resp"), body, 1005), exec).await;
assert_eq!(r2.status(), StatusCode::OK);
assert_eq!(c.load(Ordering::SeqCst), 2);
}
#[tokio::test]
async fn over_hard_limit_response_500() {
let req = mk_req(Some("overflow-1"), b"x", 1006);
let r = idempotent_run(3600, req, |_| async {
(StatusCode::OK, "z".repeat(MAX_BUFFER_BODY + 1_024))
})
.await;
assert_eq!(r.status(), StatusCode::INTERNAL_SERVER_ERROR);
assert!(body_str(r).await.contains("响应体过大"));
}
#[tokio::test]
async fn lock_held_polls_until_response_appears() {
let c = cache();
let scope = "u1007:t-:POST:/api/x";
let lock_key = format!("idem:lock:{}:/api/x:{}", scope, "poll-1");
let resp_key = format!("idem:resp:{}:/api/x:{}", scope, "poll-1");
assert!(c.set_nx(&lock_key, &"other-fp".to_string(), 30).await.unwrap());
let c2 = c.clone();
let rk = resp_key.clone();
tokio::spawn(async move {
tokio::time::sleep(Duration::from_millis(250)).await;
let cached = CachedResponse {
status: 202,
content_type: "text/plain".into(),
fingerprint: String::new(),
headers: vec![("x-foo".into(), "baz".into())],
body: CachedResponse::encode_body(b"late-resp"),
};
c2.set_ex(&rk, &cached, 60).await.unwrap();
});
let r = idempotent_run(3600, mk_req(Some("poll-1"), b"p", 1007), |_| async {
(StatusCode::OK, "SHOULD-NOT-EXECUTE")
})
.await;
assert_eq!(r.status(), StatusCode::from_u16(202).unwrap());
assert_eq!(
r.headers().get("x-foo").and_then(|v| v.to_str().ok()),
Some("baz")
);
assert!(body_str(r).await.contains("late-resp"));
let _ = c.del(&lock_key).await;
}
#[tokio::test]
async fn lock_held_poll_timeout_409() {
let c = cache();
let scope = "u1008:t-:POST:/api/x";
let lock_key = format!("idem:lock:{}:/api/x:{}", scope, "wait-1");
assert!(c.set_nx(&lock_key, &"any".to_string(), 30).await.unwrap());
let t = tokio::time::Instant::now();
let r = idempotent_run(3600, mk_req(Some("wait-1"), b"w", 1008), |_| async {
(StatusCode::OK, "SHOULD-NOT-EXECUTE")
})
.await;
assert_eq!(r.status(), StatusCode::CONFLICT);
assert!(body_str(r).await.contains("请求正在处理中"));
assert!(t.elapsed() >= Duration::from_millis(2_500));
let _ = c.del(&lock_key).await;
}
#[tokio::test]
async fn tenant_scope_isolates_idempotency() {
let c = AtomicUsize::new(0);
let exec = |_| {
c.fetch_add(1, Ordering::SeqCst);
async { (StatusCode::OK, "t") }
};
let body = b"same-bytes";
let mk = |tid: Option<i64>| {
Request::builder()
.method("POST")
.uri("/api/x")
.header("x-idempotency-key", "tenants-1")
.extension(user(1009))
.extension(TenantContext { tenant_id: tid })
.body(Body::from(body.to_vec()))
.unwrap()
};
let _ = idempotent_run(3600, mk(Some(7)), exec).await;
let _ = idempotent_run(3600, mk(Some(8)), exec).await;
assert_eq!(c.load(Ordering::SeqCst), 2);
let _ = idempotent_run(3600, mk(Some(7)), exec).await;
assert_eq!(c.load(Ordering::SeqCst), 2);
}
#[tokio::test]
async fn json_extractor_replayed_after_fingerprint() {
use axum::extract::Json;
use axum::handler::Handler;
async fn inner(Json(p): Json<serde_json::Value>) -> Json<serde_json::Value> {
Json(p)
}
let c = AtomicUsize::new(0);
let exec = |req: Request| {
c.fetch_add(1, Ordering::SeqCst);
Box::pin(async move { Handler::call(inner, req, ()).await })
as std::pin::Pin<Box<dyn Future<Output = Response> + Send>>
};
let body = br#"{"amount":7}"#;
let mk = || {
Request::builder()
.method("POST")
.uri("/api/x")
.header("x-idempotency-key", "json-1")
.header("content-type", "application/json")
.extension(user(1020))
.body(Body::from(body.to_vec()))
.unwrap()
};
let r1 = idempotent_run(3600, mk(), exec).await;
assert_eq!(r1.status(), StatusCode::OK);
assert!(body_str(r1).await.contains("\"amount\":7"));
assert_eq!(c.load(Ordering::SeqCst), 1);
let r2 = idempotent_run(3600, mk(), exec).await;
assert_eq!(c.load(Ordering::SeqCst), 1, "重放不应再次执行 Json handler");
assert!(body_str(r2).await.contains("\"amount\":7"));
}
#[test]
fn extract_idem_key_rules() {
let mk = |k: Option<&str>| {
let mut b = Request::builder().method("POST").uri("/");
if let Some(k) = k {
b = b.header("x-idempotency-key", k);
}
b.body(Body::empty()).unwrap()
};
assert_eq!(extract_idem_key(&mk(None)), None);
assert_eq!(extract_idem_key(&mk(Some(""))), None);
assert_eq!(extract_idem_key(&mk(Some("abc"))).as_deref(), Some("abc"));
assert_eq!(extract_idem_key(&mk(Some(&"k".repeat(129)))), None);
assert!(extract_idem_key(&mk(Some(&"k".repeat(128)))).is_some());
}
#[test]
fn extract_scope_format_with_tenant() {
let req = Request::builder()
.method("PUT")
.uri("/api/y?z=1")
.extension(user(42))
.extension(TenantContext { tenant_id: Some(9) })
.body(Body::empty())
.unwrap();
assert_eq!(extract_scope(&req).as_deref(), Some("u42:t9:PUT:/api/y"));
let req2 = Request::builder()
.method("POST")
.uri("/api/x")
.extension(user(1))
.body(Body::empty())
.unwrap();
assert_eq!(extract_scope(&req2).as_deref(), Some("u1:t-:POST:/api/x"));
let anon = Request::builder()
.method("POST")
.uri("/api/x")
.body(Body::empty())
.unwrap();
assert_eq!(extract_scope(&anon), None);
}
#[tokio::test]
async fn compute_fingerprint_restores_body() {
let req = mk_req(Some("fp-c"), b"ab", 1010);
let (fp, restored) = compute_fingerprint(req).await;
let fp = fp.unwrap();
let expect = wordval_utils::Crypto::sha256_parts(&[b"POST:/api/x::", b"ab"]);
assert_eq!(fp, expect);
assert_eq!(&to_bytes(restored.into_body(), 1024).await.unwrap()[..], b"ab");
}
#[tokio::test]
async fn build_response_fault_tolerant() {
let cached = CachedResponse {
status: 999,
content_type: "application/json".into(),
fingerprint: "f".into(),
headers: vec![
("bad header".into(), "v".into()),
("x-ok".into(), "yes".into()),
],
body: CachedResponse::encode_body(b"payload"),
};
let resp = build_response(cached);
assert_eq!(resp.status().as_u16(), 999);
assert_eq!(
resp.headers().get("x-ok").and_then(|v| v.to_str().ok()),
Some("yes")
);
assert!(body_str(resp).await.contains("payload"));
let zero = CachedResponse {
status: 0,
content_type: "text/plain".into(),
fingerprint: String::new(),
headers: vec![],
body: CachedResponse::encode_body(b"z"),
};
assert_eq!(build_response(zero).status(), StatusCode::OK);
let broken = CachedResponse {
status: 200,
content_type: "text/plain".into(),
fingerprint: String::new(),
headers: vec![],
body: "!!!not-base64!!!".into(),
};
assert_eq!(broken.decode_body(), Vec::<u8>::new());
}
#[tokio::test]
async fn fast_path_and_conflict_helpers() {
let c = cache();
let key = "idem:resp:unit-test-key";
let cached = CachedResponse {
status: 200,
content_type: "text/plain".into(),
fingerprint: "fpA".into(),
headers: vec![],
body: CachedResponse::encode_body(b"R"),
};
c.set_ex(key, &cached, 60).await.unwrap();
assert!(try_fast_path(c, key, "fpA").await.is_some());
assert!(try_fast_path(c, key, "fpB").await.is_none());
assert!(fingerprint_conflict(c, key, "fpB").await);
assert!(!fingerprint_conflict(c, key, "fpA").await);
assert!(!fingerprint_conflict(c, "idem:resp:none", "x").await);
assert!(try_fast_path(c, "idem:resp:none", "x").await.is_none());
let _ = c.del(key).await;
}
#[tokio::test]
async fn claim_lock_mutual_exclusion() {
let c = cache();
let lk = "idem:lock:unit-claim";
assert!(try_claim_lock(c, lk, "fp1").await.unwrap());
assert!(!try_claim_lock(c, lk, "fp2").await.unwrap());
let _ = c.del(lk).await;
assert!(try_claim_lock(c, lk, "fp3").await.unwrap());
let _ = c.del(lk).await;
}
}