use std::collections::BTreeSet;
use std::sync::Arc;
use redis::cluster_routing::{MultipleNodeRoutingInfo, RoutingInfo, SingleNodeRoutingInfo};
use redis::{ErrorKind, ServerErrorKind, Value};
use tokio::sync::OnceCell;
use crate::client::{Conn, RedisClient};
use crate::error::{NasaRedisError, Result};
use crate::keytag::redis_slot;
use super::config::{IdempotentLayoutSnapshot, IdempotentTtlMode};
use super::metrics::IdempotentCounterMetrics;
use super::result::{IdempotentCounterError, IdempotentRejection, IdempotentResultCode};
use super::wire;
const HFE_SCRIPT: &str = include_str!("lua/idempotent_counter_hfe.lua");
const BUCKET_SCRIPT: &str = include_str!("lua/idempotent_counter_bucket.lua");
type IdempotentInvocation<'a> = (Vec<Vec<u8>>, Vec<Vec<u8>>, &'static str, &'a str);
#[derive(Debug, Clone, Copy)]
pub(crate) enum Operation {
StringIncrement,
StringDecrement,
HashIncrement,
HashDecrement,
ZSetIncrement,
}
impl Operation {
fn family(self) -> u8 {
match self {
Self::StringIncrement | Self::StringDecrement => wire::FAMILY_STRING,
Self::HashIncrement | Self::HashDecrement => wire::FAMILY_HASH,
Self::ZSetIncrement => wire::FAMILY_ZSET,
}
}
fn command(self) -> &'static [u8] {
match self {
Self::StringIncrement => b"STR_INCR",
Self::StringDecrement => b"STR_DECR",
Self::HashIncrement => b"HASH_INCR",
Self::HashDecrement => b"HASH_DECR",
Self::ZSetIncrement => b"ZSET_INCR",
}
}
}
pub(crate) struct IdempotentCounterRuntime {
layout: IdempotentLayoutSnapshot,
metrics: Arc<IdempotentCounterMetrics>,
resolved: OnceCell<IdempotentTtlMode>,
hfe_sha: String,
bucket_sha: String,
}
enum ClusterProbeCoverage {
Supported,
Unsupported,
Incomplete,
}
impl IdempotentCounterRuntime {
pub(crate) fn new(layout: IdempotentLayoutSnapshot) -> Self {
Self {
layout,
metrics: Arc::new(IdempotentCounterMetrics::default()),
resolved: OnceCell::new(),
hfe_sha: redis::Script::new(HFE_SCRIPT).get_hash().to_owned(),
bucket_sha: redis::Script::new(BUCKET_SCRIPT).get_hash().to_owned(),
}
}
pub(crate) fn metrics(&self) -> &Arc<IdempotentCounterMetrics> {
&self.metrics
}
pub(crate) async fn execute_long(
&self,
client: &RedisClient,
op: Operation,
key: &str,
member: &[u8],
delta: i64,
nonce: &str,
) -> Result<i64> {
let value = self
.execute(
client,
op,
key,
member,
delta.to_string().into_bytes(),
nonce,
)
.await?;
value
.parse::<i64>()
.map_err(|_| protocol("idempotent integer script returned a non-int64 value"))
}
pub(crate) async fn execute_double(
&self,
client: &RedisClient,
key: &str,
member: &[u8],
delta: f64,
nonce: &str,
) -> Result<f64> {
let delta_text = format!("{delta}").into_bytes();
let value = self
.execute(
client,
Operation::ZSetIncrement,
key,
member,
delta_text,
nonce,
)
.await?;
parse_redis_double(&value)
}
async fn execute(
&self,
client: &RedisClient,
op: Operation,
key: &str,
member: &[u8],
delta: Vec<u8>,
nonce: &str,
) -> Result<String> {
let mode = self.resolved_mode(client).await?;
let (keys, args, script, sha) = self.invocation(mode, op, key, member, delta, nonce)?;
let mut conn = client.conn();
let raw = eval_binary(&mut conn, script, sha, &keys, &args).await?;
interpret_result(&self.metrics, &raw)
}
fn invocation(
&self,
mode: IdempotentTtlMode,
op: Operation,
key: &str,
member: &[u8],
delta: Vec<u8>,
nonce: &str,
) -> Result<IdempotentInvocation<'_>> {
if nonce.trim().is_empty() {
return Err(cfg("nonce 不能为空或全空白"));
}
let target_key = key.as_bytes();
let expected_slot = redis_slot(target_key);
let family = [op.family()];
let record_field = wire::record_field(&family, member, nonce.as_bytes());
let base = wire::ledger_base(
&self.layout.ledger_key_prefix,
target_key,
&family,
member,
self.layout.ledger_shards,
)
.ok_or_else(|| protocol("idempotent ledger_shards is not a non-zero power of two"))?;
let mut keys = vec![target_key.to_vec()];
match mode {
IdempotentTtlMode::HashField => {
keys.push(self.ledger_key(base.into_bytes(), expected_slot)?);
}
IdempotentTtlMode::HashBucket => {
for index in 0..self.layout.bucket_count {
keys.push(
self.ledger_key(format!("{base}:{index}").into_bytes(), expected_slot)?,
);
}
}
IdempotentTtlMode::Auto => {
return Err(protocol(
"idempotent runtime resolved to an unresolved AUTO mode",
));
}
}
let args = vec![
record_field.to_vec(),
op.command().to_vec(),
member.to_vec(),
delta,
self.layout.nonce_ttl_ms.to_string().into_bytes(),
self.layout.bucket_span_ms.to_string().into_bytes(),
];
let (script, sha) = match mode {
IdempotentTtlMode::HashField => (HFE_SCRIPT, self.hfe_sha.as_str()),
IdempotentTtlMode::HashBucket => (BUCKET_SCRIPT, self.bucket_sha.as_str()),
IdempotentTtlMode::Auto => unreachable!("AUTO 已在上文拒绝"),
};
Ok((keys, args, script, sha))
}
pub(crate) fn pipeline_command(
&self,
op: Operation,
key: &str,
member: &[u8],
delta: Vec<u8>,
nonce: &str,
) -> Result<redis::Cmd> {
let mode = self
.resolved
.get()
.copied()
.ok_or_else(|| cfg("idempotent pipeline 创建前必须先准备账本布局"))?;
let (keys, args, script, _) = self.invocation(mode, op, key, member, delta, nonce)?;
let mut command = redis::cmd("EVAL");
command.arg(script.as_bytes()).arg(keys.len());
for key in keys {
command.arg(key);
}
for arg in args {
command.arg(arg);
}
Ok(command)
}
fn ledger_key(&self, key: Vec<u8>, expected_slot: u16) -> Result<Vec<u8>> {
if !wire::ledger_slot_matches(&key, expected_slot) {
return Err(protocol(
"derived idempotent ledger key is not in the target Redis slot",
));
}
Ok(key)
}
pub(crate) async fn prepare(&self, client: &RedisClient) -> Result<()> {
self.resolved_mode(client).await.map(|_| ())
}
async fn resolved_mode(&self, client: &RedisClient) -> Result<IdempotentTtlMode> {
self.resolved
.get_or_try_init(|| self.resolve_layout(client))
.await
.copied()
}
async fn resolve_layout(&self, client: &RedisClient) -> Result<IdempotentTtlMode> {
let marker_key = self.layout.marker_key();
if let Some(existing) = read_marker(client, &marker_key).await? {
let mode = self.verify_marker(client, &existing).await?;
self.metrics.resolve_layout(mode, &existing);
return Ok(mode);
}
let resolved = match self.layout.ttl_mode {
IdempotentTtlMode::HashBucket => IdempotentTtlMode::HashBucket,
IdempotentTtlMode::HashField => {
if !self.all_masters_support_hpexpire(client).await? {
return Err(cfg("HASH_FIELD 要求每个 master 都支持 HPEXPIRE"));
}
IdempotentTtlMode::HashField
}
IdempotentTtlMode::Auto => {
if self.all_masters_support_hpexpire(client).await? {
IdempotentTtlMode::HashField
} else {
IdempotentTtlMode::HashBucket
}
}
};
let candidate = self.layout.marker_value(resolved);
let mut conn = client.conn();
let _created: Option<String> = redis::cmd("SET")
.arg(marker_key.as_bytes())
.arg(candidate.as_bytes())
.arg("NX")
.query_async(&mut conn)
.await
.map_err(NasaRedisError::Redis)?;
let actual = read_marker(client, &marker_key)
.await?
.ok_or_else(|| protocol("idempotent counter layout marker was not persisted"))?;
let mode = self.verify_marker(client, &actual).await?;
self.metrics.resolve_layout(mode, &actual);
Ok(mode)
}
async fn verify_marker(&self, client: &RedisClient, marker: &str) -> Result<IdempotentTtlMode> {
let parts: Vec<&str> = marker.split('|').collect();
let (mode_index, ttl_index, span_index, shards_index) = match self.layout.profile {
crate::config::CompatibilityProfile::LegacyV1 => {
if parts.len() != 5 || parts[0] != "1" {
return Err(cfg("LegacyV1 需要版本 1 idempotent counter layout marker"));
}
(1, 2, 3, 4)
}
crate::config::CompatibilityProfile::RustV2 => {
if parts.len() != 7
|| parts[0] != "2"
|| parts[1] != self.layout.ledger_key_prefix
|| parts[2] != "record-sha256-v1"
{
return Err(cfg("RustV2 idempotent counter layout marker 不兼容"));
}
(3, 4, 5, 6)
}
};
let mode = IdempotentTtlMode::resolved_from_marker(parts[mode_index])
.ok_or_else(|| cfg("layout marker 必须包含已解析的 TTL 模式"))?;
let nonce_ttl: u64 = parts[ttl_index]
.parse()
.map_err(|_| cfg("layout marker nonceTtlMs 非法"))?;
let bucket_span: u64 = parts[span_index]
.parse()
.map_err(|_| cfg("layout marker bucketSpanMs 非法"))?;
let shards: u32 = parts[shards_index]
.parse()
.map_err(|_| cfg("layout marker ledgerShards 非法"))?;
if nonce_ttl != self.layout.nonce_ttl_ms
|| bucket_span != self.layout.bucket_span_ms
|| shards != self.layout.ledger_shards
{
return Err(cfg(
"本地 idempotent counter 配置与共享 layout marker 不一致",
));
}
if self.layout.ttl_mode != IdempotentTtlMode::Auto && self.layout.ttl_mode != mode {
return Err(cfg("本地 ttl_mode 与共享 layout marker 不一致"));
}
if mode == IdempotentTtlMode::HashField
&& !self.all_masters_support_hpexpire(client).await?
{
return Err(cfg("共享 HASH_FIELD 布局要求每个 master 都支持 HPEXPIRE"));
}
Ok(mode)
}
async fn all_masters_support_hpexpire(&self, client: &RedisClient) -> Result<bool> {
let outcome = self.probe_all_masters_hpexpire(client).await;
if outcome.is_err() {
self.metrics.record_probe_failure();
}
outcome
}
async fn probe_all_masters_hpexpire(&self, client: &RedisClient) -> Result<bool> {
match client.conn() {
Conn::Single(mut conn) => redis::cmd("COMMAND")
.arg("INFO")
.arg("HPEXPIRE")
.query_async::<Value>(&mut conn)
.await
.map_err(NasaRedisError::Redis)
.and_then(|value| {
supports_hpexpire_response(&value)
.ok_or_else(|| cfg("HPEXPIRE 能力探测返回了无效的命令定义"))
}),
Conn::Cluster(mut conn) => {
const RETRY_DELAYS_MS: [u64; 5] = [0, 100, 250, 500, 1_000];
for delay_ms in RETRY_DELAYS_MS {
if delay_ms > 0 {
tokio::time::sleep(std::time::Duration::from_millis(delay_ms)).await;
}
let Some(before) = cluster_primary_addresses(&mut conn).await else {
continue;
};
let mut command = redis::cmd("COMMAND");
command.arg("INFO").arg("HPEXPIRE");
let responses = conn
.route_command(
command,
RoutingInfo::MultiNode((MultipleNodeRoutingInfo::AllMasters, None)),
)
.await;
let Some(after) = cluster_primary_addresses(&mut conn).await else {
continue;
};
if before != after {
continue;
}
let Ok(value) = responses else {
continue;
};
match cluster_response_coverage(&value, &before) {
ClusterProbeCoverage::Supported => return Ok(true),
ClusterProbeCoverage::Unsupported => return Ok(false),
ClusterProbeCoverage::Incomplete => continue,
}
}
Err(cfg("HPEXPIRE 能力探测未取得稳定拓扑的全量 master 响应"))
}
}
}
}
async fn cluster_primary_addresses(
conn: &mut redis::cluster_async::ClusterConnection,
) -> Option<BTreeSet<String>> {
let mut command = redis::cmd("CLUSTER");
command.arg("SLOTS");
let value = conn
.route_command(
command,
RoutingInfo::SingleNode(SingleNodeRoutingInfo::RandomPrimary),
)
.await
.ok()?;
let Value::Array(rows) = value else {
return None;
};
let mut primaries = BTreeSet::new();
for row in rows {
let Value::Array(fields) = row else {
return None;
};
let Some(Value::Array(primary)) = fields.get(2) else {
return None;
};
let host = redis_text(primary.first()?)?;
let port = match primary.get(1)? {
Value::Int(value) if (1..=u16::MAX as i64).contains(value) => *value as u16,
_ => return None,
};
if host == "?" {
return None;
}
let address = if host.is_empty() || matches!(host.as_str(), "0.0.0.0" | "::" | "[::]") {
format!("*:{port}")
} else {
format!("{host}:{port}")
};
primaries.insert(address);
}
(!primaries.is_empty()).then_some(primaries)
}
fn cluster_response_coverage(value: &Value, expected: &BTreeSet<String>) -> ClusterProbeCoverage {
let Value::Map(entries) = value else {
return ClusterProbeCoverage::Incomplete;
};
let mut answered = BTreeSet::new();
let mut supported = true;
for (address, response) in entries {
let Some(address) = redis_text(address) else {
return ClusterProbeCoverage::Incomplete;
};
if !answered.insert(address) {
return ClusterProbeCoverage::Incomplete;
}
let Some(node_supported) = supports_hpexpire_response(response) else {
return ClusterProbeCoverage::Incomplete;
};
supported &= node_supported;
}
if !addresses_cover(expected, &answered) {
return ClusterProbeCoverage::Incomplete;
}
if supported {
ClusterProbeCoverage::Supported
} else {
ClusterProbeCoverage::Unsupported
}
}
fn addresses_cover(expected: &BTreeSet<String>, answered: &BTreeSet<String>) -> bool {
if expected.len() != answered.len() {
return false;
}
let mut remaining = answered.clone();
for address in expected {
if remaining.remove(address) {
continue;
}
let Some(port) = address.strip_prefix("*:") else {
return false;
};
let candidates: Vec<String> = remaining
.iter()
.filter(|candidate| address_port(candidate).is_some_and(|value| value == port))
.cloned()
.collect();
if candidates.len() != 1 {
return false;
}
remaining.remove(&candidates[0]);
}
remaining.is_empty()
}
fn address_port(address: &str) -> Option<&str> {
let (_, port) = address.rsplit_once(':')?;
(!port.is_empty()).then_some(port)
}
fn supports_hpexpire_response(value: &Value) -> Option<bool> {
let Value::Array(items) = value else {
return None;
};
let [command] = items.as_slice() else {
return None;
};
match command {
Value::Nil => Some(false),
Value::Array(fields) if fields.len() >= 6 => {
let name = redis_text(&fields[0])?;
if !name.eq_ignore_ascii_case("hpexpire")
|| !matches!(fields[1], Value::Int(arity) if arity != 0)
|| !matches!(&fields[2], Value::Array(flags) if flags.iter().all(|flag| matches!(flag, Value::BulkString(_) | Value::SimpleString(_))))
|| !fields[3..6]
.iter()
.all(|field| matches!(field, Value::Int(position) if *position >= 0))
{
return None;
}
Some(true)
}
_ => None,
}
}
fn redis_text(value: &Value) -> Option<String> {
match value {
Value::BulkString(bytes) => String::from_utf8(bytes.clone()).ok(),
Value::SimpleString(text) => Some(text.clone()),
_ => None,
}
}
pub(crate) fn interpret_result(
metrics: &IdempotentCounterMetrics,
raw: &[Value],
) -> Result<String> {
if raw.len() != 3 {
return Err(protocol(
"idempotent counter script returned an invalid result",
));
}
let code_text = value_to_string(&raw[0]);
let value = value_to_string(&raw[1]);
let detail = value_to_string(&raw[2]);
let code = IdempotentResultCode::parse(&code_text)
.ok_or_else(|| protocol("idempotent counter script returned an unknown result code"))?;
match code {
IdempotentResultCode::Applied => {
metrics.record_applied();
Ok(value)
}
IdempotentResultCode::AppliedTtlMissing => {
metrics.record_applied();
metrics.record_ttl_missing();
tracing::error!(detail = %detail, "idempotent counter credential TTL was not confirmed");
Ok(value)
}
IdempotentResultCode::Duplicate => {
metrics.record_duplicate();
Ok(value)
}
IdempotentResultCode::RejectedLedgerType => {
metrics.record_rejected(IdempotentRejection::LedgerType);
Err(reject(IdempotentRejection::LedgerType, detail))
}
IdempotentResultCode::RejectedOperation => {
metrics.record_rejected(IdempotentRejection::Operation);
Err(reject(IdempotentRejection::Operation, detail))
}
IdempotentResultCode::RejectedCommand => {
metrics.record_rejected(IdempotentRejection::Command);
Err(reject(IdempotentRejection::Command, detail))
}
}
}
async fn eval_binary(
conn: &mut Conn,
script: &str,
sha: &str,
keys: &[Vec<u8>],
args: &[Vec<u8>],
) -> Result<Vec<Value>> {
match run_eval(conn, "EVALSHA", sha.as_bytes(), keys, args).await {
Ok(values) => Ok(values),
Err(error) if is_noscript(&error) => {
run_eval(conn, "EVAL", script.as_bytes(), keys, args)
.await
.map_err(script_execution_error)
}
Err(error) => Err(script_execution_error(error)),
}
}
fn script_execution_error(error: redis::RedisError) -> NasaRedisError {
if crate::error::redis_is_transient(&error) {
NasaRedisError::ExecutionUnknown(
"幂等计数脚本已发送,但连接在确认结果前中断;请使用原 nonce 重试".to_owned(),
)
} else {
NasaRedisError::Redis(error)
}
}
async fn run_eval(
conn: &mut Conn,
verb: &str,
body: &[u8],
keys: &[Vec<u8>],
args: &[Vec<u8>],
) -> std::result::Result<Vec<Value>, redis::RedisError> {
let mut cmd = redis::cmd(verb);
cmd.arg(body).arg(keys.len());
for key in keys {
cmd.arg(key.as_slice());
}
for arg in args {
cmd.arg(arg.as_slice());
}
cmd.query_async(conn).await
}
fn is_noscript(error: &redis::RedisError) -> bool {
error.kind() == ErrorKind::Server(ServerErrorKind::NoScript)
}
async fn read_marker(client: &RedisClient, marker_key: &str) -> Result<Option<String>> {
let mut conn = client.conn();
let raw: Option<Vec<u8>> = redis::cmd("GET")
.arg(marker_key.as_bytes())
.query_async(&mut conn)
.await
.map_err(NasaRedisError::Redis)?;
match raw {
None => Ok(None),
Some(bytes) => String::from_utf8(bytes)
.map(Some)
.map_err(|_| protocol("idempotent counter layout marker is not valid UTF-8")),
}
}
fn value_to_string(value: &Value) -> String {
match value {
Value::Nil => String::new(),
Value::Int(n) => n.to_string(),
Value::BulkString(bytes) => String::from_utf8_lossy(bytes).into_owned(),
Value::SimpleString(text) => text.clone(),
_ => String::new(),
}
}
fn parse_redis_double(value: &str) -> Result<f64> {
let lower = value.to_ascii_lowercase();
match lower.as_str() {
"inf" | "+inf" => Ok(f64::INFINITY),
"-inf" => Ok(f64::NEG_INFINITY),
_ => value
.parse::<f64>()
.map_err(|_| protocol("idempotent ZSET script returned a non-numeric value")),
}
}
fn cfg(message: &str) -> NasaRedisError {
NasaRedisError::Config(message.to_owned())
}
fn protocol(message: &str) -> NasaRedisError {
NasaRedisError::IdempotentProtocol(message.to_owned())
}
fn reject(code: IdempotentRejection, detail: String) -> NasaRedisError {
NasaRedisError::Idempotent(IdempotentCounterError {
code,
detail,
})
}