use std::sync::Arc;
use super::KeyLayout;
use crate::client::RedisClient;
use crate::error::{NasaRedisError, Result};
pub fn marker_key(layout: &KeyLayout, p: u32) -> String {
format!("{}:disposition:{}", layout.prefix, p)
}
pub fn quarantine_key(layout: &KeyLayout, park_id: &str) -> String {
format!("{}:quarantine:{}", layout.prefix, park_id)
}
pub fn parked_index_key(layout: &KeyLayout, p: u32) -> String {
format!("{}:parked:{}", layout.prefix, p)
}
pub struct OwnerLease<'a> {
pub partition: u32,
pub operation_id: &'a str,
pub expected_park_id: Option<&'a str>,
pub holder: &'a str,
pub lock_key: &'a str,
pub fence_key: &'a str,
pub round: u64,
pub nonce: &'a str,
pub counter: u64,
}
impl OwnerLease<'_> {
fn lua_fence_key(&self) -> &str {
if self.fence_key.is_empty() {
self.lock_key
} else {
self.fence_key
}
}
fn has_fence(&self) -> &'static str {
if self.fence_key.is_empty() {
"0"
} else {
"1"
}
}
}
const VERIFY_OWNER_LUA: &str = r#"
if redis.call('hexists', KEYS[1], ARGV[1]) == 0 then return 'NOT_HOLDER' end
if ARGV[2] == '1' then
if redis.call('HGET', KEYS[2], 'state') ~= 'Ready' then return 'FENCE_NOT_READY' end
if redis.call('HGET', KEYS[2], 'round') ~= ARGV[3] then return 'ROUND_MISMATCH' end
if redis.call('HGET', KEYS[2], 'nonce') ~= ARGV[4] then return 'NONCE_MISMATCH' end
if redis.call('HGET', KEYS[2], 'c:' .. ARGV[5]) ~= ARGV[6] then return 'STALE_STAMP' end
end
return 'OK'
"#;
async fn verify_owner_fenced(client: &Arc<RedisClient>, lease: &OwnerLease<'_>) -> Result<()> {
let tag: String = redis::Script::new(VERIFY_OWNER_LUA)
.key(lease.lock_key)
.key(lease.lua_fence_key())
.arg(lease.holder)
.arg(lease.has_fence())
.arg(lease.round.to_string())
.arg(lease.nonce)
.arg(lease.partition.to_string())
.arg(lease.counter.to_string())
.invoke_async(&mut client.conn())
.await?;
if tag == "OK" {
return Ok(());
}
Err(NasaRedisError::OwnershipChanged(format!(
"分区 {}:owner-fence 校验失败({tag}),拒绝改 disposition(留 PEL 交新 owner)",
lease.partition
)))
}
#[allow(clippy::too_many_arguments)]
async fn guarded_disposition<A: redis::ToRedisArgs + Send>(
client: &Arc<RedisClient>,
layout: &KeyLayout,
p: u32,
lease: &OwnerLease<'_>,
park_id: &str,
state: &str,
stream: &str,
body: &str,
arguments: A,
) -> Result<redis::Value> {
if lease.partition != p {
return Err(NasaRedisError::OwnershipChanged(format!(
"分区 {p} 与持有权证据分区 {} 不一致",
lease.partition
)));
}
if park_id.is_empty()
|| lease
.expected_park_id
.is_some_and(|expected| expected != park_id)
{
return Err(NasaRedisError::OperationConflict(format!(
"分区 {p} 处置批次与前序身份不一致"
)));
}
const GUARD_LUA: &str = r#"
-- KEYS 为 marker、锁、fence、隔离区、索引、目标 Stream;ARGV 前九项绑定持有权、批次、操作与状态。
-- 类型读取和全部拒绝条件先于操作正文,避免 Lua 不回滚时出现已写入却因旧证据继续失败的状态。
for i = 1, 5 do
local kind = redis.call('TYPE', KEYS[i])['ok']
local expected = 'hash'
if i == 5 then
expected = 'string'
end
if kind ~= 'none' and kind ~= expected then
return {'PROTOCOL', 'KEY_TYPE'}
end
end
if redis.call('HEXISTS', KEYS[2], ARGV[1]) == 0 then
return {'OWNERSHIP', 'NOT_HOLDER'}
end
if ARGV[2] == '1' then
if redis.call('HGET', KEYS[3], 'state') ~= 'Ready' then
return {'OWNERSHIP', 'FENCE_NOT_READY'}
end
if redis.call('HGET', KEYS[3], 'round') ~= ARGV[3] then
return {'OWNERSHIP', 'ROUND_MISMATCH'}
end
if redis.call('HGET', KEYS[3], 'nonce') ~= ARGV[4] then
return {'OWNERSHIP', 'NONCE_MISMATCH'}
end
if redis.call('HGET', KEYS[3], 'c:' .. ARGV[5]) ~= ARGV[6] then
return {'OWNERSHIP', 'STALE_STAMP'}
end
end
if redis.call('HGET', KEYS[1], 'park_id') ~= ARGV[7]
or redis.call('HGET', KEYS[1], 'state') ~= ARGV[9] then
return {'CONFLICT', 'BATCH_OR_STATE'}
end
-- 认领计数不能从负值继续累加,否则可能先改变状态、再因无符号响应解析失败留下半完成意图。
if ARGV[9] == 'Parked' or ARGV[9] == 'ResumeIndeterminate' then
local version = redis.call('HGET', KEYS[1], 'version')
if version and string.sub(version, 1, 1) == '-' then
return {'PROTOCOL', 'VERSION'}
end
end
local index = redis.call('GET', KEYS[5])
if index and index ~= ARGV[7] then
return {'CONFLICT', 'INDEX'}
end
-- 终态补清理与新转换认领由当前 owner 授权;已进入发布阶段后必须保持原操作,旧空身份保留续作兼容。
if ARGV[9] ~= 'Resumed' and ARGV[9] ~= 'Dlqed' and ARGV[9] ~= 'Dropped'
and ARGV[9] ~= 'Parked' and ARGV[9] ~= 'ResumeIndeterminate' then
local operation = redis.call('HGET', KEYS[1], 'transition_operation_id')
if operation and operation ~= '' and operation ~= ARGV[8] then
return {'CONFLICT', 'OPERATION'}
end
end
"#;
let script = redis::Script::new(&format!("{GUARD_LUA}\n{body}"));
let (tag, value): (String, redis::Value) = script
.key(marker_key(layout, p))
.key(lease.lock_key)
.key(lease.lua_fence_key())
.key(quarantine_key(layout, park_id))
.key(parked_index_key(layout, p))
.key(stream)
.arg(lease.holder)
.arg(lease.has_fence())
.arg(lease.round.to_string())
.arg(lease.nonce)
.arg(p.to_string())
.arg(lease.counter.to_string())
.arg(park_id)
.arg(lease.operation_id)
.arg(state)
.arg(arguments)
.invoke_async(&mut client.conn())
.await?;
match tag.as_str() {
"OK" => Ok(value),
"OWNERSHIP" => Err(NasaRedisError::OwnershipChanged(format!(
"分区 {p} 处置副作用被持有权校验拒绝"
))),
"CONFLICT" => Err(NasaRedisError::OperationConflict(format!(
"分区 {p} 处置副作用与当前批次、状态或操作冲突"
))),
_ => Err(NasaRedisError::ProtocolMarker(format!(
"分区 {p} 处置记录或脚本响应异常"
))),
}
}
async fn assert_op_owns(client: &Arc<RedisClient>, marker: &str, op_id: &str) -> Result<()> {
let owner: Option<String> = client.h_get(marker, "transition_operation_id").await?;
match owner.as_deref() {
None | Some("") => Ok(()),
Some(o) if o == op_id => Ok(()),
Some(o) => Err(NasaRedisError::OperationConflict(format!(
"在途转换属 operation={o},本 operation={op_id} 不得接管(交原 op 或新 op 显式重授权)"
))),
}
}
pub fn auto_op_id(p: u32, park_id: &str) -> String {
format!("auto:{p}:{park_id}")
}
pub fn is_auto_op(op_id: &str) -> bool {
op_id.starts_with("auto:")
}
pub async fn marker_version(client: &Arc<RedisClient>, layout: &KeyLayout, p: u32) -> Result<u64> {
Ok(client
.h_get::<u64>(&marker_key(layout, p), "version")
.await?
.unwrap_or(0))
}
async fn load_quarantine_payloads(
client: &Arc<RedisClient>,
qkey: &str,
p: u32,
park_id: &str,
) -> Result<Vec<Vec<u8>>> {
let n: Option<u64> = client.h_get(qkey, "n").await?;
let Some(n) = n else {
return Err(NasaRedisError::ProtocolMarker(format!(
"quarantine 损坏:分区 {p} park_id={park_id} 缺 'n' 字段,拒绝处置(fail-closed)"
)));
};
let fields: u64 = redis::cmd("HLEN")
.arg(qkey)
.query_async(&mut client.conn())
.await?;
if n.checked_add(1).is_none_or(|minimum| minimum > fields) {
return Err(NasaRedisError::ProtocolMarker(format!(
"quarantine 损坏:分区 {p} park_id={park_id} 声明 {n} 条但仅有 {fields} 个字段"
)));
}
let capacity = usize::try_from(n).map_err(|_| {
NasaRedisError::ProtocolMarker(format!(
"quarantine 条数超出本机可表示范围:分区 {p} park_id={park_id} n={n}"
))
})?;
let mut payloads = Vec::new();
payloads.try_reserve_exact(capacity).map_err(|_| {
NasaRedisError::ProtocolMarker(format!(
"quarantine 载荷索引容量不可预留:分区 {p} park_id={park_id} n={n}"
))
})?;
for i in 0..n {
let pl: Option<Vec<u8>> = client.h_get(qkey, &i.to_string()).await?;
match pl {
Some(v) => payloads.push(v),
None => {
return Err(NasaRedisError::ProtocolMarker(format!(
"quarantine 损坏:分区 {p} park_id={park_id} 缺第 {i} 条 payload(n={n}),\
拒绝处置(fail-closed,禁空字节替代)"
)))
}
}
}
Ok(payloads)
}
async fn load_quarantine_payloads_for(
client: &Arc<RedisClient>,
qkey: &str,
p: u32,
park_id: &str,
expected: usize,
) -> Result<Vec<Vec<u8>>> {
let payloads = load_quarantine_payloads(client, qkey, p, park_id).await?;
if payloads.len() != expected {
return Err(NasaRedisError::ProtocolMarker(format!(
"quarantine 损坏:分区 {p} park_id={park_id} payload 数 {} 与已持久化 planned 数 {expected} 不符",
payloads.len()
)));
}
Ok(payloads)
}
const PARK_LUA: &str = r#"
-- KEYS[1]=分区锁 KEYS[2]=源 Stream KEYS[3]=quarantine KEYS[4]=marker KEYS[5]=parked 索引 KEYS[6]=fence
-- ARGV[1]=holder ARGV[2]=group ARGV[3]=park_id ARGV[4]=墙钟ms ARGV[5]=条数 n
-- ARGV[6]=启用 fence ARGV[7]=round ARGV[8]=nonce ARGV[9]=分区 ARGV[10]=counter
-- ARGV[11..] 为 id,payload 对;所有权威和版本检查先于恢复数据写入与源消息确认。
-- ① 写前 TYPE 预检(none 容忍;非预期类型即拒,绝不进入写阶段)
local function type_bad(key, want)
local t = redis.call('TYPE', key)['ok']
return t ~= 'none' and t ~= want
end
if type_bad(KEYS[1], 'hash') then return 'PRECHECK_LOCK_TYPE' end
if type_bad(KEYS[2], 'stream') then return 'PRECHECK_STREAM_TYPE' end
if type_bad(KEYS[3], 'hash') then return 'PRECHECK_QUARANTINE_TYPE' end
if type_bad(KEYS[4], 'hash') then return 'PRECHECK_MARKER_TYPE' end
if type_bad(KEYS[5], 'string') then return 'PRECHECK_INDEX_TYPE' end
if ARGV[6] == '1' and type_bad(KEYS[6], 'hash') then
return 'PRECHECK_FENCE_TYPE'
end
-- ② holder 校验(类型已确认是 hash,hexists 安全)
if redis.call('hexists', KEYS[1], ARGV[1]) == 0 then
return 'NOT_HOLDER'
end
-- holder 仍在锁中不代表原分区任期仍有效;ACK 必须与来源捕获的 fence 一起校验。
if ARGV[6] == '1' then
if redis.call('HGET', KEYS[6], 'state') ~= 'Ready' then
return 'FENCE_NOT_READY'
end
if redis.call('HGET', KEYS[6], 'round') ~= ARGV[7] then
return 'ROUND_MISMATCH'
end
if redis.call('HGET', KEYS[6], 'nonce') ~= ARGV[8] then
return 'NONCE_MISMATCH'
end
if redis.call('HGET', KEYS[6], 'c:' .. ARGV[9]) ~= ARGV[10] then
return 'STALE_STAMP'
end
end
-- ③ marker 状态预检:幂等重入(同 park_id)直接成功;在途的异 park_id 处置拒绝覆盖
local mstate = redis.call('HGET', KEYS[4], 'state')
local mpark = redis.call('HGET', KEYS[4], 'park_id')
if mstate == 'Parked' and mpark == ARGV[3] then
return 1
end
if mstate ~= false and mstate ~= 'Resumed' and mstate ~= 'Dropped' and mstate ~= 'Dlqed' then
return 'PRECHECK_MARKER_STATE:' .. tostring(mstate)
end
if mstate == false and redis.call('EXISTS', KEYS[4]) ~= 0 then
return 'PRECHECK_MARKER_STATE_MISSING'
end
-- 版本必须在 ACK 前证明可递增;十进制字符串运算避免 Lua 浮点舍入和非法值降为零。
local next_ver = '1'
if mstate then
local previous = redis.call('HGET', KEYS[4], 'version') or '0'
if not string.match(previous, '^%d+$')
or (#previous > 1 and string.sub(previous, 1, 1) == '0')
or #previous > 19
or (#previous == 19 and previous >= '9223372036854775807') then
return 'PRECHECK_VERSION'
end
local digits = {}
local carry = 1
for i = #previous, 1, -1 do
local digit = string.byte(previous, i) - 48 + carry
if digit == 10 then
digits[i] = '0'
carry = 1
else
digits[i] = string.char(48 + digit)
carry = 0
end
end
next_ver = table.concat(digits)
if carry == 1 then
next_ver = '1' .. next_ver
end
end
-- ④ 目标 ID 仍在本组 PEL(读;不在 = 状态已变,拒绝)
local n = tonumber(ARGV[5])
for i = 0, n - 1 do
local id = ARGV[11 + i * 2]
local pend = redis.call('XPENDING', KEYS[2], ARGV[2], id, id, 1)
if #pend == 0 then
return 'NOT_IN_PEL:' .. id
end
end
-- ⑤ 转存(先于 ACK:源被确认前 payload 必有副本;同 key 重入 = 幂等覆盖)
for i = 0, n - 1 do
redis.call('HSET', KEYS[3], tostring(i), ARGV[12 + i * 2])
redis.call('HSET', KEYS[3], 'id:' .. tostring(i), ARGV[11 + i * 2])
end
redis.call('HSET', KEYS[3], 'n', ARGV[5])
-- ⑥ ACK 源(消息出 PEL;源 stream 此后可被 trim,payload 已安全)
for i = 0, n - 1 do
redis.call('XACK', KEYS[2], ARGV[2], ARGV[11 + i * 2])
end
-- 新批次不能继承终态的预约、操作字段或 TTL;版本已在写前按原值计算,跨批次继续递增。
if mstate then
redis.call('DEL', KEYS[4])
end
redis.call('HSET', KEYS[4], 'state', 'Parked', 'park_id', ARGV[3], 'n', ARGV[5], 'parked_at', ARGV[4], 'version', next_ver)
redis.call('SET', KEYS[5], ARGV[3])
return 1
"#;
pub async fn park(
client: &Arc<RedisClient>,
layout: &KeyLayout,
p: u32,
lease: &OwnerLease<'_>,
records: &[(String, Vec<u8>)],
) -> Result<String> {
if lease.partition != p {
return Err(NasaRedisError::OwnershipChanged(format!(
"分区 {p} 与 Park 持有权证据分区 {} 不一致",
lease.partition
)));
}
if records.is_empty() {
return Err(NasaRedisError::Config(format!(
"分区 {p} park 被拒:records 为空(不允许 park 空批,防分区冻结 + 毒消息静默丢失)"
)));
}
let park_id = uuid::Uuid::new_v4().simple().to_string();
let elapsed = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map_err(|_| NasaRedisError::Config("Park 墙钟早于 Unix epoch".into()))?;
let now_ms = u64::try_from(elapsed.as_millis())
.map_err(|_| NasaRedisError::Config("Park 墙钟毫秒超出 u64 范围".into()))?;
let lua = redis::Script::new(PARK_LUA);
let mut script = lua.prepare_invoke();
script
.key(lease.lock_key)
.key(layout.stream(p))
.key(quarantine_key(layout, &park_id))
.key(marker_key(layout, p))
.key(parked_index_key(layout, p))
.key(lease.lua_fence_key())
.arg(lease.holder)
.arg(layout.group())
.arg(&park_id)
.arg(now_ms)
.arg(records.len())
.arg(lease.has_fence())
.arg(lease.round)
.arg(lease.nonce)
.arg(p)
.arg(lease.counter);
for (id, payload) in records {
script.arg(id.as_str()).arg(payload.as_slice());
}
let r: redis::Value = script.invoke_async(&mut client.conn()).await?;
match r {
redis::Value::Int(1) => Ok(park_id),
other => Err(NasaRedisError::Config(format!(
"park 被拒绝(fenced 校验失败): {other:?}"
))),
}
}
pub async fn parked_id(
client: &Arc<RedisClient>,
layout: &KeyLayout,
p: u32,
) -> Result<Option<String>> {
match takeover_disposition(client, layout, p).await? {
TakeoverDisposition::NotParked => Ok(None),
TakeoverDisposition::Frozen {
park_id,
}
| TakeoverDisposition::AutoResumeDlq {
park_id,
..
} => Ok(Some(park_id)),
TakeoverDisposition::Inconsistent {
reason,
} => Err(NasaRedisError::ProtocolMarker(format!(
"分区 {p} disposition 状态不一致/损坏: {reason}"
))),
}
}
#[derive(Debug)]
pub enum TakeoverDisposition {
NotParked,
Frozen {
park_id: String,
},
AutoResumeDlq {
park_id: String,
operation_id: String,
},
Inconsistent {
reason: String,
},
}
const DISPOSITION_NON_TERMINAL: &[&str] = &[
"Parked",
"ResumePublishing",
"DlqPublishing",
"ForcePublishing",
"Dropping",
"ResumeIndeterminate",
"DlqIndeterminate",
];
const DISPOSITION_TERMINAL: &[&str] = &["Resumed", "Dlqed", "Dropped"];
const DISPOSITION_SNAPSHOT_LUA: &str = r#"
return {
redis.call('HGET', KEYS[1], 'state'),
redis.call('HGET', KEYS[1], 'park_id'),
redis.call('HGET', KEYS[1], 'transition_operation_id'),
redis.call('GET', KEYS[2])
}
"#;
pub async fn takeover_disposition(
client: &Arc<RedisClient>,
layout: &KeyLayout,
p: u32,
) -> Result<TakeoverDisposition> {
let (mstate, mpark, mop, index): (
Option<String>,
Option<String>,
Option<String>,
Option<String>,
) = redis::Script::new(DISPOSITION_SNAPSHOT_LUA)
.key(marker_key(layout, p))
.key(parked_index_key(layout, p))
.invoke_async(&mut client.conn())
.await?;
match mstate.as_deref() {
Some(s) if DISPOSITION_NON_TERMINAL.contains(&s) => {
let park_id = mpark.unwrap_or_default();
if park_id.is_empty() {
return Ok(TakeoverDisposition::Inconsistent {
reason: format!("disposition marker 非终态({s})但缺 park_id,无法恢复"),
});
}
match &index {
Some(idx) if *idx == park_id => {}
_ => tracing::warn!(
p,
marker_state = s,
?index,
marker_park = %park_id,
"disposition marker 非终态但 parked index 不一致(以 marker 为准冻结)"
),
}
if s == "DlqPublishing" {
if let Some(op) = mop.as_deref() {
if is_auto_op(op) {
return Ok(TakeoverDisposition::AutoResumeDlq {
park_id,
operation_id: op.to_string(),
});
}
}
}
Ok(TakeoverDisposition::Frozen { park_id })
}
Some(s) if DISPOSITION_TERMINAL.contains(&s) => {
if index.is_some() {
tracing::warn!(p, marker_state = s, ?index, "disposition marker 已终态但 parked index 仍在(陈旧,放行消费)");
}
Ok(TakeoverDisposition::NotParked)
}
Some(other) => Ok(TakeoverDisposition::Inconsistent {
reason: format!("未知 disposition marker 状态: {other}"),
}),
None => match index {
None => Ok(TakeoverDisposition::NotParked),
Some(idx) => Ok(TakeoverDisposition::Inconsistent {
reason: format!("parked index 存在(park_id={idx})但 disposition marker 缺失:Park 元数据可能部分丢失,需对账"),
}),
},
}
}
fn parse_sid(s: &str) -> Option<(u64, u64)> {
let (ms, seq) = s.split_once('-')?;
if ms.is_empty()
|| seq.is_empty()
|| !ms.bytes().all(|byte| byte.is_ascii_digit())
|| !seq.bytes().all(|byte| byte.is_ascii_digit())
{
return None;
}
Some((ms.parse().ok()?, seq.parse().ok()?))
}
fn plan_ids(base: (u64, u64), n: usize) -> Result<Vec<String>> {
let (mut ms, mut seq) = base;
let mut out = Vec::new();
out.try_reserve_exact(n)
.map_err(|_| NasaRedisError::ProtocolMarker("预约 ID 索引容量不可预留".into()))?;
for _ in 0..n {
if seq == u64::MAX {
if ms == u64::MAX {
return Err(NasaRedisError::Config(
"StreamIdExhausted: ID 空间耗尽".into(),
));
}
ms += 1;
seq = 0;
} else {
seq += 1;
}
out.push(format!("{ms}-{seq}"));
}
Ok(out)
}
fn xinfo_str(v: &redis::Value) -> Option<String> {
match v {
redis::Value::SimpleString(s) => Some(s.clone()),
redis::Value::BulkString(b) => std::str::from_utf8(b).ok().map(str::to_owned),
_ => None,
}
}
fn xinfo_field(v: redis::Value, field: &str) -> Option<String> {
let pairs: Vec<(redis::Value, redis::Value)> = match v {
redis::Value::Map(pairs) => pairs,
redis::Value::Array(items) => {
if items.len() % 2 != 0 {
return None;
}
let mut it = items.into_iter();
let mut out = Vec::new();
while let (Some(k), Some(val)) = (it.next(), it.next()) {
out.push((k, val));
}
out
}
_ => return None,
};
let mut found = None;
let mut names = std::collections::HashSet::new();
for (k, val) in pairs {
let name = xinfo_str(&k)?;
if !names.insert(name.clone()) {
return None;
}
if name == field {
found = Some(xinfo_str(&val)?);
}
}
found
}
async fn last_generated(client: &Arc<RedisClient>, stream: &str) -> Result<(u64, u64)> {
let v: redis::Value = match redis::cmd("XINFO")
.arg("STREAM")
.arg(stream)
.query_async(&mut client.conn())
.await
{
Ok(v) => v,
Err(e) if e.code() == Some("ERR") && e.detail() == Some("no such key") => {
return Ok((0, 0));
}
Err(e) => return Err(e.into()),
};
match &v {
redis::Value::Map(_) | redis::Value::Array(_) => {}
other => {
return Err(NasaRedisError::ProtocolMarker(format!(
"XINFO STREAM {stream} 响应形态异常(非 Map/Array): {other:?}"
)))
}
}
let last = xinfo_field(v, "last-generated-id");
match last {
Some(s) => parse_sid(&s).ok_or_else(|| {
NasaRedisError::ProtocolMarker(format!(
"XINFO STREAM {stream} 的 last-generated-id 不可解析: {s:?}"
))
}),
None => Err(NasaRedisError::ProtocolMarker(format!(
"XINFO STREAM {stream} 字段结构不完整、重复或缺少有效 last-generated-id"
))),
}
}
async fn reconcile_one(
client: &Arc<RedisClient>,
stream: &str,
planned: &str,
expect_payload: &[u8],
) -> Result<bool> {
let pl = parse_sid(planned).ok_or_else(|| {
NasaRedisError::ProtocolMarker(format!("reconcile:planned ID 不可解析: {planned:?}"))
})?;
let rows: redis::Value = redis::cmd("XRANGE")
.arg(stream)
.arg(planned)
.arg(planned)
.query_async(&mut client.conn())
.await?;
let redis::Value::Array(rows) = rows else {
return Err(NasaRedisError::ProtocolMarker(
"XRANGE 对账响应必须为记录数组".into(),
));
};
if !rows.is_empty() {
let [redis::Value::Array(pair)] = rows.as_slice() else {
return Err(NasaRedisError::ProtocolMarker(
"XRANGE 精确对账返回多条记录或非法记录结构".into(),
));
};
let [id, redis::Value::Array(fields)] = pair.as_slice() else {
return Err(NasaRedisError::ProtocolMarker(
"XRANGE 记录必须包含 ID 和完整字段列表".into(),
));
};
if xinfo_str(id).and_then(|id| parse_sid(&id)) != Some(pl)
|| fields.is_empty()
|| fields.len() % 2 != 0
{
return Err(NasaRedisError::ProtocolMarker(
"XRANGE 记录 ID 不符或字段列表不完整".into(),
));
}
let mut names = std::collections::HashSet::new();
let mut payload_matches = None;
for field in fields.chunks_exact(2) {
let name = match &field[0] {
redis::Value::BulkString(bytes) => bytes.as_slice(),
redis::Value::SimpleString(value) => value.as_bytes(),
_ => {
return Err(NasaRedisError::ProtocolMarker(
"XRANGE 字段名不是字节字符串".into(),
));
}
};
let value = match &field[1] {
redis::Value::BulkString(bytes) => bytes.as_slice(),
redis::Value::SimpleString(value) => value.as_bytes(),
_ => {
return Err(NasaRedisError::ProtocolMarker(
"XRANGE 字段值不是字节字符串".into(),
));
}
};
if !names.insert(name) {
return Err(NasaRedisError::ProtocolMarker(
"XRANGE 对账记录包含重复字段".into(),
));
}
if name == super::DATA_FIELD.as_bytes() {
payload_matches = Some(value == expect_payload);
}
}
return payload_matches
.ok_or_else(|| NasaRedisError::ProtocolMarker("XRANGE 对账记录缺少 data 字段".into()));
}
let lg = last_generated(client, stream).await?;
if lg < pl {
return Err(NasaRedisError::Config("RETRYABLE".into())); }
Ok(false)
}
pub async fn resume(
client: &Arc<RedisClient>,
layout: &KeyLayout,
p: u32,
lease: &OwnerLease<'_>,
) -> Result<Vec<String>> {
if lease.partition != p {
return Err(NasaRedisError::OwnershipChanged(format!(
"分区 {p} 与持有权证据分区 {} 不一致",
lease.partition
)));
}
verify_owner_fenced(client, lease).await?;
let marker = marker_key(layout, p);
let state: Option<String> = client.h_get(&marker, "state").await?;
let park_id: Option<String> = client.h_get(&marker, "park_id").await?;
let (Some(state), Some(park_id)) = (state, park_id) else {
return Err(NasaRedisError::Config(format!("分区 {p} 无 Park 记录")));
};
if lease
.expected_park_id
.is_some_and(|expected| expected != park_id.as_str())
{
return Err(NasaRedisError::OperationConflict(format!(
"分区 {p} 恢复续作的 Park 批次已变化"
)));
}
let stream = layout.stream(p);
let qkey = quarantine_key(layout, &park_id);
match state.as_str() {
"Parked" => {
const CLAIM_LUA: &str = r#"
redis.call('HINCRBY', KEYS[1], 'version', 1)
redis.call('HSET', KEYS[1], 'state', 'ResumePublishing', 'owner_holder', ARGV[1], 'transition_operation_id', ARGV[8])
-- 从持久字符串返回精确计数,避免 Lua 数值转换丢失大整数精度或产生科学计数法。
return {'OK', redis.call('HGET', KEYS[1], 'version')}
"#;
let _: u64 = redis::from_redis_value(guarded_disposition(
client, layout, p, lease, &park_id, "Parked", &stream, CLAIM_LUA,
&[] as &[&str],
).await?)?;
publish_step(
client, layout, p, &marker, &qkey, &park_id, &stream,
"Resumed", "ResumeIndeterminate", lease,
)
.await
}
"ResumePublishing" => {
assert_op_owns(client, &marker, lease.operation_id).await?;
publish_step(
client, layout, p, &marker, &qkey, &park_id, &stream,
"Resumed", "ResumeIndeterminate", lease,
)
.await
}
"ResumeIndeterminate" => Err(NasaRedisError::OperationInDoubt(format!(
"分区 {p} 处于 ResumeIndeterminate:普通 resume 被拒,需 force_republish(接受重复风险)或人工对账"
))),
"Resumed" => {
const CLEANUP_LUA: &str = r#"
local ids = redis.call('HGET', KEYS[1], 'new_ids') or ''
redis.call('DEL', KEYS[4], KEYS[5])
redis.call('EXPIRE', KEYS[1], 86400)
return {'OK', ids}
"#;
let ids: String = redis::from_redis_value(guarded_disposition(
client, layout, p, lease, &park_id, "Resumed", &stream, CLEANUP_LUA,
&[] as &[&str],
).await?)?;
Ok(ids.split(',').filter(|s| !s.is_empty()).map(String::from).collect())
}
other => Err(NasaRedisError::Config(format!("分区 {p} marker 状态 {other},拒绝 resume"))),
}
}
#[allow(clippy::too_many_arguments)]
async fn publish_step(
client: &Arc<RedisClient>,
layout: &KeyLayout,
p: u32,
marker: &str,
qkey: &str,
park_id: &str,
target_stream: &str,
final_state: &str,
indeterminate_state: &str,
lease: &OwnerLease<'_>,
) -> Result<Vec<String>> {
let publishing_state = match (final_state, indeterminate_state) {
("Resumed", "ResumeIndeterminate") => "ResumePublishing",
("Dlqed", "DlqIndeterminate") => "DlqPublishing",
_ => {
return Err(NasaRedisError::ProtocolMarker(
"预约准备的状态组合不合法".into(),
));
}
};
let planned_str: Option<String> = client.h_get(marker, "planned").await?;
let planned_json = match planned_str.as_deref().filter(|s| !s.is_empty()) {
Some(s) => s.to_owned(),
None => {
let payloads = load_quarantine_payloads(client, qkey, p, park_id).await?;
let planned = plan_ids(last_generated(client, target_stream).await?, payloads.len())?;
let planned_json = serde_json::to_string(&planned).unwrap();
const PERSIST_PLAN_LUA: &str = r#"
redis.call('HSETNX', KEYS[1], 'planned', ARGV[10])
return {'OK', redis.call('HGET', KEYS[1], 'planned')}
"#;
redis::from_redis_value(
guarded_disposition(
client,
layout,
p,
lease,
park_id,
publishing_state,
target_stream,
PERSIST_PLAN_LUA,
&planned_json,
)
.await?,
)?
}
};
let planned: Vec<String> = serde_json::from_str(&planned_json)
.map_err(|e| NasaRedisError::Codec(format!("planned 解析失败: {e}")))?;
let payloads = load_quarantine_payloads_for(client, qkey, p, park_id, planned.len()).await?;
publish_reserved(
client,
layout,
p,
park_id,
lease,
&planned_json,
&planned,
&payloads,
target_stream,
final_state,
indeterminate_state,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn publish_reserved(
client: &Arc<RedisClient>,
layout: &KeyLayout,
p: u32,
park_id: &str,
lease: &OwnerLease<'_>,
planned_json: &str,
planned: &[String],
payloads: &[Vec<u8>],
target_stream: &str,
final_state: &str,
indeterminate_state: &str,
) -> Result<Vec<String>> {
let stream = target_stream.to_string();
let publishing_state = match (final_state, indeterminate_state) {
("Resumed", "ResumeIndeterminate") => "ResumePublishing",
("Dlqed", "DlqIndeterminate") => "DlqPublishing",
_ => {
return Err(NasaRedisError::ProtocolMarker(
"预约发布的状态组合不合法".into(),
));
}
};
let mut uncertain: Vec<usize> = Vec::new();
for (i, (pid, payload)) in planned.iter().zip(payloads.iter()).enumerate() {
match reconcile_one(client, &stream, pid, payload).await {
Ok(true) => continue, Ok(false) => {
uncertain.push(i);
continue;
}
Err(NasaRedisError::Config(sig)) if sig == "RETRYABLE" => {
const PUBLISH_LUA: &str = r#"
if redis.call('HGET', KEYS[1], 'planned') ~= ARGV[10] then
return {'CONFLICT', 'PLANNED'}
end
return {'OK', redis.call('XADD', KEYS[6], ARGV[11], ARGV[12], ARGV[13])}
"#;
let r = guarded_disposition(
client,
layout,
p,
lease,
park_id,
publishing_state,
&stream,
PUBLISH_LUA,
(planned_json, pid, super::DATA_FIELD, payload.as_slice()),
)
.await;
match r {
Ok(value) => {
let id: String = redis::from_redis_value(value)?;
if parse_sid(&id) != parse_sid(pid) {
return Err(NasaRedisError::ProtocolMarker(
"预约发布响应与目标 ID 不一致".into(),
));
}
continue;
}
Err(NasaRedisError::Redis(e))
if e.is_io_error() || e.is_timeout() || e.is_connection_dropped() =>
{
return Err(NasaRedisError::ExecutionUnknown(format!(
"分区 {p} 发布 planned={pid} 写出后断线: {e}"
)));
}
Err(NasaRedisError::Redis(_)) => {
match reconcile_one(client, &stream, pid, payload).await {
Ok(true) => continue,
Ok(false) => {
uncertain.push(i);
continue;
}
Err(e) => return Err(e), }
}
Err(e) => return Err(e),
}
}
Err(e) => return Err(e), }
}
if !uncertain.is_empty() {
const UNCERTAIN_LUA: &str = r#"
if redis.call('HGET', KEYS[1], 'planned') ~= ARGV[10] then
return {'CONFLICT', 'PLANNED'}
end
redis.call('HSET', KEYS[1], 'uncertain', ARGV[11], 'state', ARGV[12])
return {'OK', 1}
"#;
guarded_disposition(
client,
layout,
p,
lease,
park_id,
publishing_state,
&stream,
UNCERTAIN_LUA,
(
planned_json,
serde_json::to_string(&uncertain).unwrap(),
indeterminate_state,
),
)
.await?;
return Err(NasaRedisError::PublishIndeterminate(format!(
"分区 {p} 有 {} 条 reservation 结果不确定(indexes={uncertain:?}):不自动重发,需 force_republish 或人工对账",
uncertain.len()
)));
}
const FINISH_LUA: &str = r#"
if redis.call('HGET', KEYS[1], 'planned') ~= ARGV[10] then
return {'CONFLICT', 'PLANNED'}
end
redis.call('HSET', KEYS[1], 'new_ids', ARGV[11], 'state', ARGV[12])
redis.call('DEL', KEYS[4], KEYS[5])
redis.call('EXPIRE', KEYS[1], 86400)
return {'OK', 1}
"#;
guarded_disposition(
client,
layout,
p,
lease,
park_id,
publishing_state,
&stream,
FINISH_LUA,
(planned_json, planned.join(","), final_state),
)
.await?;
Ok(planned.to_vec())
}
pub fn dlq_stream_key(layout: &KeyLayout) -> String {
format!("{}:dlq", layout.prefix)
}
pub async fn dlq_from_parked(
client: &Arc<RedisClient>,
layout: &KeyLayout,
p: u32,
lease: &OwnerLease<'_>,
) -> Result<Vec<String>> {
if lease.partition != p {
return Err(NasaRedisError::OwnershipChanged(format!(
"分区 {p} 与持有权证据分区 {} 不一致",
lease.partition
)));
}
verify_owner_fenced(client, lease).await?; let marker = marker_key(layout, p);
let state: Option<String> = client.h_get(&marker, "state").await?;
let park_id: Option<String> = client.h_get(&marker, "park_id").await?;
let (Some(state), Some(park_id)) = (state, park_id) else {
return Err(NasaRedisError::Config(format!("分区 {p} 无 Park 记录")));
};
if lease
.expected_park_id
.is_some_and(|expected| expected != park_id.as_str())
{
return Err(NasaRedisError::OperationConflict(format!(
"分区 {p} DLQ 续作的 Park 批次已变化"
)));
}
let dlq = dlq_stream_key(layout);
let qkey = quarantine_key(layout, &park_id);
match state.as_str() {
"Parked" => {
const CLAIM_LUA: &str = r#"
redis.call('HINCRBY', KEYS[1], 'version', 1)
redis.call('HSET', KEYS[1], 'state', 'DlqPublishing', 'owner_holder', ARGV[1], 'transition_operation_id', ARGV[8])
-- Redis 保存的十进制字符串保留完整整数,不能由 Lua 浮点返回值重建认领计数。
return {'OK', redis.call('HGET', KEYS[1], 'version')}
"#;
let _: u64 = redis::from_redis_value(
guarded_disposition(
client,
layout,
p,
lease,
&park_id,
"Parked",
&dlq,
CLAIM_LUA,
&[] as &[&str],
)
.await?,
)?;
publish_step(
client,
layout,
p,
&marker,
&qkey,
&park_id,
&dlq,
"Dlqed",
"DlqIndeterminate",
lease,
)
.await
}
"DlqPublishing" => {
assert_op_owns(client, &marker, lease.operation_id).await?;
publish_step(
client,
layout,
p,
&marker,
&qkey,
&park_id,
&dlq,
"Dlqed",
"DlqIndeterminate",
lease,
)
.await
}
"DlqIndeterminate" => Err(NasaRedisError::OperationInDoubt(format!(
"分区 {p} 处于 DlqIndeterminate:需人工对账(DLQ 重复风险评估后手工处置)"
))),
"Dlqed" => {
const CLEANUP_LUA: &str = r#"
local ids = redis.call('HGET', KEYS[1], 'new_ids') or ''
redis.call('DEL', KEYS[4], KEYS[5])
redis.call('EXPIRE', KEYS[1], 86400)
return {'OK', ids}
"#;
let ids: String = redis::from_redis_value(
guarded_disposition(
client,
layout,
p,
lease,
&park_id,
"Dlqed",
&dlq,
CLEANUP_LUA,
&[] as &[&str],
)
.await?,
)?;
Ok(ids
.split(',')
.filter(|s| !s.is_empty())
.map(String::from)
.collect())
}
other => Err(NasaRedisError::Config(format!(
"分区 {p} marker 状态 {other},拒绝 dlq"
))),
}
}
pub async fn force_republish(
client: &Arc<RedisClient>,
layout: &KeyLayout,
p: u32,
lease: &OwnerLease<'_>,
) -> Result<Vec<String>> {
if lease.partition != p {
return Err(NasaRedisError::OwnershipChanged(format!(
"分区 {p} 与持有权证据分区 {} 不一致",
lease.partition
)));
}
verify_owner_fenced(client, lease).await?; let marker = marker_key(layout, p);
let state: Option<String> = client.h_get(&marker, "state").await?;
let st = state.as_deref();
if !matches!(st, Some("ResumeIndeterminate") | Some("ForcePublishing")) {
return Err(NasaRedisError::Config(format!(
"分区 {p} 非 ResumeIndeterminate(当前 {st:?}),force_republish 被拒"
)));
}
if st == Some("ForcePublishing") {
assert_op_owns(client, &marker, lease.operation_id).await?;
}
let park_id: String = client.h_get(&marker, "park_id").await?.unwrap_or_default();
if park_id.is_empty() {
return Err(NasaRedisError::ProtocolMarker(format!(
"分区 {p} force: marker 缺 park_id(损坏),拒绝(fail-closed)"
)));
}
if lease
.expected_park_id
.is_some_and(|expected| expected != park_id.as_str())
{
return Err(NasaRedisError::OperationConflict(format!(
"分区 {p} 强制重发续作的 Park 批次已变化"
)));
}
let qkey = quarantine_key(layout, &park_id);
let planned_json: String = client.h_get(&marker, "planned").await?.ok_or_else(|| {
NasaRedisError::ProtocolMarker(format!("分区 {p} force: marker 缺 planned 字段"))
})?;
let planned: Vec<String> = serde_json::from_str(&planned_json).map_err(|e| {
NasaRedisError::ProtocolMarker(format!("分区 {p} force: planned 损坏不可解析: {e}"))
})?;
let uncertain_json: String = client.h_get(&marker, "uncertain").await?.ok_or_else(|| {
NasaRedisError::ProtocolMarker(format!("分区 {p} force: marker 缺 uncertain 字段"))
})?;
let uncertain: Vec<usize> = serde_json::from_str(&uncertain_json).map_err(|e| {
NasaRedisError::ProtocolMarker(format!("分区 {p} force: uncertain 损坏不可解析: {e}"))
})?;
if uncertain.is_empty() {
return Err(NasaRedisError::ProtocolMarker(format!(
"分区 {p} force: uncertain 为空,不应进入 force(marker 语义不一致)"
)));
}
let mut seen = std::collections::HashSet::with_capacity(uncertain.len());
for &i in &uncertain {
if i >= planned.len() {
return Err(NasaRedisError::ProtocolMarker(format!(
"分区 {p} force: uncertain index {i} 越界(planned.len={}),marker 损坏,拒绝",
planned.len()
)));
}
if !seen.insert(i) {
return Err(NasaRedisError::ProtocolMarker(format!(
"分区 {p} force: uncertain 含重复 index {i},marker 损坏,拒绝(fail-closed)"
)));
}
}
let mut payloads: Vec<(usize, Vec<u8>)> = Vec::with_capacity(uncertain.len());
for &i in &uncertain {
let payload: Vec<u8> = client.h_get::<Vec<u8>>(&qkey, &i.to_string()).await?.ok_or_else(|| {
NasaRedisError::ProtocolMarker(format!(
"quarantine 损坏:分区 {p} park_id={park_id} 缺第 {i} 条 payload,force 拒绝(fail-closed)"
))
})?;
payloads.push((i, payload));
}
let stream = layout.stream(p);
if st == Some("ResumeIndeterminate") {
const CLAIM_FORCE_LUA: &str = r#"
if redis.call('HGET', KEYS[1], 'planned') ~= ARGV[10]
or redis.call('HGET', KEYS[1], 'uncertain') ~= ARGV[11] then
return {'CONFLICT', 'FORCE_PLAN'}
end
redis.call('HINCRBY', KEYS[1], 'version', 1)
redis.call('HSET', KEYS[1], 'state', 'ForcePublishing', 'owner_holder', ARGV[1], 'transition_operation_id', ARGV[8])
-- 返回持久整数原文,避免大计数已认领后才因科学计数法无法解析而中断。
return {'OK', redis.call('HGET', KEYS[1], 'version')}
"#;
let _: u64 = redis::from_redis_value(
guarded_disposition(
client,
layout,
p,
lease,
&park_id,
"ResumeIndeterminate",
&stream,
CLAIM_FORCE_LUA,
(&planned_json, &uncertain_json),
)
.await?,
)?;
}
let mut final_ids = planned.clone();
let mut audit: Vec<(usize, String, String)> = Vec::new(); let mut expected_targets: Vec<(String, Option<String>)> = Vec::new();
const FORCE_PLAN_LUA: &str = r#"
if redis.call('HGET', KEYS[1], 'planned') ~= ARGV[10]
or redis.call('HGET', KEYS[1], 'uncertain') ~= ARGV[11] then
return {'CONFLICT', 'FORCE_PLAN'}
end
local current = redis.call('HGET', KEYS[1], ARGV[12])
if (ARGV[13] == '0' and current) or (ARGV[13] == '1' and current ~= ARGV[14]) then
return {'CONFLICT', 'FORCE_TARGET'}
end
redis.call('HSET', KEYS[1], ARGV[12], ARGV[15])
return {'OK', 1}
"#;
for (i, payload) in &payloads {
let i = *i;
let item_field = format!("force:{i}");
if let Some(target) = client.h_get::<String>(&marker, &item_field).await? {
match reconcile_one(client, &stream, &target, payload).await {
Ok(true) => {
expected_targets.push((item_field, Some(target.clone())));
final_ids[i] = target;
continue;
}
Err(NasaRedisError::Config(sig)) if sig == "RETRYABLE" => {
xadd_force_target(
client,
layout,
p,
lease,
&park_id,
&planned_json,
&uncertain_json,
&stream,
&target,
payload,
i,
)
.await?;
expected_targets.push((item_field, Some(target.clone())));
final_ids[i] = target;
continue;
}
Ok(false) => {
let base = last_generated(client, &stream).await?;
let fresh = plan_ids(base, 1)?.remove(0);
guarded_disposition(
client,
layout,
p,
lease,
&park_id,
"ForcePublishing",
&stream,
FORCE_PLAN_LUA,
(
&planned_json,
&uncertain_json,
&item_field,
"1",
&target,
&fresh,
),
)
.await?;
xadd_force_target(
client,
layout,
p,
lease,
&park_id,
&planned_json,
&uncertain_json,
&stream,
&fresh,
payload,
i,
)
.await?;
audit.push((i, target.clone(), fresh.clone()));
expected_targets.push((item_field, Some(fresh.clone())));
final_ids[i] = fresh;
continue;
}
Err(e) => return Err(e),
}
}
let target = match reconcile_one(client, &stream, &planned[i], payload).await {
Ok(true) => {
expected_targets.push((item_field, None));
final_ids[i] = planned[i].clone();
continue; }
Err(NasaRedisError::Config(sig)) if sig == "RETRYABLE" => planned[i].clone(),
Ok(false) => {
let base = last_generated(client, &stream).await?;
plan_ids(base, 1)?.remove(0)
}
Err(e) => return Err(e), };
guarded_disposition(
client,
layout,
p,
lease,
&park_id,
"ForcePublishing",
&stream,
FORCE_PLAN_LUA,
(
&planned_json,
&uncertain_json,
&item_field,
"0",
"",
&target,
),
)
.await?;
xadd_force_target(
client,
layout,
p,
lease,
&park_id,
&planned_json,
&uncertain_json,
&stream,
&target,
payload,
i,
)
.await?;
if target != planned[i] {
audit.push((i, planned[i].clone(), target.clone()));
}
expected_targets.push((item_field, Some(target.clone())));
final_ids[i] = target;
}
const FINISH_FORCE_LUA: &str = r#"
if redis.call('HGET', KEYS[1], 'planned') ~= ARGV[10]
or redis.call('HGET', KEYS[1], 'uncertain') ~= ARGV[11] then
return {'CONFLICT', 'FORCE_PLAN'}
end
local targets = cjson.decode(ARGV[12])
for _, target in ipairs(targets) do
local current = redis.call('HGET', KEYS[1], target[1])
if (target[2] == cjson.null and current)
or (target[2] ~= cjson.null and current ~= target[2]) then
return {'CONFLICT', 'FORCE_TARGET'}
end
end
redis.call('HSET', KEYS[1], 'forced', '1', 'force_audit', ARGV[13], 'new_ids', ARGV[14], 'state', 'Resumed')
redis.call('DEL', KEYS[4], KEYS[5])
redis.call('EXPIRE', KEYS[1], 86400)
return {'OK', 1}
"#;
guarded_disposition(
client,
layout,
p,
lease,
&park_id,
"ForcePublishing",
&stream,
FINISH_FORCE_LUA,
(
&planned_json,
&uncertain_json,
serde_json::to_string(&expected_targets).unwrap(),
serde_json::to_string(&audit).unwrap(),
final_ids.join(","),
),
)
.await?;
tracing::warn!(p, ?audit, "force_republish 完成(可能重复/变序,审计已记)");
Ok(final_ids)
}
#[allow(clippy::too_many_arguments)]
async fn xadd_force_target(
client: &Arc<RedisClient>,
layout: &KeyLayout,
p: u32,
lease: &OwnerLease<'_>,
park_id: &str,
planned_json: &str,
uncertain_json: &str,
stream: &str,
target: &str,
payload: &[u8],
i: usize,
) -> Result<()> {
const FORCE_XADD_LUA: &str = r#"
if redis.call('HGET', KEYS[1], 'planned') ~= ARGV[10]
or redis.call('HGET', KEYS[1], 'uncertain') ~= ARGV[11]
or redis.call('HGET', KEYS[1], ARGV[12]) ~= ARGV[13] then
return {'CONFLICT', 'FORCE_TARGET'}
end
return {'OK', redis.call('XADD', KEYS[6], ARGV[13], ARGV[14], ARGV[15])}
"#;
let r = guarded_disposition(
client,
layout,
p,
lease,
park_id,
"ForcePublishing",
stream,
FORCE_XADD_LUA,
(
planned_json,
uncertain_json,
format!("force:{i}"),
target,
super::DATA_FIELD,
payload,
),
)
.await;
match r {
Ok(value) => {
let id: String = redis::from_redis_value(value)?;
if parse_sid(&id) != parse_sid(target) {
return Err(NasaRedisError::ProtocolMarker(
"强制发布响应与目标 ID 不一致".into(),
));
}
Ok(())
}
Err(NasaRedisError::Redis(e)) if e.is_io_error() || e.is_timeout() || e.is_connection_dropped() => {
Err(NasaRedisError::ExecutionUnknown(format!(
"force XADD target={target}(index {i})写出后断线: {e}"
)))
}
Err(NasaRedisError::Redis(_)) => match reconcile_one(client, stream, target, payload).await {
Ok(true) => Ok(()), Ok(false) => Err(NasaRedisError::PublishIndeterminate(format!(
"force index {i}: target={target} 被越过且无法发布(真不确定)——marker 停 ForcePublishing,需 operator 介入(force_takeover / 人工对账)"
))),
Err(e) => Err(e),
},
Err(e) => Err(e),
}
}
pub async fn drop_parked(
client: &Arc<RedisClient>,
layout: &KeyLayout,
p: u32,
lease: &OwnerLease<'_>,
) -> Result<()> {
if lease.partition != p {
return Err(NasaRedisError::OwnershipChanged(format!(
"分区 {p} 与持有权证据分区 {} 不一致",
lease.partition
)));
}
let marker = marker_key(layout, p);
let park_id: Option<String> = client.h_get(&marker, "park_id").await?;
let Some(park_id) = park_id.filter(|id| !id.is_empty()) else {
return Err(NasaRedisError::Config(format!("分区 {p} 无 Park 记录")));
};
if lease
.expected_park_id
.is_some_and(|expected| expected != park_id.as_str())
{
return Err(NasaRedisError::OperationConflict(format!(
"分区 {p} 丢弃续作的 Park 批次已变化"
)));
}
const DROP_FENCED_LUA: &str = r#"
-- KEYS 为 marker、锁、fence、隔离区、索引;ARGV 为 holder、fence 标记、轮次、nonce、分区、计数、批次、操作和阶段。
-- Lua 出错不回滚,先检查全部键类型和身份,避免删除后才因损坏字段中止。
for i = 1, 5 do
local kind = redis.call('TYPE', KEYS[i])['ok']
local expected = 'hash'
if i == 5 then
expected = 'string'
end
if kind ~= 'none' and kind ~= expected then
return 'INVALID_TYPE'
end
end
if redis.call('HEXISTS', KEYS[2], ARGV[1]) == 0 then
return 'NOT_HOLDER'
end
if ARGV[2] == '1' then
if redis.call('HGET', KEYS[3], 'state') ~= 'Ready' then
return 'FENCE_NOT_READY'
end
if redis.call('HGET', KEYS[3], 'round') ~= ARGV[3] then
return 'ROUND_MISMATCH'
end
if redis.call('HGET', KEYS[3], 'nonce') ~= ARGV[4] then
return 'NONCE_MISMATCH'
end
if redis.call('HGET', KEYS[3], 'c:' .. ARGV[5]) ~= ARGV[6] then
return 'STALE_STAMP'
end
end
-- 当前批次和索引必须与捕获身份一致;缺索引允许终态清理或中断续作,异批次索引绝不能删除。
if redis.call('HGET', KEYS[1], 'park_id') ~= ARGV[7] then
return 'BATCH_CONFLICT'
end
local index = redis.call('GET', KEYS[5])
if index and index ~= ARGV[7] then
return 'INDEX_CONFLICT'
end
local state = redis.call('HGET', KEYS[1], 'state')
local operation = redis.call('HGET', KEYS[1], 'transition_operation_id')
if state ~= 'Dropping' and state ~= 'Dropped' and not (state == 'Parked' and ARGV[9] == 'claim') then
return 'STATE_CONFLICT'
end
-- 旧记录缺操作身份时保留续作兼容;已登记的在途操作必须匹配,终态允许当前 owner 补清残留。
if state == 'Dropping' and operation and operation ~= '' and operation ~= ARGV[8] then
return 'OPERATION_CONFLICT'
end
if ARGV[9] == 'claim' then
if state == 'Parked' then
-- 负计数不能被认领隐式转成合法批次,先拒绝,保留恢复数据与原状态。
local version = redis.call('HGET', KEYS[1], 'version')
if version and string.sub(version, 1, 1) == '-' then
return 'INVALID_VERSION'
end
redis.call('HINCRBY', KEYS[1], 'version', 1)
redis.call('HSET', KEYS[1], 'state', 'Dropping', 'owner_holder', ARGV[1], 'transition_operation_id', ARGV[8])
end
return 'OK'
end
if ARGV[9] ~= 'finish' then
return 'INVALID_PHASE'
end
-- 权威复验与终态、删除、TTL 同脚本执行;先写终态,避免字段写入失败发生在恢复数据删除之后。
redis.call('HSET', KEYS[1], 'state', 'Dropped')
redis.call('DEL', KEYS[4], KEYS[5])
redis.call('EXPIRE', KEYS[1], 86400)
return 'OK'
"#;
for phase in ["claim", "finish"] {
let tag: String = redis::Script::new(DROP_FENCED_LUA)
.key(&marker)
.key(lease.lock_key)
.key(lease.lua_fence_key())
.key(quarantine_key(layout, &park_id))
.key(parked_index_key(layout, p))
.arg(lease.holder)
.arg(lease.has_fence())
.arg(lease.round.to_string())
.arg(lease.nonce)
.arg(p.to_string())
.arg(lease.counter.to_string())
.arg(&park_id)
.arg(lease.operation_id)
.arg(phase)
.invoke_async(&mut client.conn())
.await?;
match tag.as_str() {
"OK" => {}
"NOT_HOLDER" | "FENCE_NOT_READY" | "ROUND_MISMATCH" | "NONCE_MISMATCH"
| "STALE_STAMP" => {
return Err(NasaRedisError::OwnershipChanged(format!(
"分区 {p} 丢弃被持有权校验拒绝({tag})"
)));
}
"BATCH_CONFLICT" | "INDEX_CONFLICT" | "STATE_CONFLICT" | "OPERATION_CONFLICT" => {
return Err(NasaRedisError::OperationConflict(format!(
"分区 {p} 丢弃与当前处置记录冲突({tag})"
)));
}
_ => {
return Err(NasaRedisError::ProtocolMarker(format!(
"分区 {p} 丢弃记录或脚本响应异常({tag})"
)));
}
}
}
tracing::warn!(p, park_id, "Parked 消息已 Drop(审计 marker 保留 24h)");
Ok(())
}
pub async fn force_takeover(
client: &Arc<RedisClient>,
layout: &KeyLayout,
p: u32,
lease: &OwnerLease<'_>,
) -> Result<Vec<String>> {
if lease.partition != p {
return Err(NasaRedisError::OwnershipChanged(format!(
"分区 {p} 与持有权证据分区 {} 不一致",
lease.partition
)));
}
verify_owner_fenced(client, lease).await?;
let marker = marker_key(layout, p);
let (state, park_id, operation): (Option<String>, Option<String>, Option<String>) =
redis::cmd("HMGET")
.arg(&marker)
.arg(&["state", "park_id", "transition_operation_id"])
.query_async(&mut client.conn())
.await?;
let (Some(state), Some(park_id)) = (state, park_id.filter(|id| !id.is_empty())) else {
return Err(NasaRedisError::Config(format!(
"分区 {p} 缺少完整 marker,无在途转换可接管"
)));
};
if lease
.expected_park_id
.is_some_and(|expected| expected != park_id.as_str())
{
return Err(NasaRedisError::OperationConflict(format!(
"分区 {p} 接管续作的 Park 批次已变化"
)));
}
if !matches!(
state.as_str(),
"ResumePublishing" | "DlqPublishing" | "ForcePublishing" | "Dropping"
) {
return Err(NasaRedisError::Config(format!(
"分区 {p} state={state} 非可接管的在途 *Publishing/Dropping\
(Parked 用 resume/drop/dlq;*Indeterminate 需人工对账)"
)));
}
const TAKEOVER_LUA: &str = r#"
if redis.call('HEXISTS', KEYS[2], ARGV[3]) == 0 then
return 'NOT_HOLDER'
end
if ARGV[4] == '1' then
if redis.call('HGET', KEYS[3], 'state') ~= 'Ready' then
return 'FENCE_NOT_READY'
end
if redis.call('HGET', KEYS[3], 'round') ~= ARGV[5] then
return 'ROUND_MISMATCH'
end
if redis.call('HGET', KEYS[3], 'nonce') ~= ARGV[6] then
return 'NONCE_MISMATCH'
end
if redis.call('HGET', KEYS[3], 'c:' .. ARGV[7]) ~= ARGV[8] then
return 'STALE_STAMP'
end
end
-- 缺字段与旧空操作身份兼容,但已知身份被替换时必须重新取得接管依据。
if redis.call('HGET', KEYS[1], 'state') ~= ARGV[1]
or redis.call('HGET', KEYS[1], 'park_id') ~= ARGV[9]
or (redis.call('HGET', KEYS[1], 'transition_operation_id') or '') ~= ARGV[10] then
return 'CONFLICT'
end
local index = redis.call('GET', KEYS[4])
if index and index ~= ARGV[9] then
return 'CONFLICT'
end
-- 所有可能失败的读取在移交之前完成,防止损坏索引导致身份已覆写却无法继续。
redis.call('HSET', KEYS[1], 'transition_operation_id', ARGV[2])
return 'OK'
"#;
let tag: String = redis::Script::new(TAKEOVER_LUA)
.key(&marker)
.key(lease.lock_key)
.key(lease.lua_fence_key())
.key(parked_index_key(layout, p))
.arg(&state)
.arg(lease.operation_id)
.arg(lease.holder)
.arg(lease.has_fence())
.arg(lease.round.to_string())
.arg(lease.nonce)
.arg(p.to_string())
.arg(lease.counter.to_string())
.arg(&park_id)
.arg(operation.as_deref().unwrap_or_default())
.invoke_async(&mut client.conn())
.await?;
match tag.as_str() {
"OK" => {}
"NOT_HOLDER" | "FENCE_NOT_READY" | "ROUND_MISMATCH" | "NONCE_MISMATCH" | "STALE_STAMP" => {
return Err(NasaRedisError::OwnershipChanged(format!(
"分区 {p} 接管被持有权校验拒绝({tag})"
)));
}
"CONFLICT" => {
return Err(NasaRedisError::OperationConflict(format!(
"分区 {p} 接管期间处置身份或索引已变化"
)));
}
_ => {
return Err(NasaRedisError::ProtocolMarker(format!(
"分区 {p} 接管脚本响应异常({tag})"
)));
}
}
tracing::warn!(p, %state, new_op = lease.operation_id, "通用 liveness-takeover:已覆写 op,续作在途转换");
let continuation = OwnerLease {
partition: lease.partition,
operation_id: lease.operation_id,
expected_park_id: Some(&park_id),
holder: lease.holder,
lock_key: lease.lock_key,
fence_key: lease.fence_key,
round: lease.round,
nonce: lease.nonce,
counter: lease.counter,
};
match state.as_str() {
"ResumePublishing" => resume(client, layout, p, &continuation).await,
"DlqPublishing" => dlq_from_parked(client, layout, p, &continuation).await,
"ForcePublishing" => force_republish(client, layout, p, &continuation).await,
"Dropping" => {
drop_parked(client, layout, p, &continuation).await?;
Ok(Vec::new())
}
_ => unreachable!(),
}
}