use std::sync::Arc;
use super::KeyLayout;
use crate::client::RedisClient;
use crate::error::{NasaRedisError, Result};
pub fn marker_key(layout: &KeyLayout, p: u32) -> String {
format!("{}:disposition:{}", layout.prefix, p)
}
pub fn quarantine_key(layout: &KeyLayout, park_id: &str) -> String {
format!("{}:quarantine:{}", layout.prefix, park_id)
}
pub fn parked_index_key(layout: &KeyLayout, p: u32) -> String {
format!("{}:parked:{}", layout.prefix, p)
}
const CLAIM_TRANSITION_FENCED_LUA: &str = r#"
if redis.call('hexists', KEYS[2], ARGV[3]) == 0 then return {'NOT_HOLDER', ''} end
if ARGV[4] == '1' then
if redis.call('HGET', KEYS[3], 'state') ~= 'Ready' then return {'FENCE_NOT_READY', ''} end
if redis.call('HGET', KEYS[3], 'round') ~= ARGV[5] then return {'ROUND_MISMATCH', ''} end
if redis.call('HGET', KEYS[3], 'nonce') ~= ARGV[6] then return {'NONCE_MISMATCH', ''} end
if redis.call('HGET', KEYS[3], 'c:' .. ARGV[7]) ~= ARGV[8] then return {'STALE_STAMP', ''} end
end
local cur = redis.call('HGET', KEYS[1], 'state')
if cur ~= ARGV[1] then return {'CONFLICT', cur or ''} end
local v = redis.call('HINCRBY', KEYS[1], 'version', 1)
redis.call('HSET', KEYS[1], 'state', ARGV[2], 'owner_holder', ARGV[3], 'transition_operation_id', ARGV[9])
return {'OK', tostring(v)}
"#;
pub struct OwnerLease<'a> {
pub partition: u32,
pub operation_id: &'a str,
pub holder: &'a str,
pub lock_key: &'a str,
pub fence_key: &'a str,
pub round: u64,
pub nonce: &'a str,
pub counter: u64,
}
impl OwnerLease<'_> {
fn lua_fence_key(&self) -> &str {
if self.fence_key.is_empty() {
self.lock_key
} else {
self.fence_key
}
}
fn has_fence(&self) -> &'static str {
if self.fence_key.is_empty() {
"0"
} else {
"1"
}
}
}
async fn claim_transition(
client: &Arc<RedisClient>,
marker: &str,
expected: &str,
new_state: &str,
lease: &OwnerLease<'_>,
) -> Result<u64> {
let v: redis::Value = redis::Script::new(CLAIM_TRANSITION_FENCED_LUA)
.key(marker)
.key(lease.lock_key)
.key(lease.lua_fence_key())
.arg(expected)
.arg(new_state)
.arg(lease.holder)
.arg(lease.has_fence())
.arg(lease.round.to_string())
.arg(lease.nonce)
.arg(lease.partition.to_string())
.arg(lease.counter.to_string())
.arg(lease.operation_id)
.invoke_async(&mut client.conn())
.await?;
let redis::Value::Array(arr) = v else {
return Err(NasaRedisError::ProtocolMarker(
"claim_transition 响应异常".into(),
));
};
let tag = match arr.first() {
Some(redis::Value::BulkString(b)) => String::from_utf8_lossy(b).into_owned(),
Some(redis::Value::SimpleString(s)) => s.clone(),
_ => String::new(),
};
if tag == "OK" {
let ver: u64 = match arr.get(1) {
Some(redis::Value::BulkString(b)) => {
String::from_utf8_lossy(b).parse().map_err(|_| {
NasaRedisError::ProtocolMarker(
"claim_transition 返回的 version 不可解析".into(),
)
})?
}
other => {
return Err(NasaRedisError::ProtocolMarker(format!(
"claim_transition OK 但缺 version 段: {other:?}"
)))
}
};
return Ok(ver);
}
if matches!(
tag.as_str(),
"NOT_HOLDER" | "FENCE_NOT_READY" | "ROUND_MISMATCH" | "NONCE_MISMATCH" | "STALE_STAMP"
) {
return Err(NasaRedisError::OwnershipChanged(format!(
"分区 {}:认领转换被 owner-fence 拒绝({tag})",
lease.partition
)));
}
let cur = match arr.get(1) {
Some(redis::Value::BulkString(b)) => String::from_utf8_lossy(b).into_owned(),
_ => String::new(),
};
Err(NasaRedisError::Config(format!(
"管理转换冲突(VersionConflict):期望 state={expected} 实际={cur}(已被并发管理操作转换)"
)))
}
const VERIFY_OWNER_LUA: &str = r#"
if redis.call('hexists', KEYS[1], ARGV[1]) == 0 then return 'NOT_HOLDER' end
if ARGV[2] == '1' then
if redis.call('HGET', KEYS[2], 'state') ~= 'Ready' then return 'FENCE_NOT_READY' end
if redis.call('HGET', KEYS[2], 'round') ~= ARGV[3] then return 'ROUND_MISMATCH' end
if redis.call('HGET', KEYS[2], 'nonce') ~= ARGV[4] then return 'NONCE_MISMATCH' end
if redis.call('HGET', KEYS[2], 'c:' .. ARGV[5]) ~= ARGV[6] then return 'STALE_STAMP' end
end
return 'OK'
"#;
async fn verify_owner_fenced(client: &Arc<RedisClient>, lease: &OwnerLease<'_>) -> Result<()> {
let tag: String = redis::Script::new(VERIFY_OWNER_LUA)
.key(lease.lock_key)
.key(lease.lua_fence_key())
.arg(lease.holder)
.arg(lease.has_fence())
.arg(lease.round.to_string())
.arg(lease.nonce)
.arg(lease.partition.to_string())
.arg(lease.counter.to_string())
.invoke_async(&mut client.conn())
.await?;
if tag == "OK" {
return Ok(());
}
Err(NasaRedisError::OwnershipChanged(format!(
"分区 {}:owner-fence 校验失败({tag}),拒绝改 disposition(留 PEL 交新 owner)",
lease.partition
)))
}
async fn assert_op_owns(client: &Arc<RedisClient>, marker: &str, op_id: &str) -> Result<()> {
let owner: Option<String> = client.h_get(marker, "transition_operation_id").await?;
match owner.as_deref() {
None | Some("") => Ok(()),
Some(o) if o == op_id => Ok(()),
Some(o) => Err(NasaRedisError::OperationConflict(format!(
"在途转换属 operation={o},本 operation={op_id} 不得接管(交原 op 或新 op 显式重授权)"
))),
}
}
pub fn auto_op_id(p: u32, park_id: &str) -> String {
format!("auto:{p}:{park_id}")
}
pub fn is_auto_op(op_id: &str) -> bool {
op_id.starts_with("auto:")
}
pub async fn marker_version(client: &Arc<RedisClient>, layout: &KeyLayout, p: u32) -> Result<u64> {
Ok(client
.h_get::<u64>(&marker_key(layout, p), "version")
.await?
.unwrap_or(0))
}
async fn load_quarantine_payloads(
client: &Arc<RedisClient>,
qkey: &str,
p: u32,
park_id: &str,
) -> Result<Vec<Vec<u8>>> {
let n: Option<u64> = client.h_get(qkey, "n").await?;
let Some(n) = n else {
return Err(NasaRedisError::ProtocolMarker(format!(
"quarantine 损坏:分区 {p} park_id={park_id} 缺 'n' 字段,拒绝处置(fail-closed)"
)));
};
let mut payloads = Vec::with_capacity(n as usize);
for i in 0..n {
let pl: Option<Vec<u8>> = client.h_get(qkey, &i.to_string()).await?;
match pl {
Some(v) => payloads.push(v),
None => {
return Err(NasaRedisError::ProtocolMarker(format!(
"quarantine 损坏:分区 {p} park_id={park_id} 缺第 {i} 条 payload(n={n}),\
拒绝处置(fail-closed,禁空字节替代)"
)))
}
}
}
Ok(payloads)
}
async fn load_quarantine_payloads_for(
client: &Arc<RedisClient>,
qkey: &str,
p: u32,
park_id: &str,
expected: usize,
) -> Result<Vec<Vec<u8>>> {
let payloads = load_quarantine_payloads(client, qkey, p, park_id).await?;
if payloads.len() != expected {
return Err(NasaRedisError::ProtocolMarker(format!(
"quarantine 损坏:分区 {p} park_id={park_id} payload 数 {} 与已持久化 planned 数 {expected} 不符",
payloads.len()
)));
}
Ok(payloads)
}
const PARK_LUA: &str = r#"
-- KEYS[1]=分区锁完整key KEYS[2]=源stream KEYS[3]=quarantine KEYS[4]=marker KEYS[5]=parked索引
-- ARGV[1]=holder ARGV[2]=group ARGV[3]=park_id ARGV[4]=墙钟ms ARGV[5]=条数 n
-- ARGV[6..6+2n-1] = id1,payload1,id2,payload2,...
-- ① 写前 TYPE 预检(none 容忍;非预期类型即拒,绝不进入写阶段)
local function type_bad(key, want)
local t = redis.call('TYPE', key)['ok']
return t ~= 'none' and t ~= want
end
if type_bad(KEYS[1], 'hash') then return 'PRECHECK_LOCK_TYPE' end
if type_bad(KEYS[2], 'stream') then return 'PRECHECK_STREAM_TYPE' end
if type_bad(KEYS[3], 'hash') then return 'PRECHECK_QUARANTINE_TYPE' end
if type_bad(KEYS[4], 'hash') then return 'PRECHECK_MARKER_TYPE' end
if type_bad(KEYS[5], 'string') then return 'PRECHECK_INDEX_TYPE' end
-- ② holder 校验(类型已确认是 hash,hexists 安全)
if redis.call('hexists', KEYS[1], ARGV[1]) == 0 then
return 'NOT_HOLDER'
end
-- ③ marker 状态预检:幂等重入(同 park_id)直接成功;在途的异 park_id 处置拒绝覆盖
local mstate = redis.call('HGET', KEYS[4], 'state')
local mpark = redis.call('HGET', KEYS[4], 'park_id')
if mstate == 'Parked' and mpark == ARGV[3] then
return 1
end
if mstate ~= false and mstate ~= 'Resumed' and mstate ~= 'Dropped' and mstate ~= 'Dlqed' then
return 'PRECHECK_MARKER_STATE:' .. tostring(mstate)
end
-- ④ 目标 ID 仍在本组 PEL(读;不在 = 状态已变,拒绝)
local n = tonumber(ARGV[5])
for i = 0, n - 1 do
local id = ARGV[6 + i * 2]
local pend = redis.call('XPENDING', KEYS[2], ARGV[2], id, id, 1)
if #pend == 0 then
return 'NOT_IN_PEL:' .. id
end
end
-- ⑤ 转存(先于 ACK:源被确认前 payload 必有副本;同 key 重入 = 幂等覆盖)
for i = 0, n - 1 do
redis.call('HSET', KEYS[3], tostring(i), ARGV[7 + i * 2])
redis.call('HSET', KEYS[3], 'id:' .. tostring(i), ARGV[6 + i * 2])
end
redis.call('HSET', KEYS[3], 'n', ARGV[5])
-- ⑥ ACK 源(消息出 PEL;源 stream 此后可被 trim,payload 已安全)
for i = 0, n - 1 do
redis.call('XACK', KEYS[2], ARGV[2], ARGV[6 + i * 2])
end
-- ⑦ marker + 索引。复用旧**终态** marker(Resumed/Dropped/Dlqed)前必须
-- **DEL 整个 marker**,否则新 Park 会继承旧 planned/uncertain/new_ids/forced/force_audit/
-- owner_holder 字段(污染本轮 resume/force),且 HSET 不清 TTL → 新 Parked marker 会继承上一轮
-- 终态的剩余 TTL 在仍 Parked 时自动消失,留下 index-only 冻结。DEL 一次性清字段+清 TTL。
-- 跨轮审计由 per-park_id marker 隔离;重建时必须消除字段污染与 TTL 继承。
-- version 跨 Park 轮**单调递增**——DEL 前先读旧 version,新 marker 用 old+1
-- (而非恒置 1)。否则慢命令读 before=旧高值后阻塞,被 DEL 重置成 1 的 marker 让其 after<before,
-- 破坏 command 审计的"after≥before"不变量(把正常执行误标可疑重放/降级)。审计是信任根(规范1)。
local next_ver = 1
if mstate == 'Resumed' or mstate == 'Dropped' or mstate == 'Dlqed' then
local old_ver = tonumber(redis.call('HGET', KEYS[4], 'version') or '0') or 0
next_ver = old_ver + 1
redis.call('DEL', KEYS[4])
end
redis.call('HSET', KEYS[4], 'state', 'Parked', 'park_id', ARGV[3], 'n', ARGV[5], 'parked_at', ARGV[4], 'version', next_ver)
redis.call('SET', KEYS[5], ARGV[3])
return 1
"#;
pub async fn park(
client: &Arc<RedisClient>,
layout: &KeyLayout,
p: u32,
lock_full_key: &str,
holder: &str,
records: &[(String, Vec<u8>)],
) -> Result<String> {
if records.is_empty() {
return Err(NasaRedisError::Config(format!(
"分区 {p} park 被拒:records 为空(不允许 park 空批,防分区冻结 + 毒消息静默丢失)"
)));
}
let park_id = uuid::Uuid::new_v4().simple().to_string();
let now_ms = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("墙钟早于 epoch")
.as_millis() as u64;
let lua = redis::Script::new(PARK_LUA);
let mut script = lua.prepare_invoke();
script
.key(lock_full_key)
.key(layout.stream(p))
.key(quarantine_key(layout, &park_id))
.key(marker_key(layout, p))
.key(parked_index_key(layout, p))
.arg(holder)
.arg(layout.group())
.arg(&park_id)
.arg(now_ms)
.arg(records.len());
for (id, payload) in records {
script.arg(id.as_str()).arg(payload.as_slice());
}
let r: redis::Value = script.invoke_async(&mut client.conn()).await?;
match r {
redis::Value::Int(1) => Ok(park_id),
other => Err(NasaRedisError::Config(format!(
"park 被拒绝(fenced 校验失败): {other:?}"
))),
}
}
pub async fn parked_id(
client: &Arc<RedisClient>,
layout: &KeyLayout,
p: u32,
) -> Result<Option<String>> {
match takeover_disposition(client, layout, p).await? {
TakeoverDisposition::NotParked => Ok(None),
TakeoverDisposition::Frozen { park_id }
| TakeoverDisposition::AutoResumeDlq { park_id, .. } => Ok(Some(park_id)),
TakeoverDisposition::Inconsistent { reason } => Err(NasaRedisError::ProtocolMarker(
format!("分区 {p} disposition 状态不一致/损坏: {reason}"),
)),
}
}
#[derive(Debug)]
pub enum TakeoverDisposition {
NotParked,
Frozen { park_id: String },
AutoResumeDlq {
park_id: String,
operation_id: String,
},
Inconsistent { reason: String },
}
const DISPOSITION_NON_TERMINAL: &[&str] = &[
"Parked",
"ResumePublishing",
"DlqPublishing",
"ForcePublishing",
"Dropping",
"ResumeIndeterminate",
"DlqIndeterminate",
];
const DISPOSITION_TERMINAL: &[&str] = &["Resumed", "Dlqed", "Dropped"];
const DISPOSITION_SNAPSHOT_LUA: &str = r#"
return {
redis.call('HGET', KEYS[1], 'state'),
redis.call('HGET', KEYS[1], 'park_id'),
redis.call('HGET', KEYS[1], 'transition_operation_id'),
redis.call('GET', KEYS[2])
}
"#;
pub async fn takeover_disposition(
client: &Arc<RedisClient>,
layout: &KeyLayout,
p: u32,
) -> Result<TakeoverDisposition> {
let (mstate, mpark, mop, index): (
Option<String>,
Option<String>,
Option<String>,
Option<String>,
) = redis::Script::new(DISPOSITION_SNAPSHOT_LUA)
.key(marker_key(layout, p))
.key(parked_index_key(layout, p))
.invoke_async(&mut client.conn())
.await?;
match mstate.as_deref() {
Some(s) if DISPOSITION_NON_TERMINAL.contains(&s) => {
let park_id = mpark.unwrap_or_default();
if park_id.is_empty() {
return Ok(TakeoverDisposition::Inconsistent {
reason: format!("disposition marker 非终态({s})但缺 park_id,无法恢复"),
});
}
match &index {
Some(idx) if *idx == park_id => {}
_ => tracing::warn!(
p,
marker_state = s,
?index,
marker_park = %park_id,
"disposition marker 非终态但 parked index 不一致(以 marker 为准冻结)"
),
}
if s == "DlqPublishing" {
if let Some(op) = mop.as_deref() {
if is_auto_op(op) {
return Ok(TakeoverDisposition::AutoResumeDlq {
park_id,
operation_id: op.to_string(),
});
}
}
}
Ok(TakeoverDisposition::Frozen { park_id })
}
Some(s) if DISPOSITION_TERMINAL.contains(&s) => {
if index.is_some() {
tracing::warn!(p, marker_state = s, ?index, "disposition marker 已终态但 parked index 仍在(陈旧,放行消费)");
}
Ok(TakeoverDisposition::NotParked)
}
Some(other) => Ok(TakeoverDisposition::Inconsistent {
reason: format!("未知 disposition marker 状态: {other}"),
}),
None => match index {
None => Ok(TakeoverDisposition::NotParked),
Some(idx) => Ok(TakeoverDisposition::Inconsistent {
reason: format!("parked index 存在(park_id={idx})但 disposition marker 缺失:Park 元数据可能部分丢失,需对账"),
}),
},
}
}
fn parse_sid(s: &str) -> Option<(u64, u64)> {
let (ms, seq) = s.split_once('-')?;
Some((ms.parse().ok()?, seq.parse().ok()?))
}
fn plan_ids(base: (u64, u64), n: usize) -> Result<Vec<String>> {
let (mut ms, mut seq) = base;
let mut out = Vec::with_capacity(n);
for _ in 0..n {
if seq == u64::MAX {
if ms == u64::MAX {
return Err(NasaRedisError::Config(
"StreamIdExhausted: ID 空间耗尽".into(),
));
}
ms += 1;
seq = 0;
} else {
seq += 1;
}
out.push(format!("{ms}-{seq}"));
}
Ok(out)
}
fn xinfo_str(v: &redis::Value) -> Option<String> {
match v {
redis::Value::SimpleString(s) => Some(s.clone()),
redis::Value::BulkString(b) => Some(String::from_utf8_lossy(b).into_owned()),
_ => None,
}
}
fn xinfo_field(v: redis::Value, field: &str) -> Option<String> {
let pairs: Vec<(redis::Value, redis::Value)> = match v {
redis::Value::Map(pairs) => pairs,
redis::Value::Array(items) => {
let mut it = items.into_iter();
let mut out = Vec::new();
while let (Some(k), Some(val)) = (it.next(), it.next()) {
out.push((k, val));
}
out
}
_ => return None,
};
let mut found = None;
for (k, val) in pairs {
if xinfo_str(&k).as_deref() == Some(field) {
found = xinfo_str(&val);
}
}
found
}
async fn last_generated(client: &Arc<RedisClient>, stream: &str) -> Result<(u64, u64)> {
let v: redis::Value = match redis::cmd("XINFO")
.arg("STREAM")
.arg(stream)
.query_async(&mut client.conn())
.await
{
Ok(v) => v,
Err(e) if e.to_string().contains("no such key") => return Ok((0, 0)),
Err(e) => return Err(e.into()),
};
match &v {
redis::Value::Map(_) | redis::Value::Array(_) => {}
other => {
return Err(NasaRedisError::ProtocolMarker(format!(
"XINFO STREAM {stream} 响应形态异常(非 Map/Array): {other:?}"
)))
}
}
let last = xinfo_field(v, "last-generated-id");
match last {
Some(s) => parse_sid(&s).ok_or_else(|| {
NasaRedisError::ProtocolMarker(format!(
"XINFO STREAM {stream} 的 last-generated-id 不可解析: {s:?}"
))
}),
None => Err(NasaRedisError::ProtocolMarker(format!(
"XINFO STREAM {stream} 缺 last-generated-id 字段(协议损坏)"
))),
}
}
async fn reconcile_one(
client: &Arc<RedisClient>,
stream: &str,
planned: &str,
expect_payload: &[u8],
) -> Result<bool> {
let rows: redis::Value = redis::cmd("XRANGE")
.arg(stream)
.arg(planned)
.arg(planned)
.query_async(&mut client.conn())
.await?;
if let redis::Value::Array(rows) = rows {
if let Some(redis::Value::Array(pair)) = rows.into_iter().next() {
if let Some(redis::Value::Array(fields)) = pair.into_iter().nth(1) {
let mut it = fields.into_iter();
while let (Some(f), Some(val)) = (it.next(), it.next()) {
if matches!(&f, redis::Value::BulkString(b) if b == b"data") {
if let redis::Value::BulkString(got) = val {
return Ok(got == expect_payload); }
}
}
}
return Ok(false); }
}
let lg = last_generated(client, stream).await?;
let pl = parse_sid(planned).ok_or_else(|| {
NasaRedisError::ProtocolMarker(format!("reconcile:planned ID 不可解析: {planned:?}"))
})?;
if lg < pl {
return Err(NasaRedisError::Config("RETRYABLE".into())); }
Ok(false)
}
pub async fn resume(
client: &Arc<RedisClient>,
layout: &KeyLayout,
p: u32,
lease: &OwnerLease<'_>,
) -> Result<Vec<String>> {
verify_owner_fenced(client, lease).await?;
let marker = marker_key(layout, p);
let state: Option<String> = client.h_get(&marker, "state").await?;
let park_id: Option<String> = client.h_get(&marker, "park_id").await?;
let (Some(state), Some(park_id)) = (state, park_id) else {
return Err(NasaRedisError::Config(format!("分区 {p} 无 Park 记录")));
};
let stream = layout.stream(p);
let qkey = quarantine_key(layout, &park_id);
match state.as_str() {
"Parked" => {
claim_transition(client, &marker, "Parked", "ResumePublishing", lease).await?;
publish_step(
client, layout, p, &marker, &qkey, &park_id, &stream,
"Resumed", "ResumeIndeterminate",
)
.await
}
"ResumePublishing" => {
assert_op_owns(client, &marker, lease.operation_id).await?;
publish_step(
client, layout, p, &marker, &qkey, &park_id, &stream,
"Resumed", "ResumeIndeterminate",
)
.await
}
"ResumeIndeterminate" => Err(NasaRedisError::OperationInDoubt(format!(
"分区 {p} 处于 ResumeIndeterminate:普通 resume 被拒,需 force_republish(接受重复风险)或人工对账"
))),
"Resumed" => {
client.del(&[&parked_index_key(layout, p)]).await?;
client.del(&[&qkey]).await?;
client
.expire(&marker, std::time::Duration::from_secs(24 * 3600))
.await?;
let ids: Option<String> = client.h_get(&marker, "new_ids").await?;
Ok(ids.unwrap_or_default().split(',').filter(|s| !s.is_empty()).map(String::from).collect())
}
other => Err(NasaRedisError::Config(format!("分区 {p} marker 状态 {other},拒绝 resume"))),
}
}
#[allow(clippy::too_many_arguments)]
async fn publish_step(
client: &Arc<RedisClient>,
layout: &KeyLayout,
p: u32,
marker: &str,
qkey: &str,
park_id: &str,
target_stream: &str,
final_state: &str,
indeterminate_state: &str,
) -> Result<Vec<String>> {
let planned_str: Option<String> = client.h_get(marker, "planned").await?;
let planned: Vec<String> = match planned_str.as_deref().filter(|s| !s.is_empty()) {
Some(s) => serde_json::from_str(s)
.map_err(|e| NasaRedisError::Codec(format!("planned 解析失败: {e}")))?,
None => {
let payloads = load_quarantine_payloads(client, qkey, p, park_id).await?;
let planned = plan_ids(last_generated(client, target_stream).await?, payloads.len())?;
let planned_json = serde_json::to_string(&planned).unwrap();
let won: i64 = redis::cmd("HSETNX")
.arg(marker)
.arg("planned")
.arg(&planned_json)
.query_async(&mut client.conn())
.await?;
if won == 1 {
planned
} else {
let existing: String = client.h_get(marker, "planned").await?.unwrap_or_default();
serde_json::from_str(&existing)
.map_err(|e| NasaRedisError::Codec(format!("planned 读回解析失败: {e}")))?
}
}
};
let payloads = load_quarantine_payloads_for(client, qkey, p, park_id, planned.len()).await?;
publish_reserved(
client,
layout,
p,
marker,
qkey,
&planned,
&payloads,
target_stream,
final_state,
indeterminate_state,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn publish_reserved(
client: &Arc<RedisClient>,
layout: &KeyLayout,
p: u32,
marker: &str,
qkey: &str,
planned: &[String],
payloads: &[Vec<u8>],
target_stream: &str,
final_state: &str,
indeterminate_state: &str,
) -> Result<Vec<String>> {
let stream = target_stream.to_string();
let mut uncertain: Vec<usize> = Vec::new();
for (i, (pid, payload)) in planned.iter().zip(payloads.iter()).enumerate() {
match reconcile_one(client, &stream, pid, payload).await {
Ok(true) => continue, Ok(false) => {
uncertain.push(i);
continue;
}
Err(NasaRedisError::Config(sig)) if sig == "RETRYABLE" => {
let r: std::result::Result<String, redis::RedisError> = redis::cmd("XADD")
.arg(&stream)
.arg(pid)
.arg(super::DATA_FIELD)
.arg(payload.as_slice())
.query_async(&mut client.conn())
.await;
match r {
Ok(_) => continue,
Err(e) if e.is_io_error() || e.is_timeout() || e.is_connection_dropped() => {
return Err(NasaRedisError::ExecutionUnknown(format!(
"分区 {p} 发布 planned={pid} 写出后断线: {e}"
)));
}
Err(_) => match reconcile_one(client, &stream, pid, payload).await {
Ok(true) => continue,
Ok(false) => {
uncertain.push(i);
continue;
}
Err(e) => return Err(e), },
}
}
Err(e) => return Err(e), }
}
if !uncertain.is_empty() {
client
.h_set(
marker,
"uncertain",
serde_json::to_string(&uncertain).unwrap().as_str(),
)
.await?;
client.h_set(marker, "state", indeterminate_state).await?;
return Err(NasaRedisError::PublishIndeterminate(format!(
"分区 {p} 有 {} 条 reservation 结果不确定(indexes={uncertain:?}):不自动重发,需 force_republish 或人工对账",
uncertain.len()
)));
}
client
.h_set(marker, "new_ids", planned.join(",").as_str())
.await?;
client.h_set(marker, "state", final_state).await?;
client.del(&[&parked_index_key(layout, p)]).await?;
client.del(&[qkey]).await?;
client
.expire(marker, std::time::Duration::from_secs(24 * 3600))
.await?;
Ok(planned.to_vec())
}
pub fn dlq_stream_key(layout: &KeyLayout) -> String {
format!("{}:dlq", layout.prefix)
}
pub async fn dlq_from_parked(
client: &Arc<RedisClient>,
layout: &KeyLayout,
p: u32,
lease: &OwnerLease<'_>,
) -> Result<Vec<String>> {
verify_owner_fenced(client, lease).await?; let marker = marker_key(layout, p);
let state: Option<String> = client.h_get(&marker, "state").await?;
let park_id: Option<String> = client.h_get(&marker, "park_id").await?;
let (Some(state), Some(park_id)) = (state, park_id) else {
return Err(NasaRedisError::Config(format!("分区 {p} 无 Park 记录")));
};
let dlq = dlq_stream_key(layout);
let qkey = quarantine_key(layout, &park_id);
match state.as_str() {
"Parked" => {
claim_transition(client, &marker, "Parked", "DlqPublishing", lease).await?;
publish_step(
client,
layout,
p,
&marker,
&qkey,
&park_id,
&dlq,
"Dlqed",
"DlqIndeterminate",
)
.await
}
"DlqPublishing" => {
assert_op_owns(client, &marker, lease.operation_id).await?;
publish_step(
client,
layout,
p,
&marker,
&qkey,
&park_id,
&dlq,
"Dlqed",
"DlqIndeterminate",
)
.await
}
"DlqIndeterminate" => Err(NasaRedisError::OperationInDoubt(format!(
"分区 {p} 处于 DlqIndeterminate:需人工对账(DLQ 重复风险评估后手工处置)"
))),
"Dlqed" => {
client.del(&[&parked_index_key(layout, p)]).await?;
client.del(&[&qkey]).await?;
client
.expire(&marker, std::time::Duration::from_secs(24 * 3600))
.await?;
let ids: Option<String> = client.h_get(&marker, "new_ids").await?;
Ok(ids
.unwrap_or_default()
.split(',')
.filter(|s| !s.is_empty())
.map(String::from)
.collect())
}
other => Err(NasaRedisError::Config(format!(
"分区 {p} marker 状态 {other},拒绝 dlq"
))),
}
}
pub async fn force_republish(
client: &Arc<RedisClient>,
layout: &KeyLayout,
p: u32,
lease: &OwnerLease<'_>,
) -> Result<Vec<String>> {
verify_owner_fenced(client, lease).await?; let marker = marker_key(layout, p);
let state: Option<String> = client.h_get(&marker, "state").await?;
let st = state.as_deref();
if !matches!(st, Some("ResumeIndeterminate") | Some("ForcePublishing")) {
return Err(NasaRedisError::Config(format!(
"分区 {p} 非 ResumeIndeterminate(当前 {st:?}),force_republish 被拒"
)));
}
if st == Some("ForcePublishing") {
assert_op_owns(client, &marker, lease.operation_id).await?;
}
let park_id: String = client.h_get(&marker, "park_id").await?.unwrap_or_default();
if park_id.is_empty() {
return Err(NasaRedisError::ProtocolMarker(format!(
"分区 {p} force: marker 缺 park_id(损坏),拒绝(fail-closed)"
)));
}
let qkey = quarantine_key(layout, &park_id);
let planned: Vec<String> = match client.h_get::<String>(&marker, "planned").await? {
Some(s) => serde_json::from_str(&s).map_err(|e| {
NasaRedisError::ProtocolMarker(format!("分区 {p} force: planned 损坏不可解析: {e}"))
})?,
None => {
return Err(NasaRedisError::ProtocolMarker(format!(
"分区 {p} force: marker 缺 planned 字段(损坏),拒绝(fail-closed)"
)))
}
};
let uncertain: Vec<usize> = match client.h_get::<String>(&marker, "uncertain").await? {
Some(s) => serde_json::from_str(&s).map_err(|e| {
NasaRedisError::ProtocolMarker(format!("分区 {p} force: uncertain 损坏不可解析: {e}"))
})?,
None => {
return Err(NasaRedisError::ProtocolMarker(format!(
"分区 {p} force: marker 缺 uncertain 字段(损坏),拒绝(fail-closed)"
)))
}
};
if uncertain.is_empty() {
return Err(NasaRedisError::ProtocolMarker(format!(
"分区 {p} force: uncertain 为空,不应进入 force(marker 语义不一致)"
)));
}
let mut seen = std::collections::HashSet::with_capacity(uncertain.len());
for &i in &uncertain {
if i >= planned.len() {
return Err(NasaRedisError::ProtocolMarker(format!(
"分区 {p} force: uncertain index {i} 越界(planned.len={}),marker 损坏,拒绝",
planned.len()
)));
}
if !seen.insert(i) {
return Err(NasaRedisError::ProtocolMarker(format!(
"分区 {p} force: uncertain 含重复 index {i},marker 损坏,拒绝(fail-closed)"
)));
}
}
let mut payloads: Vec<(usize, Vec<u8>)> = Vec::with_capacity(uncertain.len());
for &i in &uncertain {
let payload: Vec<u8> = client.h_get::<Vec<u8>>(&qkey, &i.to_string()).await?.ok_or_else(|| {
NasaRedisError::ProtocolMarker(format!(
"quarantine 损坏:分区 {p} park_id={park_id} 缺第 {i} 条 payload,force 拒绝(fail-closed)"
))
})?;
payloads.push((i, payload));
}
if st == Some("ResumeIndeterminate") {
claim_transition(
client,
&marker,
"ResumeIndeterminate",
"ForcePublishing",
lease,
)
.await?;
}
let stream = layout.stream(p);
let mut final_ids = planned.clone();
let mut audit: Vec<(usize, String, String)> = Vec::new(); for (i, payload) in &payloads {
let i = *i;
let item_field = format!("force:{i}");
if let Some(target) = client.h_get::<String>(&marker, &item_field).await? {
match reconcile_one(client, &stream, &target, payload).await {
Ok(true) => {
final_ids[i] = target;
continue;
}
Err(NasaRedisError::Config(sig)) if sig == "RETRYABLE" => {
xadd_force_target(client, &stream, &target, payload, i).await?;
final_ids[i] = target;
continue;
}
Ok(false) => {
let base = last_generated(client, &stream).await?;
let fresh = plan_ids(base, 1)?.remove(0);
client.h_set(&marker, &item_field, fresh.as_str()).await?; xadd_force_target(client, &stream, &fresh, payload, i).await?;
audit.push((i, target.clone(), fresh.clone()));
final_ids[i] = fresh;
continue;
}
Err(e) => return Err(e),
}
}
let target = match reconcile_one(client, &stream, &planned[i], payload).await {
Ok(true) => {
final_ids[i] = planned[i].clone();
continue; }
Err(NasaRedisError::Config(sig)) if sig == "RETRYABLE" => planned[i].clone(),
Ok(false) => {
let base = last_generated(client, &stream).await?;
plan_ids(base, 1)?.remove(0)
}
Err(e) => return Err(e), };
client.h_set(&marker, &item_field, target.as_str()).await?;
xadd_force_target(client, &stream, &target, payload, i).await?;
if target != planned[i] {
audit.push((i, planned[i].clone(), target.clone()));
}
final_ids[i] = target;
}
client.h_set(&marker, "forced", "1").await?;
client
.h_set(
&marker,
"force_audit",
serde_json::to_string(&audit).unwrap().as_str(),
)
.await?;
client
.h_set(&marker, "new_ids", final_ids.join(",").as_str())
.await?;
client.h_set(&marker, "state", "Resumed").await?;
client.del(&[&parked_index_key(layout, p)]).await?;
client.del(&[&qkey]).await?;
client
.expire(&marker, std::time::Duration::from_secs(24 * 3600))
.await?;
tracing::warn!(p, ?audit, "force_republish 完成(可能重复/变序,审计已记)");
Ok(final_ids)
}
async fn xadd_force_target(
client: &Arc<RedisClient>,
stream: &str,
target: &str,
payload: &[u8],
i: usize,
) -> Result<()> {
let r: std::result::Result<String, redis::RedisError> = redis::cmd("XADD")
.arg(stream)
.arg(target)
.arg(super::DATA_FIELD)
.arg(payload)
.query_async(&mut client.conn())
.await;
match r {
Ok(_) => Ok(()),
Err(e) if e.is_io_error() || e.is_timeout() || e.is_connection_dropped() => {
Err(NasaRedisError::ExecutionUnknown(format!(
"force XADD target={target}(index {i})写出后断线: {e}"
)))
}
Err(_) => match reconcile_one(client, stream, target, payload).await {
Ok(true) => Ok(()), Ok(false) => Err(NasaRedisError::PublishIndeterminate(format!(
"force index {i}: target={target} 被越过且无法发布(真不确定)——marker 停 ForcePublishing,需 operator 介入(force_takeover / 人工对账)"
))),
Err(e) => Err(e),
},
}
}
pub async fn drop_parked(
client: &Arc<RedisClient>,
layout: &KeyLayout,
p: u32,
lease: &OwnerLease<'_>,
) -> Result<()> {
verify_owner_fenced(client, lease).await?; let marker = marker_key(layout, p);
let state: Option<String> = client.h_get(&marker, "state").await?;
let park_id: Option<String> = client.h_get(&marker, "park_id").await?;
let (Some(state), Some(park_id)) = (state, park_id) else {
return Err(NasaRedisError::Config(format!("分区 {p} 无 Park 记录")));
};
match state.as_str() {
"Parked" => {
claim_transition(client, &marker, "Parked", "Dropping", lease).await?;
}
"Dropping" => assert_op_owns(client, &marker, lease.operation_id).await?,
"Dropped" => {}
other => {
return Err(NasaRedisError::Config(format!(
"分区 {p} marker 状态 {other},非 Parked,拒绝 drop"
)))
}
}
client.del(&[&quarantine_key(layout, &park_id)]).await?;
client.del(&[&parked_index_key(layout, p)]).await?;
client.h_set(&marker, "state", "Dropped").await?;
client
.expire(&marker, std::time::Duration::from_secs(24 * 3600))
.await?;
tracing::warn!(p, park_id, "Parked 消息已 Drop(审计 marker 保留 24h)");
Ok(())
}
pub async fn force_takeover(
client: &Arc<RedisClient>,
layout: &KeyLayout,
p: u32,
lease: &OwnerLease<'_>,
) -> Result<Vec<String>> {
verify_owner_fenced(client, lease).await?;
let marker = marker_key(layout, p);
let state: Option<String> = client.h_get(&marker, "state").await?;
let Some(state) = state else {
return Err(NasaRedisError::Config(format!(
"分区 {p} 无 marker,无在途转换可接管"
)));
};
if !matches!(
state.as_str(),
"ResumePublishing" | "DlqPublishing" | "ForcePublishing" | "Dropping"
) {
return Err(NasaRedisError::Config(format!(
"分区 {p} state={state} 非可接管的在途 *Publishing/Dropping\
(Parked 用 resume/drop/dlq;*Indeterminate 需人工对账)"
)));
}
const TAKEOVER_LUA: &str = r#"
if redis.call('HGET', KEYS[1], 'state') == ARGV[1] then
redis.call('HSET', KEYS[1], 'transition_operation_id', ARGV[2])
return 1
end
return 0
"#;
let won: i64 = redis::Script::new(TAKEOVER_LUA)
.key(&marker)
.arg(&state)
.arg(lease.operation_id)
.invoke_async(&mut client.conn())
.await?;
if won != 1 {
return Err(NasaRedisError::OperationConflict(format!(
"分区 {p} 接管期间 state 已从 {state} 变更(原 op 可能仍活跃),请重试或先确认原 op 已死"
)));
}
tracing::warn!(p, %state, new_op = lease.operation_id, "通用 liveness-takeover:已覆写 op,续作在途转换");
match state.as_str() {
"ResumePublishing" => resume(client, layout, p, lease).await,
"DlqPublishing" => dlq_from_parked(client, layout, p, lease).await,
"ForcePublishing" => force_republish(client, layout, p, lease).await,
"Dropping" => {
drop_parked(client, layout, p, lease).await?;
Ok(Vec::new())
}
_ => unreachable!(),
}
}