use std::sync::Arc;
use serde::{Deserialize, Serialize};
use super::KeyLayout;
use crate::client::RedisClient;
use crate::error::{NasaRedisError, Result};
pub fn bootstrap_key(namespace: &str) -> String {
format!("nasa:fence-bootstrap:{namespace}")
}
pub fn bootstrap_owner_key(namespace: &str) -> String {
format!("nasa:fence-bootstrap-owner:{namespace}")
}
const OWNER_LEASE_TTL_MS: u64 = 10_000;
const WAIT_POLL_MS: u64 = 200;
const WAIT_DEADLINE_MS: u64 = 30_000;
const RELEASE_OWNER_LUA: &str = r#"
if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) end
return 0
"#;
pub fn fence_key(layout: &KeyLayout) -> String {
format!("{}:fence", layout.prefix)
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BootstrapMarker {
pub state: String, pub round: u64,
pub owner_term: u64,
#[serde(default)]
pub nonce: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FencingStamp {
pub p: u32,
pub epoch: u64,
pub counter: u64,
}
#[derive(Debug, Clone)]
pub struct FenceMeta {
pub round: u64,
pub nonce: String,
pub epoch: u64,
}
const TAG_INIT_LUA: &str = r#"
local state = redis.call('HGET', KEYS[1], 'state')
local round = tonumber(redis.call('HGET', KEYS[1], 'round') or '0')
local input = tonumber(ARGV[1])
if state == false then
redis.call('HSET', KEYS[1], 'state', 'Initializing', 'round', ARGV[1], 'epoch', ARGV[2])
return 'OK'
end
if round == input then
if redis.call('HGET', KEYS[1], 'epoch') == false then return 'CORRUPT' end
return 'OK'
end
if round > input then return 'REJECT_OLD_OWNER' end
if state == 'Initializing' then
-- 覆盖前必须兑现"仅 counter 全零才覆盖":存在任何任期
-- counter(c:*)就拒绝 DEL 重建(否则更高 round 会静默清掉已发任期,旧 stamp 失去比对依据)
local keys = redis.call('HKEYS', KEYS[1])
for _, k in ipairs(keys) do
if string.sub(k, 1, 2) == 'c:' then return 'REJECT_HAS_COUNTERS' end
end
redis.call('DEL', KEYS[1])
redis.call('HSET', KEYS[1], 'state', 'Initializing', 'round', ARGV[1], 'epoch', ARGV[2])
return 'OK'
end
return 'REJECT_READY'
"#;
const TAG_ACTIVATE_LUA: &str = r#"
local round = redis.call('HGET', KEYS[1], 'round')
if round ~= ARGV[1] then return 'ROUND_MISMATCH' end
redis.call('HSET', KEYS[1], 'state', 'Ready', 'nonce', ARGV[2])
return 'OK'
"#;
const ACQUIRE_STAMP_LUA: &str = r#"
if redis.call('HGET', KEYS[1], 'state') ~= 'Ready' then return {'FENCE_NOT_READY'} end
if redis.call('HGET', KEYS[1], 'round') ~= ARGV[1] then return {'ROUND_MISMATCH'} end
if redis.call('HGET', KEYS[1], 'nonce') ~= ARGV[2] then return {'NONCE_MISMATCH'} end
local c = redis.call('HINCRBY', KEYS[1], 'c:' .. ARGV[3], 1)
local epoch = redis.call('HGET', KEYS[1], 'epoch')
return {'OK', epoch, tostring(c)}
"#;
const FENCED_ACK_LUA: &str = r#"
if redis.call('hexists', KEYS[1], ARGV[1]) == 0 then return 'NOT_HOLDER' end
if redis.call('HGET', KEYS[3], 'state') ~= 'Ready' then return 'FENCE_NOT_READY' end
if redis.call('HGET', KEYS[3], 'round') ~= ARGV[3] then return 'ROUND_MISMATCH' end
if redis.call('HGET', KEYS[3], 'nonce') ~= ARGV[4] then return 'NONCE_MISMATCH' end
local cur = redis.call('HGET', KEYS[3], 'c:' .. ARGV[5])
if cur ~= ARGV[6] then return 'STALE_STAMP' end
local n = 0
for i = 7, #ARGV do
n = n + redis.call('XACK', KEYS[2], ARGV[2], ARGV[i])
end
return n
"#;
pub async fn bootstrap(
client: &Arc<RedisClient>,
layout: &KeyLayout,
epoch: u64,
) -> Result<FenceMeta> {
let ns = client.config().namespace.clone();
let bkey = bootstrap_key(&ns);
let okey = bootstrap_owner_key(&ns);
let fkey = fence_key(layout);
let deadline = tokio::time::Instant::now() + std::time::Duration::from_millis(WAIT_DEADLINE_MS);
loop {
if let Some(raw) = client.get::<String>(&bkey).await? {
let marker: BootstrapMarker = serde_json::from_str(&raw)
.map_err(|e| NasaRedisError::Codec(format!("bootstrap marker 解析失败: {e}")))?;
if marker.state == "Ready" {
if verify_tag_ready(client, &fkey, marker.round, &marker.nonce)
.await
.is_ok()
{
let epoch_now: u64 = client.h_get(&fkey, "epoch").await?.ok_or_else(|| {
NasaRedisError::ProtocolMarker(format!(
"fence {fkey} 为 Ready 但缺 epoch 字段(协议损坏),fail-closed"
))
})?;
return Ok(FenceMeta {
round: marker.round,
nonce: marker.nonce,
epoch: epoch_now,
});
}
}
}
let token = uuid::Uuid::new_v4().simple().to_string();
let got: Option<String> = redis::cmd("SET")
.arg(&okey)
.arg(&token)
.arg("NX")
.arg("PX")
.arg(OWNER_LEASE_TTL_MS)
.query_async(&mut client.conn())
.await?;
if got.is_some() {
let r = bootstrap_as_owner(client, &bkey, &fkey, epoch).await;
let _: std::result::Result<i64, _> = redis::Script::new(RELEASE_OWNER_LUA)
.key(&okey)
.arg(&token)
.invoke_async(&mut client.conn())
.await;
return r;
}
if tokio::time::Instant::now() >= deadline {
return Err(NasaRedisError::ProtocolMarker(
"bootstrap 等待全局 Ready 超时(owner 持续未完成)".into(),
));
}
tokio::time::sleep(std::time::Duration::from_millis(WAIT_POLL_MS)).await;
}
}
async fn bootstrap_as_owner(
client: &Arc<RedisClient>,
bkey: &str,
fkey: &str,
epoch: u64,
) -> Result<FenceMeta> {
let (mut marker, fresh_created) = match client.get::<String>(bkey).await? {
Some(raw) => {
let m: BootstrapMarker = serde_json::from_str(&raw)
.map_err(|e| NasaRedisError::Codec(format!("bootstrap marker 解析失败: {e}")))?;
(m, false)
}
None => {
let fresh = BootstrapMarker {
state: "Initializing".into(),
round: 1,
owner_term: 1,
nonce: String::new(),
};
let payload = serde_json::to_string(&fresh).unwrap();
let created: Option<String> = redis::cmd("SET")
.arg(bkey)
.arg(&payload)
.arg("NX")
.query_async(&mut client.conn())
.await?;
if created.is_some() {
(fresh, true)
} else {
let m: BootstrapMarker =
serde_json::from_str(&client.get::<String>(bkey).await?.unwrap_or_default())
.map_err(|e| NasaRedisError::Codec(e.to_string()))?;
(m, false)
}
}
};
if marker.state == "Ready" {
if verify_tag_ready(client, fkey, marker.round, &marker.nonce)
.await
.is_err()
{
seed_tag(client, fkey, marker.round, &marker.nonce, epoch).await?;
}
let epoch_now: u64 = client.h_get(fkey, "epoch").await?.ok_or_else(|| {
NasaRedisError::ProtocolMarker(format!(
"fence {fkey} 为 Ready 但缺 epoch 字段(协议损坏),fail-closed"
))
})?;
return Ok(FenceMeta {
round: marker.round,
nonce: marker.nonce,
epoch: epoch_now,
});
}
if marker.state != "Initializing" && marker.state != "Activating" {
return Err(NasaRedisError::ProtocolMarker(format!(
"bootstrap marker 未知/损坏 state={}(非 Initializing/Activating/Ready),fail-closed",
marker.state
)));
}
if marker.state == "Activating" && marker.nonce.is_empty() {
return Err(NasaRedisError::ProtocolMarker(
"bootstrap marker 为 Activating 但 nonce 为空(协议损坏),fail-closed".into(),
));
}
if !fresh_created {
marker.owner_term += 1;
client
.set(bkey, serde_json::to_string(&marker).unwrap().as_str())
.await?;
tracing::warn!(
round = marker.round,
owner_term = marker.owner_term,
state = %marker.state,
"接管半完成 bootstrap(上任 owner 未完成)"
);
}
let r: String = redis::Script::new(TAG_INIT_LUA)
.key(fkey)
.arg(marker.round)
.arg(epoch)
.invoke_async(&mut client.conn())
.await?;
if r != "OK" {
return Err(NasaRedisError::ProtocolMarker(format!(
"fence init 被拒: {r}"
)));
}
if marker.state == "Initializing" {
marker.state = "Activating".into();
marker.nonce = uuid::Uuid::new_v4().simple().to_string();
client
.set(bkey, serde_json::to_string(&marker).unwrap().as_str())
.await?;
}
let r: String = redis::Script::new(TAG_ACTIVATE_LUA)
.key(fkey)
.arg(marker.round)
.arg(&marker.nonce)
.invoke_async(&mut client.conn())
.await?;
if r != "OK" {
return Err(NasaRedisError::ProtocolMarker(format!(
"fence activate 被拒: {r}"
)));
}
verify_tag_ready(client, fkey, marker.round, &marker.nonce).await?;
marker.state = "Ready".into();
client
.set(bkey, serde_json::to_string(&marker).unwrap().as_str())
.await?;
tracing::info!(
round = marker.round,
owner_term = marker.owner_term,
"fencing bootstrap 完成(全局 Ready)"
);
Ok(FenceMeta {
round: marker.round,
nonce: marker.nonce,
epoch,
})
}
async fn verify_tag_ready(
client: &Arc<RedisClient>,
fkey: &str,
round: u64,
nonce: &str,
) -> Result<()> {
let state: Option<String> = client.h_get(fkey, "state").await?;
let r: Option<u64> = client.h_get(fkey, "round").await?;
let n: Option<String> = client.h_get(fkey, "nonce").await?;
if state.as_deref() != Some("Ready") || r != Some(round) || n.as_deref() != Some(nonce) {
return Err(NasaRedisError::ProtocolMarker(format!(
"fence tag 未就绪/不一致: state={state:?} round={r:?}(期望 {round}) nonce 匹配={}",
n.as_deref() == Some(nonce)
)));
}
Ok(())
}
async fn seed_tag(
client: &Arc<RedisClient>,
fkey: &str,
round: u64,
nonce: &str,
epoch: u64,
) -> Result<()> {
let r: String = redis::Script::new(TAG_INIT_LUA)
.key(fkey)
.arg(round)
.arg(epoch)
.invoke_async(&mut client.conn())
.await?;
if r != "OK" {
return Err(NasaRedisError::ProtocolMarker(format!(
"fence init 被拒(seed_tag): {r}"
)));
}
let r: String = redis::Script::new(TAG_ACTIVATE_LUA)
.key(fkey)
.arg(round)
.arg(nonce)
.invoke_async(&mut client.conn())
.await?;
if r != "OK" {
return Err(NasaRedisError::ProtocolMarker(format!(
"fence activate 被拒(seed_tag): {r}"
)));
}
verify_tag_ready(client, fkey, round, nonce).await
}
pub async fn acquire_stamp(
client: &Arc<RedisClient>,
layout: &KeyLayout,
meta: &FenceMeta,
p: u32,
) -> Result<FencingStamp> {
let v: redis::Value = redis::Script::new(ACQUIRE_STAMP_LUA)
.key(fence_key(layout))
.arg(meta.round)
.arg(&meta.nonce)
.arg(p)
.invoke_async(&mut client.conn())
.await?;
let redis::Value::Array(arr) = v else {
return Err(NasaRedisError::ProtocolMarker(
"acquire_stamp 响应异常".into(),
));
};
let tag = match arr.first() {
Some(redis::Value::BulkString(b)) => String::from_utf8_lossy(b).into_owned(),
Some(redis::Value::SimpleString(s)) => s.clone(),
_ => String::new(),
};
if tag != "OK" {
return Err(NasaRedisError::ProtocolMarker(format!(
"acquire_stamp 被拒: {tag}"
)));
}
let epoch: u64 = match arr.get(1) {
Some(redis::Value::BulkString(b)) => String::from_utf8_lossy(b).parse().map_err(|_| {
NasaRedisError::ProtocolMarker(format!(
"fence epoch 字段损坏: {:?}",
String::from_utf8_lossy(b)
))
})?,
other => {
return Err(NasaRedisError::ProtocolMarker(format!(
"acquire_stamp 响应缺 epoch: {other:?}"
)))
}
};
let counter: u64 = match arr.get(2) {
Some(redis::Value::BulkString(b)) => String::from_utf8_lossy(b).parse().map_err(|_| {
NasaRedisError::ProtocolMarker(format!(
"fence counter 字段损坏: {:?}",
String::from_utf8_lossy(b)
))
})?,
other => {
return Err(NasaRedisError::ProtocolMarker(format!(
"acquire_stamp 响应缺 counter: {other:?}"
)))
}
};
if counter == 0 || counter > i64::MAX as u64 {
return Err(NasaRedisError::ProtocolMarker(format!(
"fence counter 越界(必须 1..=i64::MAX): {counter}"
)));
}
Ok(FencingStamp { p, epoch, counter })
}
#[derive(Debug)]
pub enum FencedAck {
Acked(i64),
Rejected(String),
}
#[allow(clippy::too_many_arguments)]
pub async fn fenced_ack(
client: &Arc<RedisClient>,
layout: &KeyLayout,
meta: &FenceMeta,
stamp: &FencingStamp,
lock_full_key: &str,
holder: &str,
ids: &[String],
) -> Result<FencedAck> {
let lua = redis::Script::new(FENCED_ACK_LUA);
let mut inv = lua.prepare_invoke();
inv.key(lock_full_key)
.key(layout.stream(stamp.p))
.key(fence_key(layout))
.arg(holder)
.arg(layout.group())
.arg(meta.round)
.arg(&meta.nonce)
.arg(stamp.p)
.arg(stamp.counter);
for id in ids {
inv.arg(id.as_str());
}
let v: redis::Value = inv.invoke_async(&mut client.conn()).await?;
Ok(match v {
redis::Value::Int(n) => FencedAck::Acked(n),
redis::Value::BulkString(b) => {
FencedAck::Rejected(String::from_utf8_lossy(&b).into_owned())
}
redis::Value::SimpleString(s) => FencedAck::Rejected(s),
other => FencedAck::Rejected(format!("{other:?}")),
})
}