use std::sync::Arc;
use serde::{Deserialize, Serialize};
use super::KeyLayout;
use crate::client::RedisClient;
use crate::error::{NasaRedisError, Result};
pub fn bootstrap_key(namespace: &str) -> String {
format!("nasa:fence-bootstrap:{namespace}")
}
pub fn bootstrap_owner_key(namespace: &str) -> String {
format!("nasa:fence-bootstrap-owner:{namespace}")
}
const OWNER_LEASE_TTL_MS: u64 = 10_000;
const WAIT_POLL_MS: u64 = 200;
const WAIT_DEADLINE_MS: u64 = 30_000;
const RELEASE_OWNER_LUA: &str = r#"
if redis.call('GET', KEYS[1]) == ARGV[1] then
return redis.call('DEL', KEYS[1])
end
return 0
"#;
const COMPARE_MARKER_LUA: &str = r#"
if redis.call('GET', KEYS[1]) ~= ARGV[1] then
return 0
end
redis.call('SET', KEYS[1], ARGV[2])
return 1
"#;
pub fn fence_key(layout: &KeyLayout) -> String {
format!("{}:fence", layout.prefix)
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct BootstrapMarker {
pub state: String,
pub round: u64,
pub owner_term: u64,
#[serde(default)]
pub nonce: String,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FencingStamp {
pub p: u32,
pub epoch: u64,
pub counter: u64,
}
#[derive(Debug, Clone)]
pub struct FenceMeta {
pub round: u64,
pub nonce: String,
pub epoch: u64,
}
async fn read_marker(client: &Arc<RedisClient>, bkey: &str) -> Result<Option<String>> {
let response: redis::Value = redis::cmd("GET")
.arg(bkey)
.query_async(&mut client.conn())
.await?;
match response {
redis::Value::Nil => Ok(None),
redis::Value::BulkString(raw) => String::from_utf8(raw)
.map(Some)
.map_err(|_| NasaRedisError::Codec("bootstrap marker UTF-8 编码非法".into())),
_ => Err(NasaRedisError::ProtocolMarker(
"bootstrap marker 读取响应非法".into(),
)),
}
}
fn parse_bootstrap_marker(raw: &str) -> Result<BootstrapMarker> {
let marker: BootstrapMarker = serde_json::from_str(raw)
.map_err(|_| NasaRedisError::Codec("bootstrap marker JSON 格式非法".into()))?;
if marker.round == 0
|| marker.owner_term == 0
|| match marker.state.as_str() {
"Initializing" => !marker.nonce.is_empty(),
"Activating" | "Ready" => marker.nonce.is_empty(),
_ => true,
}
{
return Err(NasaRedisError::ProtocolMarker(
"bootstrap marker 状态或身份非法".into(),
));
}
Ok(marker)
}
async fn compare_marker(
client: &Arc<RedisClient>,
bkey: &str,
expected: &str,
next: &BootstrapMarker,
) -> Result<String> {
let payload = serde_json::to_string(next)
.map_err(|_| NasaRedisError::Codec("bootstrap marker 编码失败".into()))?;
let result: redis::Value = redis::Script::new(COMPARE_MARKER_LUA)
.key(bkey)
.arg(expected)
.arg(&payload)
.invoke_async(&mut client.conn())
.await?;
match result {
redis::Value::Int(1) => Ok(payload),
redis::Value::Int(0) => Err(NasaRedisError::ProtocolMarker(
"bootstrap marker 已变化,当前初始化者停止推进".into(),
)),
_ => Err(NasaRedisError::ProtocolMarker(
"bootstrap marker 比较更新响应非法".into(),
)),
}
}
fn check_tag_reply(reply: redis::Value) -> Result<()> {
let tag = match &reply {
redis::Value::Okay => Some("OK"),
redis::Value::SimpleString(value) => Some(value.as_str()),
redis::Value::BulkString(value) => std::str::from_utf8(value).ok(),
_ => None,
};
if tag == Some("OK") {
return Ok(());
}
let reason = tag
.filter(|reason| {
matches!(
*reason,
"INVALID_INPUT"
| "CORRUPT"
| "REJECT_OLD_OWNER"
| "REJECT_HAS_COUNTERS"
| "REJECT_READY"
| "ROUND_MISMATCH"
| "NONCE_MISMATCH"
)
})
.unwrap_or("INVALID_RESPONSE");
Err(NasaRedisError::ProtocolMarker(format!(
"fence 状态推进被拒: {reason}"
)))
}
const TAG_INIT_LUA: &str = r#"
-- 输入不可表示为协议数字时不得创建任何持久状态。
if #ARGV ~= 2 then
return 'INVALID_INPUT'
end
for _, value in ipairs({ARGV[1], ARGV[2]}) do
if string.match(value, '^%d+$') == nil then
return 'INVALID_INPUT'
end
local normalized = string.gsub(value, '^0+', '')
if string.len(normalized) > 20 or
(string.len(normalized) == 20 and normalized > '18446744073709551615') then
return 'INVALID_INPUT'
end
end
if string.sub(ARGV[1], 1, 1) == '0' then
return 'INVALID_INPUT'
end
if redis.call('EXISTS', KEYS[1]) == 0 then
redis.call('HSET', KEYS[1], 'state', 'Initializing', 'round', ARGV[1], 'epoch', ARGV[2])
return 'OK'
end
local state = redis.call('HGET', KEYS[1], 'state')
local round = redis.call('HGET', KEYS[1], 'round')
local epoch = redis.call('HGET', KEYS[1], 'epoch')
local nonce = redis.call('HGET', KEYS[1], 'nonce')
-- 存在 HASH 却缺身份不能解释为新组,否则会掩盖既有任期或改变恢复依据。
if state ~= 'Initializing' and state ~= 'Ready' then
return 'CORRUPT'
end
for _, value in ipairs({round, epoch}) do
if value == false or string.match(value, '^%d+$') == nil then
return 'CORRUPT'
end
local normalized = string.gsub(value, '^0+', '')
if string.len(normalized) > 20 or
(string.len(normalized) == 20 and normalized > '18446744073709551615') then
return 'CORRUPT'
end
end
if string.sub(round, 1, 1) == '0' or
(state == 'Ready' and (nonce == false or nonce == '')) or
(state == 'Initializing' and nonce ~= false and nonce ~= '') then
return 'CORRUPT'
end
-- 先按位数再按字典序比较,不能把相邻大整数折叠成同一轮次。
local input = ARGV[1]
if string.len(round) > string.len(input) or
(string.len(round) == string.len(input) and round > input) then
return 'REJECT_OLD_OWNER'
end
if state == 'Ready' then
if round == input then
return 'OK'
end
return 'REJECT_READY'
end
-- 未就绪却已有任期表示状态不一致;即使同轮重入也不能继续激活或清空计数。
for _, key in ipairs(redis.call('HKEYS', KEYS[1])) do
if string.sub(key, 1, 2) == 'c:' then
return 'REJECT_HAS_COUNTERS'
end
end
if round == input then
return 'OK'
end
-- 仅未发出任期的旧 Initializing 可以重建,已就绪凭据不会经过这条路径。
redis.call('DEL', KEYS[1])
redis.call('HSET', KEYS[1], 'state', 'Initializing', 'round', ARGV[1], 'epoch', ARGV[2])
return 'OK'
"#;
const TAG_ACTIVATE_LUA: &str = r#"
-- 激活身份必须完整,错误参数不得写成可获取任期的 Ready 状态。
if #ARGV ~= 2 or ARGV[2] == '' or string.match(ARGV[1], '^%d+$') == nil or
string.sub(ARGV[1], 1, 1) == '0' or string.len(ARGV[1]) > 20 or
(string.len(ARGV[1]) == 20 and ARGV[1] > '18446744073709551615') then
return 'INVALID_INPUT'
end
local state = redis.call('HGET', KEYS[1], 'state')
local round = redis.call('HGET', KEYS[1], 'round')
local epoch = redis.call('HGET', KEYS[1], 'epoch')
local nonce = redis.call('HGET', KEYS[1], 'nonce')
-- 不能利用激活覆盖残缺元数据,否则后续 owner 无法判断既有凭据的归属。
if state ~= 'Initializing' and state ~= 'Ready' then
return 'CORRUPT'
end
for _, value in ipairs({round, epoch}) do
if value == false or string.match(value, '^%d+$') == nil then
return 'CORRUPT'
end
local normalized = string.gsub(value, '^0+', '')
if string.len(normalized) > 20 or
(string.len(normalized) == 20 and normalized > '18446744073709551615') then
return 'CORRUPT'
end
end
if string.sub(round, 1, 1) == '0' or
(state == 'Ready' and (nonce == false or nonce == '')) or
(state == 'Initializing' and nonce ~= false and nonce ~= '') then
return 'CORRUPT'
end
if round ~= ARGV[1] then
return 'ROUND_MISMATCH'
end
-- 已发布的 nonce 不可替换;迟到初始化者只能沿用同一激活身份。
if state == 'Ready' then
if nonce ~= ARGV[2] then
return 'NONCE_MISMATCH'
end
return 'OK'
end
for _, key in ipairs(redis.call('HKEYS', KEYS[1])) do
if string.sub(key, 1, 2) == 'c:' then
return 'REJECT_HAS_COUNTERS'
end
end
-- 元数据完整且从未发出任期时,一次原子写入发布组就绪身份。
redis.call('HSET', KEYS[1], 'state', 'Ready', 'nonce', ARGV[2])
return 'OK'
"#;
const ACQUIRE_STAMP_LUA: &str = r#"
if redis.call('HGET', KEYS[1], 'state') ~= 'Ready' then
return {'FENCE_NOT_READY'}
end
if redis.call('HGET', KEYS[1], 'round') ~= ARGV[1] then
return {'ROUND_MISMATCH'}
end
if redis.call('HGET', KEYS[1], 'nonce') ~= ARGV[2] then
return {'NONCE_MISMATCH'}
end
local epoch = redis.call('HGET', KEYS[1], 'epoch')
-- 无法建立完整凭据时不得先推进任期;十进制比较保持 u64 全宽精度。
if epoch == false or string.match(epoch, '^%d+$') == nil then
return {'INVALID_EPOCH'}
end
local normalized_epoch = string.gsub(epoch, '^0+', '')
if string.len(normalized_epoch) > 20 or
(string.len(normalized_epoch) == 20 and normalized_epoch > '18446744073709551615') then
return {'INVALID_EPOCH'}
end
local field = 'c:' .. ARGV[3]
local previous = redis.call('HGET', KEYS[1], field)
-- 负计数表示持久权威已损坏,递增不能把它重新解释成合法任期。
if previous ~= false and string.sub(previous, 1, 1) == '-' then
return {'INVALID_COUNTER'}
end
redis.call('HINCRBY', KEYS[1], field, 1)
return {'OK', epoch, redis.call('HGET', KEYS[1], field)}
"#;
const FENCED_ACK_LUA: &str = r#"
if redis.call('hexists', KEYS[1], ARGV[1]) == 0 then
return 'NOT_HOLDER'
end
if redis.call('HGET', KEYS[3], 'state') ~= 'Ready' then
return 'FENCE_NOT_READY'
end
if redis.call('HGET', KEYS[3], 'round') ~= ARGV[3] then
return 'ROUND_MISMATCH'
end
if redis.call('HGET', KEYS[3], 'nonce') ~= ARGV[4] then
return 'NONCE_MISMATCH'
end
local cur = redis.call('HGET', KEYS[3], 'c:' .. ARGV[5])
if cur ~= ARGV[6] then
return 'STALE_STAMP'
end
-- epoch 与计数共同组成凭据;缺失、损坏或其它 epoch 都不能在当前消息上行使确认权。
local epoch = redis.call('HGET', KEYS[3], 'epoch')
if epoch == false or string.match(epoch, '^%d+$') == nil then
return 'EPOCH_MISMATCH'
end
local normalized_epoch = string.gsub(epoch, '^0+', '')
if normalized_epoch == '' then
normalized_epoch = '0'
end
if normalized_epoch ~= ARGV[7] then
return 'EPOCH_MISMATCH'
end
local n = 0
for i = 8, #ARGV do
n = n + redis.call('XACK', KEYS[2], ARGV[2], ARGV[i])
end
return n
"#;
pub async fn bootstrap(
client: &Arc<RedisClient>,
layout: &KeyLayout,
epoch: u64,
) -> Result<FenceMeta> {
let ns = client.config().namespace.clone();
let bkey = bootstrap_key(&ns);
let okey = bootstrap_owner_key(&ns);
let fkey = fence_key(layout);
let deadline = tokio::time::Instant::now() + std::time::Duration::from_millis(WAIT_DEADLINE_MS);
loop {
if let Some(raw) = read_marker(client, &bkey).await? {
let marker = parse_bootstrap_marker(&raw)?;
if marker.state == "Ready" {
match verify_tag_ready(client, &fkey, marker.round, &marker.nonce).await {
Ok(epoch_now) => {
return Ok(FenceMeta {
round: marker.round,
nonce: marker.nonce,
epoch: epoch_now,
});
}
Err(NasaRedisError::ProtocolMarker(_)) => {}
Err(error) => return Err(error),
}
}
}
let token = uuid::Uuid::new_v4().simple().to_string();
let reply: redis::Value = redis::cmd("SET")
.arg(&okey)
.arg(&token)
.arg("NX")
.arg("PX")
.arg(OWNER_LEASE_TTL_MS)
.query_async(&mut client.conn())
.await?;
let acquired = match reply {
redis::Value::Okay => true,
redis::Value::SimpleString(value) if value == "OK" => true,
redis::Value::BulkString(value) if value == b"OK" => true,
redis::Value::Nil => false,
_ => {
return Err(NasaRedisError::ProtocolMarker(
"bootstrap owner 租约响应非法".into(),
));
}
};
if acquired {
let r = bootstrap_as_owner(client, &bkey, &fkey, epoch).await;
let _: std::result::Result<i64, _> = redis::Script::new(RELEASE_OWNER_LUA)
.key(&okey)
.arg(&token)
.invoke_async(&mut client.conn())
.await;
return r;
}
if tokio::time::Instant::now() >= deadline {
return Err(NasaRedisError::ProtocolMarker(
"bootstrap 等待全局 Ready 超时(owner 持续未完成)".into(),
));
}
tokio::time::sleep(std::time::Duration::from_millis(WAIT_POLL_MS)).await;
}
}
async fn bootstrap_as_owner(
client: &Arc<RedisClient>,
bkey: &str,
fkey: &str,
epoch: u64,
) -> Result<FenceMeta> {
let (mut marker, mut raw, fresh_created) = match read_marker(client, bkey).await? {
Some(raw) => {
let marker = parse_bootstrap_marker(&raw)?;
(marker, raw, false)
}
None => {
let fresh = BootstrapMarker {
state: "Initializing".into(),
round: 1,
owner_term: 1,
nonce: String::new(),
};
let payload = serde_json::to_string(&fresh)
.map_err(|_| NasaRedisError::Codec("bootstrap marker 编码失败".into()))?;
let reply: redis::Value = redis::cmd("SET")
.arg(bkey)
.arg(&payload)
.arg("NX")
.query_async(&mut client.conn())
.await?;
let created = match reply {
redis::Value::Okay => true,
redis::Value::SimpleString(value) if value == "OK" => true,
redis::Value::BulkString(value) if value == b"OK" => true,
redis::Value::Nil => false,
_ => {
return Err(NasaRedisError::ProtocolMarker(
"bootstrap marker 创建响应非法".into(),
));
}
};
if created {
(fresh, payload, true)
} else {
let raw = read_marker(client, bkey).await?.ok_or_else(|| {
NasaRedisError::ProtocolMarker("bootstrap marker 创建竞争后缺失".into())
})?;
let marker = parse_bootstrap_marker(&raw)?;
(marker, raw, false)
}
}
};
if marker.state == "Ready" {
let epoch_now = seed_tag(client, fkey, marker.round, &marker.nonce, epoch).await?;
return Ok(FenceMeta {
round: marker.round,
nonce: marker.nonce,
epoch: epoch_now,
});
}
if !fresh_created {
marker.owner_term = marker
.owner_term
.checked_add(1)
.ok_or_else(|| NasaRedisError::ProtocolMarker("bootstrap owner_term 已耗尽".into()))?;
raw = compare_marker(client, bkey, &raw, &marker).await?;
tracing::warn!(
round = marker.round,
owner_term = marker.owner_term,
state = %marker.state,
"接管半完成 bootstrap(上任 owner 未完成)"
);
}
let reply = redis::Script::new(TAG_INIT_LUA)
.key(fkey)
.arg(marker.round)
.arg(epoch)
.invoke_async(&mut client.conn())
.await?;
check_tag_reply(reply)?;
if marker.state == "Initializing" {
marker.state = "Activating".into();
marker.nonce = uuid::Uuid::new_v4().simple().to_string();
raw = compare_marker(client, bkey, &raw, &marker).await?;
}
let reply = redis::Script::new(TAG_ACTIVATE_LUA)
.key(fkey)
.arg(marker.round)
.arg(&marker.nonce)
.invoke_async(&mut client.conn())
.await?;
check_tag_reply(reply)?;
let epoch_now = verify_tag_ready(client, fkey, marker.round, &marker.nonce).await?;
marker.state = "Ready".into();
compare_marker(client, bkey, &raw, &marker).await?;
tracing::info!(
round = marker.round,
owner_term = marker.owner_term,
"fencing bootstrap 完成(全局 Ready)"
);
Ok(FenceMeta {
round: marker.round,
nonce: marker.nonce,
epoch: epoch_now,
})
}
async fn verify_tag_ready(
client: &Arc<RedisClient>,
fkey: &str,
round: u64,
nonce: &str,
) -> Result<u64> {
let response: redis::Value = redis::cmd("HMGET")
.arg(fkey)
.arg(&["state", "round", "nonce", "epoch"])
.query_async(&mut client.conn())
.await?;
let invalid = || NasaRedisError::ProtocolMarker("fence tag 未就绪或元数据非法".into());
let redis::Value::Array(fields) = response else {
return Err(invalid());
};
let [state, stored_round, stored_nonce, epoch] = fields.as_slice() else {
return Err(invalid());
};
let (
redis::Value::BulkString(state),
redis::Value::BulkString(stored_round),
redis::Value::BulkString(stored_nonce),
redis::Value::BulkString(epoch),
) = (state, stored_round, stored_nonce, epoch)
else {
return Err(invalid());
};
if round == 0
|| nonce.is_empty()
|| state != b"Ready"
|| *stored_round != round.to_string().as_bytes()
|| stored_nonce != nonce.as_bytes()
|| epoch.is_empty()
|| !epoch.iter().all(u8::is_ascii_digit)
{
return Err(invalid());
}
std::str::from_utf8(epoch)
.ok()
.and_then(|value| value.parse().ok())
.ok_or_else(invalid)
}
async fn seed_tag(
client: &Arc<RedisClient>,
fkey: &str,
round: u64,
nonce: &str,
epoch: u64,
) -> Result<u64> {
let reply = redis::Script::new(TAG_INIT_LUA)
.key(fkey)
.arg(round)
.arg(epoch)
.invoke_async(&mut client.conn())
.await?;
check_tag_reply(reply)?;
let reply = redis::Script::new(TAG_ACTIVATE_LUA)
.key(fkey)
.arg(round)
.arg(nonce)
.invoke_async(&mut client.conn())
.await?;
check_tag_reply(reply)?;
verify_tag_ready(client, fkey, round, nonce).await
}
pub async fn acquire_stamp(
client: &Arc<RedisClient>,
layout: &KeyLayout,
meta: &FenceMeta,
p: u32,
) -> Result<FencingStamp> {
let v: redis::Value = redis::Script::new(ACQUIRE_STAMP_LUA)
.key(fence_key(layout))
.arg(meta.round)
.arg(&meta.nonce)
.arg(p)
.invoke_async(&mut client.conn())
.await?;
let redis::Value::Array(arr) = v else {
return Err(NasaRedisError::ProtocolMarker(
"acquire_stamp 响应异常".into(),
));
};
let tag = match arr.first() {
Some(redis::Value::BulkString(bytes)) => std::str::from_utf8(bytes).ok(),
Some(redis::Value::SimpleString(value)) => Some(value.as_str()),
Some(redis::Value::Okay) => Some("OK"),
_ => None,
};
if tag != Some("OK") {
let reason = tag
.filter(|reason| {
matches!(
*reason,
"FENCE_NOT_READY"
| "ROUND_MISMATCH"
| "NONCE_MISMATCH"
| "INVALID_EPOCH"
| "INVALID_COUNTER"
)
})
.unwrap_or("INVALID_RESPONSE");
return Err(NasaRedisError::ProtocolMarker(format!(
"acquire_stamp 被拒: {reason}"
)));
}
if arr.len() != 3 {
return Err(NasaRedisError::ProtocolMarker(
"acquire_stamp 响应字段数异常".into(),
));
}
let number = |value: &redis::Value| -> Option<u64> {
let redis::Value::BulkString(bytes) = value else {
return None;
};
if bytes.is_empty() || !bytes.iter().all(u8::is_ascii_digit) {
return None;
}
std::str::from_utf8(bytes).ok()?.parse().ok()
};
let epoch = number(&arr[1])
.ok_or_else(|| NasaRedisError::ProtocolMarker("fence epoch 字段损坏".into()))?;
let counter = number(&arr[2])
.filter(|counter| *counter > 0 && *counter <= i64::MAX as u64)
.ok_or_else(|| NasaRedisError::ProtocolMarker("fence counter 字段损坏或越界".into()))?;
Ok(FencingStamp {
p,
epoch,
counter,
})
}
#[derive(Debug)]
pub enum FencedAck {
Acked(i64),
Rejected(String),
}
#[allow(clippy::too_many_arguments)]
pub async fn fenced_ack(
client: &Arc<RedisClient>,
layout: &KeyLayout,
meta: &FenceMeta,
stamp: &FencingStamp,
lock_full_key: &str,
holder: &str,
ids: &[String],
) -> Result<FencedAck> {
if stamp.counter == 0 || stamp.counter > i64::MAX as u64 || stamp.epoch != meta.epoch {
return Err(NasaRedisError::ProtocolMarker(
"fenced ACK 任期计数或 epoch 不合法".into(),
));
}
let complete_ids = ids.iter().all(|id| {
let Some((millis, sequence)) = id.split_once('-') else {
return false;
};
!millis.is_empty()
&& !sequence.is_empty()
&& millis.bytes().all(|byte| byte.is_ascii_digit())
&& sequence.bytes().all(|byte| byte.is_ascii_digit())
&& millis.parse::<u64>().is_ok()
&& sequence.parse::<u64>().is_ok()
});
if !complete_ids {
return Err(NasaRedisError::ProtocolMarker(
"fenced ACK 记录 ID 不完整或越界".into(),
));
}
let lua = redis::Script::new(FENCED_ACK_LUA);
let mut inv = lua.prepare_invoke();
inv.key(lock_full_key)
.key(layout.stream(stamp.p))
.key(fence_key(layout))
.arg(holder)
.arg(layout.group())
.arg(meta.round)
.arg(&meta.nonce)
.arg(stamp.p)
.arg(stamp.counter)
.arg(stamp.epoch);
for id in ids {
inv.arg(id.as_str());
}
let v: redis::Value = inv.invoke_async(&mut client.conn()).await?;
let reason = match &v {
redis::Value::Int(n) => {
if usize::try_from(*n).is_ok_and(|count| count <= ids.len()) {
return Ok(FencedAck::Acked(*n));
}
return Err(NasaRedisError::ProtocolMarker(
"fenced ACK 确认数超出请求范围".into(),
));
}
redis::Value::BulkString(bytes) => std::str::from_utf8(bytes).ok(),
redis::Value::SimpleString(value) => Some(value.as_str()),
_ => None,
};
match reason {
Some(
reason @ ("NOT_HOLDER" | "FENCE_NOT_READY" | "ROUND_MISMATCH" | "NONCE_MISMATCH"
| "STALE_STAMP" | "EPOCH_MISMATCH"),
) => Ok(FencedAck::Rejected(reason.to_owned())),
_ => Err(NasaRedisError::ProtocolMarker(
"fenced ACK 响应形态或拒绝标签异常".into(),
)),
}
}