use std::collections::BTreeSet;
use std::sync::{Arc, Mutex};
use serde::Serialize;
use crate::client::RedisClient;
use crate::error::{NasaRedisError, Result};
use crate::job::config::JobConfig;
use crate::job::definition::JobDefinition;
use crate::job::identifiers::{snapshot_digest, worker_key};
use crate::job::keyspace::JobKeyspace;
use crate::job::model::JobExecutorState;
use crate::job::names::require_name;
use crate::job::script::{eval, eval_retry_safe, value_to_string, JobScript};
static EXECUTOR_REGISTER: JobScript =
JobScript::new(include_str!("lua/executor_register_capabilities.lua"));
static EXECUTOR_HEARTBEAT: JobScript = JobScript::new(include_str!("lua/executor_heartbeat.lua"));
static EXECUTOR_UNREGISTER: JobScript = JobScript::new(include_str!("lua/executor_unregister.lua"));
static EXECUTOR_REMOVE_CAPABILITY: JobScript =
JobScript::new(include_str!("lua/executor_remove_capability.lua"));
static REGISTRY_GC: JobScript = JobScript::new(include_str!("lua/registry_gc.lua"));
static FANOUT_SNAPSHOT: JobScript = JobScript::new(include_str!("lua/fanout_snapshot.lua"));
static EXECUTOR_RECORD_FANOUT_EVIDENCE: JobScript =
JobScript::new(include_str!("lua/executor_record_fanout_evidence.lua"));
#[derive(Debug, Clone)]
pub struct ExecutorIdentity {
application_name: String,
node_identity: String,
executor_id: String,
startup_id: String,
}
impl ExecutorIdentity {
pub fn new(application_name: &str, instance_identity: &str, startup_id: &str) -> Result<Self> {
let application_name = require_name(application_name, "applicationName")?;
let instance_identity = require_name(instance_identity, "instanceIdentity")?;
let startup_id = require_name(startup_id, "startupId")?;
let node_identity = format!("{application_name}:{instance_identity}");
let executor_id = format!("{node_identity}:{startup_id}");
Ok(Self {
application_name,
node_identity,
executor_id,
startup_id,
})
}
pub fn from_config(config: &JobConfig, startup_id: &str) -> Result<Self> {
let configured = config.instance_identity.trim();
if !configured.is_empty() {
return Self::new(&config.application_name, configured, startup_id);
}
let fallback = std::env::var("HOSTNAME")
.or_else(|_| std::env::var("COMPUTERNAME"))
.map_err(|_| {
NasaRedisError::from(crate::job::JobError::Config(
"job.instance_identity 为空且开发环境没有 HOSTNAME/COMPUTERNAME".to_owned(),
))
})?;
tracing::warn!(
"独立 RedisJob runtime 使用主机名作为 instance_identity;生产部署应显式配置跨重启稳定身份"
);
Self::new(&config.application_name, fallback.trim(), startup_id)
}
pub fn application_name(&self) -> &str {
&self.application_name
}
pub fn node_identity(&self) -> &str {
&self.node_identity
}
pub fn executor_id(&self) -> &str {
&self.executor_id
}
pub fn startup_id(&self) -> &str {
&self.startup_id
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RegisterCapabilityOutcome {
Ok {
redis_now: i64,
expire_at: i64,
heartbeat_revision: i64,
},
WorkerKeyConflict,
ContractMismatch,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum HeartbeatOutcome {
Ok {
redis_now: i64,
expire_at: i64,
heartbeat_revision: i64,
},
NotFound,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum UnregisterOutcome {
Ok,
NotFound,
StaleExecutor,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ExecutorMember {
pub node_identity: String,
pub executor_id: String,
pub startup_id: String,
pub application_name: String,
pub runtime: String,
pub implementation_digest: String,
pub heartbeat_revision: i64,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct ClusterSnapshot {
pub snapshot_id: String,
pub worker_name: String,
pub selected_at: i64,
pub members: Vec<ExecutorMember>,
pub snapshot_digest: String,
}
impl ClusterSnapshot {
pub fn wire_json(&self) -> Vec<u8> {
serde_json::to_vec(self).expect("cluster snapshot 序列化不应失败")
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SnapshotOutcome {
Ok(ClusterSnapshot),
ContractMismatch,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum RecordEvidenceOutcome {
Recorded {
count: i64,
},
MarkedUnready {
count: i64,
until: i64,
},
StaleStartup,
StaleHeartbeat,
NotActive,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RegistryGc {
pub deleted: i64,
pub newly_expired: i64,
pub redis_now: i64,
}
pub struct ExecutorRegistry {
client: Arc<RedisClient>,
keyspace: JobKeyspace,
config: Arc<JobConfig>,
identity: ExecutorIdentity,
workers: Mutex<BTreeSet<String>>,
known_workers: Mutex<BTreeSet<String>>,
}
impl ExecutorRegistry {
pub fn new(
client: Arc<RedisClient>,
keyspace: JobKeyspace,
config: Arc<JobConfig>,
identity: ExecutorIdentity,
) -> Self {
Self {
client,
keyspace,
config,
identity,
workers: Mutex::new(BTreeSet::new()),
known_workers: Mutex::new(BTreeSet::new()),
}
}
pub fn identity(&self) -> &ExecutorIdentity {
&self.identity
}
pub async fn register(
&self,
definition: &JobDefinition,
state: JobExecutorState,
) -> Result<RegisterCapabilityOutcome> {
let worker_name = definition.worker_name();
let fanout_capable = definition.trigger() == crate::job::model::JobTrigger::FanoutOnly;
let codecs = definition.wire_codecs();
let implementation_digest = worker_key(&format!(
"{}:rust:{}",
self.identity.application_name(),
definition.name()
));
let capability_digest = worker_key(&format!(
"{worker_name}:{}:{}:{codecs}",
definition.contract_revision(),
definition.schema_id()
));
if fanout_capable {
self.known_workers
.lock()
.expect("registry known workers lock")
.insert(worker_name.to_owned());
}
let keys: Vec<Vec<u8>> = vec![
self.keyspace.executors().into_bytes(),
self.keyspace
.executor(self.identity.executor_id())
.into_bytes(),
self.keyspace.capability(worker_name).into_bytes(),
self.keyspace.capability_meta(worker_name).into_bytes(),
self.keyspace
.contract(worker_name, definition.contract_revision())
.into_bytes(),
self.keyspace.worker_key_binding(worker_name).into_bytes(),
];
let argv: Vec<Vec<u8>> = vec![
self.identity.executor_id().as_bytes().to_vec(),
self.identity.node_identity().as_bytes().to_vec(),
self.identity.startup_id().as_bytes().to_vec(),
self.identity.application_name().as_bytes().to_vec(),
b"rust".to_vec(),
state.wire_name().as_bytes().to_vec(),
worker_name.as_bytes().to_vec(),
definition.contract_revision().to_string().into_bytes(),
definition.schema_id().as_bytes().to_vec(),
codecs.as_bytes().to_vec(),
implementation_digest.into_bytes(),
self.config.executor_capacity.to_string().into_bytes(),
self.config.executor_expire_ms.to_string().into_bytes(),
capability_digest.clone().into_bytes(),
capability_digest.into_bytes(),
if fanout_capable {
b"1"
} else {
b"0"
}
.to_vec(),
];
let raw = eval(&self.client, &EXECUTOR_REGISTER, &keys, &argv).await?;
let outcome = interpret_register_capability(&raw)?;
if fanout_capable && matches!(outcome, RegisterCapabilityOutcome::Ok { .. }) {
self.workers
.lock()
.expect("registry workers lock")
.insert(worker_name.to_owned());
}
Ok(outcome)
}
pub async fn heartbeat(
&self,
state: JobExecutorState,
inflight: u32,
) -> Result<HeartbeatOutcome> {
self.heartbeat_inner(state, inflight, None).await
}
pub(crate) async fn heartbeat_idempotent(
&self,
state: JobExecutorState,
inflight: u32,
request_id: &str,
expected_revision: i64,
) -> Result<HeartbeatOutcome> {
self.heartbeat_inner(state, inflight, Some((request_id, expected_revision)))
.await
}
async fn heartbeat_inner(
&self,
state: JobExecutorState,
inflight: u32,
idempotency: Option<(&str, i64)>,
) -> Result<HeartbeatOutcome> {
let sorted: Vec<String> = self
.workers
.lock()
.expect("registry workers lock")
.iter()
.cloned()
.collect();
let mut keys: Vec<Vec<u8>> = vec![
self.keyspace.executors().into_bytes(),
self.keyspace
.executor(self.identity.executor_id())
.into_bytes(),
self.keyspace
.fanout_evidence(self.identity.node_identity())
.into_bytes(),
];
for worker in &sorted {
keys.push(self.keyspace.capability(worker).into_bytes());
}
let fanout_ready = matches!(state, JobExecutorState::Active);
let mut argv: Vec<Vec<u8>> = vec![
self.identity.executor_id().as_bytes().to_vec(),
state.wire_name().as_bytes().to_vec(),
self.config.executor_expire_ms.to_string().into_bytes(),
inflight.to_string().into_bytes(),
if fanout_ready {
b"true".to_vec()
} else {
b"false".to_vec()
},
];
if let Some((request_id, expected_revision)) = idempotency {
argv.push(request_id.as_bytes().to_vec());
argv.push(expected_revision.to_string().into_bytes());
}
let raw = if idempotency.is_some() {
eval_retry_safe(&self.client, &EXECUTOR_HEARTBEAT, &keys, &argv).await?
} else {
eval(&self.client, &EXECUTOR_HEARTBEAT, &keys, &argv).await?
};
interpret_heartbeat(&raw)
}
pub async fn unregister(&self) -> Result<UnregisterOutcome> {
let sorted: Vec<String> = self
.known_workers
.lock()
.expect("registry known workers lock")
.iter()
.cloned()
.collect();
let mut keys: Vec<Vec<u8>> = vec![
self.keyspace.executors().into_bytes(),
self.keyspace
.executor(self.identity.executor_id())
.into_bytes(),
self.keyspace.registry_gc().into_bytes(),
];
for worker in &sorted {
keys.push(self.keyspace.capability(worker).into_bytes());
keys.push(self.keyspace.capability_meta(worker).into_bytes());
}
let raw = eval(
&self.client,
&EXECUTOR_UNREGISTER,
&keys,
&[self.identity.executor_id().as_bytes().to_vec()],
)
.await?;
interpret_unregister(&raw)
}
pub async fn remove_capability(&self, worker_name: &str) -> Result<()> {
self.workers
.lock()
.expect("registry workers lock")
.remove(worker_name);
let keys: Vec<Vec<u8>> = vec![
self.keyspace.capability(worker_name).into_bytes(),
self.keyspace.capability_meta(worker_name).into_bytes(),
self.keyspace
.executor(self.identity.executor_id())
.into_bytes(),
];
let raw = eval(
&self.client,
&EXECUTOR_REMOVE_CAPABILITY,
&keys,
&[
self.identity.executor_id().as_bytes().to_vec(),
worker_name.as_bytes().to_vec(),
],
)
.await?;
let code = raw.first().map(value_to_string).unwrap_or_default();
if code == "OK" {
self.known_workers
.lock()
.expect("registry known workers lock")
.remove(worker_name);
Ok(())
} else {
Err(protocol("executor_remove_capability 返回未知码"))
}
}
pub async fn registry_gc(&self) -> Result<RegistryGc> {
let keys: Vec<Vec<u8>> = vec![
self.keyspace.executors().into_bytes(),
self.keyspace.registry_gc().into_bytes(),
];
let argv: Vec<Vec<u8>> = vec![
self.config.registry_gc_grace_ms.to_string().into_bytes(),
self.config.scan_batch_size.to_string().into_bytes(),
self.keyspace.registry_prefix().into_bytes(),
];
let raw = eval(&self.client, ®ISTRY_GC, &keys, &argv).await?;
interpret_registry_gc(&raw)
}
pub async fn record_fanout_evidence(
&self,
node_identity: &str,
startup_id: &str,
heartbeat_revision: i64,
fanout_id: &str,
) -> Result<RecordEvidenceOutcome> {
let keys: Vec<Vec<u8>> = vec![
self.keyspace.executors().into_bytes(),
self.keyspace.fanout_evidence(node_identity).into_bytes(),
];
let argv: Vec<Vec<u8>> = vec![
node_identity.as_bytes().to_vec(),
startup_id.as_bytes().to_vec(),
heartbeat_revision.to_string().into_bytes(),
fanout_id.as_bytes().to_vec(),
self.config
.node_unready_evidence_count
.to_string()
.into_bytes(),
self.config.executor_expire_ms.to_string().into_bytes(),
self.config.registry_gc_grace_ms.to_string().into_bytes(),
self.keyspace.registry_prefix().into_bytes(),
];
let raw = eval(&self.client, &EXECUTOR_RECORD_FANOUT_EVIDENCE, &keys, &argv).await?;
let code = raw.first().map(value_to_string).unwrap_or_default();
Ok(match code.as_str() {
"STALE_STARTUP" => RecordEvidenceOutcome::StaleStartup,
"STALE_HEARTBEAT" => RecordEvidenceOutcome::StaleHeartbeat,
"NOT_ACTIVE" => RecordEvidenceOutcome::NotActive,
"RECORDED" => RecordEvidenceOutcome::Recorded {
count: parse_i64_value(
raw.get(1)
.ok_or_else(|| protocol("record_evidence 缺少 count"))?,
"count",
)?,
},
"MARKED_UNREADY" => RecordEvidenceOutcome::MarkedUnready {
count: parse_i64_value(
raw.get(1)
.ok_or_else(|| protocol("record_evidence 缺少 count"))?,
"count",
)?,
until: parse_i64_value(
raw.get(2)
.ok_or_else(|| protocol("record_evidence 缺少 until"))?,
"until",
)?,
},
_ => return Err(protocol("executor_record_fanout_evidence 返回未知码")),
})
}
pub async fn snapshot(
&self,
worker_name: &str,
contract_revision: i64,
schema_id: &str,
codecs: &str,
) -> Result<SnapshotOutcome> {
let keys: Vec<Vec<u8>> = vec![
self.keyspace.executors().into_bytes(),
self.keyspace.capability(worker_name).into_bytes(),
self.keyspace.capability_meta(worker_name).into_bytes(),
self.keyspace
.contract(worker_name, contract_revision)
.into_bytes(),
];
let argv: Vec<Vec<u8>> = vec![
worker_name.as_bytes().to_vec(),
contract_revision.to_string().into_bytes(),
schema_id.as_bytes().to_vec(),
codecs.as_bytes().to_vec(),
self.config.fanout_max_members.to_string().into_bytes(),
self.keyspace.registry_prefix().into_bytes(),
];
let raw = eval(&self.client, &FANOUT_SNAPSHOT, &keys, &argv).await?;
interpret_snapshot(
&raw,
self.keyspace.qualifier(),
self.keyspace.namespace(),
worker_name,
)
}
}
fn interpret_snapshot(
raw: &[redis::Value],
qualifier: &str,
namespace: &str,
worker_name: &str,
) -> Result<SnapshotOutcome> {
let code = raw.first().map(value_to_string).unwrap_or_default();
if code == "CONTRACT_MISMATCH" {
return Ok(SnapshotOutcome::ContractMismatch);
}
if code != "OK" {
return Err(protocol("fanout_snapshot 返回未知码"));
}
let selected_at = parse_i64_value(
raw.get(1)
.ok_or_else(|| protocol("fanout_snapshot 缺少 selectedAt"))?,
"selectedAt",
)?;
let body = &raw[2.min(raw.len())..];
if !body.len().is_multiple_of(7) {
return Err(protocol("fanout_snapshot 成员字段不完整"));
}
let mut members = Vec::with_capacity(body.len() / 7);
for chunk in body.as_chunks::<7>().0 {
members.push(ExecutorMember {
node_identity: value_to_string(&chunk[0]),
executor_id: value_to_string(&chunk[1]),
startup_id: value_to_string(&chunk[2]),
application_name: value_to_string(&chunk[3]),
runtime: value_to_string(&chunk[4]),
implementation_digest: value_to_string(&chunk[5]),
heartbeat_revision: parse_i64_value(&chunk[6], "heartbeatRevision")?,
});
}
members.sort_by(|a, b| a.node_identity.cmp(&b.node_identity));
let canonical = members
.iter()
.flat_map(|member| {
[
member.node_identity.clone(),
member.executor_id.clone(),
member.heartbeat_revision.to_string(),
]
})
.collect::<Vec<_>>();
let digest = snapshot_digest(qualifier, namespace, worker_name, selected_at, &canonical);
let snapshot_id = digest[..32].to_owned();
Ok(SnapshotOutcome::Ok(ClusterSnapshot {
snapshot_id,
worker_name: worker_name.to_owned(),
selected_at,
members,
snapshot_digest: digest,
}))
}
fn interpret_register_capability(raw: &[redis::Value]) -> Result<RegisterCapabilityOutcome> {
let code = raw.first().map(value_to_string).unwrap_or_default();
match code.as_str() {
"WORKER_KEY_CONFLICT" => Ok(RegisterCapabilityOutcome::WorkerKeyConflict),
"CONTRACT_MISMATCH" => Ok(RegisterCapabilityOutcome::ContractMismatch),
"OK" => {
let (redis_now, expire_at, heartbeat_revision) = parse_liveness(raw)?;
Ok(RegisterCapabilityOutcome::Ok {
redis_now,
expire_at,
heartbeat_revision,
})
}
_ => Err(protocol("executor_register 返回未知码")),
}
}
fn interpret_heartbeat(raw: &[redis::Value]) -> Result<HeartbeatOutcome> {
let code = raw.first().map(value_to_string).unwrap_or_default();
match code.as_str() {
"NOT_FOUND" => Ok(HeartbeatOutcome::NotFound),
"OK" => {
let (redis_now, expire_at, heartbeat_revision) = parse_liveness(raw)?;
Ok(HeartbeatOutcome::Ok {
redis_now,
expire_at,
heartbeat_revision,
})
}
"STALE_AUTHORITY" => Err(crate::job::JobError::StaleOwner(
"executor_heartbeat 的 heartbeatRevision 已被其它权威推进".to_owned(),
)
.into()),
_ => Err(protocol("executor_heartbeat 返回未知码")),
}
}
fn interpret_unregister(raw: &[redis::Value]) -> Result<UnregisterOutcome> {
let code = raw.first().map(value_to_string).unwrap_or_default();
Ok(match code.as_str() {
"OK" => UnregisterOutcome::Ok,
"NOT_FOUND" => UnregisterOutcome::NotFound,
"STALE_EXECUTOR" => UnregisterOutcome::StaleExecutor,
_ => return Err(protocol("executor_unregister 返回未知码")),
})
}
fn interpret_registry_gc(raw: &[redis::Value]) -> Result<RegistryGc> {
if raw.len() < 3 {
return Err(protocol("registry_gc 返回结构非法"));
}
Ok(RegistryGc {
deleted: parse_i64_value(&raw[0], "registry_gc deleted")?,
newly_expired: parse_i64_value(&raw[1], "registry_gc newlyExpired")?,
redis_now: parse_i64_value(&raw[2], "registry_gc redisNow")?,
})
}
fn parse_liveness(raw: &[redis::Value]) -> Result<(i64, i64, i64)> {
Ok((
parse_i64_value(
raw.get(1)
.ok_or_else(|| protocol("存活返回缺少 redisNow"))?,
"redisNow",
)?,
parse_i64_value(
raw.get(2)
.ok_or_else(|| protocol("存活返回缺少 expireAt"))?,
"expireAt",
)?,
parse_i64_value(
raw.get(3)
.ok_or_else(|| protocol("存活返回缺少 heartbeatRevision"))?,
"heartbeatRevision",
)?,
))
}
fn parse_i64_value(value: &redis::Value, field: &str) -> Result<i64> {
value_to_string(value)
.parse::<i64>()
.map_err(|_| protocol(&format!("registry {field} 字段非法")))
}
fn protocol(message: &str) -> NasaRedisError {
crate::job::JobError::Protocol(message.to_owned()).into()
}