use std::collections::BTreeMap;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use crate::error::{NasaRedisError, Result};
use crate::job::definition::JobDefinition;
use crate::job::fanout::{FanoutRepository, FanoutRoot, FanoutShard};
use crate::job::metrics::{JobMetrics, JobMetricsSnapshot, JobSupervisorLoop};
use crate::job::model::JobDefinitionState;
use crate::job::names::require_name;
use crate::job::payload::JobPayload;
use crate::job::registry::ExecutorRegistry;
use crate::job::repository::{
CancelOutcome, DefinitionControlOutcome, DeleteOutcome, JobDefinitionRecord, JobRepository,
ManualFireOutcome, NamespaceStateOutcome,
};
use crate::job::run::JobRun;
#[derive(Debug, Clone, Default, PartialEq, Eq)]
#[non_exhaustive]
pub struct NamespaceGovernanceReport {
pub total_shards: u32,
pub enabled: u32,
pub paused: u32,
pub untouched: u32,
pub unknown: u32,
pub divergent: bool,
pub last_updated_at_ms: Option<i64>,
pub last_updated_by: Option<String>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum JobSourceHealthState {
Up,
Degraded,
NotReady,
Draining,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum JobSourceHealthReason {
RedisTransport,
ProtocolContract,
ExecutionUnknown,
ExecutionAuthority,
RuntimeConfiguration,
SubscriptionNotReady,
MetricsReservation,
CronSemantics,
RestartBudgetExhausted,
CriticalRuntime,
}
impl JobSourceHealthReason {
pub fn as_str(self) -> &'static str {
match self {
Self::RedisTransport => "redis_transport",
Self::ProtocolContract => "protocol_contract",
Self::ExecutionUnknown => "execution_unknown",
Self::ExecutionAuthority => "execution_authority",
Self::RuntimeConfiguration => "runtime_configuration",
Self::SubscriptionNotReady => "subscription_not_ready",
Self::MetricsReservation => "metrics_reservation",
Self::CronSemantics => "cron_semantics",
Self::RestartBudgetExhausted => "restart_budget_exhausted",
Self::CriticalRuntime => "critical_runtime",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct JobSourceHealthSnapshot {
pub qualifier: String,
pub accepting: bool,
pub healthy: bool,
pub state: JobSourceHealthState,
pub reason: Option<JobSourceHealthReason>,
pub failure_reason: Option<String>,
}
pub(crate) struct SourceHealth {
qualifier: String,
accepting: AtomicBool,
draining: AtomicBool,
terminal_reason: Mutex<Option<JobSourceHealthReason>>,
degraded_loops: Mutex<BTreeMap<JobSupervisorLoop, (JobSourceHealthReason, u32)>>,
metrics: Arc<JobMetrics>,
}
impl SourceHealth {
pub(crate) fn new(qualifier: impl Into<String>, metrics: Arc<JobMetrics>) -> Self {
Self {
qualifier: qualifier.into(),
accepting: AtomicBool::new(true),
draining: AtomicBool::new(false),
terminal_reason: Mutex::new(None),
degraded_loops: Mutex::new(BTreeMap::new()),
metrics,
}
}
pub(crate) fn restart_scheduled(
&self,
loop_name: JobSupervisorLoop,
reason: JobSourceHealthReason,
first_failure: bool,
) -> u64 {
if first_failure {
let mut degraded = self
.degraded_loops
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let entry = degraded.entry(loop_name).or_insert((reason, 0));
entry.0 = reason;
entry.1 = entry.1.saturating_add(1);
}
let generation = self.metrics.restart_scheduled(loop_name);
tracing::warn!(
source = %self.qualifier,
supervisor_loop = loop_name.as_str(),
reason = reason.as_str(),
generation,
"redis-job supervisor scheduled a bounded recovery attempt"
);
generation
}
pub(crate) fn recovered(&self, loop_name: JobSupervisorLoop) {
let mut degraded = self
.degraded_loops
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some((_, count)) = degraded.get_mut(&loop_name) {
*count = count.saturating_sub(1);
if *count == 0 {
degraded.remove(&loop_name);
}
}
drop(degraded);
self.metrics.recovered(loop_name);
tracing::info!(
source = %self.qualifier,
supervisor_loop = loop_name.as_str(),
"redis-job supervisor recovered"
);
}
pub(crate) fn budget_exhausted(&self, loop_name: JobSupervisorLoop) {
self.accepting.store(false, Ordering::Release);
self.metrics.budget_exhausted(loop_name);
let mut terminal = self
.terminal_reason
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
terminal.get_or_insert(JobSourceHealthReason::RestartBudgetExhausted);
tracing::error!(
source = %self.qualifier,
supervisor_loop = loop_name.as_str(),
reason = JobSourceHealthReason::RestartBudgetExhausted.as_str(),
"redis-job supervisor entered terminal state"
);
}
pub(crate) fn authority_lost(&self, loop_name: JobSupervisorLoop, error: &NasaRedisError) {
self.accepting.store(false, Ordering::Release);
self.metrics.authority_lost(loop_name);
let reason = classify_error(error);
let mut terminal = self
.terminal_reason
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
terminal.get_or_insert(reason);
tracing::error!(
source = %self.qualifier,
supervisor_loop = loop_name.as_str(),
reason = reason.as_str(),
"redis-job control authority is no longer valid"
);
}
pub(crate) fn stop_accepting(&self) {
self.accepting.store(false, Ordering::Release);
self.draining.store(true, Ordering::Release);
}
pub(crate) fn metrics(&self) -> Arc<JobMetrics> {
self.metrics.clone()
}
pub(crate) fn snapshot(&self) -> JobSourceHealthSnapshot {
let terminal = *self
.terminal_reason
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let degraded = self
.degraded_loops
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let reason = terminal.or_else(|| degraded.values().next().map(|(reason, _)| *reason));
let state = if terminal.is_some() {
JobSourceHealthState::NotReady
} else if self.draining.load(Ordering::Acquire) {
JobSourceHealthState::Draining
} else if degraded.is_empty() {
JobSourceHealthState::Up
} else {
JobSourceHealthState::Degraded
};
JobSourceHealthSnapshot {
qualifier: self.qualifier.clone(),
accepting: self.accepting.load(Ordering::Acquire),
healthy: terminal.is_none(),
state,
reason,
failure_reason: reason.map(|value| value.as_str().to_owned()),
}
}
}
fn redis_response_error_is_transient(error: &redis::RedisError) -> bool {
matches!(
error.kind(),
redis::ErrorKind::Server(redis::ServerErrorKind::ResponseError)
) && error.detail().is_some_and(|detail| {
detail
.trim()
.eq_ignore_ascii_case("max number of clients reached")
})
}
fn redis_error_is_access_denied(error: &redis::RedisError) -> bool {
error.code().is_some_and(|code| {
matches!(
code.trim().to_ascii_uppercase().as_str(),
"NOAUTH" | "WRONGPASS" | "NOPERM"
)
})
}
pub(crate) fn classify_error(error: &NasaRedisError) -> JobSourceHealthReason {
match error {
NasaRedisError::Redis(error) => match error.kind() {
redis::ErrorKind::Server(
redis::ServerErrorKind::BusyLoading
| redis::ServerErrorKind::TryAgain
| redis::ServerErrorKind::ClusterDown
| redis::ServerErrorKind::MasterDown
| redis::ServerErrorKind::Moved
| redis::ServerErrorKind::Ask
| redis::ServerErrorKind::ReadOnly,
) => JobSourceHealthReason::RedisTransport,
redis::ErrorKind::Server(redis::ServerErrorKind::ResponseError)
if redis_response_error_is_transient(error) =>
{
JobSourceHealthReason::RedisTransport
}
redis::ErrorKind::Server(redis::ServerErrorKind::NoPerm) => {
JobSourceHealthReason::RuntimeConfiguration
}
redis::ErrorKind::Server(_) => JobSourceHealthReason::ProtocolContract,
redis::ErrorKind::Io
| redis::ErrorKind::ClusterConnectionNotFound
| redis::ErrorKind::NoValidReplicasFoundBySentinel => {
JobSourceHealthReason::RedisTransport
}
redis::ErrorKind::AuthenticationFailed
| redis::ErrorKind::InvalidClientConfig
| redis::ErrorKind::Client
| redis::ErrorKind::MasterNameNotFoundBySentinel
| redis::ErrorKind::EmptySentinelList => JobSourceHealthReason::RuntimeConfiguration,
redis::ErrorKind::Extension if redis_error_is_access_denied(error) => {
JobSourceHealthReason::RuntimeConfiguration
}
redis::ErrorKind::Parse
| redis::ErrorKind::UnexpectedReturnType
| redis::ErrorKind::Extension
| redis::ErrorKind::RESP3NotSupported => JobSourceHealthReason::ProtocolContract,
_ => JobSourceHealthReason::ProtocolContract,
},
NasaRedisError::ConnectProbe { .. } => JobSourceHealthReason::RedisTransport,
NasaRedisError::JobProtocol(_) => JobSourceHealthReason::ProtocolContract,
NasaRedisError::ExecutionUnknown(_) => JobSourceHealthReason::ExecutionUnknown,
NasaRedisError::JobExecutionStopped(_) => JobSourceHealthReason::ExecutionAuthority,
NasaRedisError::Config(_) => JobSourceHealthReason::RuntimeConfiguration,
NasaRedisError::Job(error) => match error {
crate::job::JobError::ExecutionUnknown(_) => JobSourceHealthReason::ExecutionUnknown,
crate::job::JobError::ExecutionStopped(_)
| crate::job::JobError::StaleOwner(_)
| crate::job::JobError::StaleAssignment(_)
| crate::job::JobError::FencingRegression(_) => {
JobSourceHealthReason::ExecutionAuthority
}
crate::job::JobError::Protocol(_)
| crate::job::JobError::SourceMismatch(_)
| crate::job::JobError::ContractMismatch(_)
| crate::job::JobError::InvalidPayload(_) => JobSourceHealthReason::ProtocolContract,
_ => JobSourceHealthReason::RuntimeConfiguration,
},
_ => JobSourceHealthReason::CriticalRuntime,
}
}
pub(crate) struct ManagedJobControlState {
registry: Arc<ExecutorRegistry>,
definitions: Mutex<BTreeMap<String, Option<String>>>,
pending_withdrawals: Mutex<std::collections::BTreeSet<String>>,
completed_deletions: Mutex<BTreeMap<(String, i64), DeleteOutcome>>,
}
impl ManagedJobControlState {
pub(crate) fn new(
registry: Arc<ExecutorRegistry>,
definitions: BTreeMap<String, Option<String>>,
) -> Arc<Self> {
Arc::new(Self {
registry,
definitions: Mutex::new(definitions),
pending_withdrawals: Mutex::new(std::collections::BTreeSet::new()),
completed_deletions: Mutex::new(BTreeMap::new()),
})
}
fn contains_definition(&self, job_name: &str) -> bool {
self.definitions
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(job_name)
}
fn completed_deletion(&self, job_name: &str, revision: i64) -> Option<DeleteOutcome> {
self.completed_deletions
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(&(job_name.to_owned(), revision))
.cloned()
}
fn definition_deleted(&self, job_name: &str, revision: i64, outcome: DeleteOutcome) {
self.completed_deletions
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.entry((job_name.to_owned(), revision))
.or_insert(outcome);
let worker = {
let mut definitions = self
.definitions
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let Some(worker) = definitions.remove(job_name) else {
return;
};
let Some(worker) = worker else {
return;
};
if definitions
.values()
.any(|other| other.as_ref() == Some(&worker))
{
return;
}
worker
};
self.pending_withdrawals
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(worker);
}
async fn withdraw_pending(&self) -> Result<()> {
let workers: Vec<String> = self
.pending_withdrawals
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.iter()
.cloned()
.collect();
for worker in workers {
self.registry.remove_capability(&worker).await?;
self.pending_withdrawals
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.remove(&worker);
}
Ok(())
}
}
#[derive(Clone)]
pub struct JobControl {
repository: JobRepository,
fanout: FanoutRepository,
health: Option<Arc<SourceHealth>>,
managed: Option<Arc<ManagedJobControlState>>,
}
impl JobControl {
pub fn new(repository: JobRepository, fanout: FanoutRepository) -> Self {
Self {
repository,
fanout,
health: None,
managed: None,
}
}
pub(crate) fn new_with_health(
repository: JobRepository,
fanout: FanoutRepository,
health: Arc<SourceHealth>,
managed: Arc<ManagedJobControlState>,
) -> Self {
Self {
repository,
fanout,
health: Some(health),
managed: Some(managed),
}
}
fn ensure_accepting(&self) -> Result<()> {
let Some(health) = &self.health else {
return Ok(());
};
let snapshot = health.snapshot();
if snapshot.accepting {
Ok(())
} else {
Err(crate::job::JobError::ExecutionStopped(format!(
"source {} 已关闭控制准入",
snapshot.qualifier
))
.into())
}
}
pub async fn trigger(
&self,
definition: &JobDefinition,
request_id: &str,
payload: &JobPayload,
) -> Result<ManualFireOutcome> {
self.ensure_accepting()?;
let outcome = self
.repository
.manual_fire(definition, request_id, payload)
.await?;
let run_id = match &outcome {
ManualFireOutcome::Fired { run_id, .. } | ManualFireOutcome::Adopted { run_id } => {
Some(run_id.as_str())
}
ManualFireOutcome::NotFound | ManualFireOutcome::StateMismatch => None,
};
if let (Some(context), Some(run_id)) = (natelemetry::ambient(), run_id) {
tracing::info!(
trace_id = %context.trace_id_hex(),
run_id,
job = definition.name(),
"redis-job manual trigger linked to ambient trace"
);
}
Ok(outcome)
}
pub async fn pause(&self, job_name: &str) -> Result<DefinitionControlOutcome> {
self.ensure_accepting()?;
self.repository.pause(job_name).await
}
pub async fn resume(
&self,
job_name: &str,
next_fire_at: i64,
) -> Result<DefinitionControlOutcome> {
self.ensure_accepting()?;
self.repository.resume(job_name, next_fire_at).await
}
pub async fn pause_namespace(&self, actor: &str) -> Result<()> {
self.set_namespace_state(JobDefinitionState::Paused, actor)
.await
}
pub async fn resume_namespace(&self, actor: &str) -> Result<()> {
self.set_namespace_state(JobDefinitionState::Enabled, actor)
.await
}
pub async fn namespace_governance(&self) -> Result<NamespaceGovernanceReport> {
self.ensure_accepting()?;
let total = self.repository.shard_count();
let mut report = NamespaceGovernanceReport {
total_shards: total,
..Default::default()
};
for shard in 0..total {
let snapshot = self.repository.namespace_state_snapshot(shard).await?;
match snapshot.state.as_deref() {
Some("PAUSED") => report.paused = report.paused.saturating_add(1),
Some("ENABLED") => report.enabled = report.enabled.saturating_add(1),
Some(_) => report.unknown = report.unknown.saturating_add(1),
None => report.untouched = report.untouched.saturating_add(1),
}
if snapshot.updated_at_ms > report.last_updated_at_ms {
report.last_updated_at_ms = snapshot.updated_at_ms;
report.last_updated_by = snapshot.updated_by;
}
}
report.divergent =
report.unknown > 0 || (report.paused > 0 && report.paused < report.total_shards);
Ok(report)
}
async fn set_namespace_state(&self, state: JobDefinitionState, actor: &str) -> Result<()> {
self.ensure_accepting()?;
let actor = require_name(actor, "actor")?;
let total = self.repository.shard_count();
for shard in 0..total {
let outcome = self
.repository
.set_namespace_state(shard, state, &actor)
.await
.inspect_err(|_| {
tracing::warn!(
completed_shards = shard,
total_shards = total,
target_state = state.wire_name(),
"redis-job namespace state propagation stopped before convergence"
);
})?;
if matches!(outcome, NamespaceStateOutcome::Invalid) {
return Err(crate::job::JobError::Protocol(
"namespace_set_state 拒绝内部已校验状态".to_owned(),
)
.into());
}
}
Ok(())
}
pub async fn request_cancel(
&self,
definition: &JobDefinition,
run_id: &str,
) -> Result<CancelOutcome> {
self.ensure_accepting()?;
let run = self.repository.read_run(definition.name(), run_id).await?;
let outcome = self.repository.request_cancel(definition, run_id).await?;
if matches!(outcome, CancelOutcome::Ok { .. }) {
if let Some(run) = run.filter(|run| !run.fanout_id.is_empty()) {
if let Some(root) = self.fanout.read_root(&run.fanout_id).await? {
self.fanout.cancel_batch(&root, "CANCEL_REQUESTED").await?;
}
}
}
Ok(outcome)
}
pub async fn delete_definition(&self, job_name: &str, revision: i64) -> Result<DeleteOutcome> {
self.ensure_accepting()?;
if let Some(managed) = &self.managed {
managed.withdraw_pending().await?;
if let Some(outcome) = managed.completed_deletion(job_name, revision) {
return Ok(outcome);
}
if !managed.contains_definition(job_name) {
return Err(crate::job::JobError::Config(format!(
"当前 source 未登记本地定义 {job_name}"
))
.into());
}
}
let outcome = self.repository.delete(job_name, revision).await?;
if matches!(outcome, DeleteOutcome::Ok { .. }) {
if let Some(managed) = &self.managed {
managed.definition_deleted(job_name, revision, outcome.clone());
managed.withdraw_pending().await?;
}
}
Ok(outcome)
}
pub async fn resolve_conflict(
&self,
definition: &JobDefinition,
next_fire_at: i64,
) -> Result<DefinitionControlOutcome> {
self.ensure_accepting()?;
self.repository
.resolve_conflict(definition, next_fire_at)
.await
}
}
#[derive(Clone)]
pub struct JobQuery {
repository: JobRepository,
fanout: FanoutRepository,
health: Arc<SourceHealth>,
metrics: Arc<JobMetrics>,
}
impl JobQuery {
pub(crate) fn new(
repository: JobRepository,
fanout: FanoutRepository,
health: Arc<SourceHealth>,
metrics: Arc<JobMetrics>,
) -> Self {
Self {
repository,
fanout,
health,
metrics,
}
}
pub async fn definition(&self, job_name: &str) -> Result<Option<JobDefinitionRecord>> {
self.repository.definition_record(job_name).await
}
pub async fn run(&self, job_name: &str, run_id: &str) -> Result<Option<JobRun>> {
self.repository.read_run(job_name, run_id).await
}
pub async fn fanout_root(&self, fanout_id: &str) -> Result<Option<FanoutRoot>> {
self.fanout.read_root(fanout_id).await
}
pub async fn fanout_shard(&self, fanout_id: &str, seq: i64) -> Result<Option<FanoutShard>> {
self.fanout.read_shard(fanout_id, seq).await
}
pub fn health(&self) -> JobSourceHealthSnapshot {
self.health.snapshot()
}
pub fn metrics(&self) -> JobMetricsSnapshot {
self.metrics.snapshot()
}
}