use std::collections::HashMap;
use std::sync::Arc;
use crate::client::RedisClient;
use crate::error::{NasaRedisError, Result};
use crate::job::config::JobConfig;
use crate::job::definition::JobDefinition;
use crate::job::fanout::{
FanoutDeliverOutcome, FanoutReadyDeferOutcome, FanoutReadyPromoteOutcome,
FanoutReassignOutcome, FanoutReceiptRetryOutcome, FanoutRepository, FanoutRoot, FanoutShard,
FanoutWatchRootOutcome,
};
use crate::job::keyspace::JobKeyspace;
use crate::job::metrics::JobMetrics;
use crate::job::model::{JobFanoutFailurePolicy, JobState};
use crate::job::registry::{ExecutorMember, ExecutorRegistry, SnapshotOutcome};
use crate::job::repository::{FinishFanoutRootOutcome, JobReapFanoutRequest, JobRepository};
const TERMINAL_ROOT_STATES: [&str; 4] = ["SUCCEEDED", "PARTIAL_FAILED", "FAILED", "CANCELLED"];
pub struct FanoutMonitor {
client: Arc<RedisClient>,
repository: JobRepository,
fanout: FanoutRepository,
keyspace: JobKeyspace,
config: Arc<JobConfig>,
registry: Arc<ExecutorRegistry>,
root_definitions: HashMap<String, JobDefinition>,
worker_definitions: HashMap<String, JobDefinition>,
metrics: Option<Arc<JobMetrics>>,
}
impl FanoutMonitor {
pub fn new(
client: Arc<RedisClient>,
keyspace: JobKeyspace,
config: Arc<JobConfig>,
registry: Arc<ExecutorRegistry>,
) -> Self {
let repository = JobRepository::new(client.clone(), keyspace.clone(), config.clone());
let fanout = FanoutRepository::new(client.clone(), keyspace.clone(), config.clone());
Self {
client,
repository,
fanout,
keyspace,
config,
registry,
root_definitions: HashMap::new(),
worker_definitions: HashMap::new(),
metrics: None,
}
}
pub(crate) fn new_with_metrics(
client: Arc<RedisClient>,
keyspace: JobKeyspace,
config: Arc<JobConfig>,
registry: Arc<ExecutorRegistry>,
metrics: Arc<JobMetrics>,
) -> Self {
let mut monitor = Self::new(client, keyspace, config, registry);
monitor.metrics = Some(metrics);
monitor
}
pub fn register_definition(&mut self, definition: JobDefinition) {
self.worker_definitions
.insert(definition.worker_name().to_owned(), definition.clone());
self.root_definitions
.insert(definition.name().to_owned(), definition);
}
pub async fn reconcile_root(&self, fanout_id: &str) -> Result<Option<String>> {
let (state, root_run_id, root_attempt, root_job_name, cancel_reason, error_type) =
match self.fanout.watch_root(fanout_id).await? {
FanoutWatchRootOutcome::NotFound => return Ok(None),
FanoutWatchRootOutcome::Ok {
state,
root_run_id,
root_attempt,
root_job_name,
cancel_reason,
error_type,
..
} => (
state,
root_run_id,
root_attempt,
root_job_name,
cancel_reason,
error_type,
),
};
if !TERMINAL_ROOT_STATES.contains(&state.as_str()) {
return Ok(Some(state));
}
let Some(definition) = self.root_definitions.get(&root_job_name) else {
return Ok(Some(state));
};
let summary = if cancel_reason.is_empty() {
String::new()
} else {
cancel_reason.clone()
};
let terminal_error = if cancel_reason == "JOB_DELETED" {
"JOB_DELETED"
} else {
error_type.as_str()
};
match self
.repository
.finish_fanout_root(
definition,
&root_run_id,
fanout_id,
root_attempt,
&state,
terminal_error,
&summary,
)
.await?
{
FinishFanoutRootOutcome::Ok { .. }
| FinishFanoutRootOutcome::Adopted
| FinishFanoutRootOutcome::AlreadyCompleted { .. } => {}
other => return Err(protocol(&format!("finish_fanout_root 回填拒绝: {other:?}"))),
}
let _ = self.fanout.mark_reconciled(fanout_id).await?;
Ok(Some(state))
}
pub(crate) async fn drive_deleted_fanout(&self, request: &JobReapFanoutRequest) -> Result<()> {
let watched = self.fanout.watch_root(&request.fanout_id).await?;
let FanoutWatchRootOutcome::Ok {
state,
root_run_id,
root_attempt,
root_job_name,
cancel_reason,
error_type,
..
} = watched
else {
return self
.finish_deleted_root(request, "CANCELLED", "JOB_DELETED", "")
.await;
};
if root_run_id != request.run_id
|| root_attempt != request.root_attempt
|| root_job_name != request.job_name
{
return Err(protocol("删除收敛的 Fanout 根定位证据冲突"));
}
if TERMINAL_ROOT_STATES.contains(&state.as_str()) {
let terminal_error = if cancel_reason == "JOB_DELETED" {
"JOB_DELETED"
} else {
error_type.as_str()
};
self.finish_deleted_root(request, &state, terminal_error, &cancel_reason)
.await?;
let _ = self.fanout.mark_reconciled(&request.fanout_id).await?;
return Ok(());
}
let Some(root) = self.fanout.read_root(&request.fanout_id).await? else {
return Ok(());
};
let _ = self.fanout.cancel_batch(&root, "JOB_DELETED").await?;
Ok(())
}
async fn finish_deleted_root(
&self,
request: &JobReapFanoutRequest,
state: &str,
error_type: &str,
summary: &str,
) -> Result<()> {
match self
.repository
.finish_fanout_root_for_job(
&request.job_name,
&request.run_id,
&request.fanout_id,
request.root_attempt,
state,
error_type,
summary,
)
.await?
{
FinishFanoutRootOutcome::Ok { .. }
| FinishFanoutRootOutcome::Adopted
| FinishFanoutRootOutcome::AlreadyCompleted { .. } => Ok(()),
other => Err(protocol(&format!(
"删除收敛的 finish_fanout_root 回填拒绝: {other:?}"
))),
}
}
pub async fn reconcile_bucket(&self, bucket: u32) -> Result<usize> {
let due = self.due_roots(bucket).await?;
let count = due.len();
for fanout_id in due {
self.advance_root(&fanout_id).await?;
self.reconcile_root(&fanout_id).await?;
}
self.sweep_watchdogs(bucket).await?;
Ok(count)
}
async fn sweep_watchdogs(&self, bucket: u32) -> Result<()> {
let scans = self.fanout.scan_due(bucket).await?;
if let Some(metrics) = &self.metrics {
metrics.set_shard_gauge(
"redis_job_fanout_delivery_pending",
bucket,
scans.root_count,
);
metrics.set_shard_gauge(
"redis_job_fanout_gc_pending",
bucket,
i64::try_from(scans.gc.members.len()).unwrap_or(i64::MAX),
);
}
for (member, _score) in &scans.receipts.members {
let Some((fanout_id, seq)) = parse_member(member) else {
continue;
};
if let (Some(root), Some(shard)) = self.root_and_shard(&fanout_id, seq).await? {
match self.fanout.retry_receipt(&root, &shard).await? {
FanoutReceiptRetryOutcome::Ok { .. } => {
self.add_metric("redis_job_fanout_receipt_timeout_total", 1);
self.add_metric("redis_job_fanout_receipt_retry_total", 1);
}
FanoutReceiptRetryOutcome::RetryExhausted { .. } => {
self.add_metric("redis_job_fanout_receipt_timeout_total", 1);
self.handle_unavailable(&root, &shard, "RECEIPT_TIMEOUT")
.await?;
}
FanoutReceiptRetryOutcome::StaleAssignment => {
self.add_metric("redis_job_fanout_stale_assignment_total", 1);
}
FanoutReceiptRetryOutcome::Stale | FanoutReceiptRetryOutcome::NotDue { .. } => {
}
}
}
}
for (member, _score) in &scans.ready.members {
let Some((fanout_id, seq)) = parse_member(member) else {
continue;
};
if let (Some(root), Some(shard)) = self.root_and_shard(&fanout_id, seq).await? {
match self.fanout.promote_ready(&root, &shard).await? {
FanoutReadyPromoteOutcome::Ok { .. } => {
self.add_metric("redis_job_fanout_ready_wakeup_total", 1);
}
FanoutReadyPromoteOutcome::CapacityExhausted { .. } => {
self.add_metric("redis_job_fanout_ready_wakeup_total", 1);
self.handle_capacity_pressure(&root, &shard).await?;
}
FanoutReadyPromoteOutcome::WakeupExhausted { .. } => {
self.add_metric("redis_job_fanout_ready_wakeup_total", 1);
self.handle_unavailable(&root, &shard, "START_TIMEOUT")
.await?;
}
FanoutReadyPromoteOutcome::StaleAssignment => {
self.add_metric("redis_job_fanout_stale_assignment_total", 1);
}
FanoutReadyPromoteOutcome::Stale | FanoutReadyPromoteOutcome::NotDue { .. } => {
}
}
}
}
for (member, _score) in &scans.leases.members {
let Some((fanout_id, seq)) = parse_member(member) else {
continue;
};
if let (Some(root), Some(shard)) = self.root_and_shard(&fanout_id, seq).await? {
let recovered =
if let Some(definition) = self.worker_definitions.get(&shard.worker_name) {
self.fanout.recover_shard(&root, &shard, definition).await?
} else if root.state == "CANCELLING" {
self.fanout.recover_cancelling_shard(&root, &shard).await?
} else {
continue;
};
if matches!(recovered, crate::job::repository::RecoverOutcome::Ok { .. }) {
self.add_metric("redis_job_lease_expired_total", 1);
}
}
}
for (member, _score) in &scans.gc.members {
if let Some(root) = self.fanout.read_root(member).await? {
self.fanout.cleanup(&root).await?;
}
}
Ok(())
}
async fn advance_root(&self, fanout_id: &str) -> Result<()> {
let Some(root) = self.fanout.read_root(fanout_id).await? else {
return Ok(());
};
if TERMINAL_ROOT_STATES.contains(&root.state.as_str()) {
return Ok(());
}
match root.state.as_str() {
"CREATING" => {
self.fanout.fail_creating(fanout_id).await?;
return Ok(());
}
"CANCELLING" => {
let reason = if root.cancel_reason.trim().is_empty() {
"CANCEL_REQUESTED"
} else {
root.cancel_reason.as_str()
};
self.fanout.cancel_batch(&root, reason).await?;
return Ok(());
}
"COMMITTED" | "WAITING_CHILDREN" => {}
_ => return Err(protocol("Fanout 根状态不属于封闭状态集合")),
}
if root.delivery_cursor < root.shard_total {
let count = (root.shard_total - root.delivery_cursor)
.min(self.config.fanout_delivery_batch_size as i64)
.max(0);
let mut shards = Vec::with_capacity(count as usize);
for seq in root.delivery_cursor..root.delivery_cursor + count {
let shard = self
.fanout
.read_shard(fanout_id, seq)
.await?
.ok_or_else(|| protocol("首次投递恢复缺少已提交的 shard"))?;
shards.push(shard);
}
match self
.fanout
.deliver_assignments(fanout_id, &shards, true)
.await?
{
FanoutDeliverOutcome::Ok { .. } => {}
FanoutDeliverOutcome::NotCommitted => {
return Err(protocol("Fanout 根投递时不再处于已提交状态"));
}
}
}
self.recover_capabilities(&root).await
}
async fn handle_unavailable(
&self,
root: &FanoutRoot,
shard: &FanoutShard,
reason: &str,
) -> Result<()> {
let policy = JobFanoutFailurePolicy::parse(&root.failure_policy)
.ok_or_else(|| protocol("Fanout 根包含未知失败策略"))?;
if policy == JobFanoutFailurePolicy::StrictSnapshot {
return Ok(());
}
self.registry
.record_fanout_evidence(
&shard.target_node_identity,
&shard.target_startup_id,
shard.target_heartbeat_revision,
&root.fanout_id,
)
.await?;
if policy == JobFanoutFailurePolicy::BestEffort {
self.fanout
.aggregate(
&root.fanout_id,
shard.seq,
JobState::Skipped,
"SKIPPED",
reason,
)
.await?;
return Ok(());
}
let candidates = self.compatible_members(root).await?;
let target = candidates
.iter()
.find(|candidate| candidate.node_identity != shard.target_node_identity);
self.reassign_and_deliver(root, shard, target).await
}
async fn reassign_and_deliver(
&self,
root: &FanoutRoot,
shard: &FanoutShard,
target: Option<&ExecutorMember>,
) -> Result<()> {
self.reassign_and_deliver_mode(root, shard, target, "FAILURE")
.await
}
async fn reassign_and_deliver_mode(
&self,
root: &FanoutRoot,
shard: &FanoutShard,
target: Option<&ExecutorMember>,
mode: &str,
) -> Result<()> {
match self
.fanout
.reassign_shard_mode(root, shard, target, mode)
.await?
{
FanoutReassignOutcome::Ok { .. } => {
self.add_metric("redis_job_fanout_reassign_total", 1);
let reassigned = self
.fanout
.read_shard(&root.fanout_id, shard.seq)
.await?
.ok_or_else(|| protocol("重分配后 shard 不存在"))?;
match self
.fanout
.deliver_assignments(&root.fanout_id, &[reassigned], false)
.await?
{
FanoutDeliverOutcome::Ok { .. } => Ok(()),
FanoutDeliverOutcome::NotCommitted => {
Err(protocol("重分配投递时 Fanout 根不再处于已提交状态"))
}
}
}
FanoutReassignOutcome::StaleAssignment => {
self.add_metric("redis_job_fanout_stale_assignment_total", 1);
Ok(())
}
FanoutReassignOutcome::Busy | FanoutReassignOutcome::NoCapableExecutor => Ok(()),
FanoutReassignOutcome::StaleCapacity => Ok(()),
FanoutReassignOutcome::CapacityRouteExhausted => {
match self
.fanout
.defer_ready_for_capacity_mode(shard, true)
.await?
{
FanoutReadyDeferOutcome::Deferred { .. }
| FanoutReadyDeferOutcome::CapacityExhausted { .. }
| FanoutReadyDeferOutcome::Stale
| FanoutReadyDeferOutcome::StaleAssignment => Ok(()),
}
}
}
}
async fn recover_capabilities(&self, root: &FanoutRoot) -> Result<()> {
if root.shard_total <= 0 {
return Ok(());
}
let start = root.capability_cursor.rem_euclid(root.shard_total);
let count = root
.shard_total
.min(self.config.fanout_delivery_batch_size as i64)
.max(0);
let candidates = self.compatible_members(root).await?;
for offset in 0..count {
let seq = (start + offset).rem_euclid(root.shard_total);
let Some(shard) = self.fanout.read_shard(&root.fanout_id, seq).await? else {
continue;
};
if shard.state != JobState::AwaitingCapability.wire_name() {
continue;
}
let target = candidates
.iter()
.find(|candidate| candidate.node_identity != shard.target_node_identity);
if target.is_some() {
self.reassign_and_deliver(root, &shard, target).await?;
}
}
let next = (start + count).rem_euclid(root.shard_total);
self.fanout
.advance_capability_cursor(&root.fanout_id, root.capability_cursor, next)
.await?;
Ok(())
}
async fn handle_capacity_pressure(&self, root: &FanoutRoot, shard: &FanoutShard) -> Result<()> {
let target = if shard.assignment_count < i64::from(self.config.fanout_max_assignments) {
self.compatible_members(root)
.await?
.into_iter()
.find(|member| member.node_identity != shard.target_node_identity)
} else {
None
};
if let Some(target) = target {
match self
.reassign_and_deliver_mode(root, shard, Some(&target), "CAPACITY")
.await
{
Ok(()) => Ok(()),
Err(error) => Err(error),
}
} else {
match self
.fanout
.defer_ready_for_capacity_mode(shard, true)
.await?
{
FanoutReadyDeferOutcome::Deferred { .. }
| FanoutReadyDeferOutcome::CapacityExhausted { .. }
| FanoutReadyDeferOutcome::Stale
| FanoutReadyDeferOutcome::StaleAssignment => Ok(()),
}
}
}
async fn compatible_members(&self, root: &FanoutRoot) -> Result<Vec<ExecutorMember>> {
let members = match self
.registry
.snapshot(
&root.worker_name,
root.contract_revision,
&root.schema_id,
&root.wire_codec,
)
.await?
{
SnapshotOutcome::Ok(snapshot) => snapshot.members,
SnapshotOutcome::ContractMismatch => Vec::new(),
};
if let Some(metrics) = &self.metrics {
metrics.set(
"redis_job_capable_executor_count",
i64::try_from(members.len()).unwrap_or(i64::MAX),
);
}
Ok(members)
}
fn add_metric(&self, name: &'static str, delta: i64) {
if let Some(metrics) = &self.metrics {
metrics.add(name, delta);
}
}
#[allow(clippy::type_complexity)]
async fn root_and_shard(
&self,
fanout_id: &str,
seq: i64,
) -> Result<(
Option<crate::job::fanout::FanoutRoot>,
Option<crate::job::fanout::FanoutShard>,
)> {
let root = self.fanout.read_root(fanout_id).await?;
let shard = self.fanout.read_shard(fanout_id, seq).await?;
Ok((root, shard))
}
async fn due_roots(&self, bucket: u32) -> Result<Vec<String>> {
let time = redis::cmd("TIME");
let time_reply: (i64, i64) = {
let mut conn = self.client.conn();
time.query_async(&mut conn)
.await
.map_err(NasaRedisError::Redis)?
};
let now = time_reply.0 * 1000 + time_reply.1 / 1000;
let mut cmd = redis::cmd("ZRANGEBYSCORE");
cmd.arg(self.keyspace.fanout_roots_at(bucket))
.arg("-inf")
.arg(now)
.arg("LIMIT")
.arg(0)
.arg(self.config.scan_batch_size);
let mut conn = self.client.conn();
cmd.query_async::<Vec<String>>(&mut conn)
.await
.map_err(NasaRedisError::Redis)
}
}
fn parse_member(member: &str) -> Option<(String, i64)> {
let (fanout_id, seq) = member.rsplit_once(':')?;
let seq = seq.parse::<i64>().ok()?;
Some((fanout_id.to_owned(), seq))
}
fn protocol(message: &str) -> NasaRedisError {
crate::job::JobError::Protocol(message.to_owned()).into()
}