use std::collections::{HashMap, HashSet};
use std::panic::AssertUnwindSafe;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::Arc;
use base64::Engine;
use futures::FutureExt;
use tokio::sync::{Mutex, OwnedSemaphorePermit, Semaphore};
use crate::client::RedisClient;
use crate::error::{NasaRedisError, Result};
use crate::job::config::JobConfig;
use crate::job::definition::JobDefinition;
use crate::job::fanout::{
FanoutAcceptOutcome, FanoutReadyDeferOutcome, FanoutRepository, FanoutShard,
};
use crate::job::handler::{
FanoutContext, JobExecution, JobExecutionAuthority, JobHandler, JobOutcome,
};
use crate::job::keyspace::JobKeyspace;
use crate::job::metrics::JobMetrics;
use crate::job::model::JobResultCode;
use crate::job::pubsub::{FanoutSignal, JobPubSubLane};
use crate::job::repository::{FinishOutcome, RenewOutcome, StartOutcome};
const INBOX_GROUP: &str = "redis-job-fanout";
pub(crate) enum FanoutPollError {
Subscription(NasaRedisError),
Authority(NasaRedisError),
}
pub(crate) struct FanoutDispatcher {
client: Arc<RedisClient>,
keyspace: JobKeyspace,
config: Arc<JobConfig>,
repository: FanoutRepository,
executor_id: String,
node_identity: String,
definitions: HashMap<String, JobDefinition>,
handlers: HashMap<String, Arc<dyn JobHandler>>,
pubsub: Mutex<JobPubSubLane>,
notification_capacity: Arc<Semaphore>,
handler_capacity: Arc<Semaphore>,
in_flight: Mutex<HashSet<String>>,
tasks: Mutex<Vec<tokio::task::JoinHandle<()>>>,
error_sender: tokio::sync::mpsc::UnboundedSender<NasaRedisError>,
error_receiver: Mutex<tokio::sync::mpsc::UnboundedReceiver<NasaRedisError>>,
subscription_generation: AtomicU64,
metrics: Arc<JobMetrics>,
}
impl FanoutDispatcher {
pub(crate) async fn start(
client: Arc<RedisClient>,
keyspace: JobKeyspace,
config: Arc<JobConfig>,
executor_id: impl Into<String>,
node_identity: impl Into<String>,
metrics: Arc<JobMetrics>,
) -> Result<Self> {
let executor_id = executor_id.into();
let node_identity = node_identity.into();
let pubsub = JobPubSubLane::start(
client.clone(),
&keyspace,
config.pubsub_mode,
&node_identity,
&executor_id,
)
.await?;
let repository = FanoutRepository::new(client.clone(), keyspace.clone(), config.clone());
let (error_sender, error_receiver) = tokio::sync::mpsc::unbounded_channel();
Ok(Self {
client,
keyspace,
config: config.clone(),
repository,
executor_id,
node_identity,
definitions: HashMap::new(),
handlers: HashMap::new(),
pubsub: Mutex::new(pubsub),
notification_capacity: Arc::new(Semaphore::new(
(config.handler_capacity as usize).saturating_mul(4).max(16),
)),
handler_capacity: Arc::new(Semaphore::new(config.handler_capacity as usize)),
in_flight: Mutex::new(HashSet::new()),
tasks: Mutex::new(Vec::new()),
error_sender,
error_receiver: Mutex::new(error_receiver),
subscription_generation: AtomicU64::new(1),
metrics,
})
}
pub(crate) fn register_handler(
&mut self,
definition: JobDefinition,
handler: Arc<dyn JobHandler>,
) {
self.handlers
.insert(definition.worker_name().to_owned(), handler);
self.definitions
.insert(definition.worker_name().to_owned(), definition);
}
pub(crate) async fn poll(self: &Arc<Self>) -> std::result::Result<(), FanoutPollError> {
let signal = async { self.pubsub.lock().await.next_signal().await };
let background_error = async {
self.error_receiver
.lock()
.await
.recv()
.await
.ok_or_else(|| {
NasaRedisError::from(crate::job::JobError::ExecutionUnknown(
"Fanout 后台错误通道意外关闭".to_owned(),
))
})
};
tokio::select! {
result = background_error => Err(FanoutPollError::Authority(
result.map_err(FanoutPollError::Authority)?,
)),
_ = tokio::time::sleep(std::time::Duration::from_millis(
self.config.min_scan_interval_ms.max(1),
)) => Ok(()),
result = signal => {
match result.map_err(FanoutPollError::Subscription)? {
FanoutSignal::Receipt(_) => Ok(()),
FanoutSignal::Notification(envelope) => {
let Ok(permit) = self.notification_capacity.clone().try_acquire_owned() else {
return Ok(());
};
let dispatcher = self.clone();
let generation = self.subscription_generation.load(Ordering::Acquire);
let task = tokio::spawn(async move {
let result = dispatcher
.handle_notification(generation, &envelope)
.await;
drop(permit);
if let Err(error) = result {
let _ = dispatcher.error_sender.send(error);
}
});
let mut tasks = self.tasks.lock().await;
tasks.retain(|task| !task.is_finished());
tasks.push(task);
Ok(())
}
}
}
}
}
pub(crate) async fn restart_subscription(&self) -> Result<u64> {
let replacement = JobPubSubLane::start(
self.client.clone(),
&self.keyspace,
self.config.pubsub_mode,
&self.node_identity,
&self.executor_id,
)
.await?;
*self.pubsub.lock().await = replacement;
Ok(self
.subscription_generation
.fetch_add(1, Ordering::AcqRel)
.saturating_add(1))
}
pub(crate) fn stop_accepting(&self) {
self.notification_capacity.close();
self.handler_capacity.close();
}
pub(crate) async fn inflight_count(&self) -> usize {
self.in_flight.lock().await.len()
}
async fn handle_notification(self: &Arc<Self>, generation: u64, envelope: &[u8]) -> Result<()> {
if generation != self.subscription_generation.load(Ordering::Acquire) {
return Ok(());
}
let (fanout_id, seq, assignment_epoch, message_id) =
parse_envelope(envelope).inspect_err(|_| {
self.metrics.add("redis_job_invalid_payload_total", 1);
})?;
let Some(shard) = self.repository.read_shard(&fanout_id, seq).await? else {
return Ok(());
};
if shard.target_node_identity != self.node_identity
|| shard.assignment_epoch != assignment_epoch
|| shard.inbox_message_id != message_id
{
return Ok(());
}
let Some(definition) = self.definitions.get(&shard.worker_name) else {
return Ok(());
};
let Some(handler) = self.handlers.get(&shard.worker_name) else {
return Ok(());
};
self.ensure_inbox_group(&fanout_id).await?;
match self
.repository
.accept_shard(
definition,
&fanout_id,
seq,
&self.node_identity,
&self.executor_id,
assignment_epoch,
&shard.origin_executor_id,
)
.await?
{
FanoutAcceptOutcome::Ok { .. } | FanoutAcceptOutcome::Adopted => {}
FanoutAcceptOutcome::SourceMismatch => {
return Err(crate::job::JobError::SourceMismatch(
"Fanout shard 来源与当前 source 不一致".to_owned(),
)
.into());
}
_ => return Ok(()),
}
let Ok(permit) = self.handler_capacity.clone().try_acquire_owned() else {
match self.repository.defer_ready_for_capacity(&shard).await? {
FanoutReadyDeferOutcome::Deferred { .. } => {
self.metrics.add("redis_job_fanout_capacity_defer_total", 1);
}
FanoutReadyDeferOutcome::CapacityExhausted { .. } => {
self.metrics
.add("redis_job_fanout_capacity_exhausted_total", 1);
}
FanoutReadyDeferOutcome::StaleAssignment => {
self.metrics
.add("redis_job_fanout_stale_assignment_total", 1);
}
FanoutReadyDeferOutcome::Stale => {}
}
return Ok(());
};
let (attempt, attempt_token, redis_now, lease_until) = match self
.repository
.start_shard(
definition,
&fanout_id,
seq,
&self.node_identity,
&self.executor_id,
assignment_epoch,
&message_id,
)
.await?
{
StartOutcome::Started {
attempt,
attempt_token,
redis_now,
lease_until,
}
| StartOutcome::Adopted {
attempt,
attempt_token,
redis_now,
lease_until,
} => (attempt, attempt_token, redis_now, lease_until),
StartOutcome::SourceMismatch => {
return Err(crate::job::JobError::SourceMismatch(
"Fanout start 来源与当前 source 不一致".to_owned(),
)
.into());
}
_ => return Ok(()),
};
let execution_id = format!("{fanout_id}:{seq}:{attempt_token}");
if !self.in_flight.lock().await.insert(execution_id.clone()) {
return Ok(());
}
let dispatcher = self.clone();
let definition = definition.clone();
let handler = handler.clone();
let result = dispatcher
.execute(
definition,
handler,
shard,
attempt,
attempt_token,
redis_now,
lease_until,
execution_id.clone(),
permit,
)
.await;
dispatcher.in_flight.lock().await.remove(&execution_id);
result
}
pub(crate) async fn shutdown(&self, deadline: tokio::time::Instant) -> Result<()> {
self.stop_accepting();
let mut tasks = {
let mut tracked = self.tasks.lock().await;
std::mem::take(&mut *tracked)
};
let mut first_error = None;
for mut task in tasks.drain(..) {
match tokio::time::timeout_at(deadline, &mut task).await {
Ok(Ok(())) => {}
Ok(Err(error)) if first_error.is_none() => {
first_error = Some(
crate::job::JobError::ExecutionUnknown(format!(
"Fanout 本地 task 异常退出: {error}"
))
.into(),
);
}
Err(_) => {
task.abort();
let _ = task.await;
if first_error.is_none() {
first_error = Some(
crate::job::JobError::ShutdownDeadline(
"Fanout Handler 未在停机截止前收口".to_owned(),
)
.into(),
);
}
}
_ => {}
}
}
first_error.map_or(Ok(()), Err)
}
#[allow(clippy::too_many_arguments)]
async fn execute(
&self,
definition: JobDefinition,
handler: Arc<dyn JobHandler>,
shard: FanoutShard,
attempt: i64,
attempt_token: i64,
redis_now: i64,
lease_until: i64,
_execution_id: String,
_permit: OwnedSemaphorePermit,
) -> Result<()> {
let payload = base64::engine::general_purpose::STANDARD
.decode(shard.parameter_payload.as_bytes())
.map_err(|_| {
self.metrics.add("redis_job_invalid_payload_total", 1);
NasaRedisError::from(crate::job::JobError::InvalidPayload(
"Fanout payload Base64 非法".to_owned(),
))
})?;
let authority = JobExecutionAuthority::new(
redis_now,
lease_until,
self.config
.renew_rtt_allowance_ms
.saturating_add(self.config.clock_drift_allowance_ms),
);
let execution = JobExecution {
qualifier: self.keyspace.qualifier().to_owned(),
namespace: self.keyspace.namespace().to_owned(),
run_id: crate::job::identifiers::shard_run_id(&shard.fanout_id, shard.seq),
job_name: shard.worker_name.clone(),
worker_name: shard.worker_name.clone(),
logical_fire_at: 0,
triggered_at: 0,
attempt,
attempt_token,
schema_id: shard.schema_id.clone(),
wire_codec: shard.wire_codec.clone(),
parameter_payload: payload,
fanout: Some(FanoutContext {
fanout_id: shard.fanout_id.clone(),
root_run_id: shard.root_run_id.clone(),
snapshot_id: shard.snapshot_id.clone(),
shard_index: shard.shard_index,
shard_total: shard.shard_total,
seq: shard.seq,
execution_key: shard.execution_key.clone(),
target_node_identity: shard.target_node_identity.clone(),
assignment_epoch: shard.assignment_epoch,
}),
authority: authority.clone(),
fanout_service: None,
};
let _guard = FanoutAuthorityGuard(authority.clone());
self.metrics.add("redis_job_started_total", 1);
let mut handler_future =
Box::pin(AssertUnwindSafe(handler.handle(&execution)).catch_unwind());
let timeout = tokio::time::sleep(std::time::Duration::from_millis(
definition
.timeout_ms()
.min(self.config.max_run_duration_ms)
.max(1),
));
tokio::pin!(timeout);
let mut renew = tokio::time::interval_at(
tokio::time::Instant::now()
+ std::time::Duration::from_millis(self.config.lease_renew_ms),
std::time::Duration::from_millis(self.config.lease_renew_ms),
);
renew.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
let outcome = loop {
tokio::select! {
result = &mut handler_future => {
break match result {
Ok(outcome) => outcome,
Err(_) => JobOutcome::retry("Fanout Handler panic,当前 attempt 按可重试失败收口"),
};
}
_ = &mut timeout => {
break JobOutcome {
code: JobResultCode::Timeout,
summary: "Fanout Handler 超过任务执行时限".to_owned(),
};
}
_ = renew.tick() => {
let renewed = match self.repository.renew_shard(
&shard.fanout_id,
shard.seq,
&self.executor_id,
attempt_token,
).await {
Ok(outcome) => outcome,
Err(error) => {
self.metrics.add("redis_job_self_fence_total", 1);
return Err(error);
}
};
match renewed {
RenewOutcome::Ok { redis_now, deadline, cancel_requested: true } => {
let _ = (redis_now, deadline);
authority.revoke(true);
break JobOutcome {
code: JobResultCode::Cancelled,
summary: "Fanout shard 收到持久取消请求".to_owned(),
};
}
RenewOutcome::Ok { redis_now, deadline, cancel_requested: false } => {
if !authority.renew(
redis_now,
deadline,
self.config.renew_rtt_allowance_ms
.saturating_add(self.config.clock_drift_allowance_ms),
) {
self.metrics.add("redis_job_self_fence_total", 1);
return Err(crate::job::JobError::ExecutionStopped(
"Fanout shard 本地权威门禁已关闭".to_owned(),
)
.into());
}
}
RenewOutcome::StateMismatch | RenewOutcome::StaleOwner => {
self.metrics.add("redis_job_self_fence_total", 1);
return Err(crate::job::JobError::StaleOwner(
"Fanout Handler 已失去 attempt 权威,禁止完成提交".to_owned(),
)
.into());
}
}
}
}
};
let finish = self
.repository
.finish_shard(
&shard.fanout_id,
shard.seq,
&shard.worker_name,
&self.executor_id,
attempt_token,
shard.assignment_epoch,
&definition,
outcome.code,
&outcome.summary,
)
.await?;
match finish {
FinishOutcome::Ok { next_state, .. } => {
self.record_finished(next_state);
Ok(())
}
FinishOutcome::StateMismatch | FinishOutcome::StaleOwner => {
self.metrics.add("redis_job_stale_finish_total", 1);
Err(crate::job::JobError::StaleOwner(
"Fanout shard 完成提交已失去当前状态或 owner 权威".to_owned(),
)
.into())
}
FinishOutcome::StaleAssignment => {
self.metrics
.add("redis_job_fanout_stale_assignment_total", 1);
Err(crate::job::JobError::StaleAssignment(
"Fanout shard 完成提交的 assignment epoch 已失效".to_owned(),
)
.into())
}
FinishOutcome::IdentityMismatch => {
self.metrics.add("redis_job_fencing_regression_total", 1);
Err(crate::job::JobError::FencingRegression(
"Fanout shard 完成提交的稳定执行身份不一致".to_owned(),
)
.into())
}
}
}
fn record_finished(&self, next_state: crate::job::model::JobState) {
use crate::job::model::JobState;
match next_state {
JobState::Succeeded => self.metrics.add("redis_job_success_total", 1),
JobState::RetryWait => self.metrics.add("redis_job_retry_total", 1),
JobState::Dead => {
self.metrics.add("redis_job_failed_total", 1);
self.metrics.add("redis_job_dead_total", 1);
}
JobState::Failed => self.metrics.add("redis_job_failed_total", 1),
JobState::Skipped => self.metrics.add("redis_job_skipped_total", 1),
JobState::Created
| JobState::Queued
| JobState::Blocked
| JobState::Running
| JobState::FanoutCreating
| JobState::WaitingChildren
| JobState::AwaitingCapability
| JobState::AwaitingReceipt
| JobState::Received
| JobState::Cancelled => {}
}
}
async fn ensure_inbox_group(&self, fanout_id: &str) -> Result<()> {
let inbox = self.keyspace.fanout_inbox(fanout_id, &self.node_identity);
let mut cmd = redis::cmd("XGROUP");
cmd.arg("CREATE")
.arg(inbox)
.arg(INBOX_GROUP)
.arg("0")
.arg("MKSTREAM");
let mut connection = self.client.conn();
match cmd.query_async::<()>(&mut connection).await {
Ok(()) => Ok(()),
Err(error) if error.code() == Some("BUSYGROUP") => Ok(()),
Err(error) => Err(NasaRedisError::Redis(error)),
}
}
}
struct FanoutAuthorityGuard(Arc<JobExecutionAuthority>);
impl Drop for FanoutAuthorityGuard {
fn drop(&mut self) {
self.0.revoke(false);
}
}
fn parse_envelope(envelope: &[u8]) -> Result<(String, i64, i64, String)> {
let text = std::str::from_utf8(envelope)
.map_err(|_| crate::job::JobError::Protocol("Fanout 通知信封不是 UTF-8".to_owned()))?;
let fields = text.split('|').collect::<Vec<_>>();
if fields.len() != 4 || fields[0].is_empty() || fields[3].is_empty() {
return Err(
crate::job::JobError::Protocol("Fanout 通知信封字段不符合定位合同".to_owned()).into(),
);
}
let seq = fields[1]
.parse::<i64>()
.map_err(|_| crate::job::JobError::Protocol("Fanout 通知 seq 非法".to_owned()))?;
let assignment_epoch = fields[2].parse::<i64>().map_err(|_| {
crate::job::JobError::Protocol("Fanout 通知 assignment epoch 非法".to_owned())
})?;
Ok((
fields[0].to_owned(),
seq,
assignment_epoch,
fields[3].to_owned(),
))
}