use std::collections::{HashMap, HashSet};
use std::panic::AssertUnwindSafe;
use std::sync::Arc;
use base64::Engine;
use futures::{FutureExt, StreamExt};
use redis::streams::{StreamAutoClaimReply, StreamId, StreamReadReply};
use tokio::sync::{Mutex, OwnedSemaphorePermit, Semaphore};
use tokio_util::sync::CancellationToken;
use crate::client::{Conn, RedisClient};
use crate::error::{NasaRedisError, Result};
use crate::job::config::JobConfig;
use crate::job::coordinator::FanoutService;
use crate::job::definition::JobDefinition;
use crate::job::handler::{JobExecution, JobExecutionAuthority, JobHandler, JobOutcome};
use crate::job::keyspace::JobKeyspace;
use crate::job::metrics::JobMetrics;
use crate::job::model::JobResultCode;
use crate::job::repository::{FinishOutcome, JobRepository, RenewOutcome, StartOutcome};
pub(crate) enum JobDispatcherPollError {
Recoverable(NasaRedisError),
Authority(NasaRedisError),
}
struct BlockingLane {
conn: Mutex<Conn>,
close: CancellationToken,
}
pub struct JobDispatcher {
client: Arc<RedisClient>,
keyspace: JobKeyspace,
config: Arc<JobConfig>,
repository: JobRepository,
executor_id: String,
definitions: HashMap<String, JobDefinition>,
handlers: HashMap<String, Arc<dyn JobHandler>>,
capacity: Arc<Semaphore>,
handler_capacities: HashMap<String, Arc<Semaphore>>,
in_flight: Mutex<HashSet<String>>,
blocking_lanes: std::sync::Mutex<HashMap<u32, Arc<BlockingLane>>>,
fanout_service: Option<Arc<FanoutService>>,
metrics: Arc<JobMetrics>,
}
impl JobDispatcher {
pub fn new(
client: Arc<RedisClient>,
keyspace: JobKeyspace,
config: Arc<JobConfig>,
executor_id: impl Into<String>,
) -> Self {
let metrics = Arc::new(JobMetrics::new(keyspace.qualifier()));
Self::new_with_metrics(client, keyspace, config, executor_id, metrics)
}
pub(crate) fn new_with_metrics(
client: Arc<RedisClient>,
keyspace: JobKeyspace,
config: Arc<JobConfig>,
executor_id: impl Into<String>,
metrics: Arc<JobMetrics>,
) -> Self {
let repository = JobRepository::new(client.clone(), keyspace.clone(), config.clone());
let executor_capacity = config.executor_capacity as usize;
Self {
client,
keyspace,
config,
repository,
executor_id: executor_id.into(),
definitions: HashMap::new(),
handlers: HashMap::new(),
capacity: Arc::new(Semaphore::new(executor_capacity)),
handler_capacities: HashMap::new(),
in_flight: Mutex::new(HashSet::new()),
blocking_lanes: std::sync::Mutex::new(HashMap::new()),
fanout_service: None,
metrics,
}
}
pub(crate) fn set_fanout_service(&mut self, service: Arc<FanoutService>) {
self.fanout_service = Some(service);
}
pub(crate) async fn inflight_count(&self) -> usize {
self.in_flight.lock().await.len()
}
pub fn register_handler(&mut self, definition: JobDefinition, handler: Arc<dyn JobHandler>) {
self.handler_capacities
.entry(definition.worker_name().to_owned())
.or_insert_with(|| Arc::new(Semaphore::new(self.config.handler_capacity as usize)));
self.handlers.insert(definition.name().to_owned(), handler);
self.definitions
.insert(definition.name().to_owned(), definition);
}
pub async fn poll_shard(&self, shard: u32) -> Result<usize> {
self.poll_shard_supervised(shard)
.await
.map_err(|error| match error {
JobDispatcherPollError::Recoverable(error)
| JobDispatcherPollError::Authority(error) => error,
})
}
pub(crate) async fn poll_shard_supervised(
&self,
shard: u32,
) -> std::result::Result<usize, JobDispatcherPollError> {
let group = self.config.dispatch_group.clone();
let mut streams: Vec<String> = self
.definitions
.values()
.filter(|definition| self.keyspace.schedule_shard(definition.name()) == shard)
.map(|definition| self.keyspace.dispatch(shard, definition.worker_name()))
.collect();
streams.sort();
streams.dedup();
for stream in &streams {
self.ensure_group(stream, &group)
.await
.map_err(JobDispatcherPollError::Recoverable)?;
}
if streams.is_empty() || self.capacity.available_permits() == 0 {
return Ok(0);
}
let mut entries: Vec<(String, StreamId)> = Vec::new();
let mut remaining = self
.capacity
.available_permits()
.min(self.config.scan_batch_size as usize);
for stream in &streams {
if remaining == 0 {
break;
}
let claimed = self
.claim_group(shard, stream, &group, remaining)
.await
.map_err(JobDispatcherPollError::Recoverable)?;
remaining = remaining.saturating_sub(claimed.claimed.len());
entries.extend(
claimed
.claimed
.into_iter()
.map(|entry| (stream.clone(), entry)),
);
}
if remaining > 0 {
let reply = self
.read_group(shard, &streams, &group, remaining)
.await
.map_err(JobDispatcherPollError::Recoverable)?;
for key in reply.keys {
entries.extend(key.ids.into_iter().map(|entry| (key.key.clone(), entry)));
}
}
let mut seen = HashSet::new();
let mut work = Vec::new();
for (stream, entry) in entries {
if !seen.insert((stream.clone(), entry.id.clone())) {
continue;
}
let Ok(permit) = self.capacity.clone().try_acquire_owned() else {
break;
};
work.push((stream, entry, permit));
}
let results = futures::stream::iter(work)
.map(|(stream, entry, permit)| async move {
self.process_entry(&stream, entry, permit).await
})
.buffer_unordered(self.config.executor_capacity as usize)
.collect::<Vec<_>>()
.await;
let mut processed = 0;
let mut first_error = None;
for result in results {
match result {
Ok(true) => processed += 1,
Ok(false) => {}
Err(error) if first_error.is_none() => first_error = Some(error),
Err(_) => {}
}
}
match first_error {
Some(error) => Err(JobDispatcherPollError::Authority(error)),
None => Ok(processed),
}
}
async fn ensure_group(&self, stream: &str, group: &str) -> Result<()> {
let mut cmd = redis::cmd("XGROUP");
cmd.arg("CREATE")
.arg(stream)
.arg(group)
.arg("0")
.arg("MKSTREAM");
let mut conn = self.client.conn();
match cmd.query_async::<()>(&mut conn).await {
Ok(()) => Ok(()),
Err(error) if error.code() == Some("BUSYGROUP") => Ok(()),
Err(error) => Err(NasaRedisError::Redis(error)),
}
}
async fn read_group(
&self,
shard: u32,
streams: &[String],
group: &str,
limit: usize,
) -> Result<StreamReadReply> {
let mut cmd = redis::cmd("XREADGROUP");
cmd.arg("GROUP")
.arg(group)
.arg(&self.executor_id)
.arg("COUNT")
.arg(limit)
.arg("BLOCK")
.arg(self.config.min_scan_interval_ms.max(1))
.arg("STREAMS")
.arg(streams);
for _ in streams {
cmd.arg(">");
}
self.query_blocking(shard, &mut cmd).await
}
async fn claim_group(
&self,
shard: u32,
stream: &str,
group: &str,
limit: usize,
) -> Result<StreamAutoClaimReply> {
let mut cmd = redis::cmd("XAUTOCLAIM");
cmd.arg(stream)
.arg(group)
.arg(&self.executor_id)
.arg(self.config.xautoclaim_min_idle_ms)
.arg("0-0")
.arg("COUNT")
.arg(limit);
self.query_blocking(shard, &mut cmd).await
}
async fn query_blocking<T: redis::FromRedisValue>(
&self,
shard: u32,
cmd: &mut redis::Cmd,
) -> Result<T> {
let lane = self.blocking_lane(shard).await?;
let mut conn = tokio::select! {
_ = lane.close.cancelled() => {
return Err(crate::job::JobError::ExecutionStopped(
format!("Dispatch 分片 {shard} 的阻塞读取已关闭"),
).into());
}
conn = lane.conn.lock() => conn,
};
let result = tokio::select! {
_ = lane.close.cancelled() => {
return Err(crate::job::JobError::ExecutionStopped(
format!("Dispatch 分片 {shard} 的阻塞读取已关闭"),
).into());
}
result = cmd.query_async::<T>(&mut *conn) => result,
};
drop(conn);
match result {
Ok(value) => Ok(value),
Err(error) => {
self.retire_blocking_lane(shard, &lane);
Err(NasaRedisError::Redis(error))
}
}
}
async fn blocking_lane(&self, shard: u32) -> Result<Arc<BlockingLane>> {
if let Some(lane) = self
.blocking_lanes
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(&shard)
.cloned()
{
return Ok(lane);
}
let conn = self
.client
.dedicated_conn("Job Dispatch blocking lane")
.await?;
let candidate = Arc::new(BlockingLane {
conn: Mutex::new(conn),
close: CancellationToken::new(),
});
let mut lanes = self
.blocking_lanes
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
Ok(lanes.entry(shard).or_insert(candidate).clone())
}
fn retire_blocking_lane(&self, shard: u32, lane: &Arc<BlockingLane>) {
let mut lanes = self
.blocking_lanes
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if lanes
.get(&shard)
.is_some_and(|current| Arc::ptr_eq(current, lane))
{
lane.close.cancel();
lanes.remove(&shard);
}
}
pub(crate) fn close_blocking_lanes(&self) {
let mut lanes = self
.blocking_lanes
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
for lane in lanes.values() {
lane.close.cancel();
}
lanes.clear();
}
async fn process_entry(
&self,
stream: &str,
entry: StreamId,
_permit: OwnedSemaphorePermit,
) -> Result<bool> {
let job_name = field(&entry.map, "jobName");
let run_id = field(&entry.map, "runId");
if job_name.is_empty() || run_id.is_empty() {
self.metrics.add("redis_job_invalid_payload_total", 1);
return Err(crate::job::JobError::InvalidPayload(
"Dispatch 消息缺少 jobName 或 runId".to_owned(),
)
.into());
}
let source = field(&entry.map, "schedulerQualifier");
if source != self.keyspace.qualifier() {
return Err(crate::job::JobError::SourceMismatch(format!(
"expected={} observed={}",
self.keyspace.qualifier(),
source
))
.into());
}
{
let mut in_flight = self.in_flight.lock().await;
if !in_flight.insert(run_id.clone()) {
return Ok(false);
}
}
let result = self.process(stream, &job_name, &run_id, &entry.id).await;
let mut in_flight = self.in_flight.lock().await;
in_flight.remove(&run_id);
result
}
async fn process(
&self,
stream: &str,
job_name: &str,
run_id: &str,
message_id: &str,
) -> Result<bool> {
let definition = self.definitions.get(job_name);
let handler = self.handlers.get(job_name);
let (definition, handler) = match (definition, handler) {
(Some(def), Some(handler)) => (def, handler),
(Some(def), None) => {
self.repository.defer_run(def, run_id, message_id).await?;
return Ok(false);
}
_ => {
self.repository
.defer_stream(job_name, run_id, message_id, stream)
.await?;
return Ok(false);
}
};
let Some(handler_capacity) = self.handler_capacities.get(definition.worker_name()) else {
self.repository
.defer_run(definition, run_id, message_id)
.await?;
return Ok(false);
};
let Ok(_handler_permit) = handler_capacity.clone().try_acquire_owned() else {
self.repository
.defer_run(definition, run_id, message_id)
.await?;
return Ok(false);
};
let (attempt, attempt_token, redis_now, lease_until) = match self
.repository
.start_run(definition, run_id, &self.executor_id, message_id)
.await?
{
StartOutcome::Started {
attempt,
attempt_token,
redis_now,
lease_until,
}
| StartOutcome::Adopted {
attempt,
attempt_token,
redis_now,
lease_until,
} => (attempt, attempt_token, redis_now, lease_until),
StartOutcome::SourceMismatch => {
return Err(crate::job::JobError::SourceMismatch(
"Run 或定义声明与当前运行时不一致".to_owned(),
)
.into());
}
_ => return Ok(false),
};
let run = self
.repository
.read_run(job_name, run_id)
.await?
.ok_or_else(|| crate::job::JobError::Protocol("领取后 Run 记录缺失".to_owned()))?;
let payload = base64::engine::general_purpose::STANDARD
.decode(run.parameter_payload.as_bytes())
.map_err(|_| {
self.metrics.add("redis_job_invalid_payload_total", 1);
NasaRedisError::from(crate::job::JobError::InvalidPayload(
"Run 参数 Base64 非法".to_owned(),
))
})?;
let execution = JobExecution {
qualifier: self.keyspace.qualifier().to_owned(),
namespace: self.keyspace.namespace().to_owned(),
run_id: run_id.to_owned(),
job_name: job_name.to_owned(),
worker_name: run.worker_name,
logical_fire_at: run.logical_fire_at,
triggered_at: run.triggered_at,
attempt,
attempt_token,
schema_id: run.schema_id,
wire_codec: run.wire_codec,
parameter_payload: payload,
fanout: None,
authority: JobExecutionAuthority::new(
redis_now,
lease_until,
self.config
.renew_rtt_allowance_ms
.saturating_add(self.config.clock_drift_allowance_ms),
),
fanout_service: self.fanout_service.clone(),
};
self.metrics.add("redis_job_started_total", 1);
let outcome = self
.run_with_lease(definition, run_id, attempt_token, handler, &execution)
.await?;
if execution.authority.is_fanout_transferred() {
return Ok(true);
}
let finish = self
.repository
.finish_run(
definition,
run_id,
&self.executor_id,
attempt_token,
outcome.code,
&outcome.summary,
)
.await?;
match finish {
FinishOutcome::Ok { next_state, .. } => {
self.record_finished(next_state);
Ok(true)
}
FinishOutcome::StateMismatch
| FinishOutcome::StaleOwner
| FinishOutcome::StaleAssignment => {
self.metrics.add("redis_job_stale_finish_total", 1);
Err(crate::job::JobError::StaleOwner(
"attempt 完成提交已失去当前状态或 owner 权威".to_owned(),
)
.into())
}
FinishOutcome::IdentityMismatch => {
self.metrics.add("redis_job_fencing_regression_total", 1);
Err(crate::job::JobError::FencingRegression(
"attempt 完成提交的稳定执行身份不一致".to_owned(),
)
.into())
}
}
}
fn record_finished(&self, next_state: crate::job::model::JobState) {
use crate::job::model::JobState;
match next_state {
JobState::Succeeded => self.metrics.add("redis_job_success_total", 1),
JobState::RetryWait => self.metrics.add("redis_job_retry_total", 1),
JobState::Dead => {
self.metrics.add("redis_job_failed_total", 1);
self.metrics.add("redis_job_dead_total", 1);
}
JobState::Failed => self.metrics.add("redis_job_failed_total", 1),
JobState::Skipped => self.metrics.add("redis_job_skipped_total", 1),
JobState::Created
| JobState::Queued
| JobState::Blocked
| JobState::Running
| JobState::FanoutCreating
| JobState::WaitingChildren
| JobState::AwaitingCapability
| JobState::AwaitingReceipt
| JobState::Received
| JobState::Cancelled => {}
}
}
async fn run_with_lease(
&self,
definition: &JobDefinition,
run_id: &str,
attempt_token: i64,
handler: &Arc<dyn JobHandler>,
execution: &JobExecution,
) -> Result<JobOutcome> {
let authority = execution.authority.clone();
let _authority_guard = AuthorityGuard(authority.clone());
let mut handler_future =
Box::pin(AssertUnwindSafe(handler.handle(execution)).catch_unwind());
let run_timeout_ms = definition
.timeout_ms()
.min(self.config.max_run_duration_ms)
.max(1);
let timeout = tokio::time::sleep(std::time::Duration::from_millis(run_timeout_ms));
tokio::pin!(timeout);
let first_renew = tokio::time::Instant::now()
+ std::time::Duration::from_millis(self.config.lease_renew_ms);
let mut renew = tokio::time::interval_at(
first_renew,
std::time::Duration::from_millis(self.config.lease_renew_ms),
);
renew.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
loop {
tokio::select! {
result = &mut handler_future => {
return Ok(match result {
Ok(outcome) => outcome,
Err(_) => JobOutcome::retry("Handler panic,当前 attempt 按可重试失败收口"),
});
}
_ = &mut timeout => {
return Ok(JobOutcome {
code: JobResultCode::Timeout,
summary: "Handler 超过任务执行时限".to_owned(),
});
}
_ = renew.tick() => {
if execution.authority.is_fanout_transferred() {
return Ok(JobOutcome::success());
}
let renewed = match self.repository
.renew_run(definition, run_id, &self.executor_id, attempt_token)
.await
{
Ok(outcome) => outcome,
Err(error) => {
self.metrics.add("redis_job_self_fence_total", 1);
return Err(error);
}
};
match renewed {
RenewOutcome::Ok { cancel_requested: true, .. } => {
authority.revoke(true);
return Ok(JobOutcome {
code: JobResultCode::Cancelled,
summary: "收到持久取消请求".to_owned(),
});
}
RenewOutcome::Ok {
redis_now,
deadline,
cancel_requested: false,
} => {
let updated = authority.renew(
redis_now,
deadline,
self.config
.renew_rtt_allowance_ms
.saturating_add(self.config.clock_drift_allowance_ms),
);
if !updated {
self.metrics.add("redis_job_self_fence_total", 1);
return Err(crate::job::JobError::ExecutionStopped(
"当前 attempt 的本地权威门禁已关闭".to_owned(),
)
.into());
}
}
RenewOutcome::StateMismatch | RenewOutcome::StaleOwner => {
if execution.authority.is_fanout_transferred() {
return Ok(JobOutcome::success());
}
self.metrics.add("redis_job_self_fence_total", 1);
return Err(crate::job::JobError::StaleOwner(
"Handler 执行期间已失去 attempt 权威,已停止本地 future 且禁止完成提交".to_owned(),
)
.into());
}
}
}
}
}
}
}
struct AuthorityGuard(Arc<JobExecutionAuthority>);
impl Drop for AuthorityGuard {
fn drop(&mut self) {
self.0.revoke(false);
}
}
fn field(map: &HashMap<String, redis::Value>, key: &str) -> String {
match map.get(key) {
Some(redis::Value::BulkString(bytes)) => String::from_utf8_lossy(bytes).into_owned(),
Some(redis::Value::SimpleString(text)) => text.clone(),
_ => String::new(),
}
}