use std::collections::HashMap;
use std::sync::Arc;
use crate::client::RedisClient;
use crate::error::{NasaRedisError, Result};
use crate::job::config::JobConfig;
use crate::job::definition::JobDefinition;
use crate::job::fanout::{
FanoutAddShardsOutcome, FanoutBeginOutcome, FanoutCommitOutcome, FanoutContract,
FanoutDeliverOutcome, FanoutRepository,
};
use crate::job::handler::{JobExecution, JobExecutionAuthority};
use crate::job::identifiers::fanout_id;
use crate::job::keyspace::JobKeyspace;
use crate::job::metrics::JobMetrics;
use crate::job::model::{JobFanoutFailurePolicy, JobTrigger, JobWireCodec};
use crate::job::payload::JobPayload;
use crate::job::registry::{ClusterSnapshot, ExecutorRegistry, SnapshotOutcome};
use crate::job::repository::{FinishFanoutRootOutcome, JobRepository, PrepareFanoutRootOutcome};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct FanoutSubmitOutcome {
pub fanout_id: String,
pub shard_total: usize,
pub delivered: i64,
}
#[derive(Debug, Clone)]
pub struct FanoutRootContext<'a> {
pub definition: &'a JobDefinition,
pub contract: &'a FanoutContract,
pub root_run_id: &'a str,
pub root_owner: &'a str,
pub root_attempt: i64,
pub root_attempt_token: i64,
}
#[derive(Clone)]
pub struct FanoutCoordinator {
repository: JobRepository,
fanout: FanoutRepository,
origin_executor_id: String,
metrics: Option<Arc<JobMetrics>>,
}
#[derive(Clone)]
pub(crate) struct FanoutService {
registry: Arc<ExecutorRegistry>,
coordinator: FanoutCoordinator,
definitions: Arc<HashMap<String, JobDefinition>>,
config: Arc<JobConfig>,
executor_id: String,
}
impl std::fmt::Debug for FanoutService {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("FanoutService")
.finish_non_exhaustive()
}
}
impl FanoutService {
pub(crate) fn new(
registry: Arc<ExecutorRegistry>,
coordinator: FanoutCoordinator,
definitions: Arc<HashMap<String, JobDefinition>>,
config: Arc<JobConfig>,
executor_id: impl Into<String>,
) -> Arc<Self> {
Arc::new(Self {
registry,
coordinator,
definitions,
config,
executor_id: executor_id.into(),
})
}
pub(crate) fn builder(
self: &Arc<Self>,
root: &JobExecution,
worker_name: impl Into<String>,
) -> Result<FanoutBuilder> {
root.checkpoint()?;
if root.fanout_context().is_some() {
return Err(config("FANOUT_ONLY Worker 不能递归创建 Fanout".to_owned()));
}
let worker_name = worker_name.into();
if worker_name.trim().is_empty() {
return Err(config("Fanout worker_name 不能为空".to_owned()));
}
Ok(FanoutBuilder {
service: self.clone(),
root: root.clone(),
worker_name,
contract_revision: None,
schema_id: None,
wire_codec: None,
failure_policy: JobFanoutFailurePolicy::ReassignOnFailure,
input: FanoutInput::Missing,
})
}
}
enum FanoutInput {
Missing,
Shards(Vec<JobPayload>),
Partition(Box<dyn FnOnce(usize) -> Result<Vec<Vec<u8>>> + Send>),
}
pub struct FanoutBuilder {
service: Arc<FanoutService>,
root: JobExecution,
worker_name: String,
contract_revision: Option<i64>,
schema_id: Option<String>,
wire_codec: Option<JobWireCodec>,
failure_policy: JobFanoutFailurePolicy,
input: FanoutInput,
}
impl FanoutBuilder {
pub fn contract_revision(mut self, revision: i64) -> Result<Self> {
if revision <= 0 {
return Err(config("Fanout contract_revision 必须大于 0".to_owned()));
}
self.contract_revision = Some(revision);
Ok(self)
}
pub fn schema(mut self, schema_id: impl Into<String>) -> Result<Self> {
let schema_id = schema_id.into();
if schema_id.trim().is_empty() {
return Err(config("Fanout schema 不能为空"));
}
self.schema_id = Some(schema_id);
Ok(self)
}
pub fn codec(mut self, codec: JobWireCodec) -> Self {
self.wire_codec = Some(codec);
self
}
pub fn failure_policy(mut self, policy: JobFanoutFailurePolicy) -> Self {
self.failure_policy = policy;
self
}
pub fn shards(mut self, payloads: impl IntoIterator<Item = JobPayload>) -> Self {
self.input = FanoutInput::Shards(payloads.into_iter().collect());
self
}
pub fn partition<T, F>(mut self, items: Vec<T>, partitioner: F) -> Self
where
T: serde::Serialize + Send + 'static,
F: FnOnce(Vec<T>, usize) -> Vec<Vec<T>> + Send + 'static,
{
self.input = FanoutInput::Partition(Box::new(move |member_count| {
partitioner(items, member_count)
.into_iter()
.map(|partition| {
serde_json::to_vec(&partition)
.map_err(|error| config(format!("Fanout JSON 分片序列化失败: {error}")))
})
.collect()
}));
self
}
pub async fn dispatch(self) -> Result<FanoutSubmitOutcome> {
self.root.checkpoint()?;
let worker = self
.service
.definitions
.values()
.find(|definition| definition.worker_name() == self.worker_name)
.ok_or_else(|| {
config(format!(
"Fanout 未找到本地 Worker 定义: {}",
self.worker_name
))
})?;
if worker.trigger() != JobTrigger::FanoutOnly {
return Err(config(format!(
"Fanout 目标必须声明为 FANOUT_ONLY: {}",
self.worker_name
)));
}
let root_definition = self
.service
.definitions
.get(&self.root.job_name)
.ok_or_else(|| config("Fanout 根定义未冻结"))?;
let contract_revision = self.contract_revision.unwrap_or(worker.contract_revision());
let schema_id = self
.schema_id
.unwrap_or_else(|| worker.schema_id().to_owned());
let wire_codec = self.wire_codec.unwrap_or_else(|| worker.codecs()[0]);
if !worker.codecs().contains(&wire_codec) {
return Err(config("Fanout 选择的 codec 未由 Worker 声明".to_owned()));
}
let snapshot = match self
.service
.registry
.snapshot(
&self.worker_name,
contract_revision,
&schema_id,
wire_codec.wire_name(),
)
.await?
{
SnapshotOutcome::Ok(snapshot) if !snapshot.members.is_empty() => snapshot,
SnapshotOutcome::Ok(_) => {
return Err(crate::job::JobError::NoCapableExecutor(
"没有已发布 fanoutReady 的兼容成员".to_owned(),
)
.into());
}
SnapshotOutcome::ContractMismatch => {
return Err(crate::job::JobError::ContractMismatch(
"Worker 合同与 registry 不一致".to_owned(),
)
.into());
}
};
let payloads = match self.input {
FanoutInput::Missing => {
return Err(config("Fanout 未配置 shards 或 partition".to_owned()));
}
FanoutInput::Shards(payloads) => {
let mut bytes = Vec::with_capacity(payloads.len());
for payload in payloads {
if payload.schema_id() != schema_id || payload.codec() != wire_codec {
return Err(config(
"Fanout shard 的 schema 或 codec 与本批合同不一致".to_owned(),
));
}
bytes.push(payload.bytes().to_vec());
}
bytes
}
FanoutInput::Partition(materialize) => {
if wire_codec != JobWireCodec::Json {
return Err(config("非 JSON Fanout 必须使用预编码 shards".to_owned()));
}
materialize(snapshot.members.len())?
}
};
if payloads.len() != snapshot.members.len() {
return Err(config("Fanout shard 数必须与冻结成员数相同".to_owned()));
}
let mut total = 0_u64;
for payload in &payloads {
if payload.len() > self.service.config.max_parameter_bytes as usize {
return Err(config(
"Fanout 单 shard 超过 max_parameter_bytes".to_owned(),
));
}
total = total
.checked_add(payload.len() as u64)
.ok_or_else(|| config("Fanout payload 总量溢出"))?;
}
if total > self.service.config.fanout_max_total_parameter_bytes as u64 {
return Err(config("Fanout payload 总量超过配置上限".to_owned()));
}
let contract = FanoutContract {
worker_name: self.worker_name,
contract_revision,
schema_id,
wire_codec,
failure_policy: self.failure_policy,
};
let root = FanoutRootContext {
definition: root_definition,
contract: &contract,
root_run_id: &self.root.run_id,
root_owner: &self.service.executor_id,
root_attempt: self.root.attempt,
root_attempt_token: self.root.attempt_token,
};
self.service
.coordinator
.submit_with_authority(&root, &snapshot, &payloads, &self.root.authority)
.await
}
}
impl FanoutCoordinator {
pub fn new(
client: Arc<RedisClient>,
keyspace: JobKeyspace,
config: Arc<JobConfig>,
origin_executor_id: impl Into<String>,
) -> Self {
let repository = JobRepository::new(client.clone(), keyspace.clone(), config.clone());
let fanout = FanoutRepository::new(client, keyspace, config);
Self {
repository,
fanout,
origin_executor_id: origin_executor_id.into(),
metrics: None,
}
}
pub(crate) fn new_with_metrics(
client: Arc<RedisClient>,
keyspace: JobKeyspace,
config: Arc<JobConfig>,
origin_executor_id: impl Into<String>,
metrics: Arc<JobMetrics>,
) -> Self {
let mut coordinator = Self::new(client, keyspace, config, origin_executor_id);
coordinator.metrics = Some(metrics);
coordinator
}
pub async fn submit(
&self,
root: &FanoutRootContext<'_>,
snapshot: &ClusterSnapshot,
shard_payloads: &[Vec<u8>],
) -> Result<FanoutSubmitOutcome> {
self.submit_inner(root, snapshot, shard_payloads, None)
.await
}
pub(crate) async fn submit_with_authority(
&self,
root: &FanoutRootContext<'_>,
snapshot: &ClusterSnapshot,
shard_payloads: &[Vec<u8>],
authority: &Arc<JobExecutionAuthority>,
) -> Result<FanoutSubmitOutcome> {
self.submit_inner(root, snapshot, shard_payloads, Some(authority))
.await
}
async fn submit_inner(
&self,
root: &FanoutRootContext<'_>,
snapshot: &ClusterSnapshot,
shard_payloads: &[Vec<u8>],
authority: Option<&Arc<JobExecutionAuthority>>,
) -> Result<FanoutSubmitOutcome> {
if shard_payloads.len() != snapshot.members.len() {
return Err(protocol("submit 载荷数必须与快照成员数一致"));
}
let shard_total = shard_payloads.len() as i64;
let fanout = fanout_id(root.root_run_id, root.root_attempt as i32);
let prepare = self
.repository
.prepare_fanout_root(
root.definition,
root.root_run_id,
root.root_owner,
root.root_attempt_token,
&fanout,
&snapshot.snapshot_id,
shard_total,
)
.await;
let prepare = match prepare {
Ok(outcome) => outcome,
Err(error) => {
if let Some(authority) = authority {
authority.transfer_to_fanout();
}
return Err(error);
}
};
match prepare {
PrepareFanoutRootOutcome::Prepared { .. }
| PrepareFanoutRootOutcome::Adopted { .. } => {}
PrepareFanoutRootOutcome::JobDeleted => {
return Err(crate::job::JobError::ExecutionStopped(
"Fanout 根定义已删除,拒绝建立新 intent".to_owned(),
)
.into());
}
other => return Err(protocol(&format!("prepare_fanout_root 拒绝: {other:?}"))),
}
if let Some(authority) = authority {
authority.transfer_to_fanout();
}
match self
.fanout
.begin(
&fanout,
root.root_run_id,
root.root_attempt,
snapshot,
root.definition,
root.contract,
shard_total,
)
.await?
{
FanoutBeginOutcome::Ok { .. } | FanoutBeginOutcome::Adopted => {}
other => return Err(protocol(&format!("fanout_begin 拒绝: {other:?}"))),
}
let added = match self
.fanout
.add_shards(
&fanout,
snapshot,
root.contract,
root.root_run_id,
&self.origin_executor_id,
shard_payloads,
)
.await?
{
FanoutAddShardsOutcome::Ok { added, .. } => added,
other => return Err(protocol(&format!("fanout_add_shards 拒绝: {other:?}"))),
};
if let Some(metrics) = &self.metrics {
metrics.add("redis_job_fanout_shard_total", added);
}
match self.fanout.commit(&fanout).await? {
FanoutCommitOutcome::Ok { .. } | FanoutCommitOutcome::Adopted => {}
other => return Err(protocol(&format!("fanout_commit 拒绝: {other:?}"))),
}
match self
.repository
.finish_fanout_root(
root.definition,
root.root_run_id,
&fanout,
root.root_attempt,
"COMMITTED",
"",
"",
)
.await?
{
FinishFanoutRootOutcome::Ok { .. } | FinishFanoutRootOutcome::Adopted => {}
FinishFanoutRootOutcome::JobDeleted { .. } => {
let root = self
.fanout
.read_root(&fanout)
.await?
.ok_or_else(|| protocol("删除 fence 命中后缺少已提交 Fanout 根"))?;
let _ = self.fanout.cancel_batch(&root, "JOB_DELETED").await?;
return Err(crate::job::JobError::ExecutionStopped(
"Fanout 根定义已删除,已关闭桶内新执行权".to_owned(),
)
.into());
}
other => return Err(protocol(&format!("finish_fanout_root 拒绝: {other:?}"))),
}
let delivered = match self
.fanout
.deliver_batch(&fanout, snapshot, root.definition)
.await?
{
FanoutDeliverOutcome::Ok { delivered, .. } => delivered,
FanoutDeliverOutcome::NotCommitted => {
return Err(protocol("fanout_deliver_batch 拒绝: NotCommitted"))
}
};
Ok(FanoutSubmitOutcome {
fanout_id: fanout,
shard_total: shard_payloads.len(),
delivered,
})
}
}
fn protocol(message: &str) -> NasaRedisError {
crate::job::JobError::Protocol(message.to_owned()).into()
}
fn config(message: impl Into<String>) -> NasaRedisError {
crate::job::JobError::Config(message.into()).into()
}