use std::collections::BTreeMap;
use serde::{Deserialize, Serialize};
use crate::error::{NasaRedisError, Result};
use crate::job::model::{JobPubSubMode, JobSerialOverflowPolicy};
const SUPPORTED_PROTOCOL_VERSION: u32 = 1;
#[derive(Debug, Clone, Deserialize, Serialize)]
#[serde(default, deny_unknown_fields)]
pub struct JobConfig {
pub namespace: String,
pub protocol_version: u32,
pub shard_count: u32,
pub fanout_bucket_count: u32,
pub application_name: String,
pub instance_identity: String,
pub min_scan_interval_ms: u64,
pub max_scan_interval_ms: u64,
pub scan_batch_size: u32,
pub schedule_rtt_allowance_ms: u64,
pub need_recompute_max_retries: u32,
pub executor_capacity: u32,
pub handler_capacity: u32,
pub lease_ms: u64,
pub lease_renew_ms: u64,
pub renew_rtt_allowance_ms: u64,
pub clock_drift_allowance_ms: u64,
pub max_tolerated_gc_pause_ms: u64,
pub visibility_timeout_ms: u64,
pub heartbeat_ms: u64,
pub executor_expire_ms: u64,
pub xautoclaim_min_idle_ms: u64,
pub registry_gc_grace_ms: u64,
pub dispatch_group: String,
pub max_run_duration_ms: u64,
pub max_result_summary_bytes: u32,
pub max_dispatch_attempts: u32,
pub max_serial_backlog: u32,
pub serial_overflow_policy: JobSerialOverflowPolicy,
pub pubsub_mode: JobPubSubMode,
pub ready_max_wakeups: u32,
pub fanout_capacity_wait_ms: u64,
pub fanout_max_assignments: u32,
pub node_unready_evidence_count: u32,
pub run_retention_ms: u64,
pub completion_retention_ms: u64,
pub tombstone_retention_ms: u64,
pub fanout_max_members: u32,
pub fanout_delivery_batch_size: u32,
pub fanout_create_timeout_ms: u64,
pub fanout_max_wait_ms: u64,
pub fanout_retention_ms: u64,
pub fanout_cleanup_batch_size: u32,
pub max_parameter_bytes: u32,
pub max_catch_up_runs: u32,
pub max_catch_up_window_ms: u64,
pub fanout_max_total_parameter_bytes: u32,
pub completion_max_len: u64,
pub completion_trim_batch_size: u32,
pub json_default_typing: bool,
#[serde(default, skip_serializing)]
pub sources: BTreeMap<String, JobSourceConfig>,
}
#[derive(Debug, Clone, Default, Deserialize)]
#[serde(default)]
pub struct JobSourceConfig {
pub enabled: Option<bool>,
#[serde(flatten)]
pub overrides: BTreeMap<String, serde_json::Value>,
}
pub struct JobConfigBuilder {
config: JobConfig,
}
pub struct JobSourceConfigBuilder {
source: JobSourceConfig,
}
impl Default for JobConfig {
fn default() -> Self {
Self {
namespace: String::new(),
protocol_version: SUPPORTED_PROTOCOL_VERSION,
shard_count: 64,
fanout_bucket_count: 32,
application_name: "application".to_owned(),
instance_identity: String::new(),
min_scan_interval_ms: 50,
max_scan_interval_ms: 30_000,
scan_batch_size: 100,
schedule_rtt_allowance_ms: 100,
need_recompute_max_retries: 3,
executor_capacity: 128,
handler_capacity: 8,
lease_ms: 30_000,
lease_renew_ms: 10_000,
renew_rtt_allowance_ms: 1_000,
clock_drift_allowance_ms: 1_000,
max_tolerated_gc_pause_ms: 1_000,
visibility_timeout_ms: 60_000,
heartbeat_ms: 10_000,
executor_expire_ms: 30_000,
xautoclaim_min_idle_ms: 30_000,
registry_gc_grace_ms: 3_600_000,
dispatch_group: "redis-job-executor".to_owned(),
max_run_duration_ms: 3_600_000,
max_result_summary_bytes: 4_096,
max_dispatch_attempts: 20,
max_serial_backlog: 1_000,
serial_overflow_policy: JobSerialOverflowPolicy::SkipOldest,
pubsub_mode: JobPubSubMode::Sharded,
ready_max_wakeups: 3,
fanout_capacity_wait_ms: 10_000,
fanout_max_assignments: 5,
node_unready_evidence_count: 3,
run_retention_ms: 604_800_000,
completion_retention_ms: 604_800_000,
tombstone_retention_ms: 604_800_000,
fanout_max_members: 512,
fanout_delivery_batch_size: 64,
fanout_create_timeout_ms: 60_000,
fanout_max_wait_ms: 1_800_000,
fanout_retention_ms: 604_800_000,
fanout_cleanup_batch_size: 100,
max_parameter_bytes: 65_536,
max_catch_up_runs: 50,
max_catch_up_window_ms: 3_600_000,
fanout_max_total_parameter_bytes: 16 * 1024 * 1024,
completion_max_len: 100_000,
completion_trim_batch_size: 1_000,
json_default_typing: false,
sources: BTreeMap::new(),
}
}
}
impl JobConfig {
pub fn builder() -> JobConfigBuilder {
JobConfigBuilder {
config: Self::default(),
}
}
pub fn validate(&self) -> Result<()> {
for (qualifier, source) in &self.sources {
crate::job::source::JobSourceId::new(qualifier)?;
if source.overrides.contains_key("application_name")
|| source.overrides.contains_key("instance_identity")
{
return Err(cfg(
"job source 不得覆盖 application_name 或 instance_identity;执行器身份必须在进程内一致",
));
}
}
if self.protocol_version != SUPPORTED_PROTOCOL_VERSION {
return Err(cfg("job.protocol_version 首版只支持 1"));
}
if self.shard_count == 0 || self.fanout_bucket_count == 0 {
return Err(cfg("job.shard_count 与 fanout_bucket_count 必须大于 0"));
}
if self.json_default_typing {
return Err(cfg("job.json_default_typing 必须为 false"));
}
if self.executor_capacity == 0
|| self.handler_capacity == 0
|| self.handler_capacity > self.executor_capacity
{
return Err(cfg(
"job.executor_capacity 与 handler_capacity 必须满足 1 <= handler_capacity <= executor_capacity",
));
}
if self.min_scan_interval_ms == 0
|| self.max_scan_interval_ms < self.min_scan_interval_ms
|| self.scan_batch_size == 0
|| self.need_recompute_max_retries == 0
{
return Err(cfg("job 扫描间隔、批量与重新计算上限非法"));
}
if self.fanout_max_members == 0
|| self.fanout_delivery_batch_size == 0
|| self.fanout_delivery_batch_size > self.fanout_max_members
|| self.fanout_max_assignments == 0
{
return Err(cfg("job Fanout 成员、投递批量与 assignment 上限非法"));
}
if self.max_parameter_bytes == 0
|| self.fanout_max_total_parameter_bytes == 0
|| self.max_parameter_bytes > self.fanout_max_total_parameter_bytes
{
return Err(cfg("job Fanout 参数字节上限非法"));
}
if self.fanout_capacity_wait_ms == 0
|| self.node_unready_evidence_count <= 1
|| self.fanout_cleanup_batch_size == 0
{
return Err(cfg("job Fanout 容量等待、失联证据与清理批量非法"));
}
if self.fanout_create_timeout_ms == 0
|| self.fanout_max_wait_ms <= self.fanout_create_timeout_ms
|| self.fanout_retention_ms == 0
|| self.registry_gc_grace_ms <= self.fanout_max_wait_ms
{
return Err(cfg("job Fanout 时限与注册表回收宽限关系非法"));
}
if self.lease_ms == 0 || self.lease_renew_ms == 0 {
return Err(cfg("job.lease_ms 与 lease_renew_ms 必须大于 0"));
}
let two_renew = self
.lease_renew_ms
.checked_mul(2)
.and_then(|v| v.checked_add(self.max_tolerated_gc_pause_ms))
.ok_or_else(|| cfg("job 租约时间关系溢出"))?;
if self.lease_ms <= two_renew {
return Err(cfg(
"job.lease_ms 必须大于 2*lease_renew_ms + max_tolerated_gc_pause_ms",
));
}
let authority_allowance = self
.renew_rtt_allowance_ms
.checked_add(self.clock_drift_allowance_ms)
.ok_or_else(|| cfg("job 续期与时钟预算相加溢出"))?;
if authority_allowance >= self.lease_ms / 3 {
return Err(cfg("job renew_rtt+clock_drift 预算必须小于 lease_ms/3"));
}
let two_leases = self
.lease_ms
.checked_mul(2)
.ok_or_else(|| cfg("job.visibility_timeout_ms 时间关系溢出"))?;
if self.visibility_timeout_ms < two_leases {
return Err(cfg("job.visibility_timeout_ms 必须不小于 2*lease_ms"));
}
if self.heartbeat_ms == 0 {
return Err(cfg("job.heartbeat_ms 必须大于 0"));
}
let two_heartbeats = self
.heartbeat_ms
.checked_mul(2)
.ok_or_else(|| cfg("job.executor_expire_ms 时间关系溢出"))?;
if self.executor_expire_ms < two_heartbeats {
return Err(cfg("job.executor_expire_ms 必须不小于 2*heartbeat_ms"));
}
if !(self.min_scan_interval_ms < self.xautoclaim_min_idle_ms
&& self.xautoclaim_min_idle_ms < self.visibility_timeout_ms)
{
return Err(cfg(
"job 必须满足 min_scan_interval_ms < xautoclaim_min_idle_ms < visibility_timeout_ms",
));
}
if self.max_run_duration_ms == 0 {
return Err(cfg("job.max_run_duration_ms 必须大于 0"));
}
if self.max_result_summary_bytes == 0 {
return Err(cfg("job.max_result_summary_bytes 必须大于 0"));
}
if self.run_retention_ms == 0
|| self.completion_retention_ms == 0
|| self.tombstone_retention_ms == 0
|| self.max_catch_up_runs == 0
|| self.max_catch_up_window_ms == 0
{
return Err(cfg("job 的 Run 保留期与补偿边界必须大于 0"));
}
if self.max_dispatch_attempts == 0 {
return Err(cfg("job.max_dispatch_attempts 必须大于 0"));
}
if self.max_serial_backlog == 0 {
return Err(cfg("job.max_serial_backlog 必须大于 0"));
}
if self.dispatch_group.is_empty() {
return Err(cfg("job.dispatch_group 不能为空"));
}
if self.completion_retention_ms < self.fanout_max_wait_ms
|| self.completion_max_len == 0
|| self.completion_trim_batch_size == 0
{
return Err(cfg(
"job Completion 时间窗必须覆盖 fanout_max_wait_ms,长度与裁剪批量必须大于 0",
));
}
Ok(())
}
pub fn resolve_source(&self, qualifier: &str, client_namespace: &str) -> Result<Self> {
let source = self.sources.get(qualifier);
if source.and_then(|value| value.enabled) == Some(false) {
return Err(crate::job::JobError::SourceDisabled(qualifier.to_owned()).into());
}
let mut value = serde_json::to_value(self)
.map_err(|error| cfg(&format!("job 根配置无法规范化: {error}")))?;
let object = value
.as_object_mut()
.ok_or_else(|| cfg("job 根配置必须是对象"))?;
if self.namespace.trim().is_empty() {
object.insert(
"namespace".to_owned(),
serde_json::Value::String(client_namespace.to_owned()),
);
}
if let Some(source) = source {
for (key, override_value) in &source.overrides {
object.insert(key.clone(), override_value.clone());
}
}
let mut resolved: JobConfig = serde_json::from_value(value)
.map_err(|error| cfg(&format!("job source {qualifier} 配置非法: {error}")))?;
resolved.sources.clear();
resolved.validate()?;
Ok(resolved)
}
}
impl JobConfigBuilder {
pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
self.config.namespace = namespace.into();
self
}
pub fn application_name(mut self, application_name: impl Into<String>) -> Self {
self.config.application_name = application_name.into();
self
}
pub fn instance_identity(mut self, instance_identity: impl Into<String>) -> Self {
self.config.instance_identity = instance_identity.into();
self
}
pub fn source<F>(mut self, qualifier: &str, configure: F) -> Result<Self>
where
F: FnOnce(JobSourceConfigBuilder) -> JobSourceConfigBuilder,
{
let source_id = crate::job::source::JobSourceId::new(qualifier)?;
if self.config.sources.contains_key(source_id.as_str()) {
return Err(cfg(&format!("job source {} 重复配置", source_id.as_str())));
}
let source = configure(JobSourceConfigBuilder {
source: JobSourceConfig::default(),
})
.source;
self.config
.sources
.insert(source_id.as_str().to_owned(), source);
Ok(self)
}
pub fn build(self) -> Result<JobConfig> {
self.config.validate()?;
Ok(self.config)
}
}
impl JobSourceConfigBuilder {
pub fn enabled(mut self, enabled: bool) -> Self {
self.source.enabled = Some(enabled);
self
}
pub fn namespace(mut self, namespace: impl Into<String>) -> Self {
self.source.overrides.insert(
"namespace".to_owned(),
serde_json::Value::String(namespace.into()),
);
self
}
pub fn handler_capacity(mut self, capacity: u32) -> Self {
self.source.overrides.insert(
"handler_capacity".to_owned(),
serde_json::Value::from(capacity),
);
self
}
}
fn cfg(message: &str) -> NasaRedisError {
crate::job::JobError::Config(message.to_owned()).into()
}