use std::collections::BTreeMap;
use std::sync::Mutex;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum JobFireResult {
Fired,
Adopted,
Skipped,
NotDue,
NeedRecompute,
StateMismatch,
Stale,
}
impl JobFireResult {
pub fn as_str(self) -> &'static str {
match self {
Self::Fired => "fired",
Self::Adopted => "adopted",
Self::Skipped => "skipped",
Self::NotDue => "not_due",
Self::NeedRecompute => "need_recompute",
Self::StateMismatch => "state_mismatch",
Self::Stale => "stale",
}
}
pub(crate) fn from_script_code(code: &str) -> Option<Self> {
match code {
"OK" => Some(Self::Fired),
"ADOPTED" => Some(Self::Adopted),
"SKIPPED" => Some(Self::Skipped),
"NOT_DUE" => Some(Self::NotDue),
"NEED_RECOMPUTE" => Some(Self::NeedRecompute),
"STATE_MISMATCH" => Some(Self::StateMismatch),
"STALE" => Some(Self::Stale),
_ => None,
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum JobSupervisorLoop {
Subscription,
Scanner,
Dispatcher,
FanoutMonitor,
Completion,
Heartbeat,
}
impl JobSupervisorLoop {
pub fn as_str(self) -> &'static str {
match self {
Self::Subscription => "subscription",
Self::Scanner => "scanner",
Self::Dispatcher => "dispatcher",
Self::FanoutMonitor => "fanout_monitor",
Self::Completion => "completion",
Self::Heartbeat => "heartbeat",
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
pub enum JobSupervisorResult {
Running,
RestartScheduled,
Recovered,
BudgetExhausted,
AuthorityLost,
}
impl JobSupervisorResult {
pub fn as_str(self) -> &'static str {
match self {
Self::Running => "running",
Self::RestartScheduled => "restart_scheduled",
Self::Recovered => "recovered",
Self::BudgetExhausted => "budget_exhausted",
Self::AuthorityLost => "authority_lost",
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct JobSupervisorMetricsSnapshot {
pub loop_name: JobSupervisorLoop,
pub generation: u64,
pub restart_total: u64,
pub result: JobSupervisorResult,
pub result_totals: BTreeMap<JobSupervisorResult, u64>,
}
#[derive(Debug, Clone)]
struct SupervisorMetric {
generation: u64,
restart_total: u64,
result: JobSupervisorResult,
result_totals: BTreeMap<JobSupervisorResult, u64>,
}
impl Default for SupervisorMetric {
fn default() -> Self {
Self {
generation: 1,
restart_total: 0,
result: JobSupervisorResult::Running,
result_totals: ALL_SUPERVISOR_RESULTS
.iter()
.copied()
.map(|result| (result, 0))
.collect(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct JobMetricsSnapshot {
pub qualifier: String,
pub values: BTreeMap<&'static str, i64>,
pub fire_results: BTreeMap<JobFireResult, u64>,
pub supervisors: Vec<JobSupervisorMetricsSnapshot>,
pub completion_oldest_id: Option<String>,
}
#[derive(Debug)]
struct JobMetricsState {
values: BTreeMap<&'static str, i64>,
fire_results: BTreeMap<JobFireResult, u64>,
shard_gauges: BTreeMap<&'static str, BTreeMap<u32, i64>>,
supervisors: BTreeMap<JobSupervisorLoop, SupervisorMetric>,
completion_oldest_id: Option<String>,
}
#[derive(Debug)]
pub(crate) struct JobMetrics {
qualifier: String,
state: Mutex<JobMetricsState>,
}
impl JobMetrics {
pub(crate) fn new(qualifier: impl Into<String>) -> Self {
let values = JOB_METRIC_NAMES
.iter()
.copied()
.map(|name| (name, 0))
.collect();
let supervisors = ALL_SUPERVISOR_LOOPS
.iter()
.copied()
.map(|loop_name| (loop_name, SupervisorMetric::default()))
.collect();
Self {
qualifier: qualifier.into(),
state: Mutex::new(JobMetricsState {
values,
fire_results: ALL_FIRE_RESULTS
.iter()
.copied()
.map(|result| (result, 0))
.collect(),
shard_gauges: BTreeMap::new(),
supervisors,
completion_oldest_id: None,
}),
}
}
pub(crate) fn add(&self, name: &'static str, delta: i64) {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(value) = state.values.get_mut(name) {
*value = value.saturating_add(delta.max(0));
}
}
pub(crate) fn set(&self, name: &'static str, value: i64) {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(current) = state.values.get_mut(name) {
*current = value;
}
}
pub(crate) fn set_shard_gauge(&self, name: &'static str, shard: u32, value: i64) {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if !state.values.contains_key(name) {
return;
}
let aggregate = {
let shards = state.shard_gauges.entry(name).or_default();
shards.insert(shard, value.max(0));
shards.values().copied().fold(0_i64, i64::saturating_add)
};
state.values.insert(name, aggregate);
}
pub(crate) fn record_fire(&self, result: JobFireResult, delta: u64) {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(total) = state.fire_results.get_mut(&result) {
*total = total.saturating_add(delta);
}
}
pub(crate) fn restart_scheduled(&self, loop_name: JobSupervisorLoop) -> u64 {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let metric = state.supervisors.entry(loop_name).or_default();
metric.generation = metric.generation.saturating_add(1);
metric.restart_total = metric.restart_total.saturating_add(1);
metric.result = JobSupervisorResult::RestartScheduled;
if let Some(total) = metric
.result_totals
.get_mut(&JobSupervisorResult::RestartScheduled)
{
*total = total.saturating_add(1);
}
metric.generation
}
pub(crate) fn recovered(&self, loop_name: JobSupervisorLoop) {
self.set_supervisor_result(loop_name, JobSupervisorResult::Recovered);
}
pub(crate) fn budget_exhausted(&self, loop_name: JobSupervisorLoop) {
self.set_supervisor_result(loop_name, JobSupervisorResult::BudgetExhausted);
}
pub(crate) fn authority_lost(&self, loop_name: JobSupervisorLoop) {
self.set_supervisor_result(loop_name, JobSupervisorResult::AuthorityLost);
}
pub(crate) fn record_completion_trim(
&self,
length_trimmed: i64,
retention_trimmed: i64,
oldest_id: Option<&str>,
redis_now: i64,
) {
self.add("redis_job_completion_length_trimmed_total", length_trimmed);
self.add(
"redis_job_completion_retention_trimmed_total",
retention_trimmed,
);
let oldest_millis = oldest_id
.and_then(|id| id.split_once('-').map(|(millis, _)| millis))
.and_then(|millis| millis.parse::<i64>().ok());
self.set(
"redis_job_completion_oldest_age_ms",
oldest_millis.map_or(0, |oldest| redis_now.saturating_sub(oldest).max(0)),
);
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.completion_oldest_id = oldest_id.map(ToOwned::to_owned);
}
pub(crate) fn snapshot(&self) -> JobMetricsSnapshot {
let state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
JobMetricsSnapshot {
qualifier: self.qualifier.clone(),
values: state.values.clone(),
fire_results: state.fire_results.clone(),
supervisors: state
.supervisors
.iter()
.map(|(loop_name, metric)| JobSupervisorMetricsSnapshot {
loop_name: *loop_name,
generation: metric.generation,
restart_total: metric.restart_total,
result: metric.result,
result_totals: metric.result_totals.clone(),
})
.collect(),
completion_oldest_id: state.completion_oldest_id.clone(),
}
}
fn set_supervisor_result(&self, loop_name: JobSupervisorLoop, result: JobSupervisorResult) {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let metric = state.supervisors.entry(loop_name).or_default();
metric.result = result;
if let Some(total) = metric.result_totals.get_mut(&result) {
*total = total.saturating_add(1);
}
}
}
const ALL_SUPERVISOR_LOOPS: [JobSupervisorLoop; 6] = [
JobSupervisorLoop::Subscription,
JobSupervisorLoop::Scanner,
JobSupervisorLoop::Dispatcher,
JobSupervisorLoop::FanoutMonitor,
JobSupervisorLoop::Completion,
JobSupervisorLoop::Heartbeat,
];
const ALL_SUPERVISOR_RESULTS: [JobSupervisorResult; 5] = [
JobSupervisorResult::Running,
JobSupervisorResult::RestartScheduled,
JobSupervisorResult::Recovered,
JobSupervisorResult::BudgetExhausted,
JobSupervisorResult::AuthorityLost,
];
const ALL_FIRE_RESULTS: [JobFireResult; 7] = [
JobFireResult::Fired,
JobFireResult::Adopted,
JobFireResult::Skipped,
JobFireResult::NotDue,
JobFireResult::NeedRecompute,
JobFireResult::StateMismatch,
JobFireResult::Stale,
];
const JOB_METRIC_NAMES: [&str; 36] = [
"redis_job_schedule_lag_ms",
"redis_job_due_scan_total",
"redis_job_fire_total",
"redis_job_visible_size",
"redis_job_running",
"redis_job_started_total",
"redis_job_success_total",
"redis_job_failed_total",
"redis_job_retry_total",
"redis_job_dead_total",
"redis_job_skipped_total",
"redis_job_lease_expired_total",
"redis_job_stale_finish_total",
"redis_job_fencing_regression_total",
"redis_job_self_fence_total",
"redis_job_serial_wait_depth",
"redis_job_executor_capacity",
"redis_job_definition_conflict_total",
"redis_job_capable_executor_count",
"redis_job_invalid_payload_total",
"redis_job_fanout_shard_total",
"redis_job_fanout_receipt_timeout_total",
"redis_job_fanout_receipt_retry_total",
"redis_job_fanout_reassign_total",
"redis_job_fanout_stale_assignment_total",
"redis_job_fanout_delivery_pending",
"redis_job_fanout_ready_wakeup_total",
"redis_job_fanout_capacity_defer_total",
"redis_job_fanout_capacity_exhausted_total",
"redis_job_fanout_gc_pending",
"redis_job_registry_gc_total",
"redis_job_script_reload_total",
"redis_job_completion_length_trimmed_total",
"redis_job_completion_retention_trimmed_total",
"redis_job_completion_oldest_age_ms",
"redis_job_pubsub_recovery_window_ms",
];