#![allow(dead_code)]
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub enum WorkflowMetric {
QueueWaitSeconds,
TaskDurationSeconds,
TaskMemoryBytes,
CpuPercent,
RetryCount,
WorkflowDurationSeconds,
}
impl WorkflowMetric {
#[must_use]
pub fn unit(self) -> &'static str {
match self {
Self::QueueWaitSeconds | Self::TaskDurationSeconds | Self::WorkflowDurationSeconds => {
"s"
}
Self::TaskMemoryBytes => "bytes",
Self::CpuPercent => "%",
Self::RetryCount => "count",
}
}
#[must_use]
pub fn lower_is_better(self) -> bool {
!matches!(self, Self::CpuPercent)
}
}
impl std::fmt::Display for WorkflowMetric {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let s = match self {
Self::QueueWaitSeconds => "queue_wait_seconds",
Self::TaskDurationSeconds => "task_duration_seconds",
Self::TaskMemoryBytes => "task_memory_bytes",
Self::CpuPercent => "cpu_percent",
Self::RetryCount => "retry_count",
Self::WorkflowDurationSeconds => "workflow_duration_seconds",
};
write!(f, "{s}")
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct MetricSample {
pub metric: WorkflowMetric,
pub source_id: String,
pub value: f64,
}
impl MetricSample {
#[must_use]
pub fn new(metric: WorkflowMetric, source_id: impl Into<String>, value: f64) -> Self {
Self {
metric,
source_id: source_id.into(),
value,
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct MetricSummary {
pub metric: WorkflowMetric,
pub count: usize,
pub sum: f64,
pub min: f64,
pub max: f64,
}
impl MetricSummary {
#[allow(clippy::cast_precision_loss)]
#[must_use]
pub fn mean(&self) -> f64 {
if self.count == 0 {
0.0
} else {
self.sum / self.count as f64
}
}
}
#[derive(Debug, Default, Clone)]
pub struct WorkflowMetricsCollector {
samples: Vec<MetricSample>,
}
impl WorkflowMetricsCollector {
#[must_use]
pub fn new() -> Self {
Self::default()
}
pub fn record(&mut self, sample: MetricSample) {
self.samples.push(sample);
}
pub fn record_value(
&mut self,
metric: WorkflowMetric,
source_id: impl Into<String>,
value: f64,
) {
self.record(MetricSample::new(metric, source_id, value));
}
#[must_use]
pub fn sample_count(&self) -> usize {
self.samples.len()
}
#[must_use]
pub fn samples_for(&self, metric: WorkflowMetric) -> Vec<&MetricSample> {
self.samples.iter().filter(|s| s.metric == metric).collect()
}
#[must_use]
pub fn summarize(&self, metric: WorkflowMetric) -> Option<MetricSummary> {
let relevant: Vec<f64> = self
.samples
.iter()
.filter(|s| s.metric == metric)
.map(|s| s.value)
.collect();
if relevant.is_empty() {
return None;
}
let sum: f64 = relevant.iter().sum();
let min = relevant.iter().copied().fold(f64::INFINITY, f64::min);
let max = relevant.iter().copied().fold(f64::NEG_INFINITY, f64::max);
Some(MetricSummary {
metric,
count: relevant.len(),
sum,
min,
max,
})
}
#[must_use]
pub fn all_summaries(&self) -> HashMap<WorkflowMetric, MetricSummary> {
let mut map = HashMap::new();
for metric in [
WorkflowMetric::QueueWaitSeconds,
WorkflowMetric::TaskDurationSeconds,
WorkflowMetric::TaskMemoryBytes,
WorkflowMetric::CpuPercent,
WorkflowMetric::RetryCount,
WorkflowMetric::WorkflowDurationSeconds,
] {
if let Some(summary) = self.summarize(metric) {
map.insert(metric, summary);
}
}
map
}
pub fn reset(&mut self) {
self.samples.clear();
}
}
#[cfg(test)]
mod tests {
use super::*;
fn collector_with_samples() -> WorkflowMetricsCollector {
let mut c = WorkflowMetricsCollector::new();
c.record_value(WorkflowMetric::TaskDurationSeconds, "task-1", 10.0);
c.record_value(WorkflowMetric::TaskDurationSeconds, "task-2", 20.0);
c.record_value(WorkflowMetric::TaskDurationSeconds, "task-3", 30.0);
c.record_value(WorkflowMetric::CpuPercent, "task-1", 55.0);
c
}
#[test]
fn test_new_collector_empty() {
let c = WorkflowMetricsCollector::new();
assert_eq!(c.sample_count(), 0);
}
#[test]
fn test_record_value_increments_count() {
let mut c = WorkflowMetricsCollector::new();
c.record_value(WorkflowMetric::RetryCount, "wf-1", 2.0);
assert_eq!(c.sample_count(), 1);
}
#[test]
fn test_samples_for_filters_correctly() {
let c = collector_with_samples();
let dur_samples = c.samples_for(WorkflowMetric::TaskDurationSeconds);
assert_eq!(dur_samples.len(), 3);
}
#[test]
fn test_summarize_mean() {
let c = collector_with_samples();
let summary = c
.summarize(WorkflowMetric::TaskDurationSeconds)
.expect("should succeed in test");
assert!((summary.mean() - 20.0).abs() < 1e-9);
}
#[test]
fn test_summarize_min_max() {
let c = collector_with_samples();
let summary = c
.summarize(WorkflowMetric::TaskDurationSeconds)
.expect("should succeed in test");
assert!((summary.min - 10.0).abs() < 1e-9);
assert!((summary.max - 30.0).abs() < 1e-9);
}
#[test]
fn test_summarize_count() {
let c = collector_with_samples();
let summary = c
.summarize(WorkflowMetric::TaskDurationSeconds)
.expect("should succeed in test");
assert_eq!(summary.count, 3);
}
#[test]
fn test_summarize_none_for_missing_metric() {
let c = collector_with_samples();
assert!(c.summarize(WorkflowMetric::QueueWaitSeconds).is_none());
}
#[test]
fn test_all_summaries_keys() {
let c = collector_with_samples();
let summaries = c.all_summaries();
assert!(summaries.contains_key(&WorkflowMetric::TaskDurationSeconds));
assert!(summaries.contains_key(&WorkflowMetric::CpuPercent));
assert!(!summaries.contains_key(&WorkflowMetric::QueueWaitSeconds));
}
#[test]
fn test_reset_clears_samples() {
let mut c = collector_with_samples();
c.reset();
assert_eq!(c.sample_count(), 0);
assert!(c.summarize(WorkflowMetric::TaskDurationSeconds).is_none());
}
#[test]
fn test_metric_unit() {
assert_eq!(WorkflowMetric::TaskDurationSeconds.unit(), "s");
assert_eq!(WorkflowMetric::TaskMemoryBytes.unit(), "bytes");
assert_eq!(WorkflowMetric::CpuPercent.unit(), "%");
assert_eq!(WorkflowMetric::RetryCount.unit(), "count");
}
#[test]
fn test_metric_lower_is_better() {
assert!(WorkflowMetric::TaskDurationSeconds.lower_is_better());
assert!(WorkflowMetric::QueueWaitSeconds.lower_is_better());
assert!(!WorkflowMetric::CpuPercent.lower_is_better());
}
#[test]
fn test_metric_display() {
assert_eq!(
format!("{}", WorkflowMetric::TaskDurationSeconds),
"task_duration_seconds"
);
}
#[test]
fn test_metric_summary_mean_empty() {
let s = MetricSummary {
metric: WorkflowMetric::RetryCount,
count: 0,
sum: 0.0,
min: 0.0,
max: 0.0,
};
assert!((s.mean() - 0.0).abs() < 1e-9);
}
#[test]
fn test_single_sample_summary() {
let mut c = WorkflowMetricsCollector::new();
c.record_value(WorkflowMetric::WorkflowDurationSeconds, "wf-a", 42.0);
let s = c
.summarize(WorkflowMetric::WorkflowDurationSeconds)
.expect("should succeed in test");
assert_eq!(s.count, 1);
assert!((s.min - 42.0).abs() < 1e-9);
assert!((s.max - 42.0).abs() < 1e-9);
assert!((s.mean() - 42.0).abs() < 1e-9);
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct StepMetric {
pub step_id: String,
pub started_at: std::time::SystemTime,
pub duration_secs: f64,
pub success: bool,
pub retries: u32,
pub output_size_bytes: Option<u64>,
pub cpu_seconds: Option<f64>,
}
impl StepMetric {
#[must_use]
pub fn new(step_id: impl Into<String>, duration_secs: f64, success: bool) -> Self {
Self {
step_id: step_id.into(),
started_at: std::time::SystemTime::now(),
duration_secs,
success,
retries: 0,
output_size_bytes: None,
cpu_seconds: None,
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WorkflowRunMetrics {
pub workflow_id: String,
pub started_at: std::time::SystemTime,
pub completed_at: Option<std::time::SystemTime>,
pub step_metrics: Vec<StepMetric>,
pub total_duration_secs: Option<f64>,
pub success: Option<bool>,
}
impl WorkflowRunMetrics {
#[must_use]
pub fn new(workflow_id: impl Into<String>) -> Self {
Self {
workflow_id: workflow_id.into(),
started_at: std::time::SystemTime::now(),
completed_at: None,
step_metrics: Vec::new(),
total_duration_secs: None,
success: None,
}
}
pub fn finish(&mut self, success: bool) {
let now = std::time::SystemTime::now();
self.completed_at = Some(now);
self.success = Some(success);
self.total_duration_secs = now
.duration_since(self.started_at)
.map(|d| d.as_secs_f64())
.ok();
}
}
pub struct WorkflowMetricsAggregator {
history: Vec<WorkflowRunMetrics>,
max_history: usize,
}
impl WorkflowMetricsAggregator {
#[must_use]
pub fn new(max_history: usize) -> Self {
Self {
history: Vec::new(),
max_history,
}
}
pub fn record(&mut self, metrics: WorkflowRunMetrics) {
if self.max_history > 0 && self.history.len() >= self.max_history {
self.history.remove(0);
}
self.history.push(metrics);
}
#[must_use]
pub fn len(&self) -> usize {
self.history.len()
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.history.is_empty()
}
#[must_use]
pub fn avg_duration_secs(&self) -> Option<f64> {
let durations: Vec<f64> = self
.history
.iter()
.filter_map(|m| m.total_duration_secs)
.collect();
if durations.is_empty() {
None
} else {
#[allow(clippy::cast_precision_loss)]
Some(durations.iter().sum::<f64>() / durations.len() as f64)
}
}
#[must_use]
#[allow(clippy::cast_precision_loss)]
pub fn success_rate(&self) -> f64 {
let finished: Vec<bool> = self.history.iter().filter_map(|m| m.success).collect();
if finished.is_empty() {
return 0.0;
}
let successes = finished.iter().filter(|&&s| s).count();
successes as f64 / finished.len() as f64
}
#[must_use]
pub fn p95_duration_secs(&self) -> Option<f64> {
let mut durations: Vec<f64> = self
.history
.iter()
.filter_map(|m| m.total_duration_secs)
.collect();
if durations.is_empty() {
return None;
}
durations.sort_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal));
#[allow(clippy::cast_precision_loss)]
let idx = ((durations.len() as f64 * 0.95).ceil() as usize).saturating_sub(1);
let idx = idx.min(durations.len() - 1);
Some(durations[idx])
}
#[must_use]
#[allow(clippy::cast_precision_loss)]
pub fn slowest_steps(&self, n: usize) -> Vec<(String, f64)> {
let mut totals: HashMap<String, (f64, usize)> = HashMap::new();
for run in &self.history {
for step in &run.step_metrics {
let entry = totals.entry(step.step_id.clone()).or_insert((0.0, 0));
entry.0 += step.duration_secs;
entry.1 += 1;
}
}
let mut avgs: Vec<(String, f64)> = totals
.into_iter()
.map(|(id, (total, count))| (id, total / count as f64))
.collect();
avgs.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
avgs.truncate(n);
avgs
}
#[must_use]
#[allow(clippy::cast_precision_loss)]
pub fn failure_rate_by_step(&self) -> Vec<(String, f64)> {
let mut counts: HashMap<String, (usize, usize)> = HashMap::new(); for run in &self.history {
for step in &run.step_metrics {
let entry = counts.entry(step.step_id.clone()).or_insert((0, 0));
entry.0 += 1;
if !step.success {
entry.1 += 1;
}
}
}
let mut rates: Vec<(String, f64)> = counts
.into_iter()
.map(|(id, (total, failures))| (id, failures as f64 / total as f64))
.collect();
rates.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
rates
}
#[must_use]
pub fn history(&self) -> &[WorkflowRunMetrics] {
&self.history
}
}
#[cfg(test)]
mod aggregator_tests {
use super::*;
fn make_run(
id: &str,
duration: f64,
success: bool,
steps: Vec<StepMetric>,
) -> WorkflowRunMetrics {
WorkflowRunMetrics {
workflow_id: id.to_string(),
started_at: std::time::SystemTime::now(),
completed_at: Some(std::time::SystemTime::now()),
step_metrics: steps,
total_duration_secs: Some(duration),
success: Some(success),
}
}
fn make_step(id: &str, duration: f64, success: bool) -> StepMetric {
StepMetric::new(id, duration, success)
}
#[test]
fn test_aggregator_new_is_empty() {
let agg = WorkflowMetricsAggregator::new(100);
assert!(agg.is_empty());
assert_eq!(agg.len(), 0);
}
#[test]
fn test_record_increments_len() {
let mut agg = WorkflowMetricsAggregator::new(100);
agg.record(make_run("wf-1", 10.0, true, vec![]));
assert_eq!(agg.len(), 1);
}
#[test]
fn test_max_history_evicts_oldest() {
let mut agg = WorkflowMetricsAggregator::new(2);
agg.record(make_run("wf-1", 10.0, true, vec![]));
agg.record(make_run("wf-2", 20.0, true, vec![]));
agg.record(make_run("wf-3", 30.0, true, vec![]));
assert_eq!(agg.len(), 2);
assert_eq!(agg.history()[0].workflow_id, "wf-2");
}
#[test]
fn test_avg_duration_secs_correct() {
let mut agg = WorkflowMetricsAggregator::new(100);
agg.record(make_run("wf-1", 10.0, true, vec![]));
agg.record(make_run("wf-2", 30.0, true, vec![]));
let avg = agg.avg_duration_secs().expect("should have avg");
assert!((avg - 20.0).abs() < 1e-9);
}
#[test]
fn test_avg_duration_none_when_empty() {
let agg = WorkflowMetricsAggregator::new(100);
assert!(agg.avg_duration_secs().is_none());
}
#[test]
fn test_success_rate_all_success() {
let mut agg = WorkflowMetricsAggregator::new(100);
agg.record(make_run("wf-1", 10.0, true, vec![]));
agg.record(make_run("wf-2", 10.0, true, vec![]));
assert!((agg.success_rate() - 1.0).abs() < 1e-9);
}
#[test]
fn test_success_rate_half_success() {
let mut agg = WorkflowMetricsAggregator::new(100);
agg.record(make_run("wf-1", 10.0, true, vec![]));
agg.record(make_run("wf-2", 10.0, false, vec![]));
assert!((agg.success_rate() - 0.5).abs() < 1e-9);
}
#[test]
fn test_success_rate_zero_when_empty() {
let agg = WorkflowMetricsAggregator::new(100);
assert!((agg.success_rate() - 0.0).abs() < 1e-9);
}
#[test]
fn test_p95_duration_secs() {
let mut agg = WorkflowMetricsAggregator::new(100);
for i in 1u32..=20 {
agg.record(make_run(&format!("wf-{i}"), f64::from(i), true, vec![]));
}
let p95 = agg.p95_duration_secs().expect("should have p95");
assert!((p95 - 19.0).abs() < 1e-9);
}
#[test]
fn test_p95_none_when_empty() {
let agg = WorkflowMetricsAggregator::new(100);
assert!(agg.p95_duration_secs().is_none());
}
#[test]
fn test_slowest_steps_returns_top_n() {
let mut agg = WorkflowMetricsAggregator::new(100);
agg.record(make_run(
"wf-1",
100.0,
true,
vec![
make_step("transcode", 80.0, true),
make_step("ingest", 10.0, true),
make_step("deliver", 5.0, true),
],
));
let slowest = agg.slowest_steps(2);
assert_eq!(slowest.len(), 2);
assert_eq!(slowest[0].0, "transcode");
assert_eq!(slowest[1].0, "ingest");
}
#[test]
fn test_failure_rate_by_step_sorted_desc() {
let mut agg = WorkflowMetricsAggregator::new(100);
for _ in 0..2 {
agg.record(make_run(
"wf",
10.0,
false,
vec![
make_step("transcode", 5.0, false),
make_step("ingest", 2.0, true),
],
));
}
let rates = agg.failure_rate_by_step();
assert!(!rates.is_empty());
assert_eq!(rates[0].0, "transcode");
assert!((rates[0].1 - 1.0).abs() < 1e-9);
let ingest_rate = rates
.iter()
.find(|(id, _)| id == "ingest")
.map(|(_, r)| *r)
.unwrap_or(0.0);
assert!((ingest_rate - 0.0).abs() < 1e-9);
}
}