mockforge-chaos 0.3.193

//! Prometheus metrics for chaos engineering
//!
//! Provides real-time metrics that can be integrated with Grafana
//! for monitoring chaos orchestrations, scenarios, and system impact.

use once_cell::sync::Lazy;
use prometheus::{
    proto::MetricFamily, register_counter_vec, register_gauge_vec, register_histogram_vec,
    CounterVec, GaugeVec, HistogramVec, Registry,
};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;

/// Chaos orchestration metrics
pub struct ChaosMetrics {
    /// Number of scenarios executed
    pub scenarios_total: CounterVec,

    /// Number of faults injected
    pub faults_injected_total: CounterVec,

    /// Latency injected (histogram)
    pub latency_injected: HistogramVec,

    /// Jitter applied on top of base latency (histogram, milliseconds, absolute
    /// offset). Surfaces independently from `latency_injected` so users can
    /// see jitter activity even when the configured base delay is zero.
    /// Issue #79 — Srikanth's round-3 reply.
    pub jitter_applied: HistogramVec,

    /// Bandwidth-throttle delay applied to a request/response transfer
    /// (histogram, milliseconds). Records the artificial wait that the
    /// `bandwidth_limit_bps` knob produced; samples = how often we actually
    /// throttled. Issue #79.
    pub bandwidth_throttle_delay: HistogramVec,

    /// Rate limit violations
    pub rate_limit_violations_total: CounterVec,

    /// Circuit breaker state
    pub circuit_breaker_state: GaugeVec,

    /// Bulkhead concurrent requests
    pub bulkhead_concurrent: GaugeVec,

    /// Orchestration step duration
    pub orchestration_step_duration: HistogramVec,

    /// Orchestration execution status
    pub orchestration_executions_total: CounterVec,

    /// Active orchestrations
    pub active_orchestrations: GaugeVec,

    /// Assertion results
    pub assertion_results_total: CounterVec,

    /// Hook executions
    pub hook_executions_total: CounterVec,

    /// Recommendation count
    pub recommendations_total: GaugeVec,

    /// System impact score
    pub chaos_impact_score: GaugeVec,
}

impl ChaosMetrics {
    /// Create new metrics
    pub fn new() -> Result<Self, prometheus::Error> {
        Ok(Self {
            scenarios_total: register_counter_vec!(
                "mockforge_chaos_scenarios_total",
                "Total number of chaos scenarios executed",
                &["scenario_type", "status"]
            )?,

            faults_injected_total: register_counter_vec!(
                "mockforge_chaos_faults_total",
                "Total number of faults injected",
                &["fault_type", "endpoint"]
            )?,

            latency_injected: register_histogram_vec!(
                "mockforge_chaos_latency_ms",
                "Latency injected in milliseconds",
                &["endpoint"],
                vec![10.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 2500.0, 5000.0, 10000.0]
            )?,

            jitter_applied: register_histogram_vec!(
                "mockforge_chaos_jitter_ms",
                "Jitter offset applied on top of base latency, in milliseconds (absolute value)",
                &["endpoint"],
                vec![1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0]
            )?,

            bandwidth_throttle_delay: register_histogram_vec!(
                "mockforge_chaos_bandwidth_throttle_ms",
                "Bandwidth-throttle delay added to a transfer, in milliseconds",
                &["endpoint", "direction"],
                vec![1.0, 10.0, 50.0, 100.0, 500.0, 1000.0, 5000.0, 10000.0]
            )?,

            rate_limit_violations_total: register_counter_vec!(
                "mockforge_chaos_rate_limit_violations_total",
                "Total rate limit violations",
                &["endpoint"]
            )?,

            circuit_breaker_state: register_gauge_vec!(
                "mockforge_chaos_circuit_breaker_state",
                "Circuit breaker state (0=closed, 1=open, 2=half-open)",
                &["circuit_name"]
            )?,

            bulkhead_concurrent: register_gauge_vec!(
                "mockforge_chaos_bulkhead_concurrent_requests",
                "Current concurrent requests in bulkhead",
                &["bulkhead_name"]
            )?,

            orchestration_step_duration: register_histogram_vec!(
                "mockforge_chaos_orchestration_step_duration_seconds",
                "Duration of orchestration steps in seconds",
                &["orchestration", "step"],
                vec![0.1, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0, 60.0]
            )?,

            orchestration_executions_total: register_counter_vec!(
                "mockforge_chaos_orchestration_executions_total",
                "Total orchestration executions",
                &["orchestration", "status"]
            )?,

            active_orchestrations: register_gauge_vec!(
                "mockforge_chaos_active_orchestrations",
                "Number of active orchestrations",
                &["orchestration"]
            )?,

            assertion_results_total: register_counter_vec!(
                "mockforge_chaos_assertion_results_total",
                "Total assertion results",
                &["orchestration", "result"]
            )?,

            hook_executions_total: register_counter_vec!(
                "mockforge_chaos_hook_executions_total",
                "Total hook executions",
                &["hook_type", "status"]
            )?,

            recommendations_total: register_gauge_vec!(
                "mockforge_chaos_recommendations_total",
                "Number of AI recommendations",
                &["category", "severity"]
            )?,

            chaos_impact_score: register_gauge_vec!(
                "mockforge_chaos_impact_score",
                "Overall chaos impact score (0.0-1.0)",
                &["time_window"]
            )?,
        })
    }

    /// Record scenario execution
    pub fn record_scenario(&self, scenario_type: &str, success: bool) {
        self.scenarios_total
            .with_label_values(&[scenario_type, if success { "success" } else { "failure" }])
            .inc();
    }

    /// Record fault injection
    pub fn record_fault(&self, fault_type: &str, endpoint: &str) {
        self.faults_injected_total.with_label_values(&[fault_type, endpoint]).inc();
    }

    /// Record latency injection
    pub fn record_latency(&self, endpoint: &str, latency_ms: f64) {
        self.latency_injected.with_label_values(&[endpoint]).observe(latency_ms);
    }

    /// Record a jitter offset application. Independent from `record_latency`
    /// so the TUI / `/metrics` can show jitter activity even when the base
    /// delay was zero. Issue #79.
    pub fn record_jitter(&self, endpoint: &str, jitter_ms: f64) {
        self.jitter_applied.with_label_values(&[endpoint]).observe(jitter_ms);
    }

    /// Record a bandwidth-throttle delay sample. `direction` is `"request"` or
    /// `"response"` so users can tell which side of the exchange was
    /// throttled. Issue #79.
    pub fn record_bandwidth_throttle(&self, endpoint: &str, direction: &str, delay_ms: f64) {
        self.bandwidth_throttle_delay
            .with_label_values(&[endpoint, direction])
            .observe(delay_ms);
    }

    /// Record rate limit violation
    pub fn record_rate_limit_violation(&self, endpoint: &str) {
        self.rate_limit_violations_total.with_label_values(&[endpoint]).inc();
    }

    /// Update circuit breaker state
    pub fn update_circuit_breaker_state(&self, circuit_name: &str, state: f64) {
        self.circuit_breaker_state.with_label_values(&[circuit_name]).set(state);
    }

    /// Update bulkhead concurrent requests
    pub fn update_bulkhead_concurrent(&self, bulkhead_name: &str, count: f64) {
        self.bulkhead_concurrent.with_label_values(&[bulkhead_name]).set(count);
    }

    /// Record orchestration step duration
    pub fn record_step_duration(&self, orchestration: &str, step: &str, duration_secs: f64) {
        self.orchestration_step_duration
            .with_label_values(&[orchestration, step])
            .observe(duration_secs);
    }

    /// Record orchestration execution
    pub fn record_orchestration_execution(&self, orchestration: &str, success: bool) {
        self.orchestration_executions_total
            .with_label_values(&[orchestration, if success { "success" } else { "failure" }])
            .inc();
    }

    /// Update active orchestrations
    pub fn update_active_orchestrations(&self, orchestration: &str, active: bool) {
        if active {
            self.active_orchestrations.with_label_values(&[orchestration]).inc();
        } else {
            self.active_orchestrations.with_label_values(&[orchestration]).dec();
        }
    }

    /// Record assertion result
    pub fn record_assertion(&self, orchestration: &str, passed: bool) {
        self.assertion_results_total
            .with_label_values(&[orchestration, if passed { "passed" } else { "failed" }])
            .inc();
    }

    /// Record hook execution
    pub fn record_hook(&self, hook_type: &str, success: bool) {
        self.hook_executions_total
            .with_label_values(&[hook_type, if success { "success" } else { "failure" }])
            .inc();
    }

    /// Update recommendations count
    pub fn update_recommendations(&self, category: &str, severity: &str, count: f64) {
        self.recommendations_total.with_label_values(&[category, severity]).set(count);
    }

    /// Update chaos impact score
    pub fn update_impact_score(&self, time_window: &str, score: f64) {
        self.chaos_impact_score.with_label_values(&[time_window]).set(score);
    }

    /// Snapshot the active counter values as a JSON-serializable struct.
    ///
    /// Issue #79 follow-up: the prometheus counters were wired in 0.3.128 but
    /// only readable via `/metrics` (Prometheus exposition format). This gives
    /// the TUI / dashboard a structured JSON view of fault injections,
    /// rate-limit violations, and latency injection counts — keyed by
    /// fault_type and endpoint.
    pub fn snapshot(&self) -> ChaosStatsSnapshot {
        use prometheus::core::Collector;

        let mut faults_by_type: HashMap<String, HashMap<String, u64>> = HashMap::new();
        let mut faults_total_by_type: HashMap<String, u64> = HashMap::new();
        let mut faults_grand_total: u64 = 0;
        for fam in self.faults_injected_total.collect() {
            walk_counter(&fam, |labels, count| {
                let fault_type =
                    labels.get("fault_type").cloned().unwrap_or_else(|| "unknown".to_string());
                let endpoint =
                    labels.get("endpoint").cloned().unwrap_or_else(|| "unknown".to_string());
                faults_by_type.entry(fault_type.clone()).or_default().insert(endpoint, count);
                *faults_total_by_type.entry(fault_type).or_default() += count;
                faults_grand_total += count;
            });
        }

        let mut rate_limit_by_endpoint: HashMap<String, u64> = HashMap::new();
        let mut rate_limit_total: u64 = 0;
        for fam in self.rate_limit_violations_total.collect() {
            walk_counter(&fam, |labels, count| {
                let endpoint =
                    labels.get("endpoint").cloned().unwrap_or_else(|| "unknown".to_string());
                rate_limit_by_endpoint.insert(endpoint, count);
                rate_limit_total += count;
            });
        }

        let mut latency_samples_by_endpoint: HashMap<String, u64> = HashMap::new();
        let mut latency_avg_ms_by_endpoint: HashMap<String, f64> = HashMap::new();
        for fam in self.latency_injected.collect() {
            for m in fam.get_metric() {
                let endpoint = m
                    .get_label()
                    .iter()
                    .find(|l| l.name() == "endpoint")
                    .map(|l| l.value().to_string())
                    .unwrap_or_else(|| "unknown".to_string());
                let hist = m.get_histogram();
                let count = hist.sample_count();
                latency_samples_by_endpoint.insert(endpoint.clone(), count);
                if count > 0 {
                    latency_avg_ms_by_endpoint.insert(endpoint, hist.sample_sum() / count as f64);
                }
            }
        }

        let mut jitter_samples_by_endpoint: HashMap<String, u64> = HashMap::new();
        let mut jitter_avg_ms_by_endpoint: HashMap<String, f64> = HashMap::new();
        for fam in self.jitter_applied.collect() {
            for m in fam.get_metric() {
                let endpoint = m
                    .get_label()
                    .iter()
                    .find(|l| l.name() == "endpoint")
                    .map(|l| l.value().to_string())
                    .unwrap_or_else(|| "unknown".to_string());
                let hist = m.get_histogram();
                let count = hist.sample_count();
                jitter_samples_by_endpoint.insert(endpoint.clone(), count);
                if count > 0 {
                    jitter_avg_ms_by_endpoint.insert(endpoint, hist.sample_sum() / count as f64);
                }
            }
        }

        let mut bandwidth_throttle_samples: HashMap<String, u64> = HashMap::new();
        let mut bandwidth_throttle_total_ms: u64 = 0;
        for fam in self.bandwidth_throttle_delay.collect() {
            for m in fam.get_metric() {
                let direction = m
                    .get_label()
                    .iter()
                    .find(|l| l.name() == "direction")
                    .map(|l| l.value().to_string())
                    .unwrap_or_else(|| "unknown".to_string());
                let hist = m.get_histogram();
                let count = hist.sample_count();
                *bandwidth_throttle_samples.entry(direction).or_default() += count;
                bandwidth_throttle_total_ms += hist.sample_sum() as u64;
            }
        }

        ChaosStatsSnapshot {
            faults_by_type,
            faults_total_by_type,
            faults_grand_total,
            rate_limit_violations_by_endpoint: rate_limit_by_endpoint,
            rate_limit_violations_total: rate_limit_total,
            latency_samples_by_endpoint,
            latency_avg_ms_by_endpoint,
            jitter_samples_by_endpoint,
            jitter_avg_ms_by_endpoint,
            bandwidth_throttle_samples_by_direction: bandwidth_throttle_samples,
            bandwidth_throttle_total_ms,
        }
    }
}

/// Iterate counter samples in a metric family, calling `visit(labels, value)`
/// for each. Only valid for counter-typed families; histograms have a
/// different shape and we read those inline in `snapshot()`.
fn walk_counter<F>(fam: &MetricFamily, mut visit: F)
where
    F: FnMut(HashMap<String, String>, u64),
{
    for m in fam.get_metric() {
        let labels: HashMap<String, String> = m
            .get_label()
            .iter()
            .map(|l| (l.name().to_string(), l.value().to_string()))
            .collect();
        let count = m.get_counter().value() as u64;
        visit(labels, count);
    }
}

/// Structured snapshot of the chaos counter state. Returned by the
/// `/api/chaos/stats` endpoint and consumed by the TUI Chaos screen.
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct ChaosStatsSnapshot {
    /// faults_by_type[fault_type][endpoint] = count. Empty when chaos has
    /// never fired since process start.
    pub faults_by_type: HashMap<String, HashMap<String, u64>>,
    /// Total faults per fault_type, summed across endpoints.
    pub faults_total_by_type: HashMap<String, u64>,
    /// Total fault injections across all types and endpoints.
    pub faults_grand_total: u64,
    /// Rate-limit violations per endpoint.
    pub rate_limit_violations_by_endpoint: HashMap<String, u64>,
    /// Total rate-limit violations.
    pub rate_limit_violations_total: u64,
    /// Number of latency-injection samples per endpoint (histogram count).
    pub latency_samples_by_endpoint: HashMap<String, u64>,
    /// Mean injected latency per endpoint in milliseconds (sample_sum / count).
    /// Issue #79 — Srikanth's round-3 reply asked for visibility into the
    /// latency the server is actually injecting; this gives it without
    /// requiring Prometheus scraping.
    #[serde(default)]
    pub latency_avg_ms_by_endpoint: HashMap<String, f64>,
    /// Per-endpoint count of jitter applications (independent from latency).
    #[serde(default)]
    pub jitter_samples_by_endpoint: HashMap<String, u64>,
    /// Mean jitter offset per endpoint in milliseconds.
    #[serde(default)]
    pub jitter_avg_ms_by_endpoint: HashMap<String, f64>,
    /// Bandwidth-throttle activity counted by direction (`"request"` /
    /// `"response"`). Empty when the bandwidth_limit_bps knob isn't firing.
    #[serde(default)]
    pub bandwidth_throttle_samples_by_direction: HashMap<String, u64>,
    /// Total bandwidth-throttle delay accumulated across all requests and
    /// responses, in milliseconds.
    #[serde(default)]
    pub bandwidth_throttle_total_ms: u64,
}

impl Default for ChaosMetrics {
    fn default() -> Self {
        Self::new().expect("Failed to create chaos metrics")
    }
}

/// Global metrics instance
pub static CHAOS_METRICS: Lazy<ChaosMetrics> =
    Lazy::new(|| ChaosMetrics::new().expect("Failed to initialize chaos metrics"));

/// Get the default Prometheus registry
pub fn registry() -> &'static Registry {
    prometheus::default_registry()
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_metrics_creation() {
        // The global CHAOS_METRICS is already initialized, proving that metrics creation works.
        // Creating a second instance would fail with "AlreadyReg" because metrics are
        // registered with the global Prometheus registry.
        // Instead, verify the global instance is accessible.
        let _metrics = &*CHAOS_METRICS;
        // If we get here without panic, the metrics were successfully created
    }

    #[test]
    fn test_record_scenario() {
        let metrics = CHAOS_METRICS.scenarios_total.clone();
        let before = metrics.with_label_values(&["test", "success"]).get();

        CHAOS_METRICS.record_scenario("test", true);

        let after = metrics.with_label_values(&["test", "success"]).get();
        assert!(after > before);
    }

    #[test]
    fn test_record_latency() {
        CHAOS_METRICS.record_latency("/api/test", 100.0);
        // Just ensure it doesn't panic
    }

    /// Issue #79 follow-up: snapshot must reflect counter increments and the
    /// label-keyed nesting (fault_type → endpoint → count) the TUI uses.
    #[test]
    fn snapshot_reflects_counter_increments() {
        // Use a unique endpoint label so this test doesn't race with other
        // tests against the global CHAOS_METRICS singleton.
        let endpoint = "/api/test_snapshot_endpoint_unique_xyz";
        let baseline = CHAOS_METRICS.snapshot();
        let baseline_count = baseline
            .faults_by_type
            .get("http_error")
            .and_then(|m| m.get(endpoint))
            .copied()
            .unwrap_or(0);

        CHAOS_METRICS.record_fault("http_error", endpoint);
        CHAOS_METRICS.record_fault("http_error", endpoint);
        CHAOS_METRICS.record_rate_limit_violation(endpoint);
        CHAOS_METRICS.record_latency(endpoint, 42.0);

        let snap = CHAOS_METRICS.snapshot();
        assert_eq!(
            snap.faults_by_type
                .get("http_error")
                .and_then(|m| m.get(endpoint))
                .copied()
                .unwrap_or(0),
            baseline_count + 2,
            "fault count for {endpoint} did not advance by 2"
        );
        assert!(
            snap.faults_total_by_type.get("http_error").copied().unwrap_or(0) >= 2,
            "faults_total_by_type[http_error] should reflect the inc"
        );
        assert!(
            snap.rate_limit_violations_by_endpoint.get(endpoint).copied().unwrap_or(0) >= 1,
            "rate_limit_violations_by_endpoint did not record"
        );
        assert!(
            snap.latency_samples_by_endpoint.get(endpoint).copied().unwrap_or(0) >= 1,
            "latency histogram count did not record"
        );
    }
}