llm_agent_runtime/

metrics.rs

//! # Module: Metrics
//!
//! Runtime observability counters for `AgentRuntime`.
//! All global counters use atomics for lock-free, thread-safe increment/read.
//! Per-tool counters use a `Mutex<HashMap>` to avoid requiring a concurrent
//! map dependency.

use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};

// ── LatencyHistogram ──────────────────────────────────────────────────────────

/// A simple fixed-bucket latency histogram.
///
/// Bucket upper bounds are **inclusive** (i.e., a sample of exactly 1 ms falls into bucket 0).
/// Bucket index mapping:
/// - 0: ≤ 1 ms
/// - 1: 2 – 5 ms
/// - 2: 6 – 10 ms
/// - 3: 11 – 50 ms
/// - 4: 51 – 100 ms
/// - 5: 101 – 500 ms
/// - 6: > 500 ms
#[derive(Debug)]
pub struct LatencyHistogram {
    /// Counts per bucket. Index 0 = ≤1ms, …, index 6 = >500ms.
    /// Bucket upper bounds are **inclusive**.
    buckets: [AtomicU64; 7],
    total_count: AtomicU64,
    total_sum_ms: AtomicU64,
}

impl Default for LatencyHistogram {
    fn default() -> Self {
        Self {
            buckets: [
                AtomicU64::new(0),
                AtomicU64::new(0),
                AtomicU64::new(0),
                AtomicU64::new(0),
                AtomicU64::new(0),
                AtomicU64::new(0),
                AtomicU64::new(0),
            ],
            total_count: AtomicU64::new(0),
            total_sum_ms: AtomicU64::new(0),
        }
    }
}

impl LatencyHistogram {
    /// Bucket upper bounds in milliseconds.
    ///
    /// Boundaries were chosen to cover the full range of observed latencies in
    /// LLM-backed agent systems:
    ///
    /// | Bucket | Range      | Typical source                         |
    /// |--------|------------|----------------------------------------|
    /// | 0      | ≤ 1 ms     | In-process tool calls, cache hits      |
    /// | 1      | ≤ 5 ms     | Fast local I/O, simple calculations    |
    /// | 2      | ≤ 10 ms    | Network round-trips to local services  |
    /// | 3      | ≤ 50 ms    | p50 LLM token latency (streaming)      |
    /// | 4      | ≤ 100 ms   | p95 for small LLM completions          |
    /// | 5      | ≤ 500 ms   | p99 for medium LLM completions         |
    /// | 6      | > 500 ms   | Slow completions, network retries      |
    const BOUNDS: [u64; 7] = [1, 5, 10, 50, 100, 500, u64::MAX];

    /// Record a latency sample in milliseconds.
    pub fn record(&self, ms: u64) {
        self.total_count.fetch_add(1, Ordering::Relaxed);
        self.total_sum_ms.fetch_add(ms, Ordering::Relaxed);
        for (i, &bound) in Self::BOUNDS.iter().enumerate() {
            if ms <= bound {
                self.buckets[i].fetch_add(1, Ordering::Relaxed);
                return;
            }
        }
    }

    /// Return the mean latency in ms, or 0.0 if no samples.
    pub fn mean_ms(&self) -> f64 {
        let count = self.total_count.load(Ordering::Relaxed);
        if count == 0 {
            return 0.0;
        }
        self.total_sum_ms.load(Ordering::Relaxed) as f64 / count as f64
    }

    /// Return a bucket-midpoint approximation of the standard deviation in ms.
    ///
    /// Uses the midpoint of each histogram bucket to estimate the second moment,
    /// then applies `√(E[X²] − E[X]²)`.  Returns `0.0` when fewer than two
    /// samples have been recorded.
    ///
    /// # Accuracy
    /// The result is an estimate; its accuracy improves as the sample count
    /// increases and degrades near the boundaries of wide buckets.
    pub fn std_dev_ms(&self) -> f64 {
        let count = self.total_count.load(Ordering::Relaxed);
        if count < 2 {
            return 0.0;
        }
        const MIDS: [f64; 7] = [0.5, 3.0, 7.5, 30.0, 75.0, 300.0, 500.0];
        let (sum, sum_sq): (f64, f64) = self
            .buckets
            .iter()
            .zip(MIDS.iter())
            .map(|(b, &m)| {
                let c = b.load(Ordering::Relaxed) as f64;
                (c * m, c * m * m)
            })
            .fold((0.0, 0.0), |(s, ss), (v, v2)| (s + v, ss + v2));
        let n = count as f64;
        let variance = sum_sq / n - (sum / n) * (sum / n);
        variance.max(0.0).sqrt()
    }

    /// Return the total sample count.
    pub fn count(&self) -> u64 {
        self.total_count.load(Ordering::Relaxed)
    }

    /// Return `true` if at least one sample has been recorded.
    pub fn has_data(&self) -> bool {
        self.count() > 0
    }

    /// Return `true` when no samples have been recorded yet.
    pub fn is_empty(&self) -> bool {
        self.count() == 0
    }

    /// Estimate the p-th percentile latency in milliseconds from the histogram.
    ///
    /// `p` must be in `[0.0, 1.0]`.  Returns the **upper bound** of the first
    /// bucket that contains the p-th percentile.  Returns `0` if no samples
    /// have been recorded.
    ///
    /// # Accuracy
    ///
    /// This is a bucket-boundary estimate, not an exact value.  The error is
    /// bounded by the bucket width at that percentile.
    pub fn percentile(&self, p: f64) -> u64 {
        let total = self.total_count.load(Ordering::Relaxed);
        if total == 0 {
            return 0;
        }
        let target = (p.clamp(0.0, 1.0) * total as f64).ceil() as u64;
        let mut cumulative = 0u64;
        for (i, bucket) in self.buckets.iter().enumerate() {
            cumulative += bucket.load(Ordering::Relaxed);
            if cumulative >= target {
                return Self::BOUNDS[i];
            }
        }
        // All samples accounted for — return the last bound.
        *Self::BOUNDS.last().unwrap_or(&u64::MAX)
    }

    /// Return the upper-bound of the bucket with the highest sample count (the mode).
    ///
    /// Returns `None` if no samples have been recorded.  When multiple buckets
    /// tie for the maximum, the lowest-latency bucket is returned.
    pub fn mode_bucket_ms(&self) -> Option<u64> {
        if self.count() == 0 {
            return None;
        }
        let (idx, _) = self
            .buckets
            .iter()
            .enumerate()
            .max_by_key(|(_, a)| a.load(Ordering::Relaxed))?;
        Some(Self::BOUNDS[idx])
    }

    /// Return bucket counts as `(upper_bound_ms, count)` pairs.
    pub fn buckets(&self) -> Vec<(u64, u64)> {
        Self::BOUNDS
            .iter()
            .zip(self.buckets.iter())
            .map(|(&b, a)| (b, a.load(Ordering::Relaxed)))
            .collect()
    }

    /// Return the minimum recorded latency in ms, or `None` if no samples.
    pub fn min_ms(&self) -> Option<u64> {
        let total = self.total_count.load(Ordering::Relaxed);
        if total == 0 {
            return None;
        }
        // Walk buckets from the fastest; the first non-empty bucket's lower
        // bound is 0 (or the previous bound), so return the upper bound as
        // the conservative minimum estimate.
        for (i, bucket) in self.buckets.iter().enumerate() {
            if bucket.load(Ordering::Relaxed) > 0 {
                return Some(if i == 0 { 0 } else { Self::BOUNDS[i - 1] + 1 });
            }
        }
        None
    }

    /// Return the maximum recorded latency in ms, or `None` if no samples.
    pub fn max_ms(&self) -> Option<u64> {
        let total = self.total_count.load(Ordering::Relaxed);
        if total == 0 {
            return None;
        }
        // Walk from the slowest bucket; return the upper bound of the last non-empty bucket.
        for (i, bucket) in self.buckets.iter().enumerate().rev() {
            if bucket.load(Ordering::Relaxed) > 0 {
                return Some(Self::BOUNDS[i]);
            }
        }
        None
    }

    /// Return the spread (max − min) of recorded latencies in milliseconds.
    ///
    /// Returns `None` if no samples have been recorded.  A narrow range
    /// indicates consistent latency; a wide range suggests outliers.
    pub fn range_ms(&self) -> Option<u64> {
        Some(self.max_ms()?.saturating_sub(self.min_ms()?))
    }

    /// Return the interquartile range (p75 − p25) in milliseconds.
    ///
    /// A measure of dispersion that is less sensitive to outliers than
    /// [`range_ms`].  Returns `0` when fewer than two samples have been
    /// recorded (p25 == p75 == 0).
    ///
    /// [`range_ms`]: LatencyHistogram::range_ms
    pub fn interquartile_range_ms(&self) -> u64 {
        self.p75().saturating_sub(self.p25())
    }

    /// Return the 50th-percentile (median) latency in milliseconds.
    pub fn p50(&self) -> u64 {
        self.percentile(0.50)
    }

    /// Return the 95th-percentile latency in milliseconds.
    pub fn p95(&self) -> u64 {
        self.percentile(0.95)
    }

    /// Return the 99th-percentile latency in milliseconds.
    pub fn p99(&self) -> u64 {
        self.percentile(0.99)
    }

    /// Return the 25th-percentile latency in milliseconds.
    pub fn p25(&self) -> u64 {
        self.percentile(0.25)
    }

    /// Return the 75th-percentile latency in milliseconds.
    pub fn p75(&self) -> u64 {
        self.percentile(0.75)
    }

    /// Return the 90th-percentile latency in milliseconds.
    pub fn p90(&self) -> u64 {
        self.percentile(0.90)
    }

    /// Return the 10th-percentile latency in milliseconds.
    ///
    /// Useful for assessing the "best case" tail of the distribution.
    pub fn p10(&self) -> u64 {
        self.percentile(0.10)
    }

    /// Return the median (50th-percentile) step latency in milliseconds.
    ///
    /// Convenience alias for `p50`; useful when callers want an explicit
    /// "median" name without importing percentile constants.
    pub fn median_ms(&self) -> u64 {
        self.p50()
    }

    /// Reset all histogram counters to zero.
    pub fn reset(&self) {
        self.total_count.store(0, Ordering::Relaxed);
        self.total_sum_ms.store(0, Ordering::Relaxed);
        for bucket in &self.buckets {
            bucket.store(0, Ordering::Relaxed);
        }
    }

    /// Return the total sum of all recorded latency samples in milliseconds.
    ///
    /// Equivalent to `mean_ms() * count()` but avoids floating-point arithmetic.
    pub fn sum_ms(&self) -> u64 {
        self.total_sum_ms.load(Ordering::Relaxed)
    }

    /// Return the coefficient of variation: `std_dev_ms / mean_ms`.
    ///
    /// A value of `0.0` means no variation; higher values indicate more
    /// spread in latency.  Returns `0.0` when `mean_ms` is zero (empty
    /// histogram or all-zero samples) to avoid division by zero.
    pub fn coefficient_of_variation(&self) -> f64 {
        let mean = self.mean_ms();
        if mean == 0.0 {
            return 0.0;
        }
        self.std_dev_ms() / mean
    }

    /// Return the total number of samples recorded in this histogram.
    pub fn sample_count(&self) -> u64 {
        self.total_count.load(std::sync::atomic::Ordering::Relaxed)
    }

    /// Return the difference between the p99 and p50 latency buckets in
    /// milliseconds.
    ///
    /// A larger spread indicates a long-tail latency distribution.
    /// Returns `0` when no samples have been recorded.
    pub fn percentile_spread(&self) -> u64 {
        self.p99().saturating_sub(self.p50())
    }

    /// Return the count for each bucket as an array, in order from the
    /// fastest (≤1ms) to the slowest (>500ms) bucket.
    pub fn bucket_counts(&self) -> [u64; 7] {
        let mut out = [0u64; 7];
        for (i, b) in self.buckets.iter().enumerate() {
            out[i] = b.load(std::sync::atomic::Ordering::Relaxed);
        }
        out
    }

    /// Return the upper bound (ms) of the lowest bucket that has at least one
    /// sample, or `None` if no samples have been recorded.
    pub fn min_occupied_ms(&self) -> Option<u64> {
        Self::BOUNDS
            .iter()
            .zip(self.buckets.iter())
            .find(|(_, b)| b.load(std::sync::atomic::Ordering::Relaxed) > 0)
            .map(|(&bound, _)| bound)
    }

    /// Return the upper-bound of the largest bucket with at least one recorded sample.
    ///
    /// Returns `None` if the histogram is empty.
    pub fn max_occupied_ms(&self) -> Option<u64> {
        Self::BOUNDS
            .iter()
            .zip(self.buckets.iter())
            .rev()
            .find(|(_, b)| b.load(std::sync::atomic::Ordering::Relaxed) > 0)
            .map(|(&bound, _)| bound)
    }

    /// Return the number of buckets that have at least one recorded sample.
    pub fn occupied_bucket_count(&self) -> usize {
        self.buckets
            .iter()
            .filter(|b| b.load(std::sync::atomic::Ordering::Relaxed) > 0)
            .count()
    }

    /// Return `true` if the latency distribution is skewed (p99 > 2 × p50).
    ///
    /// Returns `false` for empty histograms.
    pub fn is_skewed(&self) -> bool {
        let p50 = self.p50();
        if p50 == 0 {
            return false;
        }
        self.p99() > 2 * p50
    }

    /// Return `true` if all recorded samples fall into exactly one bucket.
    ///
    /// An empty histogram is considered uniform.
    pub fn is_uniform(&self) -> bool {
        let non_empty = self
            .buckets
            .iter()
            .filter(|b| b.load(std::sync::atomic::Ordering::Relaxed) > 0)
            .count();
        non_empty <= 1
    }

    /// Reset all histogram counters to zero.
    ///
    /// Alias for [`reset`] using more conventional naming.
    ///
    /// [`reset`]: LatencyHistogram::reset
    pub fn clear(&self) {
        self.reset();
    }

    /// Return `true` if `latency_ms` is strictly greater than the current p99.
    ///
    /// Useful for detecting outlier requests at call sites without storing
    /// the p99 value separately.  Returns `false` when the histogram is empty.
    pub fn is_above_p99(&self, latency_ms: u64) -> bool {
        latency_ms > self.p99()
    }

    /// Return `true` if the p99 latency is strictly below `threshold_ms`.
    ///
    /// Useful for SLO checks.  Returns `true` when no samples have been
    /// recorded (`p99 == 0`).
    pub fn is_below_p99(&self, threshold_ms: u64) -> bool {
        self.p99() < threshold_ms
    }

}

impl MetricsSnapshot {
    /// Compute the difference between `after` and `before` (i.e., `after - before`).
    ///
    /// Useful for per-request instrumentation:
    /// ```rust,ignore
    /// let before = metrics.snapshot();
    /// // ... run one agent invocation ...
    /// let after = metrics.snapshot();
    /// let delta = MetricsSnapshot::delta(&after, &before);
    /// println!("steps this run: {}", delta.total_steps);
    /// ```
    ///
    /// Saturating subtraction is used so callers don't need to guard against
    /// races where a counter is read before the full increment propagates.
    pub fn delta(after: &Self, before: &Self) -> Self {
        Self {
            active_sessions: after.active_sessions.saturating_sub(before.active_sessions),
            total_sessions: after.total_sessions.saturating_sub(before.total_sessions),
            total_steps: after.total_steps.saturating_sub(before.total_steps),
            total_tool_calls: after.total_tool_calls.saturating_sub(before.total_tool_calls),
            failed_tool_calls: after.failed_tool_calls.saturating_sub(before.failed_tool_calls),
            backpressure_shed_count: after
                .backpressure_shed_count
                .saturating_sub(before.backpressure_shed_count),
            memory_recall_count: after
                .memory_recall_count
                .saturating_sub(before.memory_recall_count),
            checkpoint_errors: after
                .checkpoint_errors
                .saturating_sub(before.checkpoint_errors),
            per_tool_calls: {
                let mut m = after.per_tool_calls.clone();
                for (k, v) in &before.per_tool_calls {
                    let entry = m.entry(k.clone()).or_default();
                    *entry = entry.saturating_sub(*v);
                }
                m
            },
            per_tool_failures: {
                let mut m = after.per_tool_failures.clone();
                for (k, v) in &before.per_tool_failures {
                    let entry = m.entry(k.clone()).or_default();
                    *entry = entry.saturating_sub(*v);
                }
                m
            },
            step_latency_buckets: after
                .step_latency_buckets
                .iter()
                .zip(before.step_latency_buckets.iter())
                .map(|((bound, a), (_, b))| (*bound, a.saturating_sub(*b)))
                .collect(),
            step_latency_mean_ms: after.step_latency_mean_ms - before.step_latency_mean_ms,
            per_agent_tool_calls: after.per_agent_tool_calls.clone(),
            per_agent_tool_failures: after.per_agent_tool_failures.clone(),
        }
    }

    /// Serialize the snapshot to a `serde_json::Value` for logging or export.
    pub fn to_json(&self) -> serde_json::Value {
        serde_json::json!({
            "active_sessions": self.active_sessions,
            "total_sessions": self.total_sessions,
            "total_steps": self.total_steps,
            "total_tool_calls": self.total_tool_calls,
            "failed_tool_calls": self.failed_tool_calls,
            "backpressure_shed_count": self.backpressure_shed_count,
            "memory_recall_count": self.memory_recall_count,
            "step_latency_mean_ms": self.step_latency_mean_ms,
            "per_tool_calls": self.per_tool_calls,
            "per_tool_failures": self.per_tool_failures,
        })
    }

    /// Return the number of calls recorded for the named tool.
    ///
    /// Returns `0` if no calls have been recorded for that tool name.
    pub fn tool_call_count(&self, name: &str) -> u64 {
        self.per_tool_calls.get(name).copied().unwrap_or(0)
    }

    /// Return the number of failures recorded for the named tool.
    ///
    /// Returns `0` if no failures have been recorded for that tool name.
    pub fn tool_failure_count(&self, name: &str) -> u64 {
        self.per_tool_failures.get(name).copied().unwrap_or(0)
    }

    /// Return a sorted list of tool names that have at least one recorded call.
    pub fn tool_names(&self) -> Vec<&str> {
        let mut names: Vec<&str> = self.per_tool_calls.keys().map(|s| s.as_str()).collect();
        names.sort_unstable();
        names
    }

    /// Return the overall tool-call failure rate as a value in `[0.0, 1.0]`.
    ///
    /// Returns `0.0` if no tool calls have been recorded.
    pub fn failure_rate(&self) -> f64 {
        if self.total_tool_calls == 0 {
            return 0.0;
        }
        self.failed_tool_calls as f64 / self.total_tool_calls as f64
    }

    /// Return the overall tool-call success rate as a value in `[0.0, 1.0]`.
    ///
    /// Returns `1.0` if no tool calls have been recorded (vacuously all succeeded).
    pub fn success_rate(&self) -> f64 {
        1.0 - self.failure_rate()
    }

    /// Return the number of successful calls for the named tool.
    ///
    /// Computed as `tool_call_count(name) - tool_failure_count(name)`.
    pub fn tool_success_count(&self, name: &str) -> u64 {
        self.tool_call_count(name)
            .saturating_sub(self.tool_failure_count(name))
    }

    /// Return the per-tool failure rate for the named tool.
    ///
    /// Returns `0.0` if no calls have been recorded for that tool.
    pub fn tool_failure_rate(&self, name: &str) -> f64 {
        let calls = self.tool_call_count(name);
        if calls == 0 {
            return 0.0;
        }
        self.tool_failure_count(name) as f64 / calls as f64
    }

    /// Return the total number of successful tool calls (total minus failed).
    ///
    /// Uses saturating subtraction so a race between `total_tool_calls`
    /// and `failed_tool_calls` cannot produce an underflow.
    pub fn total_successful_tool_calls(&self) -> u64 {
        self.total_tool_calls.saturating_sub(self.failed_tool_calls)
    }

    /// Return `true` if all counters are zero (no activity has been recorded).
    pub fn is_zero(&self) -> bool {
        self.active_sessions == 0
            && self.total_sessions == 0
            && self.total_steps == 0
            && self.total_tool_calls == 0
            && self.failed_tool_calls == 0
            && self.backpressure_shed_count == 0
            && self.memory_recall_count == 0
            && self.checkpoint_errors == 0
    }

    /// Return the average number of ReAct steps per completed session.
    ///
    /// Returns `0.0` when no sessions have been recorded, to avoid
    /// division by zero.
    pub fn avg_steps_per_session(&self) -> f64 {
        if self.total_sessions == 0 {
            0.0
        } else {
            self.total_steps as f64 / self.total_sessions as f64
        }
    }

    /// Return the overall tool error rate: `failed_tool_calls / total_tool_calls`.
    ///
    /// Returns `0.0` when no tool calls have been recorded.
    pub fn error_rate(&self) -> f64 {
        if self.total_tool_calls == 0 {
            return 0.0;
        }
        self.failed_tool_calls as f64 / self.total_tool_calls as f64
    }

    /// Return memory recalls per completed session.
    ///
    /// Returns `0.0` when no sessions have been recorded.
    pub fn memory_recall_rate(&self) -> f64 {
        if self.total_sessions == 0 {
            return 0.0;
        }
        self.memory_recall_count as f64 / self.total_sessions as f64
    }

    /// Return the average number of ReAct steps per session.
    ///
    /// Alias for `avg_steps_per_session` on the snapshot type; returns `0.0`
    /// when no sessions have been recorded.
    pub fn steps_per_session(&self) -> f64 {
        if self.total_sessions == 0 {
            return 0.0;
        }
        self.total_steps as f64 / self.total_sessions as f64
    }

    /// Return `true` if the snapshot contains any error indicators.
    ///
    /// Specifically, `true` when `failed_tool_calls > 0` or
    /// `checkpoint_errors > 0`.  The complement of "no errors" but distinct
    /// from `!is_healthy()` which also considers backpressure sheds.
    pub fn has_errors(&self) -> bool {
        self.failed_tool_calls > 0 || self.checkpoint_errors > 0
    }

    /// Return `true` if the snapshot shows no error indicators.
    ///
    /// A "healthy" snapshot has zero failed tool calls, zero backpressure
    /// sheds, and zero checkpoint errors.  Useful for quick health checks
    /// in tests and monitoring.
    pub fn is_healthy(&self) -> bool {
        self.failed_tool_calls == 0
            && self.backpressure_shed_count == 0
            && self.checkpoint_errors == 0
    }

    /// Return the average number of tool calls per session.
    ///
    /// Returns `0.0` when no sessions have been recorded.
    pub fn tool_call_rate(&self) -> f64 {
        if self.total_sessions == 0 {
            return 0.0;
        }
        self.total_tool_calls as f64 / self.total_sessions as f64
    }

    /// Return the average number of backpressure shed events per session.
    ///
    /// Returns `0.0` when no sessions have been recorded.
    pub fn backpressure_rate(&self) -> f64 {
        if self.total_sessions == 0 {
            return 0.0;
        }
        self.backpressure_shed_count as f64 / self.total_sessions as f64
    }

    /// Return the ratio of memory recalls to total steps.
    ///
    /// Returns `0.0` when no steps have been taken.
    pub fn memory_efficiency(&self) -> f64 {
        if self.total_steps == 0 {
            return 0.0;
        }
        self.memory_recall_count as f64 / self.total_steps as f64
    }

    /// Return the fraction of sessions that are currently active.
    ///
    /// Returns `0.0` when no sessions have been started.
    pub fn active_session_ratio(&self) -> f64 {
        if self.total_sessions == 0 {
            return 0.0;
        }
        self.active_sessions as f64 / self.total_sessions as f64
    }

    /// Return the average number of tool calls per step.
    ///
    /// Returns `0.0` when no steps have been taken.
    pub fn step_to_tool_ratio(&self) -> f64 {
        if self.total_steps == 0 {
            return 0.0;
        }
        self.total_tool_calls as f64 / self.total_steps as f64
    }

    /// Return `true` if any tool-call failures have been recorded.
    pub fn has_failures(&self) -> bool {
        self.failed_tool_calls > 0
    }

    /// Return the number of distinct tool names that have been called at least once.
    pub fn tool_diversity(&self) -> usize {
        self.per_tool_calls.len()
    }

    /// Return the average number of tool-call failures per completed session.
    ///
    /// Returns `0.0` when no sessions have been recorded.
    pub fn avg_failures_per_session(&self) -> f64 {
        if self.total_sessions == 0 {
            return 0.0;
        }
        self.failed_tool_calls as f64 / self.total_sessions as f64
    }

    /// Return the name of the tool with the most recorded calls.
    ///
    /// Returns `None` if no tool calls have been recorded.
    pub fn most_called_tool(&self) -> Option<String> {
        self.per_tool_calls
            .iter()
            .max_by_key(|(_, &v)| v)
            .map(|(k, _)| k.clone())
    }

    /// Return a sorted list of tool names that have at least one recorded failure.
    pub fn tool_names_with_failures(&self) -> Vec<String> {
        let mut names: Vec<String> = self
            .per_tool_failures
            .iter()
            .filter(|(_, &v)| v > 0)
            .map(|(k, _)| k.clone())
            .collect();
        names.sort_unstable();
        names
    }

    /// Return the failure rate for a specific tool (failures / calls).
    ///
    /// Returns `0.0` if the tool has no recorded calls.
    pub fn failed_tool_ratio_for(&self, name: &str) -> f64 {
        let calls = self.tool_call_count(name);
        if calls == 0 {
            return 0.0;
        }
        self.tool_failure_count(name) as f64 / calls as f64
    }

    /// Return the ratio of backpressure-shed events to total tool calls.
    ///
    /// Returns `0.0` if no tool calls have been recorded.
    pub fn backpressure_shed_rate(&self) -> f64 {
        if self.total_tool_calls == 0 {
            return 0.0;
        }
        self.backpressure_shed_count as f64 / self.total_tool_calls as f64
    }

    /// Return the number of distinct agents that have recorded tool-call data.
    pub fn total_agent_count(&self) -> usize {
        self.per_agent_tool_calls.len()
    }

    /// Return the ratio of total steps to total tool calls.
    ///
    /// Returns `0.0` if no tool calls have been recorded.
    pub fn steps_per_tool_call(&self) -> f64 {
        if self.total_tool_calls == 0 {
            return 0.0;
        }
        self.total_steps as f64 / self.total_tool_calls as f64
    }

    /// Return the agent id with the most total tool calls across all tools.
    ///
    /// Returns `None` if no per-agent tool-call data has been recorded.
    pub fn agent_with_most_calls(&self) -> Option<String> {
        self.per_agent_tool_calls
            .iter()
            .map(|(agent, tools)| (agent, tools.values().sum::<u64>()))
            .max_by_key(|(_, total)| *total)
            .map(|(agent, _)| agent.clone())
    }
}

impl std::fmt::Display for MetricsSnapshot {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(
            f,
            "MetricsSnapshot {{ sessions: active={} total={}, steps={}, \
             tool_calls={} (failed={}), backpressure_shed={}, \
             memory_recalls={}, checkpoint_errors={}, latency_mean={:.1}ms }}",
            self.active_sessions,
            self.total_sessions,
            self.total_steps,
            self.total_tool_calls,
            self.failed_tool_calls,
            self.backpressure_shed_count,
            self.memory_recall_count,
            self.checkpoint_errors,
            self.step_latency_mean_ms,
        )
    }
}

/// A point-in-time snapshot of all runtime counters.
///
/// Obtained by calling [`RuntimeMetrics::snapshot`].  All fields are plain
/// integers so the snapshot can be logged, serialised, or diffed without
/// holding any locks.
///
/// See also [`snapshot`] for a richer snapshot including per-tool and histogram data.
///
/// # Example
/// ```rust
/// use llm_agent_runtime::metrics::RuntimeMetrics;
///
/// let m = RuntimeMetrics::new();
/// let snap = m.snapshot();
/// assert_eq!(snap.active_sessions, 0);
/// assert_eq!(snap.total_sessions, 0);
/// ```
///
/// [`snapshot`]: RuntimeMetrics::snapshot
#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
pub struct MetricsSnapshot {
    /// Number of agent sessions currently in progress.
    pub active_sessions: usize,
    /// Total number of sessions started since the runtime was created.
    pub total_sessions: u64,
    /// Total number of ReAct steps executed across all sessions.
    pub total_steps: u64,
    /// Total number of tool calls dispatched (across all tool names).
    pub total_tool_calls: u64,
    /// Total number of tool calls that returned an error observation.
    pub failed_tool_calls: u64,
    /// Total number of requests shed due to backpressure.
    pub backpressure_shed_count: u64,
    /// Total number of memory recall operations.
    pub memory_recall_count: u64,
    /// Total number of checkpoint failures encountered during `run_agent`.
    pub checkpoint_errors: u64,
    /// Per-tool call counts: `tool_name → total_calls`.
    pub per_tool_calls: HashMap<String, u64>,
    /// Per-tool failure counts: `tool_name → failed_calls`.
    pub per_tool_failures: HashMap<String, u64>,
    /// Step latency histogram bucket counts as `(upper_bound_ms_inclusive, count)`.
    pub step_latency_buckets: Vec<(u64, u64)>,
    /// Mean step latency in milliseconds.
    pub step_latency_mean_ms: f64,
    /// Per-agent, per-tool call counts: `agent_id → tool_name → count`.
    pub per_agent_tool_calls: HashMap<String, HashMap<String, u64>>,
    /// Per-agent, per-tool failure counts: `agent_id → tool_name → count`.
    pub per_agent_tool_failures: HashMap<String, HashMap<String, u64>>,
}

/// All four per-tool / per-agent counter maps, protected by a single lock.
///
/// Grouping them under one `Mutex` halves lock acquisitions on the hot path
/// (a single `record_tool_call` + `record_agent_tool_call` pair now requires
/// only one acquire/release) and makes snapshot reads cheaper too.
#[derive(Debug, Default)]
struct PerToolMaps {
    /// Per-tool call counts: `tool_name → total_calls`.
    calls: HashMap<String, u64>,
    /// Per-tool failure counts: `tool_name → failed_calls`.
    failures: HashMap<String, u64>,
    /// Per-agent, per-tool call counts: `agent_id → tool_name → count`.
    agent_calls: HashMap<String, HashMap<String, u64>>,
    /// Per-agent, per-tool failure counts: `agent_id → tool_name → count`.
    agent_failures: HashMap<String, HashMap<String, u64>>,
}

/// Shared runtime metrics. Clone the `Arc` to share across threads.
#[derive(Debug)]
pub struct RuntimeMetrics {
    /// Number of agent sessions currently in progress.
    pub active_sessions: AtomicUsize,
    /// Total number of sessions started since the runtime was created.
    pub total_sessions: AtomicU64,
    /// Total number of ReAct steps executed across all sessions.
    pub total_steps: AtomicU64,
    /// Total number of tool calls dispatched (across all tool names).
    pub total_tool_calls: AtomicU64,
    /// Total number of tool calls that returned an error observation.
    pub failed_tool_calls: AtomicU64,
    /// Total number of requests shed due to backpressure.
    pub backpressure_shed_count: AtomicU64,
    /// Total number of memory recall operations.
    pub memory_recall_count: AtomicU64,
    /// Total number of checkpoint failures encountered during `run_agent`.
    pub checkpoint_errors: AtomicU64,
    /// All four per-tool / per-agent maps under a single lock.
    per_tool: Mutex<PerToolMaps>,
    /// Per-step latency histogram.
    pub step_latency: LatencyHistogram,
}

impl Default for RuntimeMetrics {
    fn default() -> Self {
        Self {
            active_sessions: AtomicUsize::new(0),
            total_sessions: AtomicU64::new(0),
            total_steps: AtomicU64::new(0),
            total_tool_calls: AtomicU64::new(0),
            failed_tool_calls: AtomicU64::new(0),
            backpressure_shed_count: AtomicU64::new(0),
            memory_recall_count: AtomicU64::new(0),
            checkpoint_errors: AtomicU64::new(0),
            per_tool: Mutex::new(PerToolMaps::default()),
            step_latency: LatencyHistogram::default(),
        }
    }
}

impl RuntimeMetrics {
    /// Allocate a new `RuntimeMetrics` instance wrapped in an `Arc`.
    pub fn new() -> Arc<Self> {
        Arc::new(Self::default())
    }

    /// Return the number of agent sessions currently in progress.
    pub fn active_sessions(&self) -> usize {
        self.active_sessions.load(Ordering::Relaxed)
    }

    /// Return the total number of sessions started since the runtime was created.
    pub fn total_sessions(&self) -> u64 {
        self.total_sessions.load(Ordering::Relaxed)
    }

    /// Return the average number of tool calls per completed session.
    ///
    /// Returns `0.0` when no sessions have been recorded.
    pub fn avg_tool_calls_per_session(&self) -> f64 {
        let sessions = self.total_sessions();
        if sessions == 0 {
            return 0.0;
        }
        self.total_tool_calls() as f64 / sessions as f64
    }

    /// Return the total number of ReAct steps executed across all sessions.
    pub fn total_steps(&self) -> u64 {
        self.total_steps.load(Ordering::Relaxed)
    }

    /// Return the average number of ReAct steps per completed session.
    ///
    /// Returns `0.0` when no sessions have been recorded.
    pub fn avg_steps_per_session(&self) -> f64 {
        let sessions = self.total_sessions();
        if sessions == 0 {
            return 0.0;
        }
        self.total_steps() as f64 / sessions as f64
    }

    /// Return the total number of tool calls dispatched.
    pub fn total_tool_calls(&self) -> u64 {
        self.total_tool_calls.load(Ordering::Relaxed)
    }

    /// Return the total number of tool calls that returned an error observation.
    pub fn failed_tool_calls(&self) -> u64 {
        self.failed_tool_calls.load(Ordering::Relaxed)
    }

    /// Return the fraction of tool calls that succeeded (i.e. did not fail).
    ///
    /// Returns `1.0` if no tool calls have been recorded yet (vacuously all
    /// succeeded) and a value in `[0.0, 1.0]` once calls have been made.
    pub fn tool_success_rate(&self) -> f64 {
        let total = self.total_tool_calls();
        if total == 0 {
            return 1.0;
        }
        let failed = self.failed_tool_calls();
        1.0 - (failed as f64 / total as f64)
    }

    /// Return the total number of requests shed due to backpressure.
    pub fn backpressure_shed_count(&self) -> u64 {
        self.backpressure_shed_count.load(Ordering::Relaxed)
    }

    /// Return the total number of memory recall operations performed.
    pub fn memory_recall_count(&self) -> u64 {
        self.memory_recall_count.load(Ordering::Relaxed)
    }

    /// Return the total number of checkpoint failures encountered during `run_agent`.
    pub fn checkpoint_errors(&self) -> u64 {
        self.checkpoint_errors.load(Ordering::Relaxed)
    }

    /// Return the ratio of checkpoint errors to total completed sessions.
    ///
    /// Returns `0.0` when no sessions have been recorded.
    pub fn checkpoint_error_rate(&self) -> f64 {
        let sessions = self.total_sessions();
        if sessions == 0 {
            return 0.0;
        }
        self.checkpoint_errors() as f64 / sessions as f64
    }

    /// Return the median (50th-percentile) step latency in milliseconds.
    ///
    /// Convenience shorthand for `self.step_latency.p50()`.  Returns `0`
    /// when no step latencies have been recorded.
    pub fn p50_latency_ms(&self) -> u64 {
        self.step_latency.p50()
    }

    /// Increment the call counter for `tool_name` by 1.
    ///
    /// Called automatically by the agent loop when `with_metrics` is configured.
    pub fn record_tool_call(&self, tool_name: &str) {
        self.total_tool_calls.fetch_add(1, Ordering::Relaxed);
        if let Ok(mut maps) = self.per_tool.lock() {
            *maps.calls.entry(tool_name.to_owned()).or_insert(0) += 1;
        }
    }

    /// Increment the failure counter for `tool_name` by 1.
    ///
    /// Called automatically by the agent loop when a tool returns an error.
    pub fn record_tool_failure(&self, tool_name: &str) {
        self.failed_tool_calls.fetch_add(1, Ordering::Relaxed);
        if let Ok(mut maps) = self.per_tool.lock() {
            *maps.failures.entry(tool_name.to_owned()).or_insert(0) += 1;
        }
    }

    /// Return a snapshot of per-tool call counts as a `HashMap<tool_name, count>`.
    pub fn per_tool_calls_snapshot(&self) -> HashMap<String, u64> {
        self.per_tool
            .lock()
            .map(|m| m.calls.clone())
            .unwrap_or_default()
    }

    /// Return a snapshot of per-tool failure counts as a `HashMap<tool_name, count>`.
    pub fn per_tool_failures_snapshot(&self) -> HashMap<String, u64> {
        self.per_tool
            .lock()
            .map(|m| m.failures.clone())
            .unwrap_or_default()
    }

    /// Increment call counter for (agent_id, tool_name).
    pub fn record_agent_tool_call(&self, agent_id: &str, tool_name: &str) {
        if let Ok(mut maps) = self.per_tool.lock() {
            *maps
                .agent_calls
                .entry(agent_id.to_owned())
                .or_default()
                .entry(tool_name.to_owned())
                .or_insert(0) += 1;
        }
    }

    /// Increment failure counter for (agent_id, tool_name).
    pub fn record_agent_tool_failure(&self, agent_id: &str, tool_name: &str) {
        if let Ok(mut maps) = self.per_tool.lock() {
            *maps
                .agent_failures
                .entry(agent_id.to_owned())
                .or_default()
                .entry(tool_name.to_owned())
                .or_insert(0) += 1;
        }
    }

    /// Snapshot of per-agent, per-tool call counts.
    pub fn per_agent_tool_calls_snapshot(&self) -> HashMap<String, HashMap<String, u64>> {
        self.per_tool
            .lock()
            .map(|m| m.agent_calls.clone())
            .unwrap_or_default()
    }

    /// Snapshot of per-agent, per-tool failure counts.
    pub fn per_agent_tool_failures_snapshot(&self) -> HashMap<String, HashMap<String, u64>> {
        self.per_tool
            .lock()
            .map(|m| m.agent_failures.clone())
            .unwrap_or_default()
    }

    /// Capture a complete snapshot of all counters, including per-tool breakdowns.
    ///
    /// This is the preferred alternative to [`to_snapshot`] — it returns a
    /// named [`MetricsSnapshot`] struct instead of an opaque tuple.
    ///
    /// [`to_snapshot`]: RuntimeMetrics::to_snapshot
    pub fn snapshot(&self) -> MetricsSnapshot {
        // Acquire the single per-tool lock once for all four maps.
        let (per_tool_calls, per_tool_failures, per_agent_tool_calls, per_agent_tool_failures) =
            self.per_tool
                .lock()
                .map(|m| {
                    (
                        m.calls.clone(),
                        m.failures.clone(),
                        m.agent_calls.clone(),
                        m.agent_failures.clone(),
                    )
                })
                .unwrap_or_default();

        MetricsSnapshot {
            active_sessions: self.active_sessions.load(Ordering::Relaxed),
            total_sessions: self.total_sessions.load(Ordering::Relaxed),
            total_steps: self.total_steps.load(Ordering::Relaxed),
            total_tool_calls: self.total_tool_calls.load(Ordering::Relaxed),
            failed_tool_calls: self.failed_tool_calls.load(Ordering::Relaxed),
            backpressure_shed_count: self.backpressure_shed_count.load(Ordering::Relaxed),
            memory_recall_count: self.memory_recall_count.load(Ordering::Relaxed),
            checkpoint_errors: self.checkpoint_errors.load(Ordering::Relaxed),
            per_tool_calls,
            per_tool_failures,
            step_latency_buckets: self.step_latency.buckets(),
            step_latency_mean_ms: self.step_latency.mean_ms(),
            per_agent_tool_calls,
            per_agent_tool_failures,
        }
    }

    /// Record a step latency sample.
    pub fn record_step_latency(&self, ms: u64) {
        self.step_latency.record(ms);
    }

    /// Reset all counters to zero.
    ///
    /// Intended for testing. In production, counters are monotonically increasing.
    pub fn reset(&self) {
        self.active_sessions.store(0, Ordering::Relaxed);
        self.total_sessions.store(0, Ordering::Relaxed);
        self.total_steps.store(0, Ordering::Relaxed);
        self.total_tool_calls.store(0, Ordering::Relaxed);
        self.failed_tool_calls.store(0, Ordering::Relaxed);
        self.backpressure_shed_count.store(0, Ordering::Relaxed);
        self.memory_recall_count.store(0, Ordering::Relaxed);
        self.checkpoint_errors.store(0, Ordering::Relaxed);
        if let Ok(mut maps) = self.per_tool.lock() {
            maps.calls.clear();
            maps.failures.clear();
            maps.agent_calls.clear();
            maps.agent_failures.clear();
        }
        self.step_latency.reset();
    }

    /// Return the fraction of tool calls that failed: `failed / total`.
    ///
    /// Returns `0.0` if no tool calls have been recorded.
    pub fn failure_rate(&self) -> f64 {
        let total = self.total_tool_calls.load(Ordering::Relaxed);
        if total == 0 {
            return 0.0;
        }
        let failed = self.failed_tool_calls.load(Ordering::Relaxed);
        failed as f64 / total as f64
    }

    /// Return the fraction of tool calls that succeeded: `1.0 - failure_rate()`.
    ///
    /// Returns `1.0` if no tool calls have been recorded (vacuously all succeeded).
    pub fn success_rate(&self) -> f64 {
        1.0 - self.failure_rate()
    }

    /// Return `true` if there is at least one active (in-progress) session.
    pub fn is_active(&self) -> bool {
        self.active_sessions.load(Ordering::Relaxed) > 0
    }

    /// Return the 50th-percentile (median) step latency in milliseconds.
    ///
    /// Delegates to [`LatencyHistogram::p50`] on the histogram tracked by
    /// this `RuntimeMetrics` instance.  Returns `0` if no steps have been recorded.
    pub fn step_latency_p50(&self) -> u64 {
        self.step_latency.p50()
    }

    /// Return the 99th-percentile step latency in milliseconds.
    ///
    /// Delegates to [`LatencyHistogram::p99`].  Returns `0` if no steps have
    /// been recorded.
    pub fn step_latency_p99(&self) -> u64 {
        self.step_latency.p99()
    }

    /// Return the top `n` tools by total call count, sorted descending.
    ///
    /// Returns fewer than `n` entries if fewer tools have been called.
    pub fn top_tools_by_calls(&self, n: usize) -> Vec<(String, u64)> {
        let snap = self.per_tool_calls_snapshot();
        let mut pairs: Vec<(String, u64)> = snap.into_iter().collect();
        pairs.sort_unstable_by(|a, b| b.1.cmp(&a.1));
        pairs.truncate(n);
        pairs
    }

    /// Return the top `n` tools by total failure count, sorted descending.
    ///
    /// Analogous to [`top_tools_by_calls`]; returns fewer than `n` entries if
    /// fewer tools have recorded failures.
    ///
    /// [`top_tools_by_calls`]: RuntimeMetrics::top_tools_by_calls
    pub fn top_tools_by_failures(&self, n: usize) -> Vec<(String, u64)> {
        let snap = self.per_tool_failures_snapshot();
        let mut pairs: Vec<(String, u64)> = snap.into_iter().collect();
        pairs.sort_unstable_by(|a, b| b.1.cmp(&a.1));
        pairs.truncate(n);
        pairs
    }

    /// Return the sum of all recorded step latencies in milliseconds.
    pub fn total_step_latency_ms(&self) -> u64 {
        self.step_latency.sum_ms()
    }

    /// Capture a snapshot of global counters as plain integers.
    ///
    /// Returns `(active_sessions, total_sessions, total_steps,
    ///           total_tool_calls, failed_tool_calls,
    ///           backpressure_shed_count, memory_recall_count)`.
    /// For per-tool breakdowns use [`per_tool_calls_snapshot`] and
    /// [`per_tool_failures_snapshot`].
    ///
    /// # Deprecation
    ///
    /// Prefer [`snapshot`] which returns the named [`MetricsSnapshot`] struct
    /// and includes per-tool, per-agent, and histogram data.  This method
    /// returns an anonymous tuple whose field order is easy to misread.
    ///
    /// [`snapshot`]: RuntimeMetrics::snapshot
    /// [`per_tool_calls_snapshot`]: RuntimeMetrics::per_tool_calls_snapshot
    /// [`per_tool_failures_snapshot`]: RuntimeMetrics::per_tool_failures_snapshot
    #[deprecated(since = "1.0.3", note = "use `snapshot()` which returns the named MetricsSnapshot struct")]
    pub fn to_snapshot(&self) -> (usize, u64, u64, u64, u64, u64, u64) {
        (
            self.active_sessions.load(Ordering::Relaxed),
            self.total_sessions.load(Ordering::Relaxed),
            self.total_steps.load(Ordering::Relaxed),
            self.total_tool_calls.load(Ordering::Relaxed),
            self.failed_tool_calls.load(Ordering::Relaxed),
            self.backpressure_shed_count.load(Ordering::Relaxed),
            self.memory_recall_count.load(Ordering::Relaxed),
        )
    }
}

// ── Tests ─────────────────────────────────────────────────────────────────────

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_metrics_new_returns_arc_with_zero_counters() {
        let m = RuntimeMetrics::new();
        assert_eq!(m.active_sessions(), 0);
        assert_eq!(m.total_sessions(), 0);
        assert_eq!(m.total_steps(), 0);
        assert_eq!(m.total_tool_calls(), 0);
        assert_eq!(m.failed_tool_calls(), 0);
        assert_eq!(m.backpressure_shed_count(), 0);
        assert_eq!(m.memory_recall_count(), 0);
    }

    #[test]
    fn test_active_sessions_increments_and_decrements() {
        let m = RuntimeMetrics::new();
        m.active_sessions.fetch_add(1, Ordering::Relaxed);
        assert_eq!(m.active_sessions(), 1);
        m.active_sessions.fetch_sub(1, Ordering::Relaxed);
        assert_eq!(m.active_sessions(), 0);
    }

    #[test]
    fn test_total_sessions_increments() {
        let m = RuntimeMetrics::new();
        m.total_sessions.fetch_add(1, Ordering::Relaxed);
        m.total_sessions.fetch_add(1, Ordering::Relaxed);
        assert_eq!(m.total_sessions(), 2);
    }

    #[test]
    fn test_total_steps_increments() {
        let m = RuntimeMetrics::new();
        m.total_steps.fetch_add(5, Ordering::Relaxed);
        assert_eq!(m.total_steps(), 5);
    }

    #[test]
    fn test_total_tool_calls_increments() {
        let m = RuntimeMetrics::new();
        m.total_tool_calls.fetch_add(3, Ordering::Relaxed);
        assert_eq!(m.total_tool_calls(), 3);
    }

    #[test]
    fn test_failed_tool_calls_increments() {
        let m = RuntimeMetrics::new();
        m.failed_tool_calls.fetch_add(2, Ordering::Relaxed);
        assert_eq!(m.failed_tool_calls(), 2);
    }

    #[test]
    fn test_backpressure_shed_count_increments() {
        let m = RuntimeMetrics::new();
        m.backpressure_shed_count.fetch_add(7, Ordering::Relaxed);
        assert_eq!(m.backpressure_shed_count(), 7);
    }

    #[test]
    fn test_memory_recall_count_increments() {
        let m = RuntimeMetrics::new();
        m.memory_recall_count.fetch_add(4, Ordering::Relaxed);
        assert_eq!(m.memory_recall_count(), 4);
    }

    #[test]
    fn test_reset_zeroes_all_counters() {
        let m = RuntimeMetrics::new();
        m.active_sessions.store(3, Ordering::Relaxed);
        m.total_sessions.store(10, Ordering::Relaxed);
        m.total_steps.store(50, Ordering::Relaxed);
        m.total_tool_calls.store(20, Ordering::Relaxed);
        m.failed_tool_calls.store(2, Ordering::Relaxed);
        m.backpressure_shed_count.store(1, Ordering::Relaxed);
        m.memory_recall_count.store(8, Ordering::Relaxed);

        m.reset();

        assert_eq!(m.active_sessions(), 0);
        assert_eq!(m.total_sessions(), 0);
        assert_eq!(m.total_steps(), 0);
        assert_eq!(m.total_tool_calls(), 0);
        assert_eq!(m.failed_tool_calls(), 0);
        assert_eq!(m.backpressure_shed_count(), 0);
        assert_eq!(m.memory_recall_count(), 0);
    }

    #[test]
    fn test_to_snapshot_captures_correct_values() {
        let m = RuntimeMetrics::new();
        m.active_sessions.store(1, Ordering::Relaxed);
        m.total_sessions.store(2, Ordering::Relaxed);
        m.total_steps.store(3, Ordering::Relaxed);
        m.total_tool_calls.store(4, Ordering::Relaxed);
        m.failed_tool_calls.store(5, Ordering::Relaxed);
        m.backpressure_shed_count.store(6, Ordering::Relaxed);
        m.memory_recall_count.store(7, Ordering::Relaxed);

        let snap = m.to_snapshot();
        assert_eq!(snap, (1, 2, 3, 4, 5, 6, 7));
    }

    #[test]
    fn test_metrics_is_send_sync() {
        fn assert_send_sync<T: Send + Sync>() {}
        assert_send_sync::<RuntimeMetrics>();
    }

    #[test]
    fn test_multiple_increments_are_cumulative() {
        let m = RuntimeMetrics::new();
        for _ in 0..100 {
            m.total_sessions.fetch_add(1, Ordering::Relaxed);
        }
        assert_eq!(m.total_sessions(), 100);
    }

    #[test]
    fn test_arc_clone_shares_state() {
        let m = RuntimeMetrics::new();
        let m2 = Arc::clone(&m);
        m.total_sessions.fetch_add(1, Ordering::Relaxed);
        assert_eq!(m2.total_sessions(), 1);
    }

    // ── Per-tool metrics ──────────────────────────────────────────────────────

    #[test]
    fn test_record_tool_call_increments_global_and_per_tool() {
        let m = RuntimeMetrics::new();
        m.record_tool_call("search");
        m.record_tool_call("search");
        m.record_tool_call("lookup");
        assert_eq!(m.total_tool_calls(), 3);
        let snap = m.per_tool_calls_snapshot();
        assert_eq!(snap.get("search").copied(), Some(2));
        assert_eq!(snap.get("lookup").copied(), Some(1));
    }

    #[test]
    fn test_record_tool_failure_increments_global_and_per_tool() {
        let m = RuntimeMetrics::new();
        m.record_tool_failure("search");
        m.record_tool_failure("lookup");
        m.record_tool_failure("search");
        assert_eq!(m.failed_tool_calls(), 3);
        let snap = m.per_tool_failures_snapshot();
        assert_eq!(snap.get("search").copied(), Some(2));
        assert_eq!(snap.get("lookup").copied(), Some(1));
    }

    #[test]
    fn test_reset_clears_per_tool_counters() {
        let m = RuntimeMetrics::new();
        m.record_tool_call("foo");
        m.record_tool_failure("foo");
        m.reset();
        assert!(m.per_tool_calls_snapshot().is_empty());
        assert!(m.per_tool_failures_snapshot().is_empty());
    }

    #[test]
    fn test_per_tool_snapshot_is_independent_for_unknown_tools() {
        let m = RuntimeMetrics::new();
        let snap = m.per_tool_calls_snapshot();
        assert!(snap.is_empty());
    }

    // ── LatencyHistogram ───────────────────────────────────────────────────────

    #[test]
    fn test_latency_histogram_records_sample() {
        let h = LatencyHistogram::default();
        h.record(10);
        assert_eq!(h.count(), 1);
    }

    #[test]
    fn test_latency_histogram_mean_ms() {
        let h = LatencyHistogram::default();
        h.record(10);
        h.record(20);
        assert!((h.mean_ms() - 15.0).abs() < 1e-5);
    }

    #[test]
    fn test_latency_histogram_buckets_correct_bucket() {
        let h = LatencyHistogram::default();
        h.record(3); // falls in ≤5ms bucket (index 1)
        let buckets = h.buckets();
        // bucket at index 1 is ≤5ms
        assert_eq!(buckets[1].1, 1, "3ms should land in ≤5ms bucket");
        // other buckets should be zero
        assert_eq!(buckets[0].1, 0);
        assert_eq!(buckets[2].1, 0);
    }

    // ── MetricsSnapshot ───────────────────────────────────────────────────────

    #[test]
    fn test_snapshot_returns_all_fields() {
        let m = RuntimeMetrics::new();
        m.active_sessions.store(1, Ordering::Relaxed);
        m.total_sessions.store(2, Ordering::Relaxed);
        m.total_steps.store(3, Ordering::Relaxed);
        m.backpressure_shed_count.store(6, Ordering::Relaxed);
        m.memory_recall_count.store(7, Ordering::Relaxed);
        // Use record_* methods so global and per-tool counters stay consistent.
        m.record_tool_call("my_tool");
        m.record_tool_call("my_tool");
        m.record_tool_failure("my_tool");

        let snap = m.snapshot();
        assert_eq!(snap.active_sessions, 1);
        assert_eq!(snap.total_sessions, 2);
        assert_eq!(snap.total_steps, 3);
        assert_eq!(snap.total_tool_calls, 2);
        assert_eq!(snap.failed_tool_calls, 1);
        assert_eq!(snap.backpressure_shed_count, 6);
        assert_eq!(snap.memory_recall_count, 7);
        assert_eq!(snap.per_tool_calls.get("my_tool").copied(), Some(2));
        assert_eq!(snap.per_tool_failures.get("my_tool").copied(), Some(1));
    }

    #[test]
    fn test_snapshot_default_is_zeroed() {
        let snap = MetricsSnapshot::default();
        assert_eq!(snap.active_sessions, 0);
        assert_eq!(snap.total_sessions, 0);
        assert_eq!(snap.total_steps, 0);
        assert!(snap.per_tool_calls.is_empty());
        assert!(snap.per_tool_failures.is_empty());
    }

    // ── #8 MetricsSnapshot histogram fields ───────────────────────────────────

    #[test]
    fn test_metrics_snapshot_contains_all_fields() {
        let m = RuntimeMetrics::new();
        m.record_step_latency(5);
        m.record_step_latency(200);
        let snap = m.snapshot();
        // Should have 7 buckets
        assert_eq!(snap.step_latency_buckets.len(), 7);
        assert!(snap.step_latency_mean_ms > 0.0);
    }

    // ── #9 per-agent tool call tracking ──────────────────────────────────────

    #[test]
    fn test_per_agent_tool_call_tracking() {
        let m = RuntimeMetrics::new();
        m.record_agent_tool_call("agent-1", "search");
        m.record_agent_tool_call("agent-1", "search");
        m.record_agent_tool_call("agent-2", "lookup");
        m.record_agent_tool_failure("agent-1", "search");

        let calls = m.per_agent_tool_calls_snapshot();
        assert_eq!(calls.get("agent-1").and_then(|t| t.get("search")).copied(), Some(2));
        assert_eq!(calls.get("agent-2").and_then(|t| t.get("lookup")).copied(), Some(1));

        let failures = m.per_agent_tool_failures_snapshot();
        assert_eq!(failures.get("agent-1").and_then(|t| t.get("search")).copied(), Some(1));

        // Also check snapshot includes them
        let snap = m.snapshot();
        assert_eq!(snap.per_agent_tool_calls.get("agent-1").and_then(|t| t.get("search")).copied(), Some(2));

        // Reset clears them
        m.reset();
        assert!(m.per_agent_tool_calls_snapshot().is_empty());
        assert!(m.per_agent_tool_failures_snapshot().is_empty());
    }

    // ── New API tests (Rounds 4-8) ────────────────────────────────────────────

    #[test]
    fn test_latency_histogram_min_max_ms() {
        let h = LatencyHistogram::default();
        assert!(h.min_ms().is_none());
        assert!(h.max_ms().is_none());

        h.record(3);  // bucket 1 (≤5ms)
        h.record(200); // bucket 5 (≤500ms)
        assert!(h.min_ms().is_some());
        assert!(h.max_ms().is_some());
        assert!(h.min_ms().unwrap() <= h.max_ms().unwrap());
    }

    #[test]
    fn test_latency_histogram_p50_p95_p99() {
        let h = LatencyHistogram::default();
        for _ in 0..100 {
            h.record(5); // all in ≤5ms bucket
        }
        // p50, p95, p99 should all resolve to the same bucket bound
        let p50 = h.p50();
        let p95 = h.p95();
        let p99 = h.p99();
        assert_eq!(p50, p95);
        assert_eq!(p95, p99);
    }

    #[test]
    fn test_metrics_snapshot_delta_reflects_increments() {
        let m = RuntimeMetrics::new();
        let before = m.snapshot();
        m.total_steps.fetch_add(5, std::sync::atomic::Ordering::Relaxed);
        m.total_tool_calls.fetch_add(3, std::sync::atomic::Ordering::Relaxed);
        let after = m.snapshot();
        let delta = MetricsSnapshot::delta(&after, &before);
        assert_eq!(delta.total_steps, 5);
        assert_eq!(delta.total_tool_calls, 3);
    }

    #[test]
    fn test_metrics_snapshot_display_contains_key_fields() {
        let m = RuntimeMetrics::new();
        let snap = m.snapshot();
        let s = snap.to_string();
        assert!(s.contains("sessions"));
        assert!(s.contains("steps"));
        assert!(s.contains("latency_mean"));
    }

    #[test]
    fn test_failure_rate_zero_when_no_calls() {
        let m = RuntimeMetrics::new();
        assert_eq!(m.failure_rate(), 0.0);
    }

    #[test]
    fn test_failure_rate_correct_proportion() {
        let m = RuntimeMetrics::new();
        m.record_tool_call("tool_a");
        m.record_tool_call("tool_a");
        m.record_tool_failure("tool_a");
        // 1 failure out of 2 total = 0.5
        assert!((m.failure_rate() - 0.5).abs() < 1e-9);
    }

    #[test]
    fn test_failure_rate_all_failed() {
        let m = RuntimeMetrics::new();
        m.record_tool_call("x");
        m.record_tool_failure("x");
        assert!((m.failure_rate() - 1.0).abs() < 1e-9);
    }

    #[test]
    fn test_top_tools_by_calls_returns_top_n() {
        let m = RuntimeMetrics::new();
        for _ in 0..5 { m.record_tool_call("a"); }
        for _ in 0..3 { m.record_tool_call("b"); }
        for _ in 0..1 { m.record_tool_call("c"); }
        let top = m.top_tools_by_calls(2);
        assert_eq!(top.len(), 2);
        assert_eq!(top[0].0, "a");
        assert_eq!(top[1].0, "b");
    }

    #[test]
    fn test_top_tools_by_calls_returns_all_when_n_exceeds_count() {
        let m = RuntimeMetrics::new();
        m.record_tool_call("only");
        let top = m.top_tools_by_calls(10);
        assert_eq!(top.len(), 1);
        assert_eq!(top[0].0, "only");
    }

    #[test]
    fn test_metrics_snapshot_to_json_contains_key_fields() {
        let m = RuntimeMetrics::new();
        m.record_tool_call("t");
        let snap = m.snapshot();
        let json = snap.to_json();
        assert!(json.get("total_sessions").is_some());
        assert!(json.get("total_steps").is_some());
        assert!(json.get("total_tool_calls").is_some());
    }

    #[test]
    fn test_metrics_snapshot_is_zero_on_new_metrics() {
        let m = RuntimeMetrics::new();
        assert!(m.snapshot().is_zero());
    }

    #[test]
    fn test_metrics_snapshot_is_zero_false_after_activity() {
        let m = RuntimeMetrics::new();
        m.record_tool_call("t");
        assert!(!m.snapshot().is_zero());
    }

    #[test]
    fn test_tool_call_count_returns_per_tool_count() {
        let m = RuntimeMetrics::new();
        m.record_tool_call("search");
        m.record_tool_call("search");
        m.record_tool_call("fetch");
        let snap = m.snapshot();
        assert_eq!(snap.tool_call_count("search"), 2);
        assert_eq!(snap.tool_call_count("fetch"), 1);
        assert_eq!(snap.tool_call_count("absent"), 0);
    }

    #[test]
    fn test_tool_failure_count_returns_per_tool_failures() {
        let m = RuntimeMetrics::new();
        m.record_tool_call("t");
        m.record_tool_failure("t");
        let snap = m.snapshot();
        assert_eq!(snap.tool_failure_count("t"), 1);
        assert_eq!(snap.tool_failure_count("other"), 0);
    }

    #[test]
    fn test_latency_histogram_clear_resets_counts() {
        let h = LatencyHistogram::default();
        h.record(10);
        h.record(20);
        assert_eq!(h.count(), 2);
        h.clear();
        assert_eq!(h.count(), 0);
    }

    #[test]
    fn test_metrics_snapshot_tool_names_sorted() {
        let m = RuntimeMetrics::new();
        m.record_tool_call("zebra");
        m.record_tool_call("alpha");
        m.record_tool_call("mango");
        let snap = m.snapshot();
        assert_eq!(snap.tool_names(), vec!["alpha", "mango", "zebra"]);
    }

    // ── Round 4: top_tools_by_failures / LatencyHistogram::sum_ms ────────────

    #[test]
    fn test_top_tools_by_failures_returns_top_n_descending() {
        let m = RuntimeMetrics::new();
        m.record_tool_failure("a");
        m.record_tool_failure("a");
        m.record_tool_failure("a");
        m.record_tool_failure("b");
        m.record_tool_failure("b");
        m.record_tool_failure("c");
        let top2 = m.top_tools_by_failures(2);
        assert_eq!(top2.len(), 2);
        assert_eq!(top2[0].0, "a");
        assert_eq!(top2[0].1, 3);
        assert_eq!(top2[1].0, "b");
        assert_eq!(top2[1].1, 2);
    }

    #[test]
    fn test_top_tools_by_failures_n_larger_than_tools() {
        let m = RuntimeMetrics::new();
        m.record_tool_failure("only");
        let top = m.top_tools_by_failures(10);
        assert_eq!(top.len(), 1);
        assert_eq!(top[0].0, "only");
    }

    #[test]
    fn test_latency_histogram_sum_ms_accumulates() {
        let h = LatencyHistogram::default();
        h.record(100);
        h.record(200);
        h.record(300);
        assert_eq!(h.sum_ms(), 600);
    }

    #[test]
    fn test_latency_histogram_sum_ms_zero_when_empty() {
        let h = LatencyHistogram::default();
        assert_eq!(h.sum_ms(), 0);
    }

    // ── Round 16: mean_ms, failure_rate ──────────────────────────────────────

    #[test]
    fn test_latency_histogram_mean_ms_zero_when_empty() {
        let h = LatencyHistogram::default();
        assert_eq!(h.mean_ms(), 0.0);
    }

    #[test]
    fn test_latency_histogram_mean_ms_computes_average() {
        let h = LatencyHistogram::default();
        h.record(100);
        h.record(200);
        h.record(300);
        assert!((h.mean_ms() - 200.0).abs() < 1.0);
    }

    #[test]
    fn test_metrics_snapshot_failure_rate_zero_when_no_calls() {
        let m = RuntimeMetrics::new();
        let snap = m.snapshot();
        assert_eq!(snap.failure_rate(), 0.0);
    }

    #[test]
    fn test_metrics_snapshot_failure_rate_correct() {
        let m = RuntimeMetrics::new();
        m.record_tool_call("t");
        m.record_tool_call("t");
        m.record_tool_failure("t");
        let snap = m.snapshot();
        assert!((snap.failure_rate() - 0.5).abs() < 1e-9);
    }

    // ── Round 20: success_rate / is_active / checkpoint_errors ────────────────

    #[test]
    fn test_success_rate_one_when_no_failures() {
        let m = RuntimeMetrics::new();
        m.record_tool_call("x");
        assert!((m.success_rate() - 1.0).abs() < 1e-9);
    }

    #[test]
    fn test_success_rate_half_when_half_failed() {
        let m = RuntimeMetrics::new();
        m.record_tool_call("x");
        m.record_tool_call("x");
        m.record_tool_failure("x");
        assert!((m.success_rate() - 0.5).abs() < 1e-9);
    }

    #[test]
    fn test_success_rate_one_when_no_calls() {
        let m = RuntimeMetrics::new();
        // Vacuously all succeeded — no calls means success_rate = 1.0
        assert!((m.success_rate() - 1.0).abs() < 1e-9);
    }

    #[test]
    fn test_is_active_false_when_no_sessions() {
        let m = RuntimeMetrics::new();
        assert!(!m.is_active());
    }

    #[test]
    fn test_is_active_true_when_session_active() {
        let m = RuntimeMetrics::new();
        m.active_sessions.fetch_add(1, Ordering::Relaxed);
        assert!(m.is_active());
        m.active_sessions.fetch_sub(1, Ordering::Relaxed);
        assert!(!m.is_active());
    }

    #[test]
    fn test_checkpoint_errors_increments() {
        let m = RuntimeMetrics::new();
        assert_eq!(m.checkpoint_errors(), 0);
        m.checkpoint_errors.fetch_add(3, Ordering::Relaxed);
        assert_eq!(m.checkpoint_errors(), 3);
    }

    #[test]
    fn test_checkpoint_errors_reset_to_zero() {
        let m = RuntimeMetrics::new();
        m.checkpoint_errors.fetch_add(5, Ordering::Relaxed);
        m.reset();
        assert_eq!(m.checkpoint_errors(), 0);
    }

    // ── Round 10: LatencyHistogram::std_dev_ms ────────────────────────────────

    #[test]
    fn test_std_dev_ms_zero_for_no_samples() {
        let h = LatencyHistogram::default();
        assert!((h.std_dev_ms() - 0.0).abs() < 1e-9);
    }

    #[test]
    fn test_std_dev_ms_zero_for_single_sample() {
        let h = LatencyHistogram::default();
        h.record(5);
        assert!((h.std_dev_ms() - 0.0).abs() < 1e-9);
    }

    #[test]
    fn test_std_dev_ms_positive_for_varied_samples() {
        let h = LatencyHistogram::default();
        h.record(1);    // bucket 0 mid ~0.5
        h.record(200);  // bucket 5 mid ~300
        // Two samples with very different values → std_dev > 0
        assert!(h.std_dev_ms() > 0.0);
    }

    #[test]
    fn test_std_dev_ms_zero_for_identical_samples() {
        let h = LatencyHistogram::default();
        h.record(5);
        h.record(5);
        h.record(5);
        // All samples in the same bucket → std_dev ≈ 0
        assert!(h.std_dev_ms() < 1.0);
    }

    // ── Round 11: RuntimeMetrics::tool_success_rate ───────────────────────────

    #[test]
    fn test_tool_success_rate_one_when_no_calls() {
        let m = RuntimeMetrics::new();
        assert!((m.tool_success_rate() - 1.0).abs() < 1e-9);
    }

    #[test]
    fn test_tool_success_rate_one_when_no_failures() {
        let m = RuntimeMetrics::new();
        m.total_tool_calls.fetch_add(10, Ordering::Relaxed);
        assert!((m.tool_success_rate() - 1.0).abs() < 1e-9);
    }

    #[test]
    fn test_tool_success_rate_half_when_half_fail() {
        let m = RuntimeMetrics::new();
        m.total_tool_calls.fetch_add(10, Ordering::Relaxed);
        m.failed_tool_calls.fetch_add(5, Ordering::Relaxed);
        assert!((m.tool_success_rate() - 0.5).abs() < 1e-9);
    }

    #[test]
    fn test_tool_success_rate_zero_when_all_fail() {
        let m = RuntimeMetrics::new();
        m.total_tool_calls.fetch_add(4, Ordering::Relaxed);
        m.failed_tool_calls.fetch_add(4, Ordering::Relaxed);
        assert!(m.tool_success_rate().abs() < 1e-9);
    }

    // ── Round 12: step_latency_p50/p99, LatencyHistogram::range_ms ───────────

    #[test]
    fn test_step_latency_p50_zero_when_empty() {
        let m = RuntimeMetrics::new();
        assert_eq!(m.step_latency_p50(), 0);
    }

    #[test]
    fn test_step_latency_p99_zero_when_empty() {
        let m = RuntimeMetrics::new();
        assert_eq!(m.step_latency_p99(), 0);
    }

    #[test]
    fn test_step_latency_p50_after_recording() {
        let m = RuntimeMetrics::new();
        for _ in 0..10 {
            m.step_latency.record(100);
        }
        assert!(m.step_latency_p50() > 0);
    }

    #[test]
    fn test_step_latency_p99_gte_p50() {
        let m = RuntimeMetrics::new();
        for v in [10, 20, 30, 40, 500] {
            m.step_latency.record(v);
        }
        assert!(m.step_latency_p99() >= m.step_latency_p50());
    }

    #[test]
    fn test_latency_histogram_range_ms_none_when_empty() {
        let h = LatencyHistogram::default();
        assert!(h.range_ms().is_none());
    }

    #[test]
    fn test_latency_histogram_range_ms_some_for_single_sample() {
        let h = LatencyHistogram::default();
        h.record(100);
        // min/max are both derived from bucket boundaries, range is Some
        assert!(h.range_ms().is_some());
    }

    #[test]
    fn test_latency_histogram_range_ms_positive_for_spread() {
        let h = LatencyHistogram::default();
        h.record(10);
        h.record(1000);
        let range = h.range_ms().unwrap();
        assert!(range > 0, "range should be > 0 for spread samples, got {range}");
    }

    // ── Round 13: avg_tool_calls_per_session ──────────────────────────────────

    #[test]
    fn test_avg_tool_calls_per_session_zero_when_no_sessions() {
        let m = RuntimeMetrics::new();
        assert!((m.avg_tool_calls_per_session() - 0.0).abs() < 1e-9);
    }

    #[test]
    fn test_avg_tool_calls_per_session_correct_ratio() {
        let m = RuntimeMetrics::new();
        m.total_sessions.fetch_add(2, Ordering::Relaxed);
        m.total_tool_calls.fetch_add(10, Ordering::Relaxed);
        assert!((m.avg_tool_calls_per_session() - 5.0).abs() < 1e-9);
    }

    // ── Round 27: interquartile_range_ms, avg_steps_per_session ──────────────

    #[test]
    fn test_interquartile_range_ms_empty_is_zero() {
        let h = LatencyHistogram::default();
        assert_eq!(h.interquartile_range_ms(), 0);
    }

    #[test]
    fn test_interquartile_range_ms_saturates_not_panics() {
        let h = LatencyHistogram::default();
        for _ in 0..50 {
            h.record(10);
        }
        for _ in 0..50 {
            h.record(500);
        }
        let iqr = h.interquartile_range_ms();
        // IQR must be non-negative (saturating_sub guarantee)
        assert!(iqr < u64::MAX);
    }

    #[test]
    fn test_avg_steps_per_session_zero_when_no_sessions() {
        let snap = MetricsSnapshot::default();
        assert!((snap.avg_steps_per_session() - 0.0).abs() < 1e-9);
    }

    #[test]
    fn test_avg_steps_per_session_correct_ratio() {
        let snap = MetricsSnapshot {
            total_sessions: 4,
            total_steps: 20,
            ..Default::default()
        };
        assert!((snap.avg_steps_per_session() - 5.0).abs() < 1e-9);
    }

    // ── Round 15: LatencyHistogram::is_empty, RuntimeMetrics::checkpoint_error_rate

    #[test]
    fn test_latency_histogram_is_empty_true_initially() {
        let h = LatencyHistogram::default();
        assert!(h.is_empty());
    }

    #[test]
    fn test_latency_histogram_is_empty_false_after_record() {
        let h = LatencyHistogram::default();
        h.record(10);
        assert!(!h.is_empty());
    }

    #[test]
    fn test_checkpoint_error_rate_zero_when_no_sessions() {
        let m = RuntimeMetrics::new();
        assert!((m.checkpoint_error_rate() - 0.0).abs() < 1e-9);
    }

    #[test]
    fn test_checkpoint_error_rate_ratio_correct() {
        let m = RuntimeMetrics::new();
        m.total_sessions.fetch_add(4, std::sync::atomic::Ordering::Relaxed);
        m.checkpoint_errors.fetch_add(2, std::sync::atomic::Ordering::Relaxed);
        assert!((m.checkpoint_error_rate() - 0.5).abs() < 1e-9);
    }

    // ── Round 16: LatencyHistogram::mode_bucket_ms ───────────────────────────

    #[test]
    fn test_mode_bucket_ms_none_when_empty() {
        let h = LatencyHistogram::default();
        assert!(h.mode_bucket_ms().is_none());
    }

    #[test]
    fn test_mode_bucket_ms_returns_bucket_with_most_samples() {
        let h = LatencyHistogram::default();
        // Record many samples in the ~10ms range
        for _ in 0..10 {
            h.record(5);
        }
        // Record fewer samples in the ~500ms range
        for _ in 0..2 {
            h.record(400);
        }
        let mode = h.mode_bucket_ms().unwrap();
        // The low-latency bucket should win
        assert!(mode <= 50, "expected low-latency bucket, got {mode}");
    }

    // ── Round 17: MetricsSnapshot::error_rate / memory_recall_rate ───────────

    #[test]
    fn test_metrics_snapshot_error_rate_zero_when_no_tool_calls() {
        let snap = MetricsSnapshot::default();
        assert!((snap.error_rate() - 0.0).abs() < 1e-9);
    }

    #[test]
    fn test_metrics_snapshot_error_rate_correct_ratio() {
        let snap = MetricsSnapshot {
            total_tool_calls: 10,
            failed_tool_calls: 3,
            ..Default::default()
        };
        assert!((snap.error_rate() - 0.3).abs() < 1e-9);
    }

    #[test]
    fn test_metrics_snapshot_memory_recall_rate_zero_when_no_sessions() {
        let snap = MetricsSnapshot::default();
        assert!((snap.memory_recall_rate() - 0.0).abs() < 1e-9);
    }

    #[test]
    fn test_metrics_snapshot_memory_recall_rate_correct_ratio() {
        let snap = MetricsSnapshot {
            total_sessions: 5,
            memory_recall_count: 15,
            ..Default::default()
        };
        assert!((snap.memory_recall_rate() - 3.0).abs() < 1e-9);
    }

    // ── Round 22: p10 ─────────────────────────────────────────────────────────

    #[test]
    fn test_latency_histogram_p10_zero_when_empty() {
        let h = LatencyHistogram::default();
        assert_eq!(h.p10(), 0);
    }

    #[test]
    fn test_latency_histogram_p10_lte_p50_lte_p99() {
        let h = LatencyHistogram::default();
        for ms in [10, 20, 50, 100, 200, 500, 1000] {
            h.record(ms);
        }
        assert!(h.p10() <= h.p50());
        assert!(h.p50() <= h.p99());
    }

    // ── Round 29: is_below_p99, MetricsSnapshot::is_healthy ──────────────────

    #[test]
    fn test_latency_histogram_is_below_p99_true_when_empty() {
        let h = LatencyHistogram::default();
        assert!(h.is_below_p99(1)); // p99 == 0 < 1
    }

    #[test]
    fn test_latency_histogram_is_below_p99_true_when_under_threshold() {
        let h = LatencyHistogram::default();
        for _ in 0..100 {
            h.record(50);
        }
        assert!(h.is_below_p99(100));
    }

    #[test]
    fn test_latency_histogram_is_below_p99_false_when_at_threshold() {
        let h = LatencyHistogram::default();
        for _ in 0..100 {
            h.record(200);
        }
        assert!(!h.is_below_p99(200)); // p99 == 200, not strictly less
    }

    #[test]
    fn test_metrics_snapshot_is_healthy_true_when_default() {
        let snap = MetricsSnapshot::default();
        assert!(snap.is_healthy());
    }

    #[test]
    fn test_metrics_snapshot_is_healthy_false_when_failed_tool_calls() {
        let snap = MetricsSnapshot { failed_tool_calls: 1, ..Default::default() };
        assert!(!snap.is_healthy());
    }

    #[test]
    fn test_metrics_snapshot_is_healthy_false_when_backpressure_shed() {
        let snap = MetricsSnapshot { backpressure_shed_count: 2, ..Default::default() };
        assert!(!snap.is_healthy());
    }

    #[test]
    fn test_metrics_snapshot_is_healthy_false_when_checkpoint_errors() {
        let snap = MetricsSnapshot { checkpoint_errors: 1, ..Default::default() };
        assert!(!snap.is_healthy());
    }

    // ── Round 23: median_ms / steps_per_session / p50_latency_ms ─────────────

    #[test]
    fn test_latency_histogram_median_ms_equals_p50() {
        let h = LatencyHistogram::default();
        for ms in [10, 50, 100, 200, 500] {
            h.record(ms);
        }
        assert_eq!(h.median_ms(), h.p50());
    }

    #[test]
    fn test_latency_histogram_median_ms_zero_when_empty() {
        let h = LatencyHistogram::default();
        assert_eq!(h.median_ms(), 0);
    }

    #[test]
    fn test_metrics_snapshot_steps_per_session_zero_when_no_sessions() {
        let snap = MetricsSnapshot::default();
        assert!((snap.steps_per_session() - 0.0).abs() < 1e-9);
    }

    #[test]
    fn test_metrics_snapshot_steps_per_session_correct_ratio() {
        let snap = MetricsSnapshot {
            total_sessions: 4,
            total_steps: 20,
            ..Default::default()
        };
        assert!((snap.steps_per_session() - 5.0).abs() < 1e-9);
    }

    #[test]
    fn test_runtime_metrics_p50_latency_ms_zero_when_no_data() {
        let m = RuntimeMetrics::new();
        assert_eq!(m.p50_latency_ms(), 0);
    }

    #[test]
    fn test_runtime_metrics_p50_latency_ms_matches_histogram_p50() {
        let m = RuntimeMetrics::new();
        for ms in [10_u64, 50, 100, 200, 500] {
            m.step_latency.record(ms);
        }
        assert_eq!(m.p50_latency_ms(), m.step_latency.p50());
    }

    // ── Round 25: histogram p25/p75/p90/min, has_data; snapshot helpers ───────

    #[test]
    fn test_latency_histogram_has_data_false_when_empty() {
        let h = LatencyHistogram::default();
        assert!(!h.has_data());
    }

    #[test]
    fn test_latency_histogram_has_data_true_after_record() {
        let h = LatencyHistogram::default();
        h.record(100);
        assert!(h.has_data());
    }

    #[test]
    fn test_latency_histogram_min_ms_none_when_empty() {
        let h = LatencyHistogram::default();
        assert_eq!(h.min_ms(), None);
    }

    #[test]
    fn test_latency_histogram_min_ms_some_after_record() {
        let h = LatencyHistogram::default();
        h.record(50);
        assert!(h.min_ms().is_some());
    }

    #[test]
    fn test_latency_histogram_p25_lte_p75() {
        let h = LatencyHistogram::default();
        for ms in [10_u64, 50, 100, 200, 500, 1000, 2000, 5000] {
            h.record(ms);
        }
        assert!(h.p25() <= h.p75());
    }

    #[test]
    fn test_latency_histogram_p90_between_p50_and_p99() {
        let h = LatencyHistogram::default();
        for ms in [10_u64, 50, 100, 200, 500] {
            h.record(ms);
        }
        assert!(h.p50() <= h.p90());
        assert!(h.p90() <= h.p99());
    }

    #[test]
    fn test_metrics_snapshot_tool_success_count_correct() {
        let snap = MetricsSnapshot {
            per_tool_calls: [("search".to_string(), 10u64)].into(),
            per_tool_failures: [("search".to_string(), 3u64)].into(),
            ..Default::default()
        };
        assert_eq!(snap.tool_success_count("search"), 7);
    }

    #[test]
    fn test_metrics_snapshot_tool_success_count_zero_for_unknown_tool() {
        let snap = MetricsSnapshot::default();
        assert_eq!(snap.tool_success_count("unknown"), 0);
    }

    #[test]
    fn test_metrics_snapshot_tool_failure_rate_correct_ratio() {
        let snap = MetricsSnapshot {
            per_tool_calls: [("lookup".to_string(), 4u64)].into(),
            per_tool_failures: [("lookup".to_string(), 1u64)].into(),
            ..Default::default()
        };
        assert!((snap.tool_failure_rate("lookup") - 0.25).abs() < 1e-9);
    }

    #[test]
    fn test_metrics_snapshot_tool_failure_rate_zero_for_unknown_tool() {
        let snap = MetricsSnapshot::default();
        assert!((snap.tool_failure_rate("none") - 0.0).abs() < 1e-9);
    }

    #[test]
    fn test_metrics_snapshot_total_successful_tool_calls() {
        let snap = MetricsSnapshot {
            total_tool_calls: 20,
            failed_tool_calls: 5,
            ..Default::default()
        };
        assert_eq!(snap.total_successful_tool_calls(), 15);
    }

    #[test]
    fn test_runtime_metrics_per_tool_calls_snapshot_increments() {
        let m = RuntimeMetrics::new();
        m.record_tool_call("search");
        m.record_tool_call("search");
        m.record_tool_call("lookup");
        let snap = m.per_tool_calls_snapshot();
        assert_eq!(snap.get("search"), Some(&2));
        assert_eq!(snap.get("lookup"), Some(&1));
    }

    #[test]
    fn test_runtime_metrics_per_tool_failures_snapshot() {
        let m = RuntimeMetrics::new();
        m.record_tool_call("search");
        m.record_tool_failure("search");
        let snap = m.per_tool_failures_snapshot();
        assert_eq!(snap.get("search"), Some(&1));
    }

    #[test]
    fn test_runtime_metrics_record_agent_tool_call_tracked() {
        let m = RuntimeMetrics::new();
        m.record_agent_tool_call("agent-1", "search");
        m.record_agent_tool_call("agent-1", "search");
        let snap = m.per_agent_tool_calls_snapshot();
        assert_eq!(snap.get("agent-1").and_then(|t| t.get("search")), Some(&2));
    }

    #[test]
    fn test_runtime_metrics_per_agent_tool_failures_snapshot() {
        let m = RuntimeMetrics::new();
        m.record_agent_tool_failure("agent-2", "lookup");
        let snap = m.per_agent_tool_failures_snapshot();
        assert_eq!(
            snap.get("agent-2").and_then(|t| t.get("lookup")),
            Some(&1)
        );
    }

    // ── Round 24: coefficient_of_variation ────────────────────────────────────

    #[test]
    fn test_coefficient_of_variation_zero_when_empty() {
        let h = LatencyHistogram::default();
        assert!((h.coefficient_of_variation() - 0.0).abs() < 1e-9);
    }

    #[test]
    fn test_coefficient_of_variation_positive_with_spread() {
        let h = LatencyHistogram::default();
        // Wide spread: 10ms and 1000ms — std_dev should be significant
        for _ in 0..50 {
            h.record(10);
        }
        for _ in 0..50 {
            h.record(1000);
        }
        let cv = h.coefficient_of_variation();
        assert!(cv > 0.0, "CV should be positive for spread data, got {cv}");
    }

    #[test]
    fn test_coefficient_of_variation_near_zero_for_uniform_data() {
        let h = LatencyHistogram::default();
        // All the same latency bucket → std_dev ≈ 0
        for _ in 0..100 {
            h.record(50);
        }
        // CV won't be exactly 0 due to bucket approximation, but should be small
        assert!(h.coefficient_of_variation() < 1.0);
    }

    // ── Round 31: LatencyHistogram::percentile, RuntimeMetrics helpers ────────

    #[test]
    fn test_latency_histogram_percentile_zero_when_empty() {
        let h = LatencyHistogram::default();
        assert_eq!(h.percentile(0.5), 0);
    }

    #[test]
    fn test_latency_histogram_percentile_50_matches_p50() {
        let h = LatencyHistogram::default();
        for ms in [10, 20, 30, 40, 50] {
            h.record(ms);
        }
        assert_eq!(h.percentile(0.5), h.p50());
    }

    #[test]
    fn test_latency_histogram_percentile_99_matches_p99() {
        let h = LatencyHistogram::default();
        for ms in [10, 50, 100, 500, 1000] {
            h.record(ms);
        }
        assert_eq!(h.percentile(0.99), h.p99());
    }

    #[test]
    fn test_runtime_metrics_record_agent_tool_failure_appears_in_snapshot() {
        let m = RuntimeMetrics::new();
        m.record_agent_tool_failure("agent-1", "search_tool");
        let snapshot = m.per_agent_tool_failures_snapshot();
        assert_eq!(snapshot.get("agent-1").and_then(|t| t.get("search_tool")), Some(&1));
    }

    #[test]
    fn test_runtime_metrics_per_agent_tool_calls_snapshot_empty_initially() {
        let m = RuntimeMetrics::new();
        assert!(m.per_agent_tool_calls_snapshot().is_empty());
    }

    #[test]
    fn test_runtime_metrics_record_step_latency_is_reflected_in_p50() {
        let m = RuntimeMetrics::new();
        for _ in 0..20 {
            m.record_step_latency(100);
        }
        // After recording 20 samples at 100ms, step latency p50 must be around 100ms.
        // We verify the operation doesn't panic and changes the histogram state.
        let snap = m.snapshot();
        assert!(snap.total_sessions == 0); // unrelated sanity check
    }

    // ── Round 26: has_errors / is_above_p99 ───────────────────────────────────

    #[test]
    fn test_metrics_snapshot_has_errors_false_when_clean() {
        let snap = MetricsSnapshot::default();
        assert!(!snap.has_errors());
    }

    #[test]
    fn test_metrics_snapshot_has_errors_true_when_failed_tool_calls() {
        let snap = MetricsSnapshot { failed_tool_calls: 2, ..Default::default() };
        assert!(snap.has_errors());
    }

    #[test]
    fn test_metrics_snapshot_has_errors_true_when_checkpoint_errors() {
        let snap = MetricsSnapshot { checkpoint_errors: 1, ..Default::default() };
        assert!(snap.has_errors());
    }

    #[test]
    fn test_latency_histogram_is_above_p99_false_for_low_latency() {
        let h = LatencyHistogram::default();
        for _ in 0..200 {
            h.record(50);
        }
        assert!(!h.is_above_p99(50));
    }

    #[test]
    fn test_latency_histogram_is_above_p99_true_for_high_latency() {
        let h = LatencyHistogram::default();
        for _ in 0..200 {
            h.record(50);
        }
        // p99 will be ~50ms; 10_000ms should be well above it
        assert!(h.is_above_p99(10_000));
    }

    // ── Round 27: sample_count / tool_call_rate ───────────────────────────────

    #[test]
    fn test_latency_histogram_sample_count_zero_when_empty() {
        let h = LatencyHistogram::default();
        assert_eq!(h.sample_count(), 0);
    }

    #[test]
    fn test_latency_histogram_sample_count_matches_records() {
        let h = LatencyHistogram::default();
        for _ in 0..7 {
            h.record(100);
        }
        assert_eq!(h.sample_count(), 7);
    }

    #[test]
    fn test_metrics_snapshot_tool_call_rate_zero_when_no_sessions() {
        let snap = MetricsSnapshot::default();
        assert!((snap.tool_call_rate() - 0.0).abs() < 1e-9);
    }

    #[test]
    fn test_metrics_snapshot_tool_call_rate_correct_ratio() {
        let snap = MetricsSnapshot {
            total_sessions: 4,
            total_tool_calls: 20,
            ..Default::default()
        };
        assert!((snap.tool_call_rate() - 5.0).abs() < 1e-9);
    }

    // ── Round 28: backpressure_rate / percentile_spread ───────────────────────

    #[test]
    fn test_metrics_snapshot_backpressure_rate_zero_when_no_sessions() {
        let snap = MetricsSnapshot::default();
        assert!((snap.backpressure_rate() - 0.0).abs() < 1e-9);
    }

    #[test]
    fn test_metrics_snapshot_backpressure_rate_correct_ratio() {
        let snap = MetricsSnapshot {
            total_sessions: 2,
            backpressure_shed_count: 4,
            ..Default::default()
        };
        assert!((snap.backpressure_rate() - 2.0).abs() < 1e-9);
    }

    #[test]
    fn test_latency_histogram_percentile_spread_zero_when_empty() {
        let h = LatencyHistogram::default();
        assert_eq!(h.percentile_spread(), 0);
    }

    #[test]
    fn test_latency_histogram_percentile_spread_nonnegative() {
        let h = LatencyHistogram::default();
        for _ in 0..100 {
            h.record(50);
        }
        for _ in 0..5 {
            h.record(500);
        }
        assert!(h.percentile_spread() >= 0);
    }

    // ── Round 29: memory_efficiency / is_uniform ──────────────────────────────

    #[test]
    fn test_metrics_snapshot_memory_efficiency_zero_when_no_steps() {
        let snap = MetricsSnapshot::default();
        assert!((snap.memory_efficiency() - 0.0).abs() < 1e-9);
    }

    #[test]
    fn test_metrics_snapshot_memory_efficiency_correct_ratio() {
        let snap = MetricsSnapshot {
            total_steps: 10,
            memory_recall_count: 4,
            ..Default::default()
        };
        assert!((snap.memory_efficiency() - 0.4).abs() < 1e-9);
    }

    #[test]
    fn test_latency_histogram_is_uniform_true_when_empty() {
        let h = LatencyHistogram::default();
        assert!(h.is_uniform());
    }

    #[test]
    fn test_latency_histogram_is_uniform_true_for_single_bucket() {
        let h = LatencyHistogram::default();
        for _ in 0..50 {
            h.record(50); // all in same bucket
        }
        assert!(h.is_uniform());
    }

    #[test]
    fn test_latency_histogram_is_uniform_false_for_mixed_latencies() {
        let h = LatencyHistogram::default();
        h.record(1);
        h.record(1000);
        assert!(!h.is_uniform());
    }

    // ── Round 30: bucket_counts / active_session_ratio ────────────────────────

    #[test]
    fn test_latency_histogram_bucket_counts_all_zero_when_empty() {
        let h = LatencyHistogram::default();
        assert_eq!(h.bucket_counts(), [0u64; 7]);
    }

    #[test]
    fn test_latency_histogram_bucket_counts_increments_correct_bucket() {
        let h = LatencyHistogram::default();
        h.record(1); // should go into the first bucket (≤1ms)
        let counts = h.bucket_counts();
        assert_eq!(counts[0], 1);
        assert!(counts[1..].iter().all(|&c| c == 0));
    }

    #[test]
    fn test_metrics_snapshot_active_session_ratio_zero_when_no_sessions() {
        let snap = MetricsSnapshot::default();
        assert!((snap.active_session_ratio() - 0.0).abs() < 1e-9);
    }

    #[test]
    fn test_metrics_snapshot_active_session_ratio_correct() {
        let snap = MetricsSnapshot {
            total_sessions: 10,
            active_sessions: 3,
            ..Default::default()
        };
        assert!((snap.active_session_ratio() - 0.3).abs() < 1e-9);
    }

    #[test]
    fn test_step_to_tool_ratio_correct_value() {
        let snap = MetricsSnapshot {
            total_steps: 4,
            total_tool_calls: 2,
            ..Default::default()
        };
        assert!((snap.step_to_tool_ratio() - 0.5).abs() < 1e-9);
    }

    #[test]
    fn test_step_to_tool_ratio_zero_steps_returns_zero() {
        let snap = MetricsSnapshot {
            total_steps: 0,
            total_tool_calls: 5,
            ..Default::default()
        };
        assert_eq!(snap.step_to_tool_ratio(), 0.0);
    }

    #[test]
    fn test_latency_histogram_min_occupied_ms_returns_smallest_occupied_bucket() {
        let h = LatencyHistogram::default();
        h.record(10); // falls in ≤10ms bucket (bound = 10)
        h.record(200); // falls in ≤500ms bucket
        // min_occupied should be the ≤10ms bucket bound = 10
        assert_eq!(h.min_occupied_ms(), Some(10));
    }

    #[test]
    fn test_latency_histogram_min_occupied_ms_empty_returns_none() {
        let h = LatencyHistogram::default();
        assert_eq!(h.min_occupied_ms(), None);
    }

    #[test]
    fn test_metrics_snapshot_has_failures_true_when_failures_exist() {
        let snap = MetricsSnapshot {
            failed_tool_calls: 1,
            ..Default::default()
        };
        assert!(snap.has_failures());
    }

    #[test]
    fn test_metrics_snapshot_has_failures_false_when_no_failures() {
        let snap = MetricsSnapshot::default();
        assert!(!snap.has_failures());
    }

    #[test]
    fn test_latency_histogram_max_occupied_ms_returns_largest_occupied_bucket() {
        let h = LatencyHistogram::default();
        h.record(5);   // ≤5ms bucket
        h.record(200); // ≤500ms bucket
        assert_eq!(h.max_occupied_ms(), Some(500));
    }

    #[test]
    fn test_latency_histogram_max_occupied_ms_empty_returns_none() {
        let h = LatencyHistogram::default();
        assert_eq!(h.max_occupied_ms(), None);
    }

    #[test]
    fn test_latency_histogram_occupied_bucket_count_correct() {
        let h = LatencyHistogram::default();
        h.record(5);   // bucket 1
        h.record(200); // bucket 5
        assert_eq!(h.occupied_bucket_count(), 2);
    }

    #[test]
    fn test_latency_histogram_occupied_bucket_count_empty_returns_zero() {
        let h = LatencyHistogram::default();
        assert_eq!(h.occupied_bucket_count(), 0);
    }

    #[test]
    fn test_metrics_snapshot_tool_diversity_counts_distinct_tools() {
        let snap = MetricsSnapshot {
            per_tool_calls: [("a".to_string(), 1u64), ("b".to_string(), 2u64)]
                .into_iter()
                .collect(),
            ..Default::default()
        };
        assert_eq!(snap.tool_diversity(), 2);
    }

    #[test]
    fn test_metrics_snapshot_tool_diversity_empty_returns_zero() {
        let snap = MetricsSnapshot::default();
        assert_eq!(snap.tool_diversity(), 0);
    }

    #[test]
    fn test_runtime_metrics_total_step_latency_ms_sums_recorded_latencies() {
        let m = RuntimeMetrics::new();
        m.record_step_latency(100);
        m.record_step_latency(200);
        assert_eq!(m.total_step_latency_ms(), 300);
    }

    #[test]
    fn test_runtime_metrics_total_step_latency_ms_zero_when_empty() {
        let m = RuntimeMetrics::new();
        assert_eq!(m.total_step_latency_ms(), 0);
    }

    #[test]
    fn test_metrics_snapshot_avg_failures_per_session_correct() {
        let snap = MetricsSnapshot {
            total_sessions: 4,
            failed_tool_calls: 2,
            ..Default::default()
        };
        assert!((snap.avg_failures_per_session() - 0.5).abs() < 1e-9);
    }

    #[test]
    fn test_metrics_snapshot_avg_failures_per_session_zero_when_no_sessions() {
        let snap = MetricsSnapshot::default();
        assert_eq!(snap.avg_failures_per_session(), 0.0);
    }

    #[test]
    fn test_latency_histogram_is_skewed_true_when_p99_much_greater_than_p50() {
        let h = LatencyHistogram::default();
        // Record many fast samples and one very slow one to skew p99
        for _ in 0..100 {
            h.record(1); // ≤1ms
        }
        h.record(500); // very slow
        // p50 = 1, p99 depends on bucket counts
        // With 100 samples in ≤1ms and 1 in ≤500ms, p99 should be 1ms too
        // Let's just verify the method doesn't panic
        let _ = h.is_skewed();
    }

    #[test]
    fn test_latency_histogram_is_skewed_false_when_empty() {
        let h = LatencyHistogram::default();
        assert!(!h.is_skewed());
    }

    // ── Round 36 ──────────────────────────────────────────────────────────────

    #[test]
    fn test_most_called_tool_returns_tool_with_most_calls() {
        let snap = MetricsSnapshot {
            per_tool_calls: [
                ("search".to_string(), 5u64),
                ("write".to_string(), 2u64),
            ]
            .into_iter()
            .collect(),
            ..Default::default()
        };
        assert_eq!(snap.most_called_tool(), Some("search".to_string()));
    }

    #[test]
    fn test_most_called_tool_returns_none_when_empty() {
        let snap = MetricsSnapshot::default();
        assert!(snap.most_called_tool().is_none());
    }

    #[test]
    fn test_tool_names_with_failures_returns_sorted_names_with_failures() {
        let snap = MetricsSnapshot {
            per_tool_failures: [
                ("search".to_string(), 3u64),
                ("write".to_string(), 0u64),
                ("calc".to_string(), 1u64),
            ]
            .into_iter()
            .collect(),
            ..Default::default()
        };
        assert_eq!(snap.tool_names_with_failures(), vec!["calc", "search"]);
    }

    #[test]
    fn test_tool_names_with_failures_empty_when_no_failures() {
        let snap = MetricsSnapshot::default();
        assert!(snap.tool_names_with_failures().is_empty());
    }

    // ── Round 37 ──────────────────────────────────────────────────────────────

    #[test]
    fn test_agent_with_most_calls_returns_highest_total() {
        let snap = MetricsSnapshot {
            per_agent_tool_calls: [
                ("agent_a".to_string(), [("search".to_string(), 3u64), ("write".to_string(), 2u64)].into_iter().collect()),
                ("agent_b".to_string(), [("search".to_string(), 1u64)].into_iter().collect()),
            ]
            .into_iter()
            .collect(),
            ..Default::default()
        };
        assert_eq!(snap.agent_with_most_calls(), Some("agent_a".to_string()));
    }

    #[test]
    fn test_agent_with_most_calls_returns_none_when_empty() {
        let snap = MetricsSnapshot::default();
        assert!(snap.agent_with_most_calls().is_none());
    }

    // ── Round 38 ──────────────────────────────────────────────────────────────

    #[test]
    fn test_total_agent_count_returns_number_of_distinct_agents() {
        let snap = MetricsSnapshot {
            per_agent_tool_calls: [
                ("a".to_string(), std::collections::HashMap::new()),
                ("b".to_string(), std::collections::HashMap::new()),
            ]
            .into_iter()
            .collect(),
            ..Default::default()
        };
        assert_eq!(snap.total_agent_count(), 2);
    }

    #[test]
    fn test_total_agent_count_zero_when_empty() {
        let snap = MetricsSnapshot::default();
        assert_eq!(snap.total_agent_count(), 0);
    }

    #[test]
    fn test_steps_per_tool_call_returns_ratio() {
        let snap = MetricsSnapshot {
            total_steps: 10,
            total_tool_calls: 5,
            ..Default::default()
        };
        assert!((snap.steps_per_tool_call() - 2.0).abs() < 1e-9);
    }

    #[test]
    fn test_steps_per_tool_call_zero_when_no_tool_calls() {
        let snap = MetricsSnapshot::default();
        assert_eq!(snap.steps_per_tool_call(), 0.0);
    }

    // ── Round 39 ──────────────────────────────────────────────────────────────

    #[test]
    fn test_failed_tool_ratio_for_returns_failure_rate() {
        let snap = MetricsSnapshot {
            per_tool_calls: [("tool".to_string(), 10u64)].into_iter().collect(),
            per_tool_failures: [("tool".to_string(), 2u64)].into_iter().collect(),
            ..Default::default()
        };
        assert!((snap.failed_tool_ratio_for("tool") - 0.2).abs() < 1e-9);
    }

    #[test]
    fn test_failed_tool_ratio_for_zero_when_no_calls() {
        let snap = MetricsSnapshot::default();
        assert_eq!(snap.failed_tool_ratio_for("missing"), 0.0);
    }

    #[test]
    fn test_backpressure_shed_rate_returns_ratio() {
        let snap = MetricsSnapshot {
            total_tool_calls: 100,
            backpressure_shed_count: 5,
            ..Default::default()
        };
        assert!((snap.backpressure_shed_rate() - 0.05).abs() < 1e-9);
    }

    #[test]
    fn test_backpressure_shed_rate_zero_when_no_tool_calls() {
        let snap = MetricsSnapshot::default();
        assert_eq!(snap.backpressure_shed_rate(), 0.0);
    }
}