use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
#[derive(Debug)]
pub struct LatencyHistogram {
buckets: [AtomicU64; 7],
total_count: AtomicU64,
total_sum_ms: AtomicU64,
}
impl Default for LatencyHistogram {
fn default() -> Self {
Self {
buckets: [
AtomicU64::new(0),
AtomicU64::new(0),
AtomicU64::new(0),
AtomicU64::new(0),
AtomicU64::new(0),
AtomicU64::new(0),
AtomicU64::new(0),
],
total_count: AtomicU64::new(0),
total_sum_ms: AtomicU64::new(0),
}
}
}
impl LatencyHistogram {
const BOUNDS: [u64; 7] = [1, 5, 10, 50, 100, 500, u64::MAX];
pub fn record(&self, ms: u64) {
self.total_count.fetch_add(1, Ordering::Relaxed);
self.total_sum_ms.fetch_add(ms, Ordering::Relaxed);
for (i, &bound) in Self::BOUNDS.iter().enumerate() {
if ms <= bound {
self.buckets[i].fetch_add(1, Ordering::Relaxed);
return;
}
}
}
pub fn mean_ms(&self) -> f64 {
let count = self.total_count.load(Ordering::Relaxed);
if count == 0 {
return 0.0;
}
self.total_sum_ms.load(Ordering::Relaxed) as f64 / count as f64
}
pub fn std_dev_ms(&self) -> f64 {
let count = self.total_count.load(Ordering::Relaxed);
if count < 2 {
return 0.0;
}
const MIDS: [f64; 7] = [0.5, 3.0, 7.5, 30.0, 75.0, 300.0, 500.0];
let (sum, sum_sq): (f64, f64) = self
.buckets
.iter()
.zip(MIDS.iter())
.map(|(b, &m)| {
let c = b.load(Ordering::Relaxed) as f64;
(c * m, c * m * m)
})
.fold((0.0, 0.0), |(s, ss), (v, v2)| (s + v, ss + v2));
let n = count as f64;
let variance = sum_sq / n - (sum / n) * (sum / n);
variance.max(0.0).sqrt()
}
pub fn count(&self) -> u64 {
self.total_count.load(Ordering::Relaxed)
}
pub fn has_data(&self) -> bool {
self.count() > 0
}
pub fn is_empty(&self) -> bool {
self.count() == 0
}
pub fn percentile(&self, p: f64) -> u64 {
let total = self.total_count.load(Ordering::Relaxed);
if total == 0 {
return 0;
}
let target = (p.clamp(0.0, 1.0) * total as f64).ceil() as u64;
let mut cumulative = 0u64;
for (i, bucket) in self.buckets.iter().enumerate() {
cumulative += bucket.load(Ordering::Relaxed);
if cumulative >= target {
return Self::BOUNDS[i];
}
}
*Self::BOUNDS.last().unwrap_or(&u64::MAX)
}
pub fn mode_bucket_ms(&self) -> Option<u64> {
if self.count() == 0 {
return None;
}
let (idx, _) = self
.buckets
.iter()
.enumerate()
.max_by_key(|(_, a)| a.load(Ordering::Relaxed))?;
Some(Self::BOUNDS[idx])
}
pub fn buckets(&self) -> Vec<(u64, u64)> {
Self::BOUNDS
.iter()
.zip(self.buckets.iter())
.map(|(&b, a)| (b, a.load(Ordering::Relaxed)))
.collect()
}
pub fn min_ms(&self) -> Option<u64> {
let total = self.total_count.load(Ordering::Relaxed);
if total == 0 {
return None;
}
for (i, bucket) in self.buckets.iter().enumerate() {
if bucket.load(Ordering::Relaxed) > 0 {
return Some(if i == 0 { 0 } else { Self::BOUNDS[i - 1] + 1 });
}
}
None
}
pub fn max_ms(&self) -> Option<u64> {
let total = self.total_count.load(Ordering::Relaxed);
if total == 0 {
return None;
}
for (i, bucket) in self.buckets.iter().enumerate().rev() {
if bucket.load(Ordering::Relaxed) > 0 {
return Some(Self::BOUNDS[i]);
}
}
None
}
pub fn range_ms(&self) -> Option<u64> {
Some(self.max_ms()?.saturating_sub(self.min_ms()?))
}
pub fn interquartile_range_ms(&self) -> u64 {
self.p75().saturating_sub(self.p25())
}
pub fn p50(&self) -> u64 {
self.percentile(0.50)
}
pub fn p95(&self) -> u64 {
self.percentile(0.95)
}
pub fn p99(&self) -> u64 {
self.percentile(0.99)
}
pub fn p25(&self) -> u64 {
self.percentile(0.25)
}
pub fn p75(&self) -> u64 {
self.percentile(0.75)
}
pub fn p90(&self) -> u64 {
self.percentile(0.90)
}
pub fn p10(&self) -> u64 {
self.percentile(0.10)
}
pub fn median_ms(&self) -> u64 {
self.p50()
}
pub fn reset(&self) {
self.total_count.store(0, Ordering::Relaxed);
self.total_sum_ms.store(0, Ordering::Relaxed);
for bucket in &self.buckets {
bucket.store(0, Ordering::Relaxed);
}
}
pub fn sum_ms(&self) -> u64 {
self.total_sum_ms.load(Ordering::Relaxed)
}
pub fn coefficient_of_variation(&self) -> f64 {
let mean = self.mean_ms();
if mean == 0.0 {
return 0.0;
}
self.std_dev_ms() / mean
}
pub fn sample_count(&self) -> u64 {
self.total_count.load(std::sync::atomic::Ordering::Relaxed)
}
pub fn percentile_spread(&self) -> u64 {
self.p99().saturating_sub(self.p50())
}
pub fn bucket_counts(&self) -> [u64; 7] {
let mut out = [0u64; 7];
for (i, b) in self.buckets.iter().enumerate() {
out[i] = b.load(std::sync::atomic::Ordering::Relaxed);
}
out
}
pub fn min_occupied_ms(&self) -> Option<u64> {
Self::BOUNDS
.iter()
.zip(self.buckets.iter())
.find(|(_, b)| b.load(std::sync::atomic::Ordering::Relaxed) > 0)
.map(|(&bound, _)| bound)
}
pub fn max_occupied_ms(&self) -> Option<u64> {
Self::BOUNDS
.iter()
.zip(self.buckets.iter())
.rev()
.find(|(_, b)| b.load(std::sync::atomic::Ordering::Relaxed) > 0)
.map(|(&bound, _)| bound)
}
pub fn occupied_bucket_count(&self) -> usize {
self.buckets
.iter()
.filter(|b| b.load(std::sync::atomic::Ordering::Relaxed) > 0)
.count()
}
pub fn is_skewed(&self) -> bool {
let p50 = self.p50();
if p50 == 0 {
return false;
}
self.p99() > 2 * p50
}
pub fn is_uniform(&self) -> bool {
let non_empty = self
.buckets
.iter()
.filter(|b| b.load(std::sync::atomic::Ordering::Relaxed) > 0)
.count();
non_empty <= 1
}
pub fn clear(&self) {
self.reset();
}
pub fn is_above_p99(&self, latency_ms: u64) -> bool {
latency_ms > self.p99()
}
pub fn is_below_p99(&self, threshold_ms: u64) -> bool {
self.p99() < threshold_ms
}
}
impl MetricsSnapshot {
pub fn delta(after: &Self, before: &Self) -> Self {
Self {
active_sessions: after.active_sessions.saturating_sub(before.active_sessions),
total_sessions: after.total_sessions.saturating_sub(before.total_sessions),
total_steps: after.total_steps.saturating_sub(before.total_steps),
total_tool_calls: after.total_tool_calls.saturating_sub(before.total_tool_calls),
failed_tool_calls: after.failed_tool_calls.saturating_sub(before.failed_tool_calls),
backpressure_shed_count: after
.backpressure_shed_count
.saturating_sub(before.backpressure_shed_count),
memory_recall_count: after
.memory_recall_count
.saturating_sub(before.memory_recall_count),
checkpoint_errors: after
.checkpoint_errors
.saturating_sub(before.checkpoint_errors),
per_tool_calls: {
let mut m = after.per_tool_calls.clone();
for (k, v) in &before.per_tool_calls {
let entry = m.entry(k.clone()).or_default();
*entry = entry.saturating_sub(*v);
}
m
},
per_tool_failures: {
let mut m = after.per_tool_failures.clone();
for (k, v) in &before.per_tool_failures {
let entry = m.entry(k.clone()).or_default();
*entry = entry.saturating_sub(*v);
}
m
},
step_latency_buckets: after
.step_latency_buckets
.iter()
.zip(before.step_latency_buckets.iter())
.map(|((bound, a), (_, b))| (*bound, a.saturating_sub(*b)))
.collect(),
step_latency_mean_ms: after.step_latency_mean_ms - before.step_latency_mean_ms,
per_agent_tool_calls: after.per_agent_tool_calls.clone(),
per_agent_tool_failures: after.per_agent_tool_failures.clone(),
}
}
pub fn to_json(&self) -> serde_json::Value {
serde_json::json!({
"active_sessions": self.active_sessions,
"total_sessions": self.total_sessions,
"total_steps": self.total_steps,
"total_tool_calls": self.total_tool_calls,
"failed_tool_calls": self.failed_tool_calls,
"backpressure_shed_count": self.backpressure_shed_count,
"memory_recall_count": self.memory_recall_count,
"step_latency_mean_ms": self.step_latency_mean_ms,
"per_tool_calls": self.per_tool_calls,
"per_tool_failures": self.per_tool_failures,
})
}
pub fn tool_call_count(&self, name: &str) -> u64 {
self.per_tool_calls.get(name).copied().unwrap_or(0)
}
pub fn summary_line(&self) -> String {
format!(
"sessions={s}, steps={st}, tool_calls={tc}, failures={f}, latency_mean={l}ms",
s = self.total_sessions,
st = self.total_steps,
tc = self.total_tool_calls,
f = self.failed_tool_calls,
l = self.step_latency_mean_ms as u64,
)
}
pub fn tool_failure_count(&self, name: &str) -> u64 {
self.per_tool_failures.get(name).copied().unwrap_or(0)
}
pub fn tool_names(&self) -> Vec<&str> {
let mut names: Vec<&str> = self.per_tool_calls.keys().map(|s| s.as_str()).collect();
names.sort_unstable();
names
}
pub fn failure_rate(&self) -> f64 {
if self.total_tool_calls == 0 {
return 0.0;
}
self.failed_tool_calls as f64 / self.total_tool_calls as f64
}
pub fn success_rate(&self) -> f64 {
1.0 - self.failure_rate()
}
pub fn tool_success_count(&self, name: &str) -> u64 {
self.tool_call_count(name)
.saturating_sub(self.tool_failure_count(name))
}
pub fn tool_failure_rate(&self, name: &str) -> f64 {
let calls = self.tool_call_count(name);
if calls == 0 {
return 0.0;
}
self.tool_failure_count(name) as f64 / calls as f64
}
pub fn total_successful_tool_calls(&self) -> u64 {
self.total_tool_calls.saturating_sub(self.failed_tool_calls)
}
pub fn is_zero(&self) -> bool {
self.active_sessions == 0
&& self.total_sessions == 0
&& self.total_steps == 0
&& self.total_tool_calls == 0
&& self.failed_tool_calls == 0
&& self.backpressure_shed_count == 0
&& self.memory_recall_count == 0
&& self.checkpoint_errors == 0
}
pub fn avg_steps_per_session(&self) -> f64 {
if self.total_sessions == 0 {
0.0
} else {
self.total_steps as f64 / self.total_sessions as f64
}
}
pub fn error_rate(&self) -> f64 {
if self.total_tool_calls == 0 {
return 0.0;
}
self.failed_tool_calls as f64 / self.total_tool_calls as f64
}
pub fn memory_recall_rate(&self) -> f64 {
if self.total_sessions == 0 {
return 0.0;
}
self.memory_recall_count as f64 / self.total_sessions as f64
}
pub fn steps_per_session(&self) -> f64 {
if self.total_sessions == 0 {
return 0.0;
}
self.total_steps as f64 / self.total_sessions as f64
}
pub fn has_errors(&self) -> bool {
self.failed_tool_calls > 0 || self.checkpoint_errors > 0
}
pub fn is_healthy(&self) -> bool {
self.failed_tool_calls == 0
&& self.backpressure_shed_count == 0
&& self.checkpoint_errors == 0
}
pub fn is_healthy_with_latency(&self, max_latency_ms: f64) -> bool {
self.is_healthy() && self.step_latency_mean_ms <= max_latency_ms
}
pub fn is_empty(&self) -> bool {
self.total_sessions == 0 && self.total_tool_calls == 0 && self.total_steps == 0
}
pub fn is_degraded(&self, threshold: f64) -> bool {
self.failure_rate() > threshold
}
pub fn tool_call_rate(&self) -> f64 {
if self.total_sessions == 0 {
return 0.0;
}
self.total_tool_calls as f64 / self.total_sessions as f64
}
pub fn backpressure_rate(&self) -> f64 {
if self.total_sessions == 0 {
return 0.0;
}
self.backpressure_shed_count as f64 / self.total_sessions as f64
}
pub fn memory_efficiency(&self) -> f64 {
if self.total_steps == 0 {
return 0.0;
}
self.memory_recall_count as f64 / self.total_steps as f64
}
pub fn active_session_ratio(&self) -> f64 {
if self.total_sessions == 0 {
return 0.0;
}
self.active_sessions as f64 / self.total_sessions as f64
}
pub fn step_to_tool_ratio(&self) -> f64 {
if self.total_steps == 0 {
return 0.0;
}
self.total_tool_calls as f64 / self.total_steps as f64
}
pub fn has_failures(&self) -> bool {
self.failed_tool_calls > 0
}
pub fn tool_diversity(&self) -> usize {
self.per_tool_calls.len()
}
pub fn avg_failures_per_session(&self) -> f64 {
if self.total_sessions == 0 {
return 0.0;
}
self.failed_tool_calls as f64 / self.total_sessions as f64
}
pub fn most_called_tool(&self) -> Option<String> {
self.per_tool_calls
.iter()
.max_by_key(|(_, &v)| v)
.map(|(k, _)| k.clone())
}
pub fn tool_names_with_failures(&self) -> Vec<String> {
let mut names: Vec<String> = self
.per_tool_failures
.iter()
.filter(|(_, &v)| v > 0)
.map(|(k, _)| k.clone())
.collect();
names.sort_unstable();
names
}
pub fn has_any_tool_failures(&self) -> bool {
self.per_tool_failures.values().any(|&v| v > 0)
}
pub fn tools_with_zero_failures(&self) -> Vec<String> {
let mut names: Vec<String> = self
.per_tool_calls
.keys()
.filter(|name| {
self.per_tool_failures
.get(*name)
.copied()
.unwrap_or(0)
== 0
})
.cloned()
.collect();
names.sort_unstable();
names
}
pub fn total_tool_calls_count(&self) -> u64 {
self.per_tool_calls.values().sum()
}
pub fn tool_call_imbalance(&self) -> f64 {
let counts: Vec<u64> = self.per_tool_calls.values().copied().collect();
if counts.len() < 2 {
return 1.0;
}
let max = counts.iter().copied().max().unwrap_or(0);
let min = counts.iter().copied().min().unwrap_or(0);
if min == 0 {
return 1.0;
}
max as f64 / min as f64
}
pub fn failed_tool_ratio_for(&self, name: &str) -> f64 {
let calls = self.tool_call_count(name);
if calls == 0 {
return 0.0;
}
self.tool_failure_count(name) as f64 / calls as f64
}
pub fn backpressure_shed_rate(&self) -> f64 {
if self.total_tool_calls == 0 {
return 0.0;
}
self.backpressure_shed_count as f64 / self.total_tool_calls as f64
}
pub fn total_agent_count(&self) -> usize {
self.per_agent_tool_calls.len()
}
pub fn steps_per_tool_call(&self) -> f64 {
if self.total_tool_calls == 0 {
return 0.0;
}
self.total_steps as f64 / self.total_tool_calls as f64
}
pub fn agent_with_most_calls(&self) -> Option<String> {
self.per_agent_tool_calls
.iter()
.map(|(agent, tools)| (agent, tools.values().sum::<u64>()))
.max_by_key(|(_, total)| *total)
.map(|(agent, _)| agent.clone())
}
pub fn total_tool_failures(&self) -> u64 {
self.per_tool_failures.values().sum()
}
pub fn least_called_tool(&self) -> Option<String> {
self.per_tool_calls
.iter()
.min_by_key(|(_, &count)| count)
.map(|(name, _)| name.clone())
}
pub fn avg_tool_calls_per_name(&self) -> f64 {
let n = self.per_tool_calls.len();
if n == 0 {
return 0.0;
}
let total: u64 = self.per_tool_calls.values().sum();
total as f64 / n as f64
}
pub fn tool_call_count_above(&self, n: u64) -> usize {
self.per_tool_calls.values().filter(|&&count| count > n).count()
}
pub fn top_n_tools_by_calls(&self, n: usize) -> Vec<(&str, u64)> {
let mut pairs: Vec<(&str, u64)> = self
.per_tool_calls
.iter()
.map(|(name, &count)| (name.as_str(), count))
.collect();
pairs.sort_unstable_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(b.0)));
pairs.truncate(n);
pairs
}
pub fn tool_call_ratio(&self, name: &str) -> f64 {
if self.total_tool_calls == 0 {
return 0.0;
}
let count = self.per_tool_calls.get(name).copied().unwrap_or(0);
count as f64 / self.total_tool_calls as f64
}
pub fn per_tool_calls_sorted(&self) -> Vec<(String, u64)> {
let mut pairs: Vec<(String, u64)> = self
.per_tool_calls
.iter()
.map(|(k, &v)| (k.clone(), v))
.collect();
pairs.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
pairs
}
pub fn has_tool(&self, name: &str) -> bool {
self.per_tool_calls.contains_key(name)
}
pub fn tool_call_share(&self, name: &str) -> f64 {
if self.total_tool_calls == 0 {
return 0.0;
}
let count = self.per_tool_calls.get(name).copied().unwrap_or(0);
count as f64 / self.total_tool_calls as f64
}
pub fn distinct_tool_count(&self) -> usize {
self.per_tool_calls.len()
}
pub fn has_any_tool_calls(&self) -> bool {
self.total_tool_calls > 0
}
pub fn tool_names_alphabetical(&self) -> Vec<String> {
let mut names: Vec<String> = self.per_tool_calls.keys().cloned().collect();
names.sort_unstable();
names
}
pub fn avg_failures_per_tool(&self) -> f64 {
let count = self.per_tool_calls.len();
if count == 0 {
return 0.0;
}
let total_failures: u64 = self.per_tool_failures.values().sum();
total_failures as f64 / count as f64
}
pub fn tools_above_failure_ratio(&self, threshold: f64) -> Vec<String> {
let mut names: Vec<String> = self
.per_tool_calls
.keys()
.filter(|name| {
let calls = self.tool_call_count(name);
if calls == 0 {
return false;
}
let failures = self.tool_failure_count(name);
failures as f64 / calls as f64 > threshold
})
.cloned()
.collect();
names.sort_unstable();
names
}
pub fn failure_ratio_for_tool(&self, name: &str) -> f64 {
let calls = self.tool_call_count(name);
if calls == 0 {
return 0.0;
}
self.tool_failure_count(name) as f64 / calls as f64
}
pub fn any_tool_exceeds_calls(&self, threshold: u64) -> bool {
self.per_tool_calls.values().any(|&c| c > threshold)
}
pub fn total_unique_tools(&self) -> usize {
self.per_tool_calls.len()
}
pub fn tool_call_ratio_for(&self, name: &str) -> f64 {
if self.total_tool_calls == 0 {
return 0.0;
}
self.tool_call_count(name) as f64 / self.total_tool_calls as f64
}
pub fn total_failures_across_all_tools(&self) -> u64 {
self.per_tool_failures.values().sum()
}
}
impl std::fmt::Display for MetricsSnapshot {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"MetricsSnapshot {{ sessions: active={} total={}, steps={}, \
tool_calls={} (failed={}), backpressure_shed={}, \
memory_recalls={}, checkpoint_errors={}, latency_mean={:.1}ms }}",
self.active_sessions,
self.total_sessions,
self.total_steps,
self.total_tool_calls,
self.failed_tool_calls,
self.backpressure_shed_count,
self.memory_recall_count,
self.checkpoint_errors,
self.step_latency_mean_ms,
)
}
}
#[derive(Debug, Clone, PartialEq, Default, Serialize, Deserialize)]
pub struct MetricsSnapshot {
pub active_sessions: usize,
pub total_sessions: u64,
pub total_steps: u64,
pub total_tool_calls: u64,
pub failed_tool_calls: u64,
pub backpressure_shed_count: u64,
pub memory_recall_count: u64,
pub checkpoint_errors: u64,
pub per_tool_calls: HashMap<String, u64>,
pub per_tool_failures: HashMap<String, u64>,
pub step_latency_buckets: Vec<(u64, u64)>,
pub step_latency_mean_ms: f64,
pub per_agent_tool_calls: HashMap<String, HashMap<String, u64>>,
pub per_agent_tool_failures: HashMap<String, HashMap<String, u64>>,
}
#[derive(Debug, Default)]
struct PerToolMaps {
calls: HashMap<String, u64>,
failures: HashMap<String, u64>,
agent_calls: HashMap<String, HashMap<String, u64>>,
agent_failures: HashMap<String, HashMap<String, u64>>,
}
#[derive(Debug)]
pub struct RuntimeMetrics {
pub active_sessions: AtomicUsize,
pub total_sessions: AtomicU64,
pub total_steps: AtomicU64,
pub total_tool_calls: AtomicU64,
pub failed_tool_calls: AtomicU64,
pub backpressure_shed_count: AtomicU64,
pub memory_recall_count: AtomicU64,
pub checkpoint_errors: AtomicU64,
per_tool: Mutex<PerToolMaps>,
pub step_latency: LatencyHistogram,
}
impl Default for RuntimeMetrics {
fn default() -> Self {
Self {
active_sessions: AtomicUsize::new(0),
total_sessions: AtomicU64::new(0),
total_steps: AtomicU64::new(0),
total_tool_calls: AtomicU64::new(0),
failed_tool_calls: AtomicU64::new(0),
backpressure_shed_count: AtomicU64::new(0),
memory_recall_count: AtomicU64::new(0),
checkpoint_errors: AtomicU64::new(0),
per_tool: Mutex::new(PerToolMaps::default()),
step_latency: LatencyHistogram::default(),
}
}
}
impl RuntimeMetrics {
pub fn new() -> Arc<Self> {
Arc::new(Self::default())
}
pub fn active_sessions(&self) -> usize {
self.active_sessions.load(Ordering::Relaxed)
}
pub fn total_sessions(&self) -> u64 {
self.total_sessions.load(Ordering::Relaxed)
}
pub fn avg_tool_calls_per_session(&self) -> f64 {
let sessions = self.total_sessions();
if sessions == 0 {
return 0.0;
}
self.total_tool_calls() as f64 / sessions as f64
}
pub fn total_steps(&self) -> u64 {
self.total_steps.load(Ordering::Relaxed)
}
pub fn avg_steps_per_session(&self) -> f64 {
let sessions = self.total_sessions();
if sessions == 0 {
return 0.0;
}
self.total_steps() as f64 / sessions as f64
}
pub fn total_tool_calls(&self) -> u64 {
self.total_tool_calls.load(Ordering::Relaxed)
}
pub fn failed_tool_calls(&self) -> u64 {
self.failed_tool_calls.load(Ordering::Relaxed)
}
pub fn tool_success_rate(&self) -> f64 {
let total = self.total_tool_calls();
if total == 0 {
return 1.0;
}
let failed = self.failed_tool_calls();
1.0 - (failed as f64 / total as f64)
}
pub fn backpressure_shed_count(&self) -> u64 {
self.backpressure_shed_count.load(Ordering::Relaxed)
}
pub fn memory_recall_count(&self) -> u64 {
self.memory_recall_count.load(Ordering::Relaxed)
}
pub fn checkpoint_errors(&self) -> u64 {
self.checkpoint_errors.load(Ordering::Relaxed)
}
pub fn checkpoint_error_rate(&self) -> f64 {
let sessions = self.total_sessions();
if sessions == 0 {
return 0.0;
}
self.checkpoint_errors() as f64 / sessions as f64
}
pub fn p50_latency_ms(&self) -> u64 {
self.step_latency.p50()
}
pub fn record_tool_call(&self, tool_name: &str) {
self.total_tool_calls.fetch_add(1, Ordering::Relaxed);
if let Ok(mut maps) = self.per_tool.lock() {
*maps.calls.entry(tool_name.to_owned()).or_insert(0) += 1;
}
}
pub fn record_tool_failure(&self, tool_name: &str) {
self.failed_tool_calls.fetch_add(1, Ordering::Relaxed);
if let Ok(mut maps) = self.per_tool.lock() {
*maps.failures.entry(tool_name.to_owned()).or_insert(0) += 1;
}
}
pub fn per_tool_calls_snapshot(&self) -> HashMap<String, u64> {
self.per_tool
.lock()
.map(|m| m.calls.clone())
.unwrap_or_default()
}
pub fn per_tool_failures_snapshot(&self) -> HashMap<String, u64> {
self.per_tool
.lock()
.map(|m| m.failures.clone())
.unwrap_or_default()
}
pub fn record_agent_tool_call(&self, agent_id: &str, tool_name: &str) {
if let Ok(mut maps) = self.per_tool.lock() {
*maps
.agent_calls
.entry(agent_id.to_owned())
.or_default()
.entry(tool_name.to_owned())
.or_insert(0) += 1;
}
}
pub fn record_agent_tool_failure(&self, agent_id: &str, tool_name: &str) {
if let Ok(mut maps) = self.per_tool.lock() {
*maps
.agent_failures
.entry(agent_id.to_owned())
.or_default()
.entry(tool_name.to_owned())
.or_insert(0) += 1;
}
}
pub fn per_agent_tool_calls_snapshot(&self) -> HashMap<String, HashMap<String, u64>> {
self.per_tool
.lock()
.map(|m| m.agent_calls.clone())
.unwrap_or_default()
}
pub fn per_agent_tool_failures_snapshot(&self) -> HashMap<String, HashMap<String, u64>> {
self.per_tool
.lock()
.map(|m| m.agent_failures.clone())
.unwrap_or_default()
}
pub fn snapshot(&self) -> MetricsSnapshot {
let (per_tool_calls, per_tool_failures, per_agent_tool_calls, per_agent_tool_failures) =
self.per_tool
.lock()
.map(|m| {
(
m.calls.clone(),
m.failures.clone(),
m.agent_calls.clone(),
m.agent_failures.clone(),
)
})
.unwrap_or_default();
MetricsSnapshot {
active_sessions: self.active_sessions.load(Ordering::Relaxed),
total_sessions: self.total_sessions.load(Ordering::Relaxed),
total_steps: self.total_steps.load(Ordering::Relaxed),
total_tool_calls: self.total_tool_calls.load(Ordering::Relaxed),
failed_tool_calls: self.failed_tool_calls.load(Ordering::Relaxed),
backpressure_shed_count: self.backpressure_shed_count.load(Ordering::Relaxed),
memory_recall_count: self.memory_recall_count.load(Ordering::Relaxed),
checkpoint_errors: self.checkpoint_errors.load(Ordering::Relaxed),
per_tool_calls,
per_tool_failures,
step_latency_buckets: self.step_latency.buckets(),
step_latency_mean_ms: self.step_latency.mean_ms(),
per_agent_tool_calls,
per_agent_tool_failures,
}
}
pub fn record_step_latency(&self, ms: u64) {
self.step_latency.record(ms);
}
pub fn reset(&self) {
self.active_sessions.store(0, Ordering::Relaxed);
self.total_sessions.store(0, Ordering::Relaxed);
self.total_steps.store(0, Ordering::Relaxed);
self.total_tool_calls.store(0, Ordering::Relaxed);
self.failed_tool_calls.store(0, Ordering::Relaxed);
self.backpressure_shed_count.store(0, Ordering::Relaxed);
self.memory_recall_count.store(0, Ordering::Relaxed);
self.checkpoint_errors.store(0, Ordering::Relaxed);
if let Ok(mut maps) = self.per_tool.lock() {
maps.calls.clear();
maps.failures.clear();
maps.agent_calls.clear();
maps.agent_failures.clear();
}
self.step_latency.reset();
}
pub fn failure_rate(&self) -> f64 {
let total = self.total_tool_calls.load(Ordering::Relaxed);
if total == 0 {
return 0.0;
}
let failed = self.failed_tool_calls.load(Ordering::Relaxed);
failed as f64 / total as f64
}
pub fn success_rate(&self) -> f64 {
1.0 - self.failure_rate()
}
pub fn is_active(&self) -> bool {
self.active_sessions.load(Ordering::Relaxed) > 0
}
pub fn step_latency_p50(&self) -> u64 {
self.step_latency.p50()
}
pub fn step_latency_p99(&self) -> u64 {
self.step_latency.p99()
}
pub fn step_latency_p95(&self) -> u64 {
self.step_latency.p95()
}
pub fn step_latency_p75(&self) -> u64 {
self.step_latency.p75()
}
pub fn step_latency_std_dev_ms(&self) -> f64 {
self.step_latency.std_dev_ms()
}
pub fn most_used_tool(&self) -> Option<String> {
let snap = self.per_tool_calls_snapshot();
snap.into_iter()
.max_by(|a, b| a.1.cmp(&b.1).then_with(|| b.0.cmp(&a.0)))
.map(|(name, _)| name)
}
pub fn tool_call_to_failure_ratio(&self) -> f64 {
let total = self.total_tool_calls.load(Ordering::Relaxed);
if total == 0 {
return 0.0;
}
self.failed_tool_calls.load(Ordering::Relaxed) as f64 / total as f64
}
pub fn active_session_rate(&self) -> f64 {
let total = self.total_sessions.load(Ordering::Relaxed);
if total == 0 {
return 0.0;
}
self.active_sessions.load(Ordering::Relaxed) as f64 / total as f64
}
pub fn memory_recall_per_session(&self) -> f64 {
let total = self.total_sessions.load(Ordering::Relaxed);
if total == 0 {
return 0.0;
}
self.memory_recall_count.load(Ordering::Relaxed) as f64 / total as f64
}
pub fn step_error_rate(&self) -> f64 {
let steps = self.total_steps.load(Ordering::Relaxed);
if steps == 0 {
return 0.0;
}
self.failed_tool_calls.load(Ordering::Relaxed) as f64 / steps as f64
}
pub fn total_errors(&self) -> u64 {
self.failed_tool_calls.load(Ordering::Relaxed)
+ self.checkpoint_errors.load(Ordering::Relaxed)
}
pub fn tool_names_containing(&self, substr: &str) -> Vec<String> {
let snap = self.per_tool_calls_snapshot();
let mut names: Vec<String> = snap
.into_keys()
.filter(|name| name.contains(substr))
.collect();
names.sort_unstable();
names
}
pub fn has_failed_tools(&self) -> bool {
self.failed_tool_calls() > 0
}
pub fn tool_names_by_call_count(&self) -> Vec<String> {
let snap = self.per_tool_calls_snapshot();
let mut pairs: Vec<(String, u64)> = snap.into_iter().collect();
pairs.sort_unstable_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0)));
pairs.into_iter().map(|(name, _)| name).collect()
}
pub fn avg_memory_recalls_per_step(&self) -> f64 {
let steps = self.total_steps();
if steps == 0 {
return 0.0;
}
self.memory_recall_count() as f64 / steps as f64
}
pub fn avg_tool_failures_per_session(&self) -> f64 {
let sessions = self.total_sessions();
if sessions == 0 {
return 0.0;
}
self.failed_tool_calls() as f64 / sessions as f64
}
pub fn tool_calls_per_memory_recall(&self) -> f64 {
let recalls = self.memory_recall_count();
if recalls == 0 {
return 0.0;
}
self.total_tool_calls() as f64 / recalls as f64
}
pub fn memory_recalls_per_tool_call(&self) -> f64 {
let calls = self.total_tool_calls();
if calls == 0 {
return 0.0;
}
self.memory_recall_count() as f64 / calls as f64
}
pub fn step_failure_rate(&self) -> f64 {
let steps = self.total_steps.load(std::sync::atomic::Ordering::Relaxed);
if steps == 0 {
return 0.0;
}
self.failed_tool_calls() as f64 / steps as f64
}
pub fn total_backpressure_shed_pct(&self) -> f64 {
let calls = self.total_tool_calls();
if calls == 0 {
return 0.0;
}
self.backpressure_shed_count() as f64 / calls as f64
}
pub fn tool_with_highest_failure_rate(&self) -> Option<String> {
let calls = self.per_tool_calls_snapshot();
let fails = self.per_tool_failures_snapshot();
calls
.iter()
.filter(|(_, &c)| c > 0)
.map(|(name, &c)| {
let f = fails.get(name).copied().unwrap_or(0);
(name.clone(), f as f64 / c as f64)
})
.max_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal))
.map(|(name, _)| name)
}
pub fn tool_call_count_for(&self, name: &str) -> u64 {
self.per_tool_calls_snapshot()
.get(name)
.copied()
.unwrap_or(0)
}
pub fn top_called_tool(&self) -> Option<String> {
self.per_tool_calls_snapshot()
.into_iter()
.max_by_key(|(_, c)| *c)
.map(|(name, _)| name)
}
pub fn avg_step_latency_ms(&self) -> f64 {
self.step_latency.mean_ms()
}
pub fn distinct_tools_called(&self) -> usize {
self.per_tool_calls_snapshot().len()
}
pub fn failure_rate_for(&self, name: &str) -> f64 {
let calls = self.tool_call_count_for(name);
if calls == 0 {
return 0.0;
}
let failures = self
.per_tool_failures_snapshot()
.get(name)
.copied()
.unwrap_or(0);
failures as f64 / calls as f64
}
pub fn checkpoint_errors_count(&self) -> u64 {
self.checkpoint_errors.load(std::sync::atomic::Ordering::Relaxed)
}
pub fn agents_with_failures(&self) -> Vec<String> {
self.per_agent_tool_failures_snapshot()
.into_iter()
.filter(|(_, tools)| tools.values().any(|&c| c > 0))
.map(|(agent_id, _)| agent_id)
.collect()
}
pub fn total_agent_failures(&self) -> u64 {
self.per_agent_tool_failures_snapshot()
.values()
.flat_map(|m| m.values())
.sum()
}
pub fn per_step_tool_call_rate(&self) -> f64 {
let steps = self.total_steps();
if steps == 0 {
return 0.0;
}
let calls: u64 = self.per_tool_calls_snapshot().values().sum();
calls as f64 / steps as f64
}
pub fn agents_with_no_failures(&self) -> Vec<String> {
let calls = self.per_agent_tool_calls_snapshot();
let failures = self.per_agent_tool_failures_snapshot();
let mut result: Vec<String> = calls
.keys()
.filter(|agent| {
let total_failures: u64 = failures
.get(*agent)
.map(|m| m.values().sum())
.unwrap_or(0);
total_failures == 0
})
.cloned()
.collect();
result.sort_unstable();
result
}
pub fn tools_with_calls_above(&self, threshold: u64) -> Vec<String> {
let snap = self.per_tool_calls_snapshot();
let mut names: Vec<String> = snap
.into_iter()
.filter(|(_, count)| *count > threshold)
.map(|(name, _)| name)
.collect();
names.sort_unstable();
names
}
pub fn agent_tool_call_count(&self, agent_id: &str) -> u64 {
let snap = self.per_agent_tool_calls_snapshot();
snap.get(agent_id)
.map(|m| m.values().sum())
.unwrap_or(0)
}
pub fn tool_calls_per_session(&self) -> f64 {
let sessions = self.total_sessions();
if sessions == 0 {
return 0.0;
}
self.total_tool_calls() as f64 / sessions as f64
}
pub fn failure_free_tools(&self) -> Vec<String> {
let calls = self.per_tool_calls_snapshot();
let failures = self.per_tool_failures_snapshot();
calls
.into_keys()
.filter(|name| failures.get(name).copied().unwrap_or(0) == 0)
.collect()
}
pub fn top_tools_by_calls(&self, n: usize) -> Vec<(String, u64)> {
let snap = self.per_tool_calls_snapshot();
let mut pairs: Vec<(String, u64)> = snap.into_iter().collect();
pairs.sort_unstable_by(|a, b| b.1.cmp(&a.1));
pairs.truncate(n);
pairs
}
pub fn top_tools_by_failures(&self, n: usize) -> Vec<(String, u64)> {
let snap = self.per_tool_failures_snapshot();
let mut pairs: Vec<(String, u64)> = snap.into_iter().collect();
pairs.sort_unstable_by(|a, b| b.1.cmp(&a.1));
pairs.truncate(n);
pairs
}
pub fn total_step_latency_ms(&self) -> u64 {
self.step_latency.sum_ms()
}
pub fn avg_calls_per_step(&self) -> f64 {
let steps = self.total_steps.load(Ordering::Relaxed);
if steps == 0 {
return 0.0;
}
self.total_tool_calls.load(Ordering::Relaxed) as f64 / steps as f64
}
pub fn memory_pressure_ratio(&self) -> f64 {
let steps = self.total_steps.load(Ordering::Relaxed);
if steps == 0 {
return 0.0;
}
self.memory_recall_count.load(Ordering::Relaxed) as f64 / steps as f64
}
pub fn backpressure_ratio(&self) -> f64 {
let steps = self.total_steps.load(Ordering::Relaxed);
if steps == 0 {
return 0.0;
}
self.backpressure_shed_count.load(Ordering::Relaxed) as f64 / steps as f64
}
pub fn sessions_per_step(&self) -> f64 {
let steps = self.total_steps.load(Ordering::Relaxed);
if steps == 0 {
return 0.0;
}
self.total_sessions.load(Ordering::Relaxed) as f64 / steps as f64
}
pub fn has_latency_data(&self) -> bool {
self.total_steps.load(Ordering::Relaxed) > 0
}
pub fn global_failure_rate(&self) -> f64 {
let total = self.total_tool_calls.load(Ordering::Relaxed);
if total == 0 {
return 0.0;
}
self.failed_tool_calls.load(Ordering::Relaxed) as f64 / total as f64
}
pub fn total_agent_tool_calls(&self) -> u64 {
self.per_agent_tool_calls_snapshot()
.values()
.flat_map(|tool_map| tool_map.values())
.sum()
}
pub fn agent_tool_count(&self) -> usize {
self.per_agent_tool_calls_snapshot().len()
}
pub fn has_recorded_agent_calls(&self) -> bool {
!self.per_agent_tool_calls_snapshot().is_empty()
}
pub fn active_session_count(&self) -> usize {
self.active_sessions.load(Ordering::Relaxed)
}
pub fn memory_to_session_ratio(&self) -> f64 {
let sessions = self.total_sessions.load(Ordering::Relaxed);
if sessions == 0 {
return 0.0;
}
self.memory_recall_count.load(Ordering::Relaxed) as f64 / sessions as f64
}
pub fn total_latency_per_session(&self) -> f64 {
let sessions = self.total_sessions.load(Ordering::Relaxed);
if sessions == 0 {
return 0.0;
}
self.step_latency.sum_ms() as f64 / sessions as f64
}
#[deprecated(since = "1.0.3", note = "use `snapshot()` which returns the named MetricsSnapshot struct")]
pub fn to_snapshot(&self) -> (usize, u64, u64, u64, u64, u64, u64) {
(
self.active_sessions.load(Ordering::Relaxed),
self.total_sessions.load(Ordering::Relaxed),
self.total_steps.load(Ordering::Relaxed),
self.total_tool_calls.load(Ordering::Relaxed),
self.failed_tool_calls.load(Ordering::Relaxed),
self.backpressure_shed_count.load(Ordering::Relaxed),
self.memory_recall_count.load(Ordering::Relaxed),
)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_metrics_new_returns_arc_with_zero_counters() {
let m = RuntimeMetrics::new();
assert_eq!(m.active_sessions(), 0);
assert_eq!(m.total_sessions(), 0);
assert_eq!(m.total_steps(), 0);
assert_eq!(m.total_tool_calls(), 0);
assert_eq!(m.failed_tool_calls(), 0);
assert_eq!(m.backpressure_shed_count(), 0);
assert_eq!(m.memory_recall_count(), 0);
}
#[test]
fn test_active_sessions_increments_and_decrements() {
let m = RuntimeMetrics::new();
m.active_sessions.fetch_add(1, Ordering::Relaxed);
assert_eq!(m.active_sessions(), 1);
m.active_sessions.fetch_sub(1, Ordering::Relaxed);
assert_eq!(m.active_sessions(), 0);
}
#[test]
fn test_total_sessions_increments() {
let m = RuntimeMetrics::new();
m.total_sessions.fetch_add(1, Ordering::Relaxed);
m.total_sessions.fetch_add(1, Ordering::Relaxed);
assert_eq!(m.total_sessions(), 2);
}
#[test]
fn test_total_steps_increments() {
let m = RuntimeMetrics::new();
m.total_steps.fetch_add(5, Ordering::Relaxed);
assert_eq!(m.total_steps(), 5);
}
#[test]
fn test_total_tool_calls_increments() {
let m = RuntimeMetrics::new();
m.total_tool_calls.fetch_add(3, Ordering::Relaxed);
assert_eq!(m.total_tool_calls(), 3);
}
#[test]
fn test_failed_tool_calls_increments() {
let m = RuntimeMetrics::new();
m.failed_tool_calls.fetch_add(2, Ordering::Relaxed);
assert_eq!(m.failed_tool_calls(), 2);
}
#[test]
fn test_backpressure_shed_count_increments() {
let m = RuntimeMetrics::new();
m.backpressure_shed_count.fetch_add(7, Ordering::Relaxed);
assert_eq!(m.backpressure_shed_count(), 7);
}
#[test]
fn test_memory_recall_count_increments() {
let m = RuntimeMetrics::new();
m.memory_recall_count.fetch_add(4, Ordering::Relaxed);
assert_eq!(m.memory_recall_count(), 4);
}
#[test]
fn test_reset_zeroes_all_counters() {
let m = RuntimeMetrics::new();
m.active_sessions.store(3, Ordering::Relaxed);
m.total_sessions.store(10, Ordering::Relaxed);
m.total_steps.store(50, Ordering::Relaxed);
m.total_tool_calls.store(20, Ordering::Relaxed);
m.failed_tool_calls.store(2, Ordering::Relaxed);
m.backpressure_shed_count.store(1, Ordering::Relaxed);
m.memory_recall_count.store(8, Ordering::Relaxed);
m.reset();
assert_eq!(m.active_sessions(), 0);
assert_eq!(m.total_sessions(), 0);
assert_eq!(m.total_steps(), 0);
assert_eq!(m.total_tool_calls(), 0);
assert_eq!(m.failed_tool_calls(), 0);
assert_eq!(m.backpressure_shed_count(), 0);
assert_eq!(m.memory_recall_count(), 0);
}
#[test]
fn test_to_snapshot_captures_correct_values() {
let m = RuntimeMetrics::new();
m.active_sessions.store(1, Ordering::Relaxed);
m.total_sessions.store(2, Ordering::Relaxed);
m.total_steps.store(3, Ordering::Relaxed);
m.total_tool_calls.store(4, Ordering::Relaxed);
m.failed_tool_calls.store(5, Ordering::Relaxed);
m.backpressure_shed_count.store(6, Ordering::Relaxed);
m.memory_recall_count.store(7, Ordering::Relaxed);
let snap = m.to_snapshot();
assert_eq!(snap, (1, 2, 3, 4, 5, 6, 7));
}
#[test]
fn test_metrics_is_send_sync() {
fn assert_send_sync<T: Send + Sync>() {}
assert_send_sync::<RuntimeMetrics>();
}
#[test]
fn test_multiple_increments_are_cumulative() {
let m = RuntimeMetrics::new();
for _ in 0..100 {
m.total_sessions.fetch_add(1, Ordering::Relaxed);
}
assert_eq!(m.total_sessions(), 100);
}
#[test]
fn test_arc_clone_shares_state() {
let m = RuntimeMetrics::new();
let m2 = Arc::clone(&m);
m.total_sessions.fetch_add(1, Ordering::Relaxed);
assert_eq!(m2.total_sessions(), 1);
}
#[test]
fn test_record_tool_call_increments_global_and_per_tool() {
let m = RuntimeMetrics::new();
m.record_tool_call("search");
m.record_tool_call("search");
m.record_tool_call("lookup");
assert_eq!(m.total_tool_calls(), 3);
let snap = m.per_tool_calls_snapshot();
assert_eq!(snap.get("search").copied(), Some(2));
assert_eq!(snap.get("lookup").copied(), Some(1));
}
#[test]
fn test_record_tool_failure_increments_global_and_per_tool() {
let m = RuntimeMetrics::new();
m.record_tool_failure("search");
m.record_tool_failure("lookup");
m.record_tool_failure("search");
assert_eq!(m.failed_tool_calls(), 3);
let snap = m.per_tool_failures_snapshot();
assert_eq!(snap.get("search").copied(), Some(2));
assert_eq!(snap.get("lookup").copied(), Some(1));
}
#[test]
fn test_reset_clears_per_tool_counters() {
let m = RuntimeMetrics::new();
m.record_tool_call("foo");
m.record_tool_failure("foo");
m.reset();
assert!(m.per_tool_calls_snapshot().is_empty());
assert!(m.per_tool_failures_snapshot().is_empty());
}
#[test]
fn test_per_tool_snapshot_is_independent_for_unknown_tools() {
let m = RuntimeMetrics::new();
let snap = m.per_tool_calls_snapshot();
assert!(snap.is_empty());
}
#[test]
fn test_latency_histogram_records_sample() {
let h = LatencyHistogram::default();
h.record(10);
assert_eq!(h.count(), 1);
}
#[test]
fn test_latency_histogram_mean_ms() {
let h = LatencyHistogram::default();
h.record(10);
h.record(20);
assert!((h.mean_ms() - 15.0).abs() < 1e-5);
}
#[test]
fn test_latency_histogram_buckets_correct_bucket() {
let h = LatencyHistogram::default();
h.record(3); let buckets = h.buckets();
assert_eq!(buckets[1].1, 1, "3ms should land in ≤5ms bucket");
assert_eq!(buckets[0].1, 0);
assert_eq!(buckets[2].1, 0);
}
#[test]
fn test_snapshot_returns_all_fields() {
let m = RuntimeMetrics::new();
m.active_sessions.store(1, Ordering::Relaxed);
m.total_sessions.store(2, Ordering::Relaxed);
m.total_steps.store(3, Ordering::Relaxed);
m.backpressure_shed_count.store(6, Ordering::Relaxed);
m.memory_recall_count.store(7, Ordering::Relaxed);
m.record_tool_call("my_tool");
m.record_tool_call("my_tool");
m.record_tool_failure("my_tool");
let snap = m.snapshot();
assert_eq!(snap.active_sessions, 1);
assert_eq!(snap.total_sessions, 2);
assert_eq!(snap.total_steps, 3);
assert_eq!(snap.total_tool_calls, 2);
assert_eq!(snap.failed_tool_calls, 1);
assert_eq!(snap.backpressure_shed_count, 6);
assert_eq!(snap.memory_recall_count, 7);
assert_eq!(snap.per_tool_calls.get("my_tool").copied(), Some(2));
assert_eq!(snap.per_tool_failures.get("my_tool").copied(), Some(1));
}
#[test]
fn test_snapshot_default_is_zeroed() {
let snap = MetricsSnapshot::default();
assert_eq!(snap.active_sessions, 0);
assert_eq!(snap.total_sessions, 0);
assert_eq!(snap.total_steps, 0);
assert!(snap.per_tool_calls.is_empty());
assert!(snap.per_tool_failures.is_empty());
}
#[test]
fn test_metrics_snapshot_contains_all_fields() {
let m = RuntimeMetrics::new();
m.record_step_latency(5);
m.record_step_latency(200);
let snap = m.snapshot();
assert_eq!(snap.step_latency_buckets.len(), 7);
assert!(snap.step_latency_mean_ms > 0.0);
}
#[test]
fn test_per_agent_tool_call_tracking() {
let m = RuntimeMetrics::new();
m.record_agent_tool_call("agent-1", "search");
m.record_agent_tool_call("agent-1", "search");
m.record_agent_tool_call("agent-2", "lookup");
m.record_agent_tool_failure("agent-1", "search");
let calls = m.per_agent_tool_calls_snapshot();
assert_eq!(calls.get("agent-1").and_then(|t| t.get("search")).copied(), Some(2));
assert_eq!(calls.get("agent-2").and_then(|t| t.get("lookup")).copied(), Some(1));
let failures = m.per_agent_tool_failures_snapshot();
assert_eq!(failures.get("agent-1").and_then(|t| t.get("search")).copied(), Some(1));
let snap = m.snapshot();
assert_eq!(snap.per_agent_tool_calls.get("agent-1").and_then(|t| t.get("search")).copied(), Some(2));
m.reset();
assert!(m.per_agent_tool_calls_snapshot().is_empty());
assert!(m.per_agent_tool_failures_snapshot().is_empty());
}
#[test]
fn test_latency_histogram_min_max_ms() {
let h = LatencyHistogram::default();
assert!(h.min_ms().is_none());
assert!(h.max_ms().is_none());
h.record(3); h.record(200); assert!(h.min_ms().is_some());
assert!(h.max_ms().is_some());
assert!(h.min_ms().unwrap() <= h.max_ms().unwrap());
}
#[test]
fn test_latency_histogram_p50_p95_p99() {
let h = LatencyHistogram::default();
for _ in 0..100 {
h.record(5); }
let p50 = h.p50();
let p95 = h.p95();
let p99 = h.p99();
assert_eq!(p50, p95);
assert_eq!(p95, p99);
}
#[test]
fn test_metrics_snapshot_delta_reflects_increments() {
let m = RuntimeMetrics::new();
let before = m.snapshot();
m.total_steps.fetch_add(5, std::sync::atomic::Ordering::Relaxed);
m.total_tool_calls.fetch_add(3, std::sync::atomic::Ordering::Relaxed);
let after = m.snapshot();
let delta = MetricsSnapshot::delta(&after, &before);
assert_eq!(delta.total_steps, 5);
assert_eq!(delta.total_tool_calls, 3);
}
#[test]
fn test_metrics_snapshot_display_contains_key_fields() {
let m = RuntimeMetrics::new();
let snap = m.snapshot();
let s = snap.to_string();
assert!(s.contains("sessions"));
assert!(s.contains("steps"));
assert!(s.contains("latency_mean"));
}
#[test]
fn test_failure_rate_zero_when_no_calls() {
let m = RuntimeMetrics::new();
assert_eq!(m.failure_rate(), 0.0);
}
#[test]
fn test_failure_rate_correct_proportion() {
let m = RuntimeMetrics::new();
m.record_tool_call("tool_a");
m.record_tool_call("tool_a");
m.record_tool_failure("tool_a");
assert!((m.failure_rate() - 0.5).abs() < 1e-9);
}
#[test]
fn test_failure_rate_all_failed() {
let m = RuntimeMetrics::new();
m.record_tool_call("x");
m.record_tool_failure("x");
assert!((m.failure_rate() - 1.0).abs() < 1e-9);
}
#[test]
fn test_top_tools_by_calls_returns_top_n() {
let m = RuntimeMetrics::new();
for _ in 0..5 { m.record_tool_call("a"); }
for _ in 0..3 { m.record_tool_call("b"); }
for _ in 0..1 { m.record_tool_call("c"); }
let top = m.top_tools_by_calls(2);
assert_eq!(top.len(), 2);
assert_eq!(top[0].0, "a");
assert_eq!(top[1].0, "b");
}
#[test]
fn test_top_tools_by_calls_returns_all_when_n_exceeds_count() {
let m = RuntimeMetrics::new();
m.record_tool_call("only");
let top = m.top_tools_by_calls(10);
assert_eq!(top.len(), 1);
assert_eq!(top[0].0, "only");
}
#[test]
fn test_metrics_snapshot_to_json_contains_key_fields() {
let m = RuntimeMetrics::new();
m.record_tool_call("t");
let snap = m.snapshot();
let json = snap.to_json();
assert!(json.get("total_sessions").is_some());
assert!(json.get("total_steps").is_some());
assert!(json.get("total_tool_calls").is_some());
}
#[test]
fn test_metrics_snapshot_is_zero_on_new_metrics() {
let m = RuntimeMetrics::new();
assert!(m.snapshot().is_zero());
}
#[test]
fn test_metrics_snapshot_is_zero_false_after_activity() {
let m = RuntimeMetrics::new();
m.record_tool_call("t");
assert!(!m.snapshot().is_zero());
}
#[test]
fn test_tool_call_count_returns_per_tool_count() {
let m = RuntimeMetrics::new();
m.record_tool_call("search");
m.record_tool_call("search");
m.record_tool_call("fetch");
let snap = m.snapshot();
assert_eq!(snap.tool_call_count("search"), 2);
assert_eq!(snap.tool_call_count("fetch"), 1);
assert_eq!(snap.tool_call_count("absent"), 0);
}
#[test]
fn test_tool_failure_count_returns_per_tool_failures() {
let m = RuntimeMetrics::new();
m.record_tool_call("t");
m.record_tool_failure("t");
let snap = m.snapshot();
assert_eq!(snap.tool_failure_count("t"), 1);
assert_eq!(snap.tool_failure_count("other"), 0);
}
#[test]
fn test_latency_histogram_clear_resets_counts() {
let h = LatencyHistogram::default();
h.record(10);
h.record(20);
assert_eq!(h.count(), 2);
h.clear();
assert_eq!(h.count(), 0);
}
#[test]
fn test_metrics_snapshot_tool_names_sorted() {
let m = RuntimeMetrics::new();
m.record_tool_call("zebra");
m.record_tool_call("alpha");
m.record_tool_call("mango");
let snap = m.snapshot();
assert_eq!(snap.tool_names(), vec!["alpha", "mango", "zebra"]);
}
#[test]
fn test_top_tools_by_failures_returns_top_n_descending() {
let m = RuntimeMetrics::new();
m.record_tool_failure("a");
m.record_tool_failure("a");
m.record_tool_failure("a");
m.record_tool_failure("b");
m.record_tool_failure("b");
m.record_tool_failure("c");
let top2 = m.top_tools_by_failures(2);
assert_eq!(top2.len(), 2);
assert_eq!(top2[0].0, "a");
assert_eq!(top2[0].1, 3);
assert_eq!(top2[1].0, "b");
assert_eq!(top2[1].1, 2);
}
#[test]
fn test_top_tools_by_failures_n_larger_than_tools() {
let m = RuntimeMetrics::new();
m.record_tool_failure("only");
let top = m.top_tools_by_failures(10);
assert_eq!(top.len(), 1);
assert_eq!(top[0].0, "only");
}
#[test]
fn test_latency_histogram_sum_ms_accumulates() {
let h = LatencyHistogram::default();
h.record(100);
h.record(200);
h.record(300);
assert_eq!(h.sum_ms(), 600);
}
#[test]
fn test_latency_histogram_sum_ms_zero_when_empty() {
let h = LatencyHistogram::default();
assert_eq!(h.sum_ms(), 0);
}
#[test]
fn test_latency_histogram_mean_ms_zero_when_empty() {
let h = LatencyHistogram::default();
assert_eq!(h.mean_ms(), 0.0);
}
#[test]
fn test_latency_histogram_mean_ms_computes_average() {
let h = LatencyHistogram::default();
h.record(100);
h.record(200);
h.record(300);
assert!((h.mean_ms() - 200.0).abs() < 1.0);
}
#[test]
fn test_metrics_snapshot_failure_rate_zero_when_no_calls() {
let m = RuntimeMetrics::new();
let snap = m.snapshot();
assert_eq!(snap.failure_rate(), 0.0);
}
#[test]
fn test_metrics_snapshot_failure_rate_correct() {
let m = RuntimeMetrics::new();
m.record_tool_call("t");
m.record_tool_call("t");
m.record_tool_failure("t");
let snap = m.snapshot();
assert!((snap.failure_rate() - 0.5).abs() < 1e-9);
}
#[test]
fn test_success_rate_one_when_no_failures() {
let m = RuntimeMetrics::new();
m.record_tool_call("x");
assert!((m.success_rate() - 1.0).abs() < 1e-9);
}
#[test]
fn test_success_rate_half_when_half_failed() {
let m = RuntimeMetrics::new();
m.record_tool_call("x");
m.record_tool_call("x");
m.record_tool_failure("x");
assert!((m.success_rate() - 0.5).abs() < 1e-9);
}
#[test]
fn test_success_rate_one_when_no_calls() {
let m = RuntimeMetrics::new();
assert!((m.success_rate() - 1.0).abs() < 1e-9);
}
#[test]
fn test_is_active_false_when_no_sessions() {
let m = RuntimeMetrics::new();
assert!(!m.is_active());
}
#[test]
fn test_is_active_true_when_session_active() {
let m = RuntimeMetrics::new();
m.active_sessions.fetch_add(1, Ordering::Relaxed);
assert!(m.is_active());
m.active_sessions.fetch_sub(1, Ordering::Relaxed);
assert!(!m.is_active());
}
#[test]
fn test_checkpoint_errors_increments() {
let m = RuntimeMetrics::new();
assert_eq!(m.checkpoint_errors(), 0);
m.checkpoint_errors.fetch_add(3, Ordering::Relaxed);
assert_eq!(m.checkpoint_errors(), 3);
}
#[test]
fn test_checkpoint_errors_reset_to_zero() {
let m = RuntimeMetrics::new();
m.checkpoint_errors.fetch_add(5, Ordering::Relaxed);
m.reset();
assert_eq!(m.checkpoint_errors(), 0);
}
#[test]
fn test_std_dev_ms_zero_for_no_samples() {
let h = LatencyHistogram::default();
assert!((h.std_dev_ms() - 0.0).abs() < 1e-9);
}
#[test]
fn test_std_dev_ms_zero_for_single_sample() {
let h = LatencyHistogram::default();
h.record(5);
assert!((h.std_dev_ms() - 0.0).abs() < 1e-9);
}
#[test]
fn test_std_dev_ms_positive_for_varied_samples() {
let h = LatencyHistogram::default();
h.record(1); h.record(200); assert!(h.std_dev_ms() > 0.0);
}
#[test]
fn test_std_dev_ms_zero_for_identical_samples() {
let h = LatencyHistogram::default();
h.record(5);
h.record(5);
h.record(5);
assert!(h.std_dev_ms() < 1.0);
}
#[test]
fn test_tool_success_rate_one_when_no_calls() {
let m = RuntimeMetrics::new();
assert!((m.tool_success_rate() - 1.0).abs() < 1e-9);
}
#[test]
fn test_tool_success_rate_one_when_no_failures() {
let m = RuntimeMetrics::new();
m.total_tool_calls.fetch_add(10, Ordering::Relaxed);
assert!((m.tool_success_rate() - 1.0).abs() < 1e-9);
}
#[test]
fn test_tool_success_rate_half_when_half_fail() {
let m = RuntimeMetrics::new();
m.total_tool_calls.fetch_add(10, Ordering::Relaxed);
m.failed_tool_calls.fetch_add(5, Ordering::Relaxed);
assert!((m.tool_success_rate() - 0.5).abs() < 1e-9);
}
#[test]
fn test_tool_success_rate_zero_when_all_fail() {
let m = RuntimeMetrics::new();
m.total_tool_calls.fetch_add(4, Ordering::Relaxed);
m.failed_tool_calls.fetch_add(4, Ordering::Relaxed);
assert!(m.tool_success_rate().abs() < 1e-9);
}
#[test]
fn test_step_latency_p50_zero_when_empty() {
let m = RuntimeMetrics::new();
assert_eq!(m.step_latency_p50(), 0);
}
#[test]
fn test_step_latency_p99_zero_when_empty() {
let m = RuntimeMetrics::new();
assert_eq!(m.step_latency_p99(), 0);
}
#[test]
fn test_step_latency_p50_after_recording() {
let m = RuntimeMetrics::new();
for _ in 0..10 {
m.step_latency.record(100);
}
assert!(m.step_latency_p50() > 0);
}
#[test]
fn test_step_latency_p99_gte_p50() {
let m = RuntimeMetrics::new();
for v in [10, 20, 30, 40, 500] {
m.step_latency.record(v);
}
assert!(m.step_latency_p99() >= m.step_latency_p50());
}
#[test]
fn test_latency_histogram_range_ms_none_when_empty() {
let h = LatencyHistogram::default();
assert!(h.range_ms().is_none());
}
#[test]
fn test_latency_histogram_range_ms_some_for_single_sample() {
let h = LatencyHistogram::default();
h.record(100);
assert!(h.range_ms().is_some());
}
#[test]
fn test_latency_histogram_range_ms_positive_for_spread() {
let h = LatencyHistogram::default();
h.record(10);
h.record(1000);
let range = h.range_ms().unwrap();
assert!(range > 0, "range should be > 0 for spread samples, got {range}");
}
#[test]
fn test_avg_tool_calls_per_session_zero_when_no_sessions() {
let m = RuntimeMetrics::new();
assert!((m.avg_tool_calls_per_session() - 0.0).abs() < 1e-9);
}
#[test]
fn test_avg_tool_calls_per_session_correct_ratio() {
let m = RuntimeMetrics::new();
m.total_sessions.fetch_add(2, Ordering::Relaxed);
m.total_tool_calls.fetch_add(10, Ordering::Relaxed);
assert!((m.avg_tool_calls_per_session() - 5.0).abs() < 1e-9);
}
#[test]
fn test_interquartile_range_ms_empty_is_zero() {
let h = LatencyHistogram::default();
assert_eq!(h.interquartile_range_ms(), 0);
}
#[test]
fn test_interquartile_range_ms_saturates_not_panics() {
let h = LatencyHistogram::default();
for _ in 0..50 {
h.record(10);
}
for _ in 0..50 {
h.record(500);
}
let iqr = h.interquartile_range_ms();
assert!(iqr < u64::MAX);
}
#[test]
fn test_avg_steps_per_session_zero_when_no_sessions() {
let snap = MetricsSnapshot::default();
assert!((snap.avg_steps_per_session() - 0.0).abs() < 1e-9);
}
#[test]
fn test_avg_steps_per_session_correct_ratio() {
let snap = MetricsSnapshot {
total_sessions: 4,
total_steps: 20,
..Default::default()
};
assert!((snap.avg_steps_per_session() - 5.0).abs() < 1e-9);
}
#[test]
fn test_latency_histogram_is_empty_true_initially() {
let h = LatencyHistogram::default();
assert!(h.is_empty());
}
#[test]
fn test_latency_histogram_is_empty_false_after_record() {
let h = LatencyHistogram::default();
h.record(10);
assert!(!h.is_empty());
}
#[test]
fn test_checkpoint_error_rate_zero_when_no_sessions() {
let m = RuntimeMetrics::new();
assert!((m.checkpoint_error_rate() - 0.0).abs() < 1e-9);
}
#[test]
fn test_checkpoint_error_rate_ratio_correct() {
let m = RuntimeMetrics::new();
m.total_sessions.fetch_add(4, std::sync::atomic::Ordering::Relaxed);
m.checkpoint_errors.fetch_add(2, std::sync::atomic::Ordering::Relaxed);
assert!((m.checkpoint_error_rate() - 0.5).abs() < 1e-9);
}
#[test]
fn test_mode_bucket_ms_none_when_empty() {
let h = LatencyHistogram::default();
assert!(h.mode_bucket_ms().is_none());
}
#[test]
fn test_mode_bucket_ms_returns_bucket_with_most_samples() {
let h = LatencyHistogram::default();
for _ in 0..10 {
h.record(5);
}
for _ in 0..2 {
h.record(400);
}
let mode = h.mode_bucket_ms().unwrap();
assert!(mode <= 50, "expected low-latency bucket, got {mode}");
}
#[test]
fn test_metrics_snapshot_error_rate_zero_when_no_tool_calls() {
let snap = MetricsSnapshot::default();
assert!((snap.error_rate() - 0.0).abs() < 1e-9);
}
#[test]
fn test_metrics_snapshot_error_rate_correct_ratio() {
let snap = MetricsSnapshot {
total_tool_calls: 10,
failed_tool_calls: 3,
..Default::default()
};
assert!((snap.error_rate() - 0.3).abs() < 1e-9);
}
#[test]
fn test_metrics_snapshot_memory_recall_rate_zero_when_no_sessions() {
let snap = MetricsSnapshot::default();
assert!((snap.memory_recall_rate() - 0.0).abs() < 1e-9);
}
#[test]
fn test_metrics_snapshot_memory_recall_rate_correct_ratio() {
let snap = MetricsSnapshot {
total_sessions: 5,
memory_recall_count: 15,
..Default::default()
};
assert!((snap.memory_recall_rate() - 3.0).abs() < 1e-9);
}
#[test]
fn test_latency_histogram_p10_zero_when_empty() {
let h = LatencyHistogram::default();
assert_eq!(h.p10(), 0);
}
#[test]
fn test_latency_histogram_p10_lte_p50_lte_p99() {
let h = LatencyHistogram::default();
for ms in [10, 20, 50, 100, 200, 500, 1000] {
h.record(ms);
}
assert!(h.p10() <= h.p50());
assert!(h.p50() <= h.p99());
}
#[test]
fn test_latency_histogram_is_below_p99_true_when_empty() {
let h = LatencyHistogram::default();
assert!(h.is_below_p99(1)); }
#[test]
fn test_latency_histogram_is_below_p99_true_when_under_threshold() {
let h = LatencyHistogram::default();
for _ in 0..100 {
h.record(50);
}
assert!(h.is_below_p99(100));
}
#[test]
fn test_latency_histogram_is_below_p99_false_when_at_threshold() {
let h = LatencyHistogram::default();
for _ in 0..100 {
h.record(200);
}
assert!(!h.is_below_p99(200)); }
#[test]
fn test_metrics_snapshot_is_healthy_true_when_default() {
let snap = MetricsSnapshot::default();
assert!(snap.is_healthy());
}
#[test]
fn test_metrics_snapshot_is_healthy_false_when_failed_tool_calls() {
let snap = MetricsSnapshot { failed_tool_calls: 1, ..Default::default() };
assert!(!snap.is_healthy());
}
#[test]
fn test_metrics_snapshot_is_healthy_false_when_backpressure_shed() {
let snap = MetricsSnapshot { backpressure_shed_count: 2, ..Default::default() };
assert!(!snap.is_healthy());
}
#[test]
fn test_metrics_snapshot_is_healthy_false_when_checkpoint_errors() {
let snap = MetricsSnapshot { checkpoint_errors: 1, ..Default::default() };
assert!(!snap.is_healthy());
}
#[test]
fn test_latency_histogram_median_ms_equals_p50() {
let h = LatencyHistogram::default();
for ms in [10, 50, 100, 200, 500] {
h.record(ms);
}
assert_eq!(h.median_ms(), h.p50());
}
#[test]
fn test_latency_histogram_median_ms_zero_when_empty() {
let h = LatencyHistogram::default();
assert_eq!(h.median_ms(), 0);
}
#[test]
fn test_metrics_snapshot_steps_per_session_zero_when_no_sessions() {
let snap = MetricsSnapshot::default();
assert!((snap.steps_per_session() - 0.0).abs() < 1e-9);
}
#[test]
fn test_metrics_snapshot_steps_per_session_correct_ratio() {
let snap = MetricsSnapshot {
total_sessions: 4,
total_steps: 20,
..Default::default()
};
assert!((snap.steps_per_session() - 5.0).abs() < 1e-9);
}
#[test]
fn test_runtime_metrics_p50_latency_ms_zero_when_no_data() {
let m = RuntimeMetrics::new();
assert_eq!(m.p50_latency_ms(), 0);
}
#[test]
fn test_runtime_metrics_p50_latency_ms_matches_histogram_p50() {
let m = RuntimeMetrics::new();
for ms in [10_u64, 50, 100, 200, 500] {
m.step_latency.record(ms);
}
assert_eq!(m.p50_latency_ms(), m.step_latency.p50());
}
#[test]
fn test_latency_histogram_has_data_false_when_empty() {
let h = LatencyHistogram::default();
assert!(!h.has_data());
}
#[test]
fn test_latency_histogram_has_data_true_after_record() {
let h = LatencyHistogram::default();
h.record(100);
assert!(h.has_data());
}
#[test]
fn test_latency_histogram_min_ms_none_when_empty() {
let h = LatencyHistogram::default();
assert_eq!(h.min_ms(), None);
}
#[test]
fn test_latency_histogram_min_ms_some_after_record() {
let h = LatencyHistogram::default();
h.record(50);
assert!(h.min_ms().is_some());
}
#[test]
fn test_latency_histogram_p25_lte_p75() {
let h = LatencyHistogram::default();
for ms in [10_u64, 50, 100, 200, 500, 1000, 2000, 5000] {
h.record(ms);
}
assert!(h.p25() <= h.p75());
}
#[test]
fn test_latency_histogram_p90_between_p50_and_p99() {
let h = LatencyHistogram::default();
for ms in [10_u64, 50, 100, 200, 500] {
h.record(ms);
}
assert!(h.p50() <= h.p90());
assert!(h.p90() <= h.p99());
}
#[test]
fn test_metrics_snapshot_tool_success_count_correct() {
let snap = MetricsSnapshot {
per_tool_calls: [("search".to_string(), 10u64)].into(),
per_tool_failures: [("search".to_string(), 3u64)].into(),
..Default::default()
};
assert_eq!(snap.tool_success_count("search"), 7);
}
#[test]
fn test_metrics_snapshot_tool_success_count_zero_for_unknown_tool() {
let snap = MetricsSnapshot::default();
assert_eq!(snap.tool_success_count("unknown"), 0);
}
#[test]
fn test_metrics_snapshot_tool_failure_rate_correct_ratio() {
let snap = MetricsSnapshot {
per_tool_calls: [("lookup".to_string(), 4u64)].into(),
per_tool_failures: [("lookup".to_string(), 1u64)].into(),
..Default::default()
};
assert!((snap.tool_failure_rate("lookup") - 0.25).abs() < 1e-9);
}
#[test]
fn test_metrics_snapshot_tool_failure_rate_zero_for_unknown_tool() {
let snap = MetricsSnapshot::default();
assert!((snap.tool_failure_rate("none") - 0.0).abs() < 1e-9);
}
#[test]
fn test_metrics_snapshot_total_successful_tool_calls() {
let snap = MetricsSnapshot {
total_tool_calls: 20,
failed_tool_calls: 5,
..Default::default()
};
assert_eq!(snap.total_successful_tool_calls(), 15);
}
#[test]
fn test_runtime_metrics_per_tool_calls_snapshot_increments() {
let m = RuntimeMetrics::new();
m.record_tool_call("search");
m.record_tool_call("search");
m.record_tool_call("lookup");
let snap = m.per_tool_calls_snapshot();
assert_eq!(snap.get("search"), Some(&2));
assert_eq!(snap.get("lookup"), Some(&1));
}
#[test]
fn test_runtime_metrics_per_tool_failures_snapshot() {
let m = RuntimeMetrics::new();
m.record_tool_call("search");
m.record_tool_failure("search");
let snap = m.per_tool_failures_snapshot();
assert_eq!(snap.get("search"), Some(&1));
}
#[test]
fn test_runtime_metrics_record_agent_tool_call_tracked() {
let m = RuntimeMetrics::new();
m.record_agent_tool_call("agent-1", "search");
m.record_agent_tool_call("agent-1", "search");
let snap = m.per_agent_tool_calls_snapshot();
assert_eq!(snap.get("agent-1").and_then(|t| t.get("search")), Some(&2));
}
#[test]
fn test_runtime_metrics_per_agent_tool_failures_snapshot() {
let m = RuntimeMetrics::new();
m.record_agent_tool_failure("agent-2", "lookup");
let snap = m.per_agent_tool_failures_snapshot();
assert_eq!(
snap.get("agent-2").and_then(|t| t.get("lookup")),
Some(&1)
);
}
#[test]
fn test_coefficient_of_variation_zero_when_empty() {
let h = LatencyHistogram::default();
assert!((h.coefficient_of_variation() - 0.0).abs() < 1e-9);
}
#[test]
fn test_coefficient_of_variation_positive_with_spread() {
let h = LatencyHistogram::default();
for _ in 0..50 {
h.record(10);
}
for _ in 0..50 {
h.record(1000);
}
let cv = h.coefficient_of_variation();
assert!(cv > 0.0, "CV should be positive for spread data, got {cv}");
}
#[test]
fn test_coefficient_of_variation_near_zero_for_uniform_data() {
let h = LatencyHistogram::default();
for _ in 0..100 {
h.record(50);
}
assert!(h.coefficient_of_variation() < 1.0);
}
#[test]
fn test_latency_histogram_percentile_zero_when_empty() {
let h = LatencyHistogram::default();
assert_eq!(h.percentile(0.5), 0);
}
#[test]
fn test_latency_histogram_percentile_50_matches_p50() {
let h = LatencyHistogram::default();
for ms in [10, 20, 30, 40, 50] {
h.record(ms);
}
assert_eq!(h.percentile(0.5), h.p50());
}
#[test]
fn test_latency_histogram_percentile_99_matches_p99() {
let h = LatencyHistogram::default();
for ms in [10, 50, 100, 500, 1000] {
h.record(ms);
}
assert_eq!(h.percentile(0.99), h.p99());
}
#[test]
fn test_runtime_metrics_record_agent_tool_failure_appears_in_snapshot() {
let m = RuntimeMetrics::new();
m.record_agent_tool_failure("agent-1", "search_tool");
let snapshot = m.per_agent_tool_failures_snapshot();
assert_eq!(snapshot.get("agent-1").and_then(|t| t.get("search_tool")), Some(&1));
}
#[test]
fn test_runtime_metrics_per_agent_tool_calls_snapshot_empty_initially() {
let m = RuntimeMetrics::new();
assert!(m.per_agent_tool_calls_snapshot().is_empty());
}
#[test]
fn test_runtime_metrics_record_step_latency_is_reflected_in_p50() {
let m = RuntimeMetrics::new();
for _ in 0..20 {
m.record_step_latency(100);
}
let snap = m.snapshot();
assert!(snap.total_sessions == 0); }
#[test]
fn test_metrics_snapshot_has_errors_false_when_clean() {
let snap = MetricsSnapshot::default();
assert!(!snap.has_errors());
}
#[test]
fn test_metrics_snapshot_has_errors_true_when_failed_tool_calls() {
let snap = MetricsSnapshot { failed_tool_calls: 2, ..Default::default() };
assert!(snap.has_errors());
}
#[test]
fn test_metrics_snapshot_has_errors_true_when_checkpoint_errors() {
let snap = MetricsSnapshot { checkpoint_errors: 1, ..Default::default() };
assert!(snap.has_errors());
}
#[test]
fn test_latency_histogram_is_above_p99_false_for_low_latency() {
let h = LatencyHistogram::default();
for _ in 0..200 {
h.record(50);
}
assert!(!h.is_above_p99(50));
}
#[test]
fn test_latency_histogram_is_above_p99_true_for_high_latency() {
let h = LatencyHistogram::default();
for _ in 0..200 {
h.record(50);
}
assert!(h.is_above_p99(10_000));
}
#[test]
fn test_latency_histogram_sample_count_zero_when_empty() {
let h = LatencyHistogram::default();
assert_eq!(h.sample_count(), 0);
}
#[test]
fn test_latency_histogram_sample_count_matches_records() {
let h = LatencyHistogram::default();
for _ in 0..7 {
h.record(100);
}
assert_eq!(h.sample_count(), 7);
}
#[test]
fn test_metrics_snapshot_tool_call_rate_zero_when_no_sessions() {
let snap = MetricsSnapshot::default();
assert!((snap.tool_call_rate() - 0.0).abs() < 1e-9);
}
#[test]
fn test_metrics_snapshot_tool_call_rate_correct_ratio() {
let snap = MetricsSnapshot {
total_sessions: 4,
total_tool_calls: 20,
..Default::default()
};
assert!((snap.tool_call_rate() - 5.0).abs() < 1e-9);
}
#[test]
fn test_metrics_snapshot_backpressure_rate_zero_when_no_sessions() {
let snap = MetricsSnapshot::default();
assert!((snap.backpressure_rate() - 0.0).abs() < 1e-9);
}
#[test]
fn test_metrics_snapshot_backpressure_rate_correct_ratio() {
let snap = MetricsSnapshot {
total_sessions: 2,
backpressure_shed_count: 4,
..Default::default()
};
assert!((snap.backpressure_rate() - 2.0).abs() < 1e-9);
}
#[test]
fn test_latency_histogram_percentile_spread_zero_when_empty() {
let h = LatencyHistogram::default();
assert_eq!(h.percentile_spread(), 0);
}
#[test]
fn test_latency_histogram_percentile_spread_nonnegative() {
let h = LatencyHistogram::default();
for _ in 0..100 {
h.record(50);
}
for _ in 0..5 {
h.record(500);
}
assert!(h.percentile_spread() >= 0);
}
#[test]
fn test_metrics_snapshot_memory_efficiency_zero_when_no_steps() {
let snap = MetricsSnapshot::default();
assert!((snap.memory_efficiency() - 0.0).abs() < 1e-9);
}
#[test]
fn test_metrics_snapshot_memory_efficiency_correct_ratio() {
let snap = MetricsSnapshot {
total_steps: 10,
memory_recall_count: 4,
..Default::default()
};
assert!((snap.memory_efficiency() - 0.4).abs() < 1e-9);
}
#[test]
fn test_latency_histogram_is_uniform_true_when_empty() {
let h = LatencyHistogram::default();
assert!(h.is_uniform());
}
#[test]
fn test_latency_histogram_is_uniform_true_for_single_bucket() {
let h = LatencyHistogram::default();
for _ in 0..50 {
h.record(50); }
assert!(h.is_uniform());
}
#[test]
fn test_latency_histogram_is_uniform_false_for_mixed_latencies() {
let h = LatencyHistogram::default();
h.record(1);
h.record(1000);
assert!(!h.is_uniform());
}
#[test]
fn test_latency_histogram_bucket_counts_all_zero_when_empty() {
let h = LatencyHistogram::default();
assert_eq!(h.bucket_counts(), [0u64; 7]);
}
#[test]
fn test_latency_histogram_bucket_counts_increments_correct_bucket() {
let h = LatencyHistogram::default();
h.record(1); let counts = h.bucket_counts();
assert_eq!(counts[0], 1);
assert!(counts[1..].iter().all(|&c| c == 0));
}
#[test]
fn test_metrics_snapshot_active_session_ratio_zero_when_no_sessions() {
let snap = MetricsSnapshot::default();
assert!((snap.active_session_ratio() - 0.0).abs() < 1e-9);
}
#[test]
fn test_metrics_snapshot_active_session_ratio_correct() {
let snap = MetricsSnapshot {
total_sessions: 10,
active_sessions: 3,
..Default::default()
};
assert!((snap.active_session_ratio() - 0.3).abs() < 1e-9);
}
#[test]
fn test_step_to_tool_ratio_correct_value() {
let snap = MetricsSnapshot {
total_steps: 4,
total_tool_calls: 2,
..Default::default()
};
assert!((snap.step_to_tool_ratio() - 0.5).abs() < 1e-9);
}
#[test]
fn test_step_to_tool_ratio_zero_steps_returns_zero() {
let snap = MetricsSnapshot {
total_steps: 0,
total_tool_calls: 5,
..Default::default()
};
assert_eq!(snap.step_to_tool_ratio(), 0.0);
}
#[test]
fn test_latency_histogram_min_occupied_ms_returns_smallest_occupied_bucket() {
let h = LatencyHistogram::default();
h.record(10); h.record(200); assert_eq!(h.min_occupied_ms(), Some(10));
}
#[test]
fn test_latency_histogram_min_occupied_ms_empty_returns_none() {
let h = LatencyHistogram::default();
assert_eq!(h.min_occupied_ms(), None);
}
#[test]
fn test_metrics_snapshot_has_failures_true_when_failures_exist() {
let snap = MetricsSnapshot {
failed_tool_calls: 1,
..Default::default()
};
assert!(snap.has_failures());
}
#[test]
fn test_metrics_snapshot_has_failures_false_when_no_failures() {
let snap = MetricsSnapshot::default();
assert!(!snap.has_failures());
}
#[test]
fn test_latency_histogram_max_occupied_ms_returns_largest_occupied_bucket() {
let h = LatencyHistogram::default();
h.record(5); h.record(200); assert_eq!(h.max_occupied_ms(), Some(500));
}
#[test]
fn test_latency_histogram_max_occupied_ms_empty_returns_none() {
let h = LatencyHistogram::default();
assert_eq!(h.max_occupied_ms(), None);
}
#[test]
fn test_latency_histogram_occupied_bucket_count_correct() {
let h = LatencyHistogram::default();
h.record(5); h.record(200); assert_eq!(h.occupied_bucket_count(), 2);
}
#[test]
fn test_latency_histogram_occupied_bucket_count_empty_returns_zero() {
let h = LatencyHistogram::default();
assert_eq!(h.occupied_bucket_count(), 0);
}
#[test]
fn test_metrics_snapshot_tool_diversity_counts_distinct_tools() {
let snap = MetricsSnapshot {
per_tool_calls: [("a".to_string(), 1u64), ("b".to_string(), 2u64)]
.into_iter()
.collect(),
..Default::default()
};
assert_eq!(snap.tool_diversity(), 2);
}
#[test]
fn test_metrics_snapshot_tool_diversity_empty_returns_zero() {
let snap = MetricsSnapshot::default();
assert_eq!(snap.tool_diversity(), 0);
}
#[test]
fn test_runtime_metrics_total_step_latency_ms_sums_recorded_latencies() {
let m = RuntimeMetrics::new();
m.record_step_latency(100);
m.record_step_latency(200);
assert_eq!(m.total_step_latency_ms(), 300);
}
#[test]
fn test_runtime_metrics_total_step_latency_ms_zero_when_empty() {
let m = RuntimeMetrics::new();
assert_eq!(m.total_step_latency_ms(), 0);
}
#[test]
fn test_metrics_snapshot_avg_failures_per_session_correct() {
let snap = MetricsSnapshot {
total_sessions: 4,
failed_tool_calls: 2,
..Default::default()
};
assert!((snap.avg_failures_per_session() - 0.5).abs() < 1e-9);
}
#[test]
fn test_metrics_snapshot_avg_failures_per_session_zero_when_no_sessions() {
let snap = MetricsSnapshot::default();
assert_eq!(snap.avg_failures_per_session(), 0.0);
}
#[test]
fn test_latency_histogram_is_skewed_true_when_p99_much_greater_than_p50() {
let h = LatencyHistogram::default();
for _ in 0..100 {
h.record(1); }
h.record(500); let _ = h.is_skewed();
}
#[test]
fn test_latency_histogram_is_skewed_false_when_empty() {
let h = LatencyHistogram::default();
assert!(!h.is_skewed());
}
#[test]
fn test_most_called_tool_returns_tool_with_most_calls() {
let snap = MetricsSnapshot {
per_tool_calls: [
("search".to_string(), 5u64),
("write".to_string(), 2u64),
]
.into_iter()
.collect(),
..Default::default()
};
assert_eq!(snap.most_called_tool(), Some("search".to_string()));
}
#[test]
fn test_most_called_tool_returns_none_when_empty() {
let snap = MetricsSnapshot::default();
assert!(snap.most_called_tool().is_none());
}
#[test]
fn test_tool_names_with_failures_returns_sorted_names_with_failures() {
let snap = MetricsSnapshot {
per_tool_failures: [
("search".to_string(), 3u64),
("write".to_string(), 0u64),
("calc".to_string(), 1u64),
]
.into_iter()
.collect(),
..Default::default()
};
assert_eq!(snap.tool_names_with_failures(), vec!["calc", "search"]);
}
#[test]
fn test_tool_names_with_failures_empty_when_no_failures() {
let snap = MetricsSnapshot::default();
assert!(snap.tool_names_with_failures().is_empty());
}
#[test]
fn test_agent_with_most_calls_returns_highest_total() {
let snap = MetricsSnapshot {
per_agent_tool_calls: [
("agent_a".to_string(), [("search".to_string(), 3u64), ("write".to_string(), 2u64)].into_iter().collect()),
("agent_b".to_string(), [("search".to_string(), 1u64)].into_iter().collect()),
]
.into_iter()
.collect(),
..Default::default()
};
assert_eq!(snap.agent_with_most_calls(), Some("agent_a".to_string()));
}
#[test]
fn test_agent_with_most_calls_returns_none_when_empty() {
let snap = MetricsSnapshot::default();
assert!(snap.agent_with_most_calls().is_none());
}
#[test]
fn test_total_agent_count_returns_number_of_distinct_agents() {
let snap = MetricsSnapshot {
per_agent_tool_calls: [
("a".to_string(), std::collections::HashMap::new()),
("b".to_string(), std::collections::HashMap::new()),
]
.into_iter()
.collect(),
..Default::default()
};
assert_eq!(snap.total_agent_count(), 2);
}
#[test]
fn test_total_agent_count_zero_when_empty() {
let snap = MetricsSnapshot::default();
assert_eq!(snap.total_agent_count(), 0);
}
#[test]
fn test_steps_per_tool_call_returns_ratio() {
let snap = MetricsSnapshot {
total_steps: 10,
total_tool_calls: 5,
..Default::default()
};
assert!((snap.steps_per_tool_call() - 2.0).abs() < 1e-9);
}
#[test]
fn test_steps_per_tool_call_zero_when_no_tool_calls() {
let snap = MetricsSnapshot::default();
assert_eq!(snap.steps_per_tool_call(), 0.0);
}
#[test]
fn test_failed_tool_ratio_for_returns_failure_rate() {
let snap = MetricsSnapshot {
per_tool_calls: [("tool".to_string(), 10u64)].into_iter().collect(),
per_tool_failures: [("tool".to_string(), 2u64)].into_iter().collect(),
..Default::default()
};
assert!((snap.failed_tool_ratio_for("tool") - 0.2).abs() < 1e-9);
}
#[test]
fn test_failed_tool_ratio_for_zero_when_no_calls() {
let snap = MetricsSnapshot::default();
assert_eq!(snap.failed_tool_ratio_for("missing"), 0.0);
}
#[test]
fn test_backpressure_shed_rate_returns_ratio() {
let snap = MetricsSnapshot {
total_tool_calls: 100,
backpressure_shed_count: 5,
..Default::default()
};
assert!((snap.backpressure_shed_rate() - 0.05).abs() < 1e-9);
}
#[test]
fn test_backpressure_shed_rate_zero_when_no_tool_calls() {
let snap = MetricsSnapshot::default();
assert_eq!(snap.backpressure_shed_rate(), 0.0);
}
#[test]
fn test_step_latency_p95_zero_when_empty() {
let m = RuntimeMetrics::new();
assert_eq!(m.step_latency_p95(), 0);
}
#[test]
fn test_step_latency_p75_zero_when_empty() {
let m = RuntimeMetrics::new();
assert_eq!(m.step_latency_p75(), 0);
}
#[test]
fn test_step_latency_p95_gte_p75_after_recording() {
let m = RuntimeMetrics::new();
for ms in [1, 5, 10, 50, 100, 500, 1000] {
m.record_step_latency(ms);
}
assert!(m.step_latency_p95() >= m.step_latency_p75());
}
#[test]
fn test_step_latency_p99_gte_p95_after_recording() {
let m = RuntimeMetrics::new();
for ms in [1, 5, 10, 50, 100, 500, 1000] {
m.record_step_latency(ms);
}
assert!(m.step_latency_p99() >= m.step_latency_p95());
}
#[test]
fn test_snapshot_is_empty_true_for_fresh_snapshot() {
let m = RuntimeMetrics::new();
let snap = m.snapshot();
assert!(snap.is_empty());
}
#[test]
fn test_snapshot_is_healthy_with_latency_true_when_below_threshold() {
let m = RuntimeMetrics::new();
let snap = m.snapshot();
assert!(snap.is_healthy_with_latency(1000.0));
}
#[test]
fn test_snapshot_is_healthy_with_latency_false_when_has_failures() {
let m = RuntimeMetrics::new();
m.record_tool_failure("search");
let snap = m.snapshot();
assert!(!snap.is_healthy_with_latency(9999.0));
}
#[test]
fn test_snapshot_is_empty_false_after_recording_step() {
let m = RuntimeMetrics::new();
m.record_step_latency(5);
let _ = m.snapshot().is_empty();
}
#[test]
fn test_step_latency_std_dev_ms_zero_when_empty() {
let m = RuntimeMetrics::new();
assert_eq!(m.step_latency_std_dev_ms(), 0.0);
}
#[test]
fn test_step_latency_std_dev_ms_positive_after_diverse_recording() {
let m = RuntimeMetrics::new();
m.record_step_latency(1);
m.record_step_latency(1000);
assert!(m.step_latency_std_dev_ms() > 0.0);
}
#[test]
fn test_most_used_tool_returns_tool_with_most_calls() {
let m = RuntimeMetrics::new();
m.record_tool_call("search");
m.record_tool_call("search");
m.record_tool_call("lookup");
assert_eq!(m.most_used_tool(), Some("search".to_string()));
}
#[test]
fn test_most_used_tool_returns_none_when_no_calls() {
let m = RuntimeMetrics::new();
assert_eq!(m.most_used_tool(), None);
}
#[test]
fn test_tool_call_to_failure_ratio_zero_when_no_calls() {
let m = RuntimeMetrics::new();
assert_eq!(m.tool_call_to_failure_ratio(), 0.0);
}
#[test]
fn test_tool_call_to_failure_ratio_computed_correctly() {
let m = RuntimeMetrics::new();
m.record_tool_call("t");
m.record_tool_call("t");
m.record_tool_failure("t");
assert!((m.tool_call_to_failure_ratio() - 0.5).abs() < 1e-9);
}
#[test]
fn test_metrics_snapshot_total_tool_failures_sums_all_failures() {
let snap = MetricsSnapshot {
per_tool_failures: [
("search".to_string(), 3u64),
("write".to_string(), 2u64),
].into_iter().collect(),
..Default::default()
};
assert_eq!(snap.total_tool_failures(), 5);
}
#[test]
fn test_metrics_snapshot_total_tool_failures_zero_when_empty() {
let snap = MetricsSnapshot::default();
assert_eq!(snap.total_tool_failures(), 0);
}
#[test]
fn test_metrics_snapshot_least_called_tool_returns_tool_with_fewest_calls() {
let snap = MetricsSnapshot {
per_tool_calls: [
("search".to_string(), 10u64),
("lookup".to_string(), 2u64),
("write".to_string(), 5u64),
].into_iter().collect(),
..Default::default()
};
assert_eq!(snap.least_called_tool(), Some("lookup".to_string()));
}
#[test]
fn test_metrics_snapshot_least_called_tool_returns_none_when_empty() {
let snap = MetricsSnapshot::default();
assert_eq!(snap.least_called_tool(), None);
}
#[test]
fn test_metrics_snapshot_summary_line_format() {
let m = RuntimeMetrics::new();
let snap = m.snapshot();
let line = snap.summary_line();
assert!(line.contains("sessions="));
assert!(line.contains("steps="));
assert!(line.contains("tool_calls="));
assert!(line.contains("failures="));
assert!(line.contains("latency_mean="));
}
#[test]
fn test_metrics_snapshot_summary_line_reflects_zero_values() {
let snap = MetricsSnapshot::default();
let line = snap.summary_line();
assert!(line.contains("sessions=0"));
assert!(line.contains("failures=0"));
}
#[test]
fn test_active_session_rate_zero_when_no_sessions() {
let m = RuntimeMetrics::new();
assert_eq!(m.active_session_rate(), 0.0);
}
#[test]
fn test_active_session_rate_one_when_all_sessions_active() {
let m = RuntimeMetrics::new();
m.active_sessions.fetch_add(2, Ordering::Relaxed);
m.total_sessions.fetch_add(2, Ordering::Relaxed);
assert!((m.active_session_rate() - 1.0).abs() < 1e-9);
}
#[test]
fn test_avg_tool_calls_per_name_computed_correctly() {
let snap = MetricsSnapshot {
per_tool_calls: [
("search".to_string(), 6u64),
("write".to_string(), 4u64),
].into_iter().collect(),
..Default::default()
};
assert!((snap.avg_tool_calls_per_name() - 5.0).abs() < 1e-9);
}
#[test]
fn test_avg_tool_calls_per_name_zero_when_no_tools() {
let snap = MetricsSnapshot::default();
assert_eq!(snap.avg_tool_calls_per_name(), 0.0);
}
#[test]
fn test_tool_call_count_above_counts_tools_exceeding_threshold() {
let snap = MetricsSnapshot {
per_tool_calls: [
("search".to_string(), 10u64),
("write".to_string(), 2u64),
("read".to_string(), 5u64),
].into_iter().collect(),
..Default::default()
};
assert_eq!(snap.tool_call_count_above(4), 2); }
#[test]
fn test_tool_call_count_above_returns_zero_when_none_exceed() {
let snap = MetricsSnapshot {
per_tool_calls: [("t".to_string(), 3u64)].into_iter().collect(),
..Default::default()
};
assert_eq!(snap.tool_call_count_above(10), 0);
}
#[test]
fn test_tool_call_count_above_zero_for_empty_snapshot() {
let snap = MetricsSnapshot::default();
assert_eq!(snap.tool_call_count_above(0), 0);
}
#[test]
fn test_memory_recall_per_session_returns_ratio() {
use std::sync::atomic::Ordering;
let m = RuntimeMetrics::default();
m.total_sessions.store(4, Ordering::Relaxed);
m.memory_recall_count.store(8, Ordering::Relaxed);
assert!((m.memory_recall_per_session() - 2.0).abs() < 1e-9);
}
#[test]
fn test_memory_recall_per_session_zero_when_no_sessions() {
let m = RuntimeMetrics::default();
assert_eq!(m.memory_recall_per_session(), 0.0);
}
#[test]
fn test_tool_call_ratio_returns_fraction_for_named_tool() {
let snap = MetricsSnapshot {
total_tool_calls: 10,
per_tool_calls: [
("search".to_string(), 4u64),
("write".to_string(), 6u64),
].into_iter().collect(),
..Default::default()
};
assert!((snap.tool_call_ratio("search") - 0.4).abs() < 1e-9);
assert!((snap.tool_call_ratio("write") - 0.6).abs() < 1e-9);
}
#[test]
fn test_tool_call_ratio_returns_zero_for_unknown_tool() {
let snap = MetricsSnapshot {
total_tool_calls: 5,
per_tool_calls: [("a".to_string(), 5u64)].into_iter().collect(),
..Default::default()
};
assert_eq!(snap.tool_call_ratio("unknown"), 0.0);
}
#[test]
fn test_tool_call_ratio_returns_zero_when_no_calls_recorded() {
let snap = MetricsSnapshot::default();
assert_eq!(snap.tool_call_ratio("any"), 0.0);
}
#[test]
fn test_top_n_tools_by_calls_returns_n_descending() {
let snap = MetricsSnapshot {
per_tool_calls: [
("a".to_string(), 10),
("b".to_string(), 5),
("c".to_string(), 20),
]
.into_iter()
.collect(),
..Default::default()
};
let top = snap.top_n_tools_by_calls(2);
assert_eq!(top.len(), 2);
assert_eq!(top[0], ("c", 20));
assert_eq!(top[1], ("a", 10));
}
#[test]
fn test_top_n_tools_by_calls_empty_for_empty_snapshot() {
let snap = MetricsSnapshot::default();
assert!(snap.top_n_tools_by_calls(5).is_empty());
}
#[test]
fn test_top_n_tools_by_calls_returns_all_when_n_exceeds_count() {
let snap = MetricsSnapshot {
per_tool_calls: [("only".to_string(), 3)].into_iter().collect(),
..Default::default()
};
assert_eq!(snap.top_n_tools_by_calls(100).len(), 1);
}
#[test]
fn test_step_error_rate_returns_ratio() {
use std::sync::atomic::Ordering;
let m = RuntimeMetrics::default();
m.total_steps.store(10, Ordering::Relaxed);
m.failed_tool_calls.store(2, Ordering::Relaxed);
assert!((m.step_error_rate() - 0.2).abs() < 1e-9);
}
#[test]
fn test_step_error_rate_zero_when_no_steps() {
let m = RuntimeMetrics::default();
assert_eq!(m.step_error_rate(), 0.0);
}
#[test]
fn test_is_degraded_true_when_failure_rate_exceeds_threshold() {
let snap = MetricsSnapshot {
total_tool_calls: 10,
failed_tool_calls: 3,
..Default::default()
};
assert!(snap.is_degraded(0.2)); }
#[test]
fn test_is_degraded_false_when_failure_rate_at_or_below_threshold() {
let snap = MetricsSnapshot {
total_tool_calls: 10,
failed_tool_calls: 2,
..Default::default()
};
assert!(!snap.is_degraded(0.2)); }
#[test]
fn test_is_degraded_false_for_zero_failures() {
let snap = MetricsSnapshot {
total_tool_calls: 5,
failed_tool_calls: 0,
..Default::default()
};
assert!(!snap.is_degraded(0.05));
}
#[test]
fn test_is_degraded_false_for_empty_snapshot() {
let snap = MetricsSnapshot::default();
assert!(!snap.is_degraded(0.1));
}
#[test]
fn test_total_errors_sums_failed_tool_calls_and_checkpoint_errors() {
use std::sync::atomic::Ordering;
let m = RuntimeMetrics::default();
m.failed_tool_calls.store(5, Ordering::Relaxed);
m.checkpoint_errors.store(3, Ordering::Relaxed);
assert_eq!(m.total_errors(), 8);
}
#[test]
fn test_total_errors_zero_when_no_errors() {
let m = RuntimeMetrics::default();
assert_eq!(m.total_errors(), 0);
}
#[test]
fn test_has_tool_true_for_recorded_tool() {
let snap = MetricsSnapshot {
per_tool_calls: [("my_tool".to_string(), 3)].into_iter().collect(),
..Default::default()
};
assert!(snap.has_tool("my_tool"));
}
#[test]
fn test_has_tool_false_for_unrecorded_tool() {
let snap = MetricsSnapshot::default();
assert!(!snap.has_tool("anything"));
}
#[test]
fn test_tool_call_share_returns_fraction() {
let snap = MetricsSnapshot {
total_tool_calls: 10,
per_tool_calls: [("a".to_string(), 4)].into_iter().collect(),
..Default::default()
};
assert!((snap.tool_call_share("a") - 0.4).abs() < 1e-9);
}
#[test]
fn test_tool_call_share_zero_when_no_calls() {
let snap = MetricsSnapshot::default();
assert_eq!(snap.tool_call_share("any"), 0.0);
}
#[test]
fn test_tool_names_containing_returns_matching_names() {
let m = RuntimeMetrics::default();
m.record_tool_call("search_web");
m.record_tool_call("search_db");
m.record_tool_call("write_file");
let mut names = m.tool_names_containing("search");
names.sort_unstable();
assert_eq!(names, vec!["search_db", "search_web"]);
}
#[test]
fn test_tool_names_containing_empty_when_no_match() {
let m = RuntimeMetrics::default();
m.record_tool_call("read");
assert!(m.tool_names_containing("write").is_empty());
}
#[test]
fn test_avg_memory_recalls_per_step_computes_ratio() {
use std::sync::atomic::Ordering;
let m = RuntimeMetrics::default();
m.total_steps.store(2, Ordering::Relaxed);
m.memory_recall_count.store(1, Ordering::Relaxed);
assert!((m.avg_memory_recalls_per_step() - 0.5).abs() < 1e-9);
}
#[test]
fn test_avg_memory_recalls_per_step_zero_when_no_steps() {
let m = RuntimeMetrics::default();
assert_eq!(m.avg_memory_recalls_per_step(), 0.0);
}
#[test]
fn test_avg_tool_failures_per_session_computes_ratio() {
use std::sync::atomic::Ordering;
let m = RuntimeMetrics::default();
m.total_sessions.store(4, Ordering::Relaxed);
m.failed_tool_calls.store(2, Ordering::Relaxed);
assert!((m.avg_tool_failures_per_session() - 0.5).abs() < 1e-9);
}
#[test]
fn test_avg_tool_failures_per_session_zero_when_no_sessions() {
let m = RuntimeMetrics::default();
assert_eq!(m.avg_tool_failures_per_session(), 0.0);
}
#[test]
fn test_has_any_tool_failures_false_when_no_failures() {
let m = RuntimeMetrics::new();
m.record_tool_call("search");
let snap = m.snapshot();
assert!(!snap.has_any_tool_failures());
}
#[test]
fn test_has_any_tool_failures_true_when_failure_recorded() {
let m = RuntimeMetrics::new();
m.record_tool_call("search");
m.record_tool_failure("search");
let snap = m.snapshot();
assert!(snap.has_any_tool_failures());
}
#[test]
fn test_total_tool_calls_count_sums_all_per_tool_calls() {
let m = RuntimeMetrics::new();
m.record_tool_call("search");
m.record_tool_call("search");
m.record_tool_call("lookup");
let snap = m.snapshot();
assert_eq!(snap.total_tool_calls_count(), 3);
}
#[test]
fn test_total_tool_calls_count_zero_for_no_calls() {
let m = RuntimeMetrics::new();
let snap = m.snapshot();
assert_eq!(snap.total_tool_calls_count(), 0);
}
#[test]
fn test_tool_call_imbalance_one_for_single_tool() {
let m = RuntimeMetrics::new();
m.record_tool_call("search");
m.record_tool_call("search");
let snap = m.snapshot();
assert!((snap.tool_call_imbalance() - 1.0).abs() < 1e-9);
}
#[test]
fn test_tool_call_imbalance_computes_max_over_min() {
let m = RuntimeMetrics::new();
m.record_tool_call("a");
m.record_tool_call("a");
m.record_tool_call("a");
m.record_tool_call("b");
let snap = m.snapshot();
assert!((snap.tool_call_imbalance() - 3.0).abs() < 1e-9);
}
#[test]
fn test_tool_call_imbalance_one_for_empty_snapshot() {
let m = RuntimeMetrics::new();
let snap = m.snapshot();
assert!((snap.tool_call_imbalance() - 1.0).abs() < 1e-9);
}
#[test]
fn test_has_failed_tools_true_when_failure_recorded() {
let m = RuntimeMetrics::new();
m.record_tool_failure("search");
assert!(m.has_failed_tools());
}
#[test]
fn test_has_failed_tools_false_when_no_failures() {
let m = RuntimeMetrics::new();
m.record_tool_call("search");
assert!(!m.has_failed_tools());
}
#[test]
fn test_distinct_tool_count_reflects_unique_tools() {
let snap = MetricsSnapshot {
per_tool_calls: [
("tool_a".to_string(), 3),
("tool_b".to_string(), 1),
]
.into_iter()
.collect(),
..Default::default()
};
assert_eq!(snap.distinct_tool_count(), 2);
}
#[test]
fn test_distinct_tool_count_zero_for_empty_snapshot() {
let snap = MetricsSnapshot::default();
assert_eq!(snap.distinct_tool_count(), 0);
}
#[test]
fn test_tools_with_zero_failures_returns_tools_without_failures() {
let m = RuntimeMetrics::new();
m.record_tool_call("search");
m.record_tool_call("lookup");
m.record_tool_failure("search");
let snap = m.snapshot();
let zero_fail = snap.tools_with_zero_failures();
assert_eq!(zero_fail, vec!["lookup"]);
}
#[test]
fn test_tools_with_zero_failures_empty_when_all_have_failures() {
let m = RuntimeMetrics::new();
m.record_tool_call("a");
m.record_tool_failure("a");
let snap = m.snapshot();
assert!(snap.tools_with_zero_failures().is_empty());
}
#[test]
fn test_tool_names_by_call_count_orders_highest_first() {
let m = RuntimeMetrics::new();
m.record_tool_call("alpha");
m.record_tool_call("beta");
m.record_tool_call("beta");
m.record_tool_call("gamma");
m.record_tool_call("gamma");
m.record_tool_call("gamma");
let names = m.tool_names_by_call_count();
assert_eq!(names[0], "gamma");
assert_eq!(names[1], "beta");
assert_eq!(names[2], "alpha");
}
#[test]
fn test_tool_names_by_call_count_empty_when_no_calls() {
let m = RuntimeMetrics::new();
assert!(m.tool_names_by_call_count().is_empty());
}
#[test]
fn test_has_any_tool_calls_false_when_no_calls() {
let m = RuntimeMetrics::new();
assert!(!m.snapshot().has_any_tool_calls());
}
#[test]
fn test_has_any_tool_calls_true_after_recording() {
let m = RuntimeMetrics::new();
m.record_tool_call("search");
assert!(m.snapshot().has_any_tool_calls());
}
#[test]
fn test_tool_names_alphabetical_sorted() {
let m = RuntimeMetrics::new();
m.record_tool_call("zebra");
m.record_tool_call("alpha");
m.record_tool_call("mango");
let names = m.snapshot().tool_names_alphabetical();
assert_eq!(names, vec!["alpha", "mango", "zebra"]);
}
#[test]
fn test_tool_names_alphabetical_empty_when_no_calls() {
let m = RuntimeMetrics::new();
assert!(m.snapshot().tool_names_alphabetical().is_empty());
}
#[test]
fn test_tool_calls_per_memory_recall_returns_ratio() {
let m = RuntimeMetrics::new();
m.memory_recall_count.store(2, std::sync::atomic::Ordering::Relaxed);
m.record_tool_call("a");
m.record_tool_call("b");
m.record_tool_call("c");
m.record_tool_call("d");
assert_eq!(m.tool_calls_per_memory_recall(), 2.0);
}
#[test]
fn test_tool_calls_per_memory_recall_zero_when_no_recalls() {
let m = RuntimeMetrics::new();
m.record_tool_call("a");
assert_eq!(m.tool_calls_per_memory_recall(), 0.0);
}
#[test]
fn test_tool_calls_per_memory_recall_zero_for_empty_metrics() {
let m = RuntimeMetrics::new();
assert_eq!(m.tool_calls_per_memory_recall(), 0.0);
}
#[test]
fn test_memory_recalls_per_tool_call_returns_ratio() {
let m = RuntimeMetrics::new();
m.record_tool_call("a");
m.record_tool_call("b");
m.memory_recall_count.store(4, std::sync::atomic::Ordering::Relaxed);
assert_eq!(m.memory_recalls_per_tool_call(), 2.0);
}
#[test]
fn test_memory_recalls_per_tool_call_zero_when_no_tool_calls() {
let m = RuntimeMetrics::new();
m.memory_recall_count.store(5, std::sync::atomic::Ordering::Relaxed);
assert_eq!(m.memory_recalls_per_tool_call(), 0.0);
}
#[test]
fn test_memory_recalls_per_tool_call_zero_for_empty_metrics() {
let m = RuntimeMetrics::new();
assert_eq!(m.memory_recalls_per_tool_call(), 0.0);
}
#[test]
fn test_avg_failures_per_tool_zero_when_no_calls() {
let m = RuntimeMetrics::new();
assert_eq!(m.snapshot().avg_failures_per_tool(), 0.0);
}
#[test]
fn test_avg_failures_per_tool_correct_value() {
let m = RuntimeMetrics::new();
m.record_tool_call("search");
m.record_tool_failure("search");
m.record_tool_call("write");
let avg = m.snapshot().avg_failures_per_tool();
assert!((avg - 0.5).abs() < 1e-9);
}
#[test]
fn test_memory_pressure_ratio_correct_ratio() {
use std::sync::atomic::Ordering;
let m = RuntimeMetrics::new();
m.total_steps.store(4, Ordering::Relaxed);
m.memory_recall_count.store(2, Ordering::Relaxed);
assert!((m.memory_pressure_ratio() - 0.5).abs() < 1e-9);
}
#[test]
fn test_memory_pressure_ratio_zero_when_no_steps() {
let m = RuntimeMetrics::new();
assert_eq!(m.memory_pressure_ratio(), 0.0);
}
#[test]
fn test_sessions_per_step_correct_ratio() {
use std::sync::atomic::Ordering;
let m = RuntimeMetrics::new();
m.total_steps.store(10, Ordering::Relaxed);
m.total_sessions.store(2, Ordering::Relaxed);
assert!((m.sessions_per_step() - 0.2).abs() < 1e-9);
}
#[test]
fn test_sessions_per_step_zero_when_no_steps() {
let m = RuntimeMetrics::new();
assert_eq!(m.sessions_per_step(), 0.0);
}
#[test]
fn test_step_failure_rate_returns_ratio() {
let m = RuntimeMetrics::new();
m.total_steps.store(4, std::sync::atomic::Ordering::Relaxed);
m.record_tool_failure("a");
m.record_tool_failure("b");
assert_eq!(m.step_failure_rate(), 0.5);
}
#[test]
fn test_step_failure_rate_zero_when_no_steps() {
let m = RuntimeMetrics::new();
assert_eq!(m.step_failure_rate(), 0.0);
}
#[test]
fn test_step_failure_rate_zero_when_no_failures() {
let m = RuntimeMetrics::new();
m.total_steps.store(3, std::sync::atomic::Ordering::Relaxed);
assert_eq!(m.step_failure_rate(), 0.0);
}
#[test]
fn test_avg_calls_per_step_correct_ratio() {
use std::sync::atomic::Ordering;
let m = RuntimeMetrics::new();
m.total_steps.store(4, Ordering::Relaxed);
m.total_tool_calls.store(8, Ordering::Relaxed);
assert!((m.avg_calls_per_step() - 2.0).abs() < 1e-9);
}
#[test]
fn test_avg_calls_per_step_zero_when_no_steps() {
let m = RuntimeMetrics::new();
assert_eq!(m.avg_calls_per_step(), 0.0);
}
#[test]
fn test_tools_above_failure_ratio_returns_failing_tools() {
let m = RuntimeMetrics::new();
m.record_tool_call("search");
m.record_tool_failure("search");
m.record_tool_call("write");
let above = m.snapshot().tools_above_failure_ratio(0.5);
assert_eq!(above, vec!["search"]);
}
#[test]
fn test_tools_above_failure_ratio_empty_when_no_calls() {
let m = RuntimeMetrics::new();
assert!(m.snapshot().tools_above_failure_ratio(0.1).is_empty());
}
#[test]
fn test_total_backpressure_shed_pct_returns_ratio() {
let m = RuntimeMetrics::new();
m.record_tool_call("a");
m.record_tool_call("b");
m.record_tool_call("c");
m.record_tool_call("d");
m.backpressure_shed_count.store(1, std::sync::atomic::Ordering::Relaxed);
assert_eq!(m.total_backpressure_shed_pct(), 0.25);
}
#[test]
fn test_total_backpressure_shed_pct_zero_when_no_calls() {
let m = RuntimeMetrics::new();
assert_eq!(m.total_backpressure_shed_pct(), 0.0);
}
#[test]
fn test_backpressure_ratio_correct() {
use std::sync::atomic::Ordering;
let m = RuntimeMetrics::new();
m.total_steps.store(4, Ordering::Relaxed);
m.backpressure_shed_count.store(1, Ordering::Relaxed);
assert!((m.backpressure_ratio() - 0.25).abs() < 1e-9);
}
#[test]
fn test_backpressure_ratio_zero_when_no_steps() {
let m = RuntimeMetrics::new();
assert_eq!(m.backpressure_ratio(), 0.0);
}
#[test]
fn test_tool_with_highest_failure_rate_returns_most_failing_tool() {
let m = RuntimeMetrics::new();
m.record_tool_call("a");
m.record_tool_failure("a");
m.record_tool_call("b");
m.record_tool_call("b");
m.record_tool_failure("b");
assert_eq!(m.tool_with_highest_failure_rate().as_deref(), Some("a"));
}
#[test]
fn test_tool_with_highest_failure_rate_none_when_no_calls() {
let m = RuntimeMetrics::new();
assert!(m.tool_with_highest_failure_rate().is_none());
}
#[test]
fn test_has_latency_data_true_after_step() {
use std::sync::atomic::Ordering;
let m = RuntimeMetrics::new();
m.total_steps.store(1, Ordering::Relaxed);
assert!(m.has_latency_data());
}
#[test]
fn test_has_latency_data_false_for_new_metrics() {
let m = RuntimeMetrics::new();
assert!(!m.has_latency_data());
}
#[test]
fn test_global_failure_rate_correct() {
let m = RuntimeMetrics::new();
m.total_tool_calls.store(10, Ordering::Relaxed);
m.failed_tool_calls.store(2, Ordering::Relaxed);
assert!((m.global_failure_rate() - 0.2).abs() < 1e-9);
}
#[test]
fn test_global_failure_rate_zero_when_no_calls() {
let m = RuntimeMetrics::new();
assert_eq!(m.global_failure_rate(), 0.0);
}
#[test]
fn test_agent_tool_count_correct() {
let m = RuntimeMetrics::new();
m.record_agent_tool_call("agent-A", "tool1");
m.record_agent_tool_call("agent-B", "tool2");
m.record_agent_tool_call("agent-A", "tool3");
assert_eq!(m.agent_tool_count(), 2);
}
#[test]
fn test_agent_tool_count_zero_when_no_calls() {
let m = RuntimeMetrics::new();
assert_eq!(m.agent_tool_count(), 0);
}
#[test]
fn test_active_session_count_correct() {
let m = RuntimeMetrics::new();
m.active_sessions.store(3, Ordering::Relaxed);
assert_eq!(m.active_session_count(), 3);
}
#[test]
fn test_active_session_count_zero_initially() {
let m = RuntimeMetrics::new();
assert_eq!(m.active_session_count(), 0);
}
#[test]
fn test_memory_to_session_ratio_correct() {
let m = RuntimeMetrics::new();
m.total_sessions.store(4, Ordering::Relaxed);
m.memory_recall_count.store(8, Ordering::Relaxed);
assert!((m.memory_to_session_ratio() - 2.0).abs() < 1e-9);
}
#[test]
fn test_memory_to_session_ratio_zero_when_no_sessions() {
let m = RuntimeMetrics::new();
assert_eq!(m.memory_to_session_ratio(), 0.0);
}
#[test]
fn test_total_latency_per_session_correct() {
let m = RuntimeMetrics::new();
m.record_step_latency(100);
m.record_step_latency(200);
m.total_sessions.store(2, Ordering::Relaxed);
assert!((m.total_latency_per_session() - 150.0).abs() < 1e-9);
}
#[test]
fn test_total_latency_per_session_zero_when_no_sessions() {
let m = RuntimeMetrics::new();
assert_eq!(m.total_latency_per_session(), 0.0);
}
#[test]
fn test_failure_ratio_for_tool_correct_ratio() {
let m = RuntimeMetrics::new();
m.record_tool_call("search");
m.record_tool_call("search");
m.record_tool_failure("search");
let snap = m.snapshot();
assert!((snap.failure_ratio_for_tool("search") - 0.5).abs() < 1e-9);
}
#[test]
fn test_failure_ratio_for_tool_zero_for_unknown_tool() {
let m = RuntimeMetrics::new();
let snap = m.snapshot();
assert_eq!(snap.failure_ratio_for_tool("unknown"), 0.0);
}
#[test]
fn test_any_tool_exceeds_calls_true_when_above_threshold() {
let m = RuntimeMetrics::new();
m.record_tool_call("a");
m.record_tool_call("a");
m.record_tool_call("a");
let snap = m.snapshot();
assert!(snap.any_tool_exceeds_calls(2));
}
#[test]
fn test_any_tool_exceeds_calls_false_when_all_at_or_below_threshold() {
let m = RuntimeMetrics::new();
m.record_tool_call("a");
m.record_tool_call("a");
let snap = m.snapshot();
assert!(!snap.any_tool_exceeds_calls(2));
}
#[test]
fn test_tool_call_count_for_returns_correct_count() {
let m = RuntimeMetrics::new();
m.record_tool_call("grep");
m.record_tool_call("grep");
m.record_tool_call("grep");
assert_eq!(m.tool_call_count_for("grep"), 3);
}
#[test]
fn test_tool_call_count_for_returns_zero_for_unknown_tool() {
let m = RuntimeMetrics::new();
assert_eq!(m.tool_call_count_for("nonexistent"), 0);
}
#[test]
fn test_total_unique_tools_counts_distinct_tools() {
let m = RuntimeMetrics::new();
m.record_tool_call("search");
m.record_tool_call("search");
m.record_tool_call("browse");
let snap = m.snapshot();
assert_eq!(snap.total_unique_tools(), 2);
}
#[test]
fn test_total_unique_tools_zero_for_no_calls() {
let m = RuntimeMetrics::new();
let snap = m.snapshot();
assert_eq!(snap.total_unique_tools(), 0);
}
#[test]
fn test_total_agent_tool_calls_sums_all_agents() {
let m = RuntimeMetrics::new();
m.record_agent_tool_call("agent-1", "search");
m.record_agent_tool_call("agent-1", "browse");
m.record_agent_tool_call("agent-2", "search");
assert_eq!(m.total_agent_tool_calls(), 3);
}
#[test]
fn test_total_agent_tool_calls_zero_for_new_metrics() {
let m = RuntimeMetrics::new();
assert_eq!(m.total_agent_tool_calls(), 0);
}
#[test]
fn test_top_called_tool_returns_most_called() {
let m = RuntimeMetrics::new();
m.record_tool_call("search");
m.record_tool_call("search");
m.record_tool_call("browse");
assert_eq!(m.top_called_tool().as_deref(), Some("search"));
}
#[test]
fn test_top_called_tool_none_for_new_metrics() {
let m = RuntimeMetrics::new();
assert!(m.top_called_tool().is_none());
}
#[test]
fn test_avg_step_latency_ms_correct() {
let m = RuntimeMetrics::new();
m.record_step_latency(100);
m.record_step_latency(200);
m.total_steps.store(2, Ordering::Relaxed);
assert!((m.avg_step_latency_ms() - 150.0).abs() < 1e-9);
}
#[test]
fn test_avg_step_latency_ms_zero_for_new_metrics() {
let m = RuntimeMetrics::new();
assert_eq!(m.avg_step_latency_ms(), 0.0);
}
#[test]
fn test_distinct_tools_called_counts_unique_tools() {
let m = RuntimeMetrics::new();
m.record_tool_call("search");
m.record_tool_call("search");
m.record_tool_call("browse");
assert_eq!(m.distinct_tools_called(), 2);
}
#[test]
fn test_distinct_tools_called_zero_for_new_metrics() {
let m = RuntimeMetrics::new();
assert_eq!(m.distinct_tools_called(), 0);
}
#[test]
fn test_agent_tool_call_count_sums_correctly() {
let m = RuntimeMetrics::new();
m.record_agent_tool_call("agent-1", "search");
m.record_agent_tool_call("agent-1", "browse");
m.record_agent_tool_call("agent-2", "search");
assert_eq!(m.agent_tool_call_count("agent-1"), 2);
}
#[test]
fn test_agent_tool_call_count_zero_for_unknown_agent() {
let m = RuntimeMetrics::new();
assert_eq!(m.agent_tool_call_count("nobody"), 0);
}
#[test]
fn test_tool_call_ratio_for_returns_correct_fraction() {
let m = RuntimeMetrics::new();
m.record_tool_call("a");
m.record_tool_call("a");
m.record_tool_call("b");
let snap = m.snapshot();
assert!((snap.tool_call_ratio_for("a") - 2.0 / 3.0).abs() < 1e-9);
}
#[test]
fn test_tool_call_ratio_for_zero_when_no_calls() {
let m = RuntimeMetrics::new();
let snap = m.snapshot();
assert_eq!(snap.tool_call_ratio_for("search"), 0.0);
}
#[test]
fn test_has_recorded_agent_calls_true_after_recording() {
let m = RuntimeMetrics::new();
m.record_agent_tool_call("agent-1", "search");
assert!(m.has_recorded_agent_calls());
}
#[test]
fn test_has_recorded_agent_calls_false_for_new_metrics() {
let m = RuntimeMetrics::new();
assert!(!m.has_recorded_agent_calls());
}
#[test]
fn test_failure_rate_for_returns_correct_ratio() {
let m = RuntimeMetrics::new();
m.record_tool_call("search");
m.record_tool_call("search");
m.record_tool_failure("search");
assert!((m.failure_rate_for("search") - 0.5).abs() < 1e-9);
}
#[test]
fn test_failure_rate_for_zero_for_unknown_tool() {
let m = RuntimeMetrics::new();
assert_eq!(m.failure_rate_for("unknown"), 0.0);
}
#[test]
fn test_failure_rate_for_zero_when_no_failures() {
let m = RuntimeMetrics::new();
m.record_tool_call("browse");
assert_eq!(m.failure_rate_for("browse"), 0.0);
}
#[test]
fn test_tool_calls_per_session_returns_correct_ratio() {
let m = RuntimeMetrics::new();
m.total_sessions
.fetch_add(2, std::sync::atomic::Ordering::Relaxed);
m.record_tool_call("search");
m.record_tool_call("browse");
m.record_tool_call("search");
assert!((m.tool_calls_per_session() - 1.5).abs() < 1e-9);
}
#[test]
fn test_tool_calls_per_session_zero_when_no_sessions() {
let m = RuntimeMetrics::new();
assert_eq!(m.tool_calls_per_session(), 0.0);
}
#[test]
fn test_failure_free_tools_returns_tools_without_failures() {
let m = RuntimeMetrics::new();
m.record_tool_call("search");
m.record_tool_call("browse");
m.record_tool_failure("search");
let tools = m.failure_free_tools();
assert!(tools.contains(&"browse".to_string()));
assert!(!tools.contains(&"search".to_string()));
}
#[test]
fn test_failure_free_tools_empty_when_all_failed() {
let m = RuntimeMetrics::new();
m.record_tool_call("a");
m.record_tool_failure("a");
let tools = m.failure_free_tools();
assert!(!tools.contains(&"a".to_string()));
}
#[test]
fn test_total_failures_across_all_tools_sums_all_failures() {
let m = RuntimeMetrics::new();
m.record_tool_call("a");
m.record_tool_failure("a");
m.record_tool_call("b");
m.record_tool_failure("b");
m.record_tool_failure("b");
let snap = m.snapshot();
assert_eq!(snap.total_failures_across_all_tools(), 3);
}
#[test]
fn test_total_failures_across_all_tools_zero_when_none() {
let m = RuntimeMetrics::new();
m.record_tool_call("a");
let snap = m.snapshot();
assert_eq!(snap.total_failures_across_all_tools(), 0);
}
#[test]
fn test_tools_with_calls_above_returns_tools_exceeding_threshold() {
let m = RuntimeMetrics::new();
for _ in 0..5 { m.record_tool_call("busy"); }
m.record_tool_call("idle");
let result = m.tools_with_calls_above(3);
assert!(result.contains(&"busy".to_string()));
assert!(!result.contains(&"idle".to_string()));
}
#[test]
fn test_tools_with_calls_above_empty_when_none_qualify() {
let m = RuntimeMetrics::new();
m.record_tool_call("once");
assert!(m.tools_with_calls_above(5).is_empty());
}
#[test]
fn test_tools_with_calls_above_returns_sorted_names() {
let m = RuntimeMetrics::new();
for _ in 0..3 { m.record_tool_call("zebra"); }
for _ in 0..3 { m.record_tool_call("apple"); }
let result = m.tools_with_calls_above(2);
assert_eq!(result, vec!["apple", "zebra"]);
}
#[test]
fn test_checkpoint_errors_count_zero_for_new_metrics() {
let m = RuntimeMetrics::new();
assert_eq!(m.checkpoint_errors_count(), 0);
}
#[test]
fn test_checkpoint_errors_count_reflects_incremented_value() {
let m = RuntimeMetrics::new();
m.checkpoint_errors
.fetch_add(3, std::sync::atomic::Ordering::Relaxed);
assert_eq!(m.checkpoint_errors_count(), 3);
}
#[test]
fn test_agents_with_failures_returns_agents_with_failures() {
let m = RuntimeMetrics::new();
m.record_agent_tool_call("agent-x", "search");
m.record_agent_tool_failure("agent-x", "search");
m.record_agent_tool_call("agent-y", "browse");
let agents = m.agents_with_failures();
assert!(agents.contains(&"agent-x".to_string()));
assert!(!agents.contains(&"agent-y".to_string()));
}
#[test]
fn test_total_agent_failures_sums_all_failures() {
let m = RuntimeMetrics::new();
m.record_agent_tool_failure("a", "tool1");
m.record_agent_tool_failure("a", "tool2");
m.record_agent_tool_failure("b", "tool1");
assert_eq!(m.total_agent_failures(), 3);
}
#[test]
fn test_total_agent_failures_zero_for_new_metrics() {
let m = RuntimeMetrics::new();
assert_eq!(m.total_agent_failures(), 0);
}
#[test]
fn test_per_step_tool_call_rate_zero_when_no_steps() {
let m = RuntimeMetrics::new();
assert_eq!(m.per_step_tool_call_rate(), 0.0);
}
#[test]
fn test_per_step_tool_call_rate_computed_correctly() {
let m = RuntimeMetrics::new();
m.total_steps.store(2, Ordering::Relaxed);
m.record_tool_call("search");
m.record_tool_call("browse");
m.record_tool_call("search");
assert!((m.per_step_tool_call_rate() - 1.5).abs() < 1e-9);
}
#[test]
fn test_agents_with_no_failures_returns_clean_agents() {
let m = RuntimeMetrics::new();
m.record_agent_tool_call("agent-clean", "search");
m.record_agent_tool_call("agent-fail", "search");
m.record_agent_tool_failure("agent-fail", "search");
let clean = m.agents_with_no_failures();
assert!(clean.contains(&"agent-clean".to_string()));
assert!(!clean.contains(&"agent-fail".to_string()));
}
#[test]
fn test_agents_with_no_failures_empty_for_new_metrics() {
let m = RuntimeMetrics::new();
assert!(m.agents_with_no_failures().is_empty());
}
}