use core::fmt;
use std::collections::BTreeMap;
use astrs_time::HlcTimestamp;
use oxicode::{Decode, Encode};
use serde::{Deserialize, Serialize};
use crate::common::duration::DurationMs;
use crate::ids::{DataId, DataflowId, NodeId};
pub const MAX_METRIC_POINTS: usize = 4096;
#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize, Encode, Decode)]
pub struct DaemonStats {
pub uptime: DurationMs,
pub node_count: u32,
pub dataflow_count: u32,
pub cpu_percent: f32,
pub rss_bytes: u64,
pub shm_bytes_mapped: u64,
pub shm_fallback_total: u64,
pub frames_sent: u64,
pub frames_received: u64,
pub bytes_sent: u64,
pub bytes_received: u64,
}
impl DaemonStats {
#[must_use]
pub const fn has_shm_fallbacks(&self) -> bool {
self.shm_fallback_total > 0
}
#[must_use]
pub fn bitwise_eq(&self, other: &Self) -> bool {
self.uptime == other.uptime
&& self.node_count == other.node_count
&& self.dataflow_count == other.dataflow_count
&& self.cpu_percent.to_bits() == other.cpu_percent.to_bits()
&& self.rss_bytes == other.rss_bytes
&& self.shm_bytes_mapped == other.shm_bytes_mapped
&& self.shm_fallback_total == other.shm_fallback_total
&& self.frames_sent == other.frames_sent
&& self.frames_received == other.frames_received
&& self.bytes_sent == other.bytes_sent
&& self.bytes_received == other.bytes_received
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode)]
pub struct NodeMetricsSample {
pub node: NodeId,
pub timestamp: HlcTimestamp,
pub cpu_percent: f32,
pub rss_bytes: u64,
pub queue_depths: BTreeMap<DataId, u32>,
pub sent_total: BTreeMap<DataId, u64>,
pub received_total: BTreeMap<DataId, u64>,
pub dropped_total: BTreeMap<DataId, u64>,
pub shm_slots_in_use: u32,
pub timer_jitter_p99_us: u64,
}
impl NodeMetricsSample {
#[must_use]
pub fn new(node: NodeId, timestamp: HlcTimestamp) -> Self {
Self {
node,
timestamp,
cpu_percent: 0.0,
rss_bytes: 0,
queue_depths: BTreeMap::new(),
sent_total: BTreeMap::new(),
received_total: BTreeMap::new(),
dropped_total: BTreeMap::new(),
shm_slots_in_use: 0,
timer_jitter_p99_us: 0,
}
}
#[must_use]
pub fn deepest_queue(&self) -> Option<(&DataId, u32)> {
self.queue_depths
.iter()
.max_by_key(|(_, depth)| **depth)
.map(|(id, depth)| (id, *depth))
}
#[must_use]
pub fn total_dropped(&self) -> u64 {
self.dropped_total.values().copied().sum()
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Encode, Decode)]
pub struct NodeIoSample {
pub node: NodeId,
pub timestamp: HlcTimestamp,
pub sent_bytes_total: BTreeMap<DataId, u64>,
pub received_bytes_total: BTreeMap<DataId, u64>,
pub shm_slots_in_use: u32,
pub shm_slots_total: u32,
pub shm_fallback_total: u64,
}
impl NodeIoSample {
#[must_use]
pub fn new(node: NodeId, timestamp: HlcTimestamp) -> Self {
Self {
node,
timestamp,
sent_bytes_total: BTreeMap::new(),
received_bytes_total: BTreeMap::new(),
shm_slots_in_use: 0,
shm_slots_total: 0,
shm_fallback_total: 0,
}
}
#[must_use]
pub fn total_sent_bytes(&self) -> u64 {
self.sent_bytes_total.values().copied().sum()
}
#[must_use]
pub fn total_received_bytes(&self) -> u64 {
self.received_bytes_total.values().copied().sum()
}
#[must_use]
pub fn busiest_output(&self) -> Option<(&DataId, u64)> {
self.sent_bytes_total
.iter()
.max_by_key(|(_, bytes)| **bytes)
.map(|(id, bytes)| (id, *bytes))
}
#[must_use]
pub fn busiest_input(&self) -> Option<(&DataId, u64)> {
self.received_bytes_total
.iter()
.max_by_key(|(_, bytes)| **bytes)
.map(|(id, bytes)| (id, *bytes))
}
#[must_use]
pub fn shm_slot_occupancy(&self) -> Option<f64> {
(self.shm_slots_total > 0)
.then(|| f64::from(self.shm_slots_in_use) / f64::from(self.shm_slots_total))
}
#[must_use]
pub fn delta_since(&self, earlier: &Self) -> Option<IoDelta> {
if earlier.node != self.node {
return None;
}
let nanos = self
.timestamp
.physical_ns()
.checked_sub(earlier.timestamp.physical_ns())?;
if nanos == 0 {
return None;
}
Some(IoDelta {
nanos,
sent_bytes: self
.total_sent_bytes()
.saturating_sub(earlier.total_sent_bytes()),
received_bytes: self
.total_received_bytes()
.saturating_sub(earlier.total_received_bytes()),
})
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct IoDelta {
pub nanos: u64,
pub sent_bytes: u64,
pub received_bytes: u64,
}
impl IoDelta {
#[must_use]
pub fn sent_bytes_per_second(&self) -> f64 {
Self::per_second(self.sent_bytes, self.nanos)
}
#[must_use]
pub fn received_bytes_per_second(&self) -> f64 {
Self::per_second(self.received_bytes, self.nanos)
}
fn per_second(bytes: u64, nanos: u64) -> f64 {
if nanos == 0 {
return 0.0;
}
#[expect(
clippy::cast_precision_loss,
reason = "a rate is a display figure; f64 holds byte counts exactly to 2^53"
)]
let seconds = nanos as f64 / 1e9;
#[expect(
clippy::cast_precision_loss,
reason = "same: the quotient is rendered, never compared for equality"
)]
let bytes = bytes as f64;
bytes / seconds
}
}
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize, Encode, Decode)]
pub struct HistogramBucket {
pub upper_bound: f64,
pub cumulative_count: u64,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode)]
#[serde(rename_all = "snake_case")]
#[non_exhaustive]
pub enum MetricValue {
#[oxicode(variant = 0)]
Counter(u64),
#[oxicode(variant = 1)]
Gauge(f64),
#[oxicode(variant = 2)]
Histogram {
count: u64,
sum: f64,
buckets: Vec<HistogramBucket>,
},
}
impl MetricValue {
#[must_use]
pub fn as_f64(&self) -> f64 {
match self {
Self::Counter(value) => *value as f64,
Self::Gauge(value) => *value,
Self::Histogram { count, .. } => *count as f64,
}
}
#[must_use]
pub const fn kind_name(&self) -> &'static str {
match self {
Self::Counter(_) => "counter",
Self::Gauge(_) => "gauge",
Self::Histogram { .. } => "histogram",
}
}
#[must_use]
pub fn bitwise_eq(&self, other: &Self) -> bool {
match (self, other) {
(Self::Gauge(left), Self::Gauge(right)) => left.to_bits() == right.to_bits(),
(
Self::Histogram {
count: lc,
sum: ls,
buckets: lb,
},
Self::Histogram {
count: rc,
sum: rs,
buckets: rb,
},
) => {
lc == rc
&& ls.to_bits() == rs.to_bits()
&& lb.len() == rb.len()
&& lb.iter().zip(rb.iter()).all(|(a, b)| {
a.upper_bound.to_bits() == b.upper_bound.to_bits()
&& a.cumulative_count == b.cumulative_count
})
}
_ => self == other,
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode)]
pub struct MetricPoint {
pub name: String,
pub value: MetricValue,
pub labels: BTreeMap<String, String>,
}
impl MetricPoint {
#[must_use]
pub fn counter(name: impl Into<String>, value: u64) -> Self {
Self {
name: name.into(),
value: MetricValue::Counter(value),
labels: BTreeMap::new(),
}
}
#[must_use]
pub fn gauge(name: impl Into<String>, value: f64) -> Self {
Self {
name: name.into(),
value: MetricValue::Gauge(value),
labels: BTreeMap::new(),
}
}
#[must_use]
pub fn with_label(mut self, key: impl Into<String>, value: impl Into<String>) -> Self {
self.labels.insert(key.into(), value.into());
self
}
#[must_use]
pub fn label(&self, key: &str) -> Option<&str> {
self.labels.get(key).map(String::as_str)
}
}
impl fmt::Display for MetricPoint {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(&self.name)?;
if !self.labels.is_empty() {
f.write_str("{")?;
for (index, (key, value)) in self.labels.iter().enumerate() {
if index > 0 {
f.write_str(",")?;
}
write!(f, "{key}={value}")?;
}
f.write_str("}")?;
}
match &self.value {
MetricValue::Counter(value) => write!(f, " {value}"),
MetricValue::Gauge(value) => write!(f, " {value}"),
MetricValue::Histogram { count, sum, .. } => write!(f, " count={count} sum={sum}"),
}
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, Encode, Decode)]
pub struct MetricBatch {
pub timestamp: HlcTimestamp,
pub scope: String,
pub dataflow: Option<DataflowId>,
pub points: Vec<MetricPoint>,
}
impl MetricBatch {
#[must_use]
pub fn new(timestamp: HlcTimestamp, scope: impl Into<String>) -> Self {
Self {
timestamp,
scope: scope.into(),
dataflow: None,
points: Vec::new(),
}
}
#[must_use]
pub fn with_point(mut self, point: MetricPoint) -> Self {
self.points.push(point);
self
}
#[must_use]
pub fn within_limits(&self) -> bool {
self.points.len() <= MAX_METRIC_POINTS
}
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
use crate::codec::{WireDecode, WireEncode};
fn node() -> NodeId {
NodeId::new("camera").unwrap()
}
#[test]
fn daemon_stats_default_to_zero() {
let stats = DaemonStats::default();
assert_eq!(stats.node_count, 0);
assert_eq!(stats.uptime, DurationMs::ZERO);
assert!(!stats.has_shm_fallbacks());
assert_eq!(stats.cpu_percent, 0.0);
}
#[test]
fn daemon_stats_round_trip() {
let stats = DaemonStats {
uptime: DurationMs::from_secs(3_600),
node_count: 12,
dataflow_count: 2,
cpu_percent: 37.5,
rss_bytes: 128 * 1024 * 1024,
shm_bytes_mapped: 64 * 1024 * 1024,
shm_fallback_total: 3,
frames_sent: 1_000,
frames_received: 999,
bytes_sent: 1 << 30,
bytes_received: 1 << 29,
};
let bytes = stats.encode_to_vec().unwrap();
assert_eq!(DaemonStats::decode_exact(&bytes).unwrap(), stats);
assert!(stats.has_shm_fallbacks());
}
#[test]
fn daemon_stats_survive_a_nan_cpu_reading() {
let stats = DaemonStats {
cpu_percent: f32::NAN,
..DaemonStats::default()
};
let bytes = stats.encode_to_vec().unwrap();
let decoded = DaemonStats::decode_exact(&bytes).unwrap();
assert_ne!(decoded, stats, "PartialEq cannot see NaN as equal");
assert!(decoded.bitwise_eq(&stats));
}
#[test]
fn node_metrics_round_trip_and_summarise() {
let mut sample = NodeMetricsSample::new(node(), HlcTimestamp::new(5, 1));
sample.cpu_percent = 12.5;
sample.rss_bytes = 1 << 20;
sample
.queue_depths
.insert(DataId::new("frames").unwrap(), 3);
sample
.queue_depths
.insert(DataId::new("control").unwrap(), 9);
sample
.dropped_total
.insert(DataId::new("frames").unwrap(), 4);
sample
.dropped_total
.insert(DataId::new("control").unwrap(), 1);
sample.timer_jitter_p99_us = 250;
let deepest = sample.deepest_queue().unwrap();
assert_eq!(deepest.0.as_str(), "control");
assert_eq!(deepest.1, 9);
assert_eq!(sample.total_dropped(), 5);
let bytes = sample.encode_to_vec().unwrap();
assert_eq!(NodeMetricsSample::decode_exact(&bytes).unwrap(), sample);
}
#[test]
fn an_empty_sample_has_no_deepest_queue() {
let sample = NodeMetricsSample::new(node(), HlcTimestamp::EPOCH);
assert_eq!(sample.deepest_queue(), None);
assert_eq!(sample.total_dropped(), 0);
}
#[test]
fn metric_values_round_trip_and_classify() {
let values = [
MetricValue::Counter(0),
MetricValue::Counter(u64::MAX),
MetricValue::Gauge(-1.5),
MetricValue::Histogram {
count: 10,
sum: 55.0,
buckets: vec![
HistogramBucket {
upper_bound: 1.0,
cumulative_count: 2,
},
HistogramBucket {
upper_bound: f64::INFINITY,
cumulative_count: 10,
},
],
},
];
let mut names = std::collections::BTreeSet::new();
for value in values {
let bytes = value.encode_to_vec().unwrap();
let decoded = MetricValue::decode_exact(&bytes).unwrap();
assert!(value.bitwise_eq(&decoded));
names.insert(value.kind_name());
}
assert_eq!(names.len(), 3);
assert_eq!(MetricValue::Counter(7).as_f64(), 7.0);
assert_eq!(MetricValue::Gauge(0.25).as_f64(), 0.25);
assert_eq!(
MetricValue::Histogram {
count: 3,
sum: 0.0,
buckets: vec![]
}
.as_f64(),
3.0
);
}
#[test]
fn metric_values_survive_nan() {
let value = MetricValue::Gauge(f64::NAN);
let bytes = value.encode_to_vec().unwrap();
let decoded = MetricValue::decode_exact(&bytes).unwrap();
assert!(value.bitwise_eq(&decoded));
assert!(!value.bitwise_eq(&MetricValue::Gauge(0.0)));
assert!(!MetricValue::Counter(1).bitwise_eq(&MetricValue::Gauge(1.0)));
}
#[test]
fn metric_points_carry_labels() {
let point = MetricPoint::gauge("queue_depth", 4.0)
.with_label("node", "camera")
.with_label("input", "frames");
assert_eq!(point.label("node"), Some("camera"));
assert_eq!(point.label("absent"), None);
assert_eq!(point.to_string(), "queue_depth{input=frames,node=camera} 4");
let bare = MetricPoint::counter("total", 1);
assert_eq!(bare.to_string(), "total 1");
let histogram = MetricPoint {
name: "latency".to_owned(),
value: MetricValue::Histogram {
count: 2,
sum: 3.0,
buckets: vec![],
},
labels: BTreeMap::new(),
};
assert_eq!(histogram.to_string(), "latency count=2 sum=3");
}
#[test]
fn metric_batches_round_trip_and_bound_themselves() {
let batch = MetricBatch::new(HlcTimestamp::new(1, 0), "astrs_daemon")
.with_point(MetricPoint::counter("a", 1))
.with_point(MetricPoint::gauge("b", 2.0));
assert!(batch.within_limits());
let bytes = batch.encode_to_vec().unwrap();
assert_eq!(MetricBatch::decode_exact(&bytes).unwrap(), batch);
let mut oversized = MetricBatch::new(HlcTimestamp::EPOCH, "s");
oversized.points = (0..=MAX_METRIC_POINTS)
.map(|index| MetricPoint::counter(format!("m{index}"), 0))
.collect();
assert!(!oversized.within_limits());
}
}