rivet/
latency.rs

1//! Latency histograms (plan.md Phase 12).
2//!
3//! Four fixed-size, zero-allocation histograms — 16 log2-scaled buckets
4//! each (`[AtomicU32; 16]`), bucket `b` covers `[2^b, 2^(b+1))` cycles
5//! (bucket 0 covers 0-1) — tracking:
6//!
7//! - [`Kind::IrqEntry`]: cycles from an interrupt firing to
8//!   [`crate::preempt::on_tick`] actually running. Recorded by the arch
9//!   trap entry (whichever `rivet-arch-*` crate captures a cycle stamp as
10//!   early as possible in the handler).
11//! - [`Kind::DispatchDecision`]: cycles spent *inside* `on_tick` itself —
12//!   the scheduling decision's own cost.
13//! - [`Kind::CriticalSection`]: cycles held between
14//!   [`crate::critical::enter`]'s entry and exit — a proxy for
15//!   interrupt-latency impact (nothing can preempt the calling hart while
16//!   held).
17//! - [`Kind::SchedulingWake`]: cycles from a task becoming ready
18//!   (`sched::unblock` / `ready_add`) to actually being dispatched
19//!   Running.
20//!
21//! Gated behind the `latency-histograms` feature (off by default — see
22//! `rivet/Cargo.toml`): recording a sample is one `cycle_count()` read
23//! plus one atomic increment, cheap but not free, and a cost-sensitive
24//! board that never reads the histograms shouldn't pay it unasked.
25
26use crate::sync::atomic::{AtomicU32, Ordering};
27
28pub const BUCKETS: usize = 16;
29
30#[derive(Debug, Clone, Copy, PartialEq, Eq)]
31pub enum Kind {
32    IrqEntry,
33    DispatchDecision,
34    CriticalSection,
35    SchedulingWake,
36}
37
38const KINDS: usize = 4;
39
40fn index(kind: Kind) -> usize {
41    match kind {
42        Kind::IrqEntry => 0,
43        Kind::DispatchDecision => 1,
44        Kind::CriticalSection => 2,
45        Kind::SchedulingWake => 3,
46    }
47}
48
49#[cfg(not(loom))]
50static HISTOGRAMS: [[AtomicU32; BUCKETS]; KINDS] =
51    [const { [const { AtomicU32::new(0) }; BUCKETS] }; KINDS];
52#[cfg(loom)]
53loom::lazy_static! {
54    static ref HISTOGRAMS: [[AtomicU32; BUCKETS]; KINDS] =
55        core::array::from_fn(|_| core::array::from_fn(|_| AtomicU32::new(0)));
56}
57
58/// Exact worst-case-observed cycle count per `Kind`, alongside the
59/// bucketed histogram above — the top histogram bucket (`[2^15, ∞)`) is
60/// unbounded above by construction, so it alone cannot answer "what was
61/// the actual worst cycle count seen," only "at least 32768." Needed for
62/// WCET reporting, where an open-ended top bucket isn't an exact figure.
63#[cfg(not(loom))]
64static MAX_CYCLES: [AtomicU32; KINDS] = [const { AtomicU32::new(0) }; KINDS];
65#[cfg(loom)]
66loom::lazy_static! {
67    static ref MAX_CYCLES: [AtomicU32; KINDS] = core::array::from_fn(|_| AtomicU32::new(0));
68}
69
70#[cfg(any(feature = "latency-histograms", test))]
71fn bucket_of(cycles: u64) -> usize {
72    if cycles == 0 {
73        return 0;
74    }
75    // 63 - leading_zeros gives floor(log2(cycles)); clamp to the last
76    // bucket for anything huge (a stalled/very-first sample) rather than
77    // panicking or silently discarding it.
78    let b = 63 - cycles.leading_zeros() as usize;
79    b.min(BUCKETS - 1)
80}
81
82/// Record one sample of `cycles` duration for `kind`. No-op unless the
83/// `latency-histograms` feature is enabled.
84#[cfg(feature = "latency-histograms")]
85pub fn record(kind: Kind, cycles: u64) {
86    HISTOGRAMS[index(kind)][bucket_of(cycles)].fetch_add(1, Ordering::Relaxed);
87    let capped = cycles.min(u32::MAX as u64) as u32;
88    MAX_CYCLES[index(kind)].fetch_max(capped, Ordering::Relaxed);
89}
90
91/// See the feature-gated [`record`] above; a no-op stub keeps call sites
92/// (arch crates, `on_tick`, `critical::enter`, `sched::unblock`) free of
93/// `#[cfg]` clutter when the feature is off.
94#[cfg(not(feature = "latency-histograms"))]
95#[inline(always)]
96pub fn record(_kind: Kind, _cycles: u64) {}
97
98/// Snapshot of one histogram's 16 bucket counts.
99pub fn snapshot(kind: Kind) -> [u32; BUCKETS] {
100    core::array::from_fn(|b| HISTOGRAMS[index(kind)][b].load(Ordering::Relaxed))
101}
102
103/// Exact worst-case-observed cycle count for `kind` (0 if never recorded)
104/// — see [`MAX_CYCLES`]'s own doc for why this exists alongside the
105/// bucketed histogram.
106pub fn max_cycles(kind: Kind) -> u32 {
107    MAX_CYCLES[index(kind)].load(Ordering::Relaxed)
108}
109
110/// Human-readable name, for `report()`.
111pub fn name(kind: Kind) -> &'static str {
112    match kind {
113        Kind::IrqEntry => "irq_entry",
114        Kind::DispatchDecision => "dispatch",
115        Kind::CriticalSection => "critsec",
116        Kind::SchedulingWake => "sched_wake",
117    }
118}
119
120pub const ALL_KINDS: [Kind; KINDS] = [
121    Kind::IrqEntry,
122    Kind::DispatchDecision,
123    Kind::CriticalSection,
124    Kind::SchedulingWake,
125];
126
127#[cfg(feature = "test-support")]
128pub(crate) fn reset_for_test() {
129    for hist in HISTOGRAMS.iter() {
130        for b in hist.iter() {
131            b.store(0, Ordering::Relaxed);
132        }
133    }
134}
135
136#[cfg(test)]
137mod tests {
138    use super::*;
139
140    #[test]
141    fn bucket_of_boundaries() {
142        assert_eq!(bucket_of(0), 0);
143        assert_eq!(bucket_of(1), 0);
144        assert_eq!(bucket_of(2), 1);
145        assert_eq!(bucket_of(3), 1);
146        assert_eq!(bucket_of(4), 2);
147        assert_eq!(bucket_of(u64::MAX), BUCKETS - 1);
148    }
149
150    #[cfg(feature = "latency-histograms")]
151    #[test]
152    fn record_and_snapshot() {
153        crate::kernel_test! {
154            record(Kind::IrqEntry, 5);
155            record(Kind::IrqEntry, 5);
156            record(Kind::IrqEntry, 100);
157            let snap = snapshot(Kind::IrqEntry);
158            assert_eq!(snap[bucket_of(5)], 2);
159            assert_eq!(snap[bucket_of(100)], 1);
160        }
161    }
162}
rivet/latency.rs

rivet/
latency.rs
