use std::sync::OnceLock;
use std::sync::atomic::{
AtomicU64,
Ordering::{Acquire, Relaxed, Release},
};
use std::time::Instant;
const N_STAGES: usize = 4;
const N_COUNTERS: usize = 1;
const HIST_BUCKETS: usize = 48;
#[derive(Copy, Clone)]
#[repr(usize)]
pub(crate) enum Stage {
TunRead = 0,
TunToMeshQueueWait = 1,
MeshSend = 2,
TunWrite = 3,
}
impl Stage {
fn name(self) -> &'static str {
match self {
Stage::TunRead => "nvpn_tun_read",
Stage::TunToMeshQueueWait => "nvpn_tun_to_mesh_queue_wait",
Stage::MeshSend => "nvpn_mesh_send",
Stage::TunWrite => "nvpn_tun_write",
}
}
}
#[derive(Copy, Clone)]
#[repr(usize)]
pub(crate) enum Counter {
TunToMeshBulkDropped = 0,
}
impl Counter {
fn name(self) -> &'static str {
match self {
Counter::TunToMeshBulkDropped => "nvpn_tun_to_mesh_bulk_dropped",
}
}
}
fn counter_from_index(idx: usize) -> Counter {
match idx {
0 => Counter::TunToMeshBulkDropped,
_ => unreachable!(),
}
}
fn stage_from_index(idx: usize) -> Stage {
match idx {
0 => Stage::TunRead,
1 => Stage::TunToMeshQueueWait,
2 => Stage::MeshSend,
3 => Stage::TunWrite,
_ => unreachable!(),
}
}
static TOTAL_NS: [AtomicU64; N_STAGES] = [const { AtomicU64::new(0) }; N_STAGES];
static COUNT: [AtomicU64; N_STAGES] = [const { AtomicU64::new(0) }; N_STAGES];
static MAX_NS: [AtomicU64; N_STAGES] = [const { AtomicU64::new(0) }; N_STAGES];
static HIST: [AtomicU64; N_STAGES * HIST_BUCKETS] =
[const { AtomicU64::new(0) }; N_STAGES * HIST_BUCKETS];
static COUNTERS: [AtomicU64; N_COUNTERS] = [const { AtomicU64::new(0) }; N_COUNTERS];
pub(crate) fn enabled() -> bool {
static ENABLED: OnceLock<bool> = OnceLock::new();
*ENABLED.get_or_init(|| {
["NVPN_PIPELINE_TRACE", "FIPS_PIPELINE_TRACE"]
.into_iter()
.any(|key| {
std::env::var(key)
.map(|s| s == "1" || s.eq_ignore_ascii_case("true"))
.unwrap_or(false)
})
})
}
#[inline]
pub(crate) fn stamp() -> Option<Instant> {
enabled().then(Instant::now)
}
#[inline]
pub(crate) fn record_since(stage: Stage, start: Option<Instant>) {
if let Some(start) = start {
record(stage, start.elapsed().as_nanos() as u64);
}
}
pub(crate) fn record(stage: Stage, elapsed_ns: u64) {
if !enabled() {
return;
}
let idx = stage as usize;
let elapsed_ns = elapsed_ns.max(1);
TOTAL_NS[idx].fetch_add(elapsed_ns, Relaxed);
MAX_NS[idx].fetch_max(elapsed_ns, Relaxed);
HIST[(idx * HIST_BUCKETS) + bucket_for_ns(elapsed_ns)].fetch_add(1, Relaxed);
COUNT[idx].fetch_add(1, Release);
}
pub(crate) fn increment_counter_by(counter: Counter, amount: u64) {
if amount > 0 && enabled() {
COUNTERS[counter as usize].fetch_add(amount, Relaxed);
}
}
pub(crate) struct Timer {
stage: Stage,
start: Option<Instant>,
}
impl Timer {
#[inline]
pub(crate) fn start(stage: Stage) -> Self {
Self {
stage,
start: stamp(),
}
}
}
impl Drop for Timer {
fn drop(&mut self) {
record_since(self.stage, self.start);
}
}
pub(crate) fn maybe_spawn_reporter() {
if !enabled() {
return;
}
static STARTED: OnceLock<()> = OnceLock::new();
if STARTED.set(()).is_err() {
return;
}
let interval = std::env::var("NVPN_PIPELINE_INTERVAL_SECS")
.ok()
.or_else(|| std::env::var("FIPS_PERF_INTERVAL_SECS").ok())
.and_then(|s| s.parse::<u64>().ok())
.unwrap_or(5)
.max(1);
tokio::spawn(async move {
let mut prev_total = [0u64; N_STAGES];
let mut prev_count = [0u64; N_STAGES];
let mut prev_hist = [0u64; N_STAGES * HIST_BUCKETS];
let mut prev_counters = [0u64; N_COUNTERS];
loop {
tokio::time::sleep(std::time::Duration::from_secs(interval)).await;
let mut line = format!("[nvpn-pipe {}s]", interval);
for i in 0..N_STAGES {
let count = COUNT[i].load(Acquire);
let dc = count.saturating_sub(prev_count[i]);
if dc == 0 {
continue;
}
let total = TOTAL_NS[i].load(Relaxed);
let dt = total.saturating_sub(prev_total[i]);
prev_total[i] = total;
prev_count[i] = count;
let base = i * HIST_BUCKETS;
let mut hist_delta = [0u64; HIST_BUCKETS];
for (bucket, slot) in hist_delta.iter_mut().enumerate() {
let idx = base + bucket;
let current = HIST[idx].load(Relaxed);
*slot = current.saturating_sub(prev_hist[idx]);
prev_hist[idx] = current;
}
let stage = stage_from_index(i);
let avg_ns = dt / dc;
let pps = dc / interval;
let p50 = percentile_ns(&hist_delta, dc, 50);
let p95 = percentile_ns(&hist_delta, dc, 95);
let p99 = percentile_ns(&hist_delta, dc, 99);
let approx_max = interval_max_ns(&hist_delta);
let lifetime_max = MAX_NS[i].load(Relaxed);
line.push_str(&format!(
" {}={}/s avg={} p50<={} p95<={} p99<={} max<={} allmax={}",
stage.name(),
pps,
fmt_ns(avg_ns),
fmt_ns(p50),
fmt_ns(p95),
fmt_ns(p99),
fmt_ns(approx_max),
fmt_ns(lifetime_max),
));
}
for i in 0..N_COUNTERS {
let current = COUNTERS[i].load(Relaxed);
let delta = current.saturating_sub(prev_counters[i]);
prev_counters[i] = current;
if delta == 0 {
continue;
}
line.push_str(&format!(
" {}={}/s total={}",
counter_from_index(i).name(),
delta / interval,
current,
));
}
eprintln!("{line}");
}
});
}
fn bucket_for_ns(ns: u64) -> usize {
if ns <= 1 {
return 0;
}
((u64::BITS - (ns - 1).leading_zeros()) as usize).min(HIST_BUCKETS - 1)
}
fn bucket_upper_ns(bucket: usize) -> u64 {
if bucket == 0 {
1
} else if bucket >= 63 {
u64::MAX
} else {
1u64 << bucket
}
}
fn percentile_ns(hist_delta: &[u64; HIST_BUCKETS], total: u64, pct: u64) -> u64 {
let observed_total = hist_delta.iter().copied().sum::<u64>();
let total = total.min(observed_total);
if total == 0 {
return 0;
}
let target = total.saturating_mul(pct).saturating_add(99) / 100;
let mut seen = 0u64;
for (idx, count) in hist_delta.iter().enumerate() {
seen = seen.saturating_add(*count);
if seen >= target {
return bucket_upper_ns(idx);
}
}
interval_max_ns(hist_delta)
}
fn interval_max_ns(hist_delta: &[u64; HIST_BUCKETS]) -> u64 {
for idx in (0..HIST_BUCKETS).rev() {
if hist_delta[idx] != 0 {
return bucket_upper_ns(idx);
}
}
0
}
fn fmt_ns(ns: u64) -> String {
if ns >= 1_000_000_000 {
format!("{:.1}s", ns as f64 / 1_000_000_000.0)
} else if ns >= 1_000_000 {
format!("{:.1}ms", ns as f64 / 1_000_000.0)
} else if ns >= 1_000 {
format!("{:.1}us", ns as f64 / 1_000.0)
} else {
format!("{ns}ns")
}
}
#[cfg(test)]
mod tests {
use super::{HIST_BUCKETS, bucket_upper_ns, percentile_ns};
#[test]
fn percentile_uses_observed_histogram_count_when_stage_count_leads() {
let mut hist = [0u64; HIST_BUCKETS];
hist[10] = 1;
assert_eq!(percentile_ns(&hist, 2, 99), bucket_upper_ns(10));
assert_eq!(percentile_ns(&[0u64; HIST_BUCKETS], 1, 99), 0);
}
}