#[cfg(not(feature = "hdr"))]
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;
#[cfg(not(feature = "hdr"))]
const LINEAR_BUCKETS: usize = 1024;
#[cfg(not(feature = "hdr"))]
const LOG_BUCKETS: usize = 64;
#[cfg(not(feature = "hdr"))]
const MEMORY_ORDER: Ordering = Ordering::Relaxed;
#[cfg(not(feature = "hdr"))]
#[derive(Debug)]
pub struct FastHistogram {
linear_buckets: [AtomicU64; LINEAR_BUCKETS],
log_buckets: [AtomicU64; LOG_BUCKETS],
min_value: AtomicU64,
max_value: AtomicU64,
total_count: AtomicU64,
sum: AtomicU64,
}
#[cfg(not(feature = "hdr"))]
impl FastHistogram {
pub fn new() -> Self {
Self {
linear_buckets: std::array::from_fn(|_| AtomicU64::new(0)),
log_buckets: std::array::from_fn(|_| AtomicU64::new(0)),
min_value: AtomicU64::new(u64::MAX),
max_value: AtomicU64::new(0),
total_count: AtomicU64::new(0),
sum: AtomicU64::new(0),
}
}
#[inline]
pub fn record(&self, value_ns: u64) {
self.update_min(value_ns);
self.update_max(value_ns);
self.total_count.fetch_add(1, MEMORY_ORDER);
self.sum
.fetch_add(value_ns.min(u64::MAX - 1000), MEMORY_ORDER);
if value_ns < LINEAR_BUCKETS as u64 {
#[allow(clippy::cast_possible_truncation)]
{
self.linear_buckets[value_ns as usize].fetch_add(1, MEMORY_ORDER);
}
} else {
let bucket_index = Self::log_bucket_index(value_ns);
if bucket_index < LOG_BUCKETS {
self.log_buckets[bucket_index].fetch_add(1, MEMORY_ORDER);
}
}
}
#[inline]
pub fn record_duration(&self, duration: Duration) {
let nanos = duration.as_nanos();
let clamped_nanos = if nanos > u128::from(u64::MAX) {
u64::MAX
} else {
u64::try_from(nanos).unwrap_or(u64::MAX)
};
self.record(clamped_nanos);
}
#[inline]
pub fn min(&self) -> Option<u64> {
let min = self.min_value.load(MEMORY_ORDER);
if min == u64::MAX {
None
} else {
Some(min)
}
}
#[inline]
pub fn max(&self) -> Option<u64> {
let count = self.total_count.load(MEMORY_ORDER);
if count == 0 {
None
} else {
Some(self.max_value.load(MEMORY_ORDER))
}
}
#[inline]
pub fn mean(&self) -> Option<f64> {
let count = self.total_count.load(MEMORY_ORDER);
if count == 0 {
None
} else {
let sum = self.sum.load(MEMORY_ORDER);
#[allow(clippy::cast_precision_loss)]
{
Some(sum as f64 / count as f64)
}
}
}
#[inline]
pub fn count(&self) -> u64 {
self.total_count.load(MEMORY_ORDER)
}
#[inline]
pub fn is_empty(&self) -> bool {
self.total_count.load(MEMORY_ORDER) == 0
}
#[inline]
pub fn percentile(&self, percentile: f64) -> Option<u64> {
let total_count = self.total_count.load(MEMORY_ORDER);
if total_count == 0 {
return None;
}
let p = percentile.clamp(0.0, 1.0);
#[allow(clippy::float_cmp)]
if p == 0.0 {
return self.min();
}
#[allow(clippy::float_cmp)]
if p == 1.0 {
return self.max();
}
let target_count = {
#[allow(
clippy::cast_possible_truncation,
clippy::cast_sign_loss,
clippy::cast_precision_loss
)]
{
(p * total_count as f64).ceil() as u64
}
};
let mut current_count = 0u64;
let min_v = self.min()?;
let max_v = self.max()?;
for (value, bucket) in self.linear_buckets.iter().enumerate() {
let count = bucket.load(MEMORY_ORDER);
if count == 0 {
continue;
}
current_count += count;
if current_count >= target_count {
let v = value as u64;
return Some(v.clamp(min_v, max_v));
}
}
for (bucket_idx, bucket) in self.log_buckets.iter().enumerate() {
let count = bucket.load(MEMORY_ORDER);
if count == 0 {
continue;
}
let bucket_start = Self::bucket_start(bucket_idx);
let bucket_end = Self::bucket_end(bucket_idx);
if current_count + count >= target_count {
let position_in_bucket = target_count.saturating_sub(current_count);
let bucket_width = bucket_end.saturating_sub(bucket_start);
if count > 0 && bucket_width > 0 {
let num = (u128::from(position_in_bucket.saturating_sub(1)))
* u128::from(bucket_width);
let den = u128::from(count);
let interpolated_offset = u64::try_from(num / den).unwrap_or(u64::MAX);
let v = bucket_start.saturating_add(interpolated_offset);
return Some(v.clamp(min_v, max_v));
}
return Some(bucket_start.clamp(min_v, max_v));
}
current_count += count;
}
self.max()
}
#[inline]
pub fn median(&self) -> Option<u64> {
self.percentile(0.5)
}
#[inline]
pub fn median_duration(&self) -> Option<Duration> {
self.median().map(Duration::from_nanos)
}
#[inline]
pub fn percentile_duration(&self, percentile: f64) -> Option<Duration> {
self.percentile(percentile).map(Duration::from_nanos)
}
#[inline]
pub fn percentiles(&self, percentiles: &[f64]) -> Vec<Option<u64>> {
let total_count = self.total_count.load(MEMORY_ORDER);
if total_count == 0 {
return vec![None; percentiles.len()];
}
let mut results = vec![None; percentiles.len()];
let mut targets: Vec<(usize, u64)> = percentiles
.iter()
.enumerate()
.map(|(i, &p_in)| {
let p = p_in.clamp(0.0, 1.0);
let target = if p == 0.0 {
0
} else if (p - 1.0).abs() < f64::EPSILON {
total_count
} else {
#[allow(
clippy::cast_possible_truncation,
clippy::cast_sign_loss,
clippy::cast_precision_loss
)]
{
(p * total_count as f64).ceil() as u64
}
};
(i, target)
})
.collect();
targets.sort_by_key(|&(_, count)| count);
let mut current_count = 0u64;
let mut target_idx = 0;
while target_idx < targets.len() && targets[target_idx].1 == 0 {
results[targets[target_idx].0] = self.min();
target_idx += 1;
}
let min_v = self.min();
let max_v = self.max();
for (value, bucket) in self.linear_buckets.iter().enumerate() {
let count = bucket.load(MEMORY_ORDER);
if count == 0 || target_idx >= targets.len() {
continue;
}
current_count += count;
while target_idx < targets.len() && current_count >= targets[target_idx].1 {
let v = value as u64;
results[targets[target_idx].0] = Some(v.clamp(min_v.unwrap(), max_v.unwrap()));
target_idx += 1;
}
}
for (bucket_idx, bucket) in self.log_buckets.iter().enumerate() {
let count = bucket.load(MEMORY_ORDER);
if count == 0 || target_idx >= targets.len() {
continue;
}
let bucket_start = Self::bucket_start(bucket_idx);
let bucket_end = Self::bucket_end(bucket_idx);
while target_idx < targets.len() && current_count + count >= targets[target_idx].1 {
let position_in_bucket = targets[target_idx].1.saturating_sub(current_count);
let bucket_width = bucket_end.saturating_sub(bucket_start);
let interpolated_value = if count > 0 && bucket_width > 0 {
let num = (u128::from(position_in_bucket.saturating_sub(1)))
* u128::from(bucket_width);
let den = u128::from(count);
let interpolated_offset = u64::try_from(num / den).unwrap_or(u64::MAX);
bucket_start.saturating_add(interpolated_offset)
} else {
bucket_start
};
let v = interpolated_value.clamp(min_v.unwrap(), max_v.unwrap());
results[targets[target_idx].0] = Some(v);
target_idx += 1;
}
current_count += count;
}
while target_idx < targets.len() {
results[targets[target_idx].0] = self.max();
target_idx += 1;
}
for (i, &p_in) in percentiles.iter().enumerate() {
let p = p_in.clamp(0.0, 1.0);
if (p - 1.0).abs() < f64::EPSILON {
results[i] = self.max();
}
}
results
}
pub fn reset(&self) {
for bucket in &self.linear_buckets {
bucket.store(0, MEMORY_ORDER);
}
for bucket in &self.log_buckets {
bucket.store(0, MEMORY_ORDER);
}
self.min_value.store(u64::MAX, MEMORY_ORDER);
self.max_value.store(0, MEMORY_ORDER);
self.total_count.store(0, MEMORY_ORDER);
self.sum.store(0, MEMORY_ORDER);
}
#[inline]
fn update_min(&self, value: u64) {
let mut current_min = self.min_value.load(MEMORY_ORDER);
while value < current_min {
match self.min_value.compare_exchange_weak(
current_min,
value,
MEMORY_ORDER,
MEMORY_ORDER,
) {
Ok(_) => break,
Err(actual) => current_min = actual,
}
}
}
#[inline]
fn update_max(&self, value: u64) {
let mut current_max = self.max_value.load(MEMORY_ORDER);
while value > current_max {
match self.max_value.compare_exchange_weak(
current_max,
value,
MEMORY_ORDER,
MEMORY_ORDER,
) {
Ok(_) => break,
Err(actual) => current_max = actual,
}
}
}
#[inline]
fn log_bucket_index(value: u64) -> usize {
if value < LINEAR_BUCKETS as u64 {
0 } else {
63 - value.leading_zeros() as usize
}
}
#[inline]
fn bucket_start(bucket_idx: usize) -> u64 {
if bucket_idx == 0 {
LINEAR_BUCKETS as u64
} else {
(1u64 << bucket_idx).max(LINEAR_BUCKETS as u64)
}
}
#[inline]
fn bucket_end(bucket_idx: usize) -> u64 {
if bucket_idx >= 63 {
u64::MAX
} else {
1u64 << (bucket_idx + 1)
}
}
}
#[cfg(not(feature = "hdr"))]
impl Default for FastHistogram {
fn default() -> Self {
Self::new()
}
}
#[cfg(feature = "hdr")]
type BackendHistogram = crate::hist_hdr::Histogram;
#[cfg(not(feature = "hdr"))]
type BackendHistogram = FastHistogram;
#[derive(Debug)]
pub struct Histogram {
inner: BackendHistogram,
}
impl Histogram {
#[inline]
pub fn new() -> Self {
Self {
inner: BackendHistogram::new(),
}
}
#[inline]
pub fn record(&self, value_ns: u64) {
self.inner.record(value_ns);
}
#[inline]
pub fn record_duration(&self, duration: Duration) {
self.inner.record_duration(duration);
}
#[inline]
pub fn min(&self) -> Option<u64> {
self.inner.min()
}
#[inline]
pub fn max(&self) -> Option<u64> {
self.inner.max()
}
#[inline]
pub fn mean(&self) -> Option<f64> {
self.inner.mean()
}
#[inline]
pub fn count(&self) -> u64 {
self.inner.count()
}
#[inline]
pub fn is_empty(&self) -> bool {
self.inner.is_empty()
}
#[inline]
pub fn percentile(&self, percentile: f64) -> Option<u64> {
self.inner.percentile(percentile)
}
#[inline]
pub fn median(&self) -> Option<u64> {
self.inner.median()
}
#[inline]
pub fn median_duration(&self) -> Option<Duration> {
self.inner.median_duration()
}
#[inline]
pub fn percentile_duration(&self, percentile: f64) -> Option<Duration> {
self.inner.percentile_duration(percentile)
}
#[inline]
pub fn percentiles(&self, percentiles: &[f64]) -> Vec<Option<u64>> {
self.inner.percentiles(percentiles)
}
pub fn reset(&self) {
self.inner.reset();
}
}
impl Default for Histogram {
fn default() -> Self {
Self::new()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[cfg(not(feature = "hdr"))]
use std::sync::Arc;
#[cfg(not(feature = "hdr"))]
use std::thread;
#[inline]
fn perf_enabled() -> bool {
std::env::var_os("PERF_TESTS").is_some()
}
#[test]
fn test_empty_histogram() {
let hist = Histogram::new();
assert!(hist.is_empty());
assert_eq!(hist.count(), 0);
assert_eq!(hist.min(), None);
assert_eq!(hist.max(), None);
assert_eq!(hist.mean(), None);
assert_eq!(hist.percentile(0.5), None);
assert_eq!(hist.median(), None);
}
#[test]
fn test_basic_statistics() {
let hist = Histogram::new();
hist.record(100);
hist.record(200);
hist.record(300);
assert!(!hist.is_empty());
assert_eq!(hist.count(), 3);
assert_eq!(hist.min(), Some(100));
assert_eq!(hist.max(), Some(300));
assert_eq!(hist.mean(), Some(200.0));
assert_eq!(hist.median(), Some(200));
}
#[cfg(not(feature = "hdr"))]
#[test]
fn test_percentiles() {
let hist = Histogram::new();
for i in 1..=100 {
hist.record(i);
}
assert_eq!(hist.percentile(0.0), Some(1));
assert_eq!(hist.percentile(0.5), Some(50));
assert_eq!(hist.percentile(0.99), Some(99));
assert_eq!(hist.percentile(1.0), Some(100));
assert_eq!(hist.percentile(-0.1), Some(1));
assert_eq!(hist.percentile(1.1), Some(100));
}
#[cfg(not(feature = "hdr"))]
#[test]
fn test_multiple_percentiles() {
let hist = Histogram::new();
for i in 1..=1000 {
hist.record(i);
}
let percentiles = hist.percentiles(&[0.0, 0.25, 0.5, 0.75, 0.95, 0.99, 1.0]);
assert_eq!(percentiles[0], Some(1)); assert_eq!(percentiles[1], Some(250)); assert_eq!(percentiles[2], Some(500)); assert_eq!(percentiles[3], Some(750)); assert_eq!(percentiles[4], Some(950)); assert_eq!(percentiles[5], Some(990)); assert_eq!(percentiles[6], Some(1000)); }
#[cfg(not(feature = "hdr"))]
#[test]
fn test_large_values() {
let hist = Histogram::new();
hist.record(1_000_000); hist.record(1_000_000_000); hist.record(500);
assert_eq!(hist.min(), Some(500));
assert_eq!(hist.max(), Some(1_000_000_000));
assert_eq!(hist.count(), 3);
}
#[cfg(not(feature = "hdr"))]
#[test]
fn test_duration_api() {
let hist = Histogram::new();
hist.record_duration(Duration::from_nanos(100));
hist.record_duration(Duration::from_micros(1)); hist.record_duration(Duration::from_millis(1));
assert_eq!(hist.count(), 3);
assert_eq!(hist.min(), Some(100));
assert_eq!(hist.max(), Some(1_000_000));
let median_duration = hist.median_duration().unwrap();
assert_eq!(median_duration, Duration::from_nanos(1000));
}
#[test]
fn test_reset() {
let hist = Histogram::new();
hist.record(100);
hist.record(200);
assert_eq!(hist.count(), 2);
hist.reset();
assert!(hist.is_empty());
assert_eq!(hist.count(), 0);
assert_eq!(hist.min(), None);
assert_eq!(hist.max(), None);
}
#[cfg(not(feature = "hdr"))]
#[test]
fn test_overflow_protection() {
let hist = Histogram::new();
hist.record(u64::MAX);
hist.record(u64::MAX - 1);
assert_eq!(hist.count(), 2);
assert_eq!(hist.max(), Some(u64::MAX));
let mean = hist.mean().unwrap();
assert!(mean > 0.0);
}
#[cfg(not(feature = "hdr"))]
#[test]
fn test_thread_safety() {
let hist = Arc::new(Histogram::new());
let mut handles = vec![];
for thread_id in 0..10 {
let hist_clone = Arc::clone(&hist);
let handle = thread::spawn(move || {
for i in 0..1000 {
hist_clone.record(thread_id * 1000 + i);
}
});
handles.push(handle);
}
for handle in handles {
handle.join().unwrap();
}
assert_eq!(hist.count(), 10_000);
assert_eq!(hist.min(), Some(0));
assert_eq!(hist.max(), Some(9_999));
}
#[cfg(not(feature = "hdr"))]
#[test]
fn test_concurrent_statistics() {
let hist = Arc::new(Histogram::new());
let mut handles = vec![];
let hist_writer = Arc::clone(&hist);
let writer_handle = thread::spawn(move || {
for i in 1..=10000 {
hist_writer.record(i);
if i % 100 == 0 {
thread::yield_now(); }
}
});
for _ in 0..5 {
let hist_reader = Arc::clone(&hist);
let reader_handle = thread::spawn(move || {
for _ in 0..100 {
let _ = hist_reader.count();
let _ = hist_reader.min();
let _ = hist_reader.max();
let _ = hist_reader.mean();
let _ = hist_reader.median();
let _ = hist_reader.percentile(0.95);
thread::yield_now();
}
});
handles.push(reader_handle);
}
writer_handle.join().unwrap();
for handle in handles {
handle.join().unwrap();
}
assert_eq!(hist.count(), 10_000);
assert_eq!(hist.min(), Some(1));
assert_eq!(hist.max(), Some(10_000));
assert_eq!(hist.median(), Some(5_000));
}
#[cfg(not(feature = "hdr"))]
#[test]
fn test_precision_linear_buckets() {
let hist = Histogram::new();
for i in 0u32..1024 {
hist.record(u64::from(i));
}
for i in 0u32..1024 {
let percentile = f64::from(i) / 1023.0;
let value = hist.percentile(percentile).unwrap();
assert!(value <= u64::from(i), "Value {value} should be <= {i}");
}
}
#[cfg(not(feature = "hdr"))]
#[test]
fn test_logarithmic_buckets() {
let hist = Histogram::new();
let large_values = vec![
2_000, 10_000, 100_000, 1_000_000, 10_000_000, ];
for &value in &large_values {
hist.record(value);
}
assert_eq!(hist.count(), 5);
assert_eq!(hist.min(), Some(2_000));
assert_eq!(hist.max(), Some(10_000_000));
let median = hist.median().unwrap();
assert!((50_000..=150_000).contains(&median));
}
#[cfg(not(feature = "hdr"))]
#[test]
fn test_edge_cases() {
let hist = Histogram::new();
hist.record(0);
assert_eq!(hist.min(), Some(0));
assert_eq!(hist.max(), Some(0));
assert_eq!(hist.median(), Some(0));
hist.reset();
hist.record(42);
assert_eq!(hist.count(), 1);
assert_eq!(hist.percentile(0.0), Some(42));
assert_eq!(hist.percentile(0.5), Some(42));
assert_eq!(hist.percentile(1.0), Some(42));
}
#[cfg_attr(
not(feature = "perf-tests"),
ignore = "perf tests are opt-in; set PERF_TESTS=1 to enable"
)]
#[test]
fn test_performance_characteristics() {
if !perf_enabled() {
eprintln!("skipping perf test: set PERF_TESTS=1 to enable");
return;
}
let hist = Histogram::new();
let start = std::time::Instant::now();
for i in 0..100_000 {
hist.record(i % 10_000);
}
let record_duration = start.elapsed();
assert!(
record_duration.as_millis() < 10,
"Recording 100k values took {}ms, expected <10ms",
record_duration.as_millis()
);
let start = std::time::Instant::now();
for _ in 0..1000 {
let _ = hist.percentile(0.95);
}
let percentile_duration = start.elapsed();
assert!(
percentile_duration.as_millis() < 1,
"1000 percentile calculations took {}ms, expected <1ms",
percentile_duration.as_millis()
);
}
#[cfg_attr(
not(feature = "perf-tests"),
ignore = "perf tests are opt-in; set PERF_TESTS=1 to enable"
)]
#[test]
fn test_memory_efficiency() {
let hist = Histogram::new();
for i in 0..1_000_000 {
hist.record(i);
}
assert_eq!(hist.count(), 1_000_000);
#[cfg(not(feature = "hdr"))]
{
assert_eq!(hist.min(), Some(0));
}
#[cfg(feature = "hdr")]
{
assert!(matches!(hist.min(), Some(0 | 1)));
}
#[cfg(not(feature = "hdr"))]
{
assert_eq!(hist.max(), Some(999_999));
}
#[cfg(feature = "hdr")]
{
assert!(matches!(hist.max(), Some(m) if m >= 999_999));
}
let median = hist.median().unwrap();
assert!((400_000..=600_000).contains(&median));
}
}
#[cfg(all(feature = "benchmark", test))]
mod benches {
use super::*;
use std::sync::Arc;
use std::thread;
#[inline]
fn perf_enabled() -> bool {
std::env::var_os("PERF_TESTS").is_some()
}
#[cfg_attr(
not(feature = "perf-tests"),
ignore = "perf tests are opt-in; set PERF_TESTS=1 to enable"
)]
#[test]
fn bench_record_single_thread() {
if !perf_enabled() {
eprintln!("skipping perf bench: set PERF_TESTS=1 to enable");
return;
}
let hist = Histogram::new();
let iterations: u64 = 10_000_000;
let start = std::time::Instant::now();
for i in 0..iterations {
hist.record(i % 100_000);
}
let duration = start.elapsed();
let ns_per_op = duration.as_nanos() / u128::from(iterations);
println!("Single-thread record: {ns_per_op} ns/op");
assert!(
ns_per_op < 20,
"Record operation too slow: {ns_per_op} ns/op",
);
}
#[cfg_attr(
not(feature = "perf-tests"),
ignore = "perf tests are opt-in; set PERF_TESTS=1 to enable"
)]
#[test]
fn bench_record_multi_thread() {
if !perf_enabled() {
eprintln!("skipping perf bench: set PERF_TESTS=1 to enable");
return;
}
let hist = Arc::new(Histogram::new());
let threads: u64 = 8;
let iterations_per_thread: u64 = 1_000_000;
let start = std::time::Instant::now();
let handles: Vec<_> = (0..threads)
.map(|thread_id| {
let hist_clone = Arc::clone(&hist);
thread::spawn(move || {
for i in 0..iterations_per_thread {
hist_clone.record(thread_id * 1_000_000 + i);
}
})
})
.collect();
for handle in handles {
handle.join().unwrap();
}
let duration = start.elapsed();
let total_ops: u64 = threads * iterations_per_thread;
let ns_per_op = duration.as_nanos() / u128::from(total_ops);
println!("Multi-thread record ({threads} threads): {ns_per_op} ns/op");
assert!(
ns_per_op < 50,
"Multi-thread record too slow: {ns_per_op} ns/op",
);
assert_eq!(hist.count(), total_ops);
}
#[cfg_attr(
not(feature = "perf-tests"),
ignore = "perf tests are opt-in; set PERF_TESTS=1 to enable"
)]
#[test]
fn bench_percentile_calculation() {
if !perf_enabled() {
eprintln!("skipping perf bench: set PERF_TESTS=1 to enable");
return;
}
let hist = Histogram::new();
for i in 0..100_000 {
hist.record(i % 10_000);
}
let iterations: u64 = 100_000;
let start = std::time::Instant::now();
for i in 0..iterations {
let percentile = f64::from((i % 100) as u32) / 100.0;
let _ = hist.percentile(percentile);
}
let duration = start.elapsed();
let ns_per_op = duration.as_nanos() / u128::from(iterations);
println!("Percentile calculation: {ns_per_op} ns/op");
assert!(
ns_per_op < 1000,
"Percentile calculation too slow: {ns_per_op} ns/op",
);
}
}