mod bytes;
mod copy;
mod utils;
use bytes::ToBytes;
pub use copy::DerefCopy;
use num_traits::{Float, FromPrimitive};
use std::cell::RefCell;
use std::cmp::{self, Eq};
use std::collections::{BTreeSet, HashMap};
use std::f64;
use std::hash::{Hash, Hasher};
use std::iter::{self, FromIterator};
use std::ops::AddAssign;
pub use utils::StatsError;
#[derive(Debug)]
pub struct SummStats<T: Float + FromPrimitive + AddAssign> {
non_nan: bool,
count: u64,
mean: T,
ssd: T,
min: T,
max: T,
}
impl<T: Float + FromPrimitive + AddAssign> SummStats<T> {
pub fn new() -> Self {
SummStats {
non_nan: false, count: 0,
mean: T::zero(),
ssd: T::zero(),
min: T::infinity(),
max: T::neg_infinity(),
}
}
pub fn add(&mut self, bval: impl DerefCopy<Output = T>) {
self.checked_add(bval).unwrap();
}
pub fn checked_add(&mut self, rval: impl DerefCopy<Output = T>) -> Result<(), StatsError> {
let count = T::from_u64(self.count + 1).ok_or("can't convert from count to float type")?;
let val = rval.deref_copy();
self.non_nan |= !val.is_nan();
self.count += 1;
let delta = val - self.mean;
self.mean += delta / count;
self.ssd += (val - self.mean) * delta;
if val < self.min {
self.min = val;
}
if self.max < val {
self.max = val;
}
Ok(())
}
pub fn count(&self) -> u64 {
self.count
}
fn tcount(&self) -> T {
T::from_u64(self.count).unwrap()
}
pub fn min(&self) -> Option<T> {
if self.non_nan {
Some(self.min)
} else {
None
}
}
pub fn max(&self) -> Option<T> {
if self.non_nan {
Some(self.max)
} else {
None
}
}
pub fn mean(&self) -> Option<T> {
match self.count {
0 => None,
_ => Some(self.mean),
}
}
pub fn sum(&self) -> T {
self.tcount() * self.mean
}
pub fn standard_deviation(&self) -> Option<T> {
self.variance().map(T::sqrt)
}
pub fn variance(&self) -> Option<T> {
match self.count {
0 | 1 => None,
_ => Some(self.ssd / T::from_u64(self.count - 1).unwrap()),
}
}
pub fn standard_error(&self) -> Option<T> {
self.standard_deviation().map(|d| d / self.tcount().sqrt())
}
}
impl<T: Float + FromPrimitive + AddAssign> Default for SummStats<T> {
fn default() -> Self {
SummStats::new()
}
}
impl<T: Float + FromPrimitive + AddAssign, V: DerefCopy<Output = T>> FromIterator<V>
for SummStats<T>
{
fn from_iter<I>(iter: I) -> Self
where
I: IntoIterator<Item = V>,
{
let mut stats = SummStats::new();
for val in iter {
stats.add(val);
}
stats
}
}
pub fn mean<T, V, I>(data: I) -> Option<T>
where
T: Float + FromPrimitive + AddAssign,
V: DerefCopy<Output = T>,
I: IntoIterator<Item = V>,
{
data.into_iter().collect::<SummStats<_>>().mean()
}
#[derive(Debug)]
struct CachedOrdering<T: Float + FromPrimitive> {
data: Vec<T>,
in_order: BTreeSet<usize>,
}
impl<T: Float + FromPrimitive> CachedOrdering<T> {
fn new() -> Self {
CachedOrdering {
data: Vec::new(),
in_order: BTreeSet::new(),
}
}
fn add(&mut self, val: T) {
self.data.push(val);
self.in_order.clear();
}
fn order_index(&mut self, index: usize) -> T {
if self.in_order.insert(index) {
let start = match self.in_order.range(..index).next_back() {
Some(ind) => ind + 1,
None => 0,
};
let end = match self.in_order.range(index + 1..).next() {
Some(&ind) => ind,
None => self.data.len(),
};
self.data[start..end].select_nth_unstable_by(index - start, |a, b| {
a.partial_cmp(b).unwrap()
});
}
self.data[index]
}
fn len(&self) -> usize {
self.data.len()
}
}
#[derive(Debug)]
pub struct Percentiles<T: Float + FromPrimitive> {
data: RefCell<CachedOrdering<T>>,
nan_count: usize,
}
impl<T: Float + FromPrimitive> Percentiles<T> {
pub fn new() -> Self {
Percentiles {
data: RefCell::new(CachedOrdering::new()),
nan_count: 0,
}
}
pub fn add(&mut self, rval: impl DerefCopy<Output = T>) {
let val = rval.deref_copy();
if val.is_nan() {
self.nan_count += 1;
} else {
self.data.borrow_mut().add(val);
}
}
pub fn count(&self) -> usize {
self.data.borrow().len() + self.nan_count
}
pub fn percentiles<P, I>(&self, percentiles: I) -> Result<Option<Vec<T>>, StatsError>
where
P: DerefCopy<Output = f64>,
I: IntoIterator<Item = P>,
{
let len = self.data.borrow().len();
match len {
0 => Ok(None),
_ => {
let mut indexed: Vec<(usize, f64)> = percentiles
.into_iter()
.map(DerefCopy::deref_copy)
.enumerate()
.collect();
if indexed.iter().any(|(_, e)| e.is_nan()) {
Err(StatsError::from("percentiles can't be nan"))?
}
indexed.sort_unstable_by(|(_, a), (_, b)| a.partial_cmp(b).unwrap());
let mut result: Vec<Option<T>> = iter::repeat(None).take(indexed.len()).collect();
for &(ind, perc) in utils::inside_out(&indexed)? {
result[ind] = Some(self.percentile(perc)?.unwrap());
}
let checked_result: Option<Vec<_>> = result.iter().copied().collect();
Ok(Some(checked_result.unwrap()))
}
}
}
pub fn percentile(
&self,
percentile: impl DerefCopy<Output = f64>,
) -> Result<Option<T>, StatsError> {
let perc = percentile.deref_copy();
if perc < 0.0 || 1.0 < perc {
Err(StatsError::new(format!(
"all percentiles must be between 0 and 1, but got: {}",
perc
)))
} else {
let mut ordering = self.data.borrow_mut();
match ordering.len() {
0 => Ok(None),
_ => {
let p_index = (ordering.len() - 1) as f64 * perc;
let low_index = p_index.floor() as usize;
let high_index = p_index.ceil() as usize;
let low = ordering.order_index(low_index);
let high = ordering.order_index(high_index);
let weight = p_index - low_index as f64;
let perc = utils::weighted_average(low, high, weight)
.ok_or("can't convert from weight to float")?;
Ok(Some(perc))
}
}
}
}
pub fn median(&self) -> Option<T> {
self.percentile(0.5).expect("0.5 is a valid percentile")
}
}
impl<T: Float + FromPrimitive> Default for Percentiles<T> {
fn default() -> Self {
Percentiles::new()
}
}
impl<T: Float + FromPrimitive, V: DerefCopy<Output = T>> FromIterator<V> for Percentiles<T> {
fn from_iter<I>(iter: I) -> Self
where
I: IntoIterator<Item = V>,
{
let mut percs = Percentiles::new();
for val in iter {
percs.add(val);
}
percs
}
}
pub fn median<T, V, I>(data: I) -> Option<T>
where
T: Float + FromPrimitive,
V: DerefCopy<Output = T>,
I: IntoIterator<Item = V>,
{
data.into_iter().collect::<Percentiles<T>>().median()
}
#[derive(Debug, PartialEq)]
struct HashFloat<T: Float + ToBytes>(T);
impl<T: Float + ToBytes> Eq for HashFloat<T> {}
impl<T: Float + ToBytes> Hash for HashFloat<T> {
fn hash<H: Hasher>(&self, state: &mut H) {
self.0.to_bytes().hash(state);
}
}
#[derive(Debug)]
pub struct Mode<T: Float + ToBytes> {
counts: HashMap<HashFloat<T>, usize>,
count: usize,
nan_count: usize,
mode: Vec<T>,
mode_count: usize,
}
impl<T: Float + ToBytes> Mode<T> {
pub fn new() -> Self {
Mode {
counts: HashMap::new(),
count: 0,
nan_count: 0,
mode: Vec::new(),
mode_count: 0,
}
}
pub fn add(&mut self, rval: impl DerefCopy<Output = T>) {
let val = rval.deref_copy();
self.count += 1;
if val.is_nan() {
self.nan_count += 1;
} else {
let val_count = self.counts.entry(HashFloat(val)).or_insert(0);
*val_count += 1;
if *val_count > self.mode_count {
self.mode.clear();
self.mode.push(val);
self.mode_count += 1;
} else if *val_count == self.mode_count {
self.mode.push(val);
}
}
}
pub fn count(&self) -> usize {
self.count
}
pub fn count_distinct(&self) -> usize {
self.counts.len()
}
pub fn count_distinct_nan(&self) -> usize {
self.counts.len() + self.nan_count
}
pub fn modes(&self) -> impl Iterator<Item = T> + '_ {
self.mode.iter().copied()
}
fn nan_mode(&self) -> Option<T> {
if self.nan_count > 0 && self.nan_count >= self.mode_count {
Some(T::nan())
} else {
None
}
}
pub fn modes_nan(&self) -> impl Iterator<Item = T> + '_ {
self.modes().chain(self.nan_mode())
}
pub fn mode(&self) -> Option<T> {
self.modes().next()
}
pub fn mode_nan(&self) -> Option<T> {
if self.nan_count > self.mode_count {
Some(T::nan())
} else {
self.mode()
}
}
pub fn mode_count(&self) -> usize {
self.mode_count
}
pub fn mode_count_nan(&self) -> usize {
cmp::max(self.mode_count, self.nan_count)
}
}
impl<T: Float + ToBytes, V: DerefCopy<Output = T>> FromIterator<V> for Mode<T> {
fn from_iter<I>(iter: I) -> Self
where
I: IntoIterator<Item = V>,
{
let mut mode = Mode::new();
for val in iter {
mode.add(val);
}
mode
}
}
pub fn mode<T, V, I>(data: I) -> Option<T>
where
T: Float + ToBytes,
V: DerefCopy<Output = T>,
I: IntoIterator<Item = V>,
{
data.into_iter().collect::<Mode<T>>().mode()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn f32_mean_test() {
let avg: f32 = mean(&[0.0, 1.0, 2.0]).unwrap();
assert!((avg - 1.0).abs() < 1e-6);
}
#[test]
fn f32_median_test() {
let avg: f32 = median(&[0.0, 1.0, 2.0, 3.0]).unwrap();
assert!((avg - 1.5).abs() < 1e-6);
}
#[test]
fn nan_percentile_test() {
let percs: Percentiles<_> = [f64::NAN].iter().collect();
assert_eq!(1, percs.count());
assert_eq!(None, percs.median());
}
#[test]
fn nan_mode_test() {
let avg: Mode<_> = [f64::NAN].iter().collect();
assert!(avg.mode_nan().unwrap().is_nan());
}
#[test]
fn cached_ordering_test() {
let mut ord = CachedOrdering::new();
ord.add(0.0);
ord.add(1.0);
ord.add(2.0);
assert_eq!(1.0, ord.order_index(1));
assert!(ord.in_order.contains(&1));
}
}