use std::fmt;
use std::str::FromStr;
use crate::error::{Result, UcciError};
use crate::num::{check_finite, check_finite_scalar, check_same_len, pairwise_sum};
use crate::policy::linspace01;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
#[cfg_attr(feature = "json", derive(serde::Serialize, serde::Deserialize))]
#[cfg_attr(feature = "json", serde(rename_all = "lowercase"))]
pub enum BinStrategy {
#[default]
Uniform,
Quantile,
}
impl BinStrategy {
pub fn as_str(self) -> &'static str {
match self {
BinStrategy::Uniform => "uniform",
BinStrategy::Quantile => "quantile",
}
}
}
impl fmt::Display for BinStrategy {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.as_str())
}
}
impl FromStr for BinStrategy {
type Err = UcciError;
fn from_str(s: &str) -> Result<Self> {
match s {
"uniform" => Ok(BinStrategy::Uniform),
"quantile" => Ok(BinStrategy::Quantile),
other => Err(UcciError::InvalidParameter {
name: "strategy",
reason: format!("must be 'uniform' or 'quantile', got {other:?}"),
}),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq)]
#[cfg_attr(feature = "json", derive(serde::Serialize, serde::Deserialize))]
pub struct ReliabilityRow {
pub bin_lower: f64,
pub bin_upper: f64,
pub count: usize,
pub mean_forecast: f64,
pub observed_frequency: f64,
}
fn forecast_inputs(
p: &[f64],
y: &[f64],
sample_weight: Option<&[f64]>,
) -> Result<(Vec<f64>, Vec<f64>, Vec<f64>)> {
if p.is_empty() {
return Err(UcciError::Empty { what: "p" });
}
check_finite(p, "p")?;
check_unit_interval(p, "p")?;
if y.is_empty() {
return Err(UcciError::Empty { what: "y" });
}
check_finite(y, "y")?;
check_unit_interval(y, "y")?;
check_same_len(p.len(), y.len(), "p and y")?;
match sample_weight {
None => Ok((p.to_vec(), y.to_vec(), vec![1.0; p.len()])),
Some(w) => {
check_same_len(p.len(), w.len(), "p and sample_weight")?;
check_finite(w, "sample_weight")?;
if let Some(index) = w.iter().position(|&v| v < 0.0) {
return Err(UcciError::OutOfRange {
what: "sample_weight",
index,
value: w[index],
expected: "non-negative",
});
}
if !w.iter().any(|&v| v > 0.0) {
return Err(UcciError::InvalidParameter {
name: "sample_weight",
reason: "sums to zero; at least one weight must be positive".to_string(),
});
}
let keep: Vec<usize> = (0..w.len()).filter(|&i| w[i] > 0.0).collect();
Ok((
keep.iter().map(|&i| p[i]).collect(),
keep.iter().map(|&i| y[i]).collect(),
keep.iter().map(|&i| w[i]).collect(),
))
}
}
}
fn check_unit_interval(v: &[f64], what: &'static str) -> Result<()> {
match v.iter().position(|x| !(0.0..=1.0).contains(x)) {
Some(index) => Err(UcciError::OutOfRange {
what,
index,
value: v[index],
expected: "in [0, 1]",
}),
None => Ok(()),
}
}
fn np_quantile_sorted(sorted: &[f64], q: f64) -> f64 {
let n = sorted.len();
let last = (n - 1) as f64;
let v = last * q;
if v >= last {
return sorted[n - 1];
}
if v < 0.0 {
return sorted[0];
}
let prev = v.floor();
let gamma = v - prev;
let i = prev as usize;
let (a, b) = (sorted[i], sorted[i + 1]);
let diff = b - a;
if gamma >= 0.5 {
b - diff * (1.0 - gamma)
} else {
a + diff * gamma
}
}
fn bin_edges(p: &[f64], n_bins: usize, strategy: BinStrategy) -> Vec<f64> {
let q = linspace01(n_bins);
match strategy {
BinStrategy::Uniform => q,
BinStrategy::Quantile => {
let mut sorted = p.to_vec();
sorted.sort_by(f64::total_cmp);
let mut edges: Vec<f64> = q
.iter()
.map(|&qi| np_quantile_sorted(&sorted, qi))
.collect();
edges.sort_by(f64::total_cmp);
edges.dedup_by(|a, b| a == b);
edges
}
}
}
fn binned(
p: &[f64],
y: &[f64],
n_bins: usize,
strategy: BinStrategy,
sample_weight: Option<&[f64]>,
) -> Result<(Vec<ReliabilityRow>, Vec<f64>, f64)> {
if n_bins < 1 {
return Err(UcciError::InvalidParameter {
name: "n_bins",
reason: format!("must be at least 1, got {n_bins}"),
});
}
let (pp, yy, w) = forecast_inputs(p, y, sample_weight)?;
let edges = bin_edges(&pp, n_bins, strategy);
let n_edges = edges.len();
let (ids, lowers, uppers): (Vec<usize>, Vec<f64>, Vec<f64>) = if n_edges == 1 {
(vec![0; pp.len()], edges.clone(), edges.clone())
} else {
let ids = pp
.iter()
.map(|&x| {
let pos = edges.partition_point(|&e| e < x);
pos.saturating_sub(1).min(n_edges - 2)
})
.collect();
(ids, edges[..n_edges - 1].to_vec(), edges[1..].to_vec())
};
let n_groups = lowers.len();
let mut counts = vec![0usize; n_groups];
let mut w_bin = vec![0.0f64; n_groups];
let mut wp_bin = vec![0.0f64; n_groups];
let mut wy_bin = vec![0.0f64; n_groups];
for (i, &b) in ids.iter().enumerate() {
counts[b] += 1;
w_bin[b] += w[i];
wp_bin[b] += w[i] * pp[i];
wy_bin[b] += w[i] * yy[i];
}
let mut rows = Vec::new();
let mut weights = Vec::new();
for b in 0..n_groups {
if counts[b] == 0 {
continue;
}
rows.push(ReliabilityRow {
bin_lower: lowers[b],
bin_upper: uppers[b],
count: counts[b],
mean_forecast: wp_bin[b] / w_bin[b],
observed_frequency: wy_bin[b] / w_bin[b],
});
weights.push(w_bin[b]);
}
Ok((rows, weights, pairwise_sum(&w)))
}
pub fn reliability_table(
p: &[f64],
y: &[f64],
n_bins: usize,
strategy: BinStrategy,
sample_weight: Option<&[f64]>,
) -> Result<Vec<ReliabilityRow>> {
Ok(binned(p, y, n_bins, strategy, sample_weight)?.0)
}
pub fn ece(
p: &[f64],
y: &[f64],
n_bins: usize,
strategy: BinStrategy,
sample_weight: Option<&[f64]>,
) -> Result<f64> {
let (rows, weights, total) = binned(p, y, n_bins, strategy, sample_weight)?;
let terms: Vec<f64> = rows
.iter()
.zip(&weights)
.map(|(r, &wb)| wb / total * (r.mean_forecast - r.observed_frequency).abs())
.collect();
Ok(pairwise_sum(&terms))
}
pub fn brier_score(p: &[f64], y: &[f64], sample_weight: Option<&[f64]>) -> Result<f64> {
let (pp, yy, w) = forecast_inputs(p, y, sample_weight)?;
let terms: Vec<f64> = (0..pp.len())
.map(|i| {
let d = pp[i] - yy[i];
w[i] * (d * d)
})
.collect();
Ok(pairwise_sum(&terms) / pairwise_sum(&w))
}
pub fn micro_f1(tp: f64, fp: f64, fn_: f64, zero_division: f64) -> Result<f64> {
for (name, v) in [("tp", tp), ("fp", fp), ("fn", fn_)] {
check_finite_scalar(v, name)?;
if v < 0.0 {
return Err(UcciError::InvalidParameter {
name,
reason: format!("counts must be non-negative, got {v}"),
});
}
}
let denom = 2.0 * tp + fp + fn_;
if denom == 0.0 {
check_finite_scalar(zero_division, "zero_division")?;
return Ok(zero_division);
}
Ok(2.0 * tp / denom)
}
#[derive(Debug, Clone, PartialEq)]
pub struct RoutedMicroF1 {
small: Vec<[f64; 3]>,
large: Vec<[f64; 3]>,
zero_division: f64,
}
impl RoutedMicroF1 {
pub fn new(
small_counts: &[[f64; 3]],
large_counts: &[[f64; 3]],
zero_division: f64,
) -> Result<Self> {
for (name, counts) in [
("small_counts", small_counts),
("large_counts", large_counts),
] {
if counts.is_empty() {
return Err(UcciError::Empty { what: name });
}
if counts.iter().flatten().any(|v| !v.is_finite() || *v < 0.0) {
return Err(UcciError::InvalidParameter {
name,
reason: "counts must be finite and non-negative".to_string(),
});
}
}
check_same_len(
small_counts.len(),
large_counts.len(),
"small_counts and large_counts",
)?;
check_finite_scalar(zero_division, "zero_division")?;
Ok(Self {
small: small_counts.to_vec(),
large: large_counts.to_vec(),
zero_division,
})
}
pub fn len(&self) -> usize {
self.small.len()
}
pub fn is_empty(&self) -> bool {
self.small.is_empty()
}
pub fn score(&self, esc: &[bool]) -> Result<f64> {
check_same_len(esc.len(), self.small.len(), "esc and counts")?;
let mut small = [0.0f64; 3];
let mut large = [0.0f64; 3];
for (i, &e) in esc.iter().enumerate() {
let (acc, row) = if e {
(&mut large, &self.large[i])
} else {
(&mut small, &self.small[i])
};
for k in 0..3 {
acc[k] += row[k];
}
}
micro_f1(
small[0] + large[0],
small[1] + large[1],
small[2] + large[2],
self.zero_division,
)
}
pub fn metric(&self) -> impl FnMut(&[bool]) -> f64 + '_ {
move |esc: &[bool]| self.score(esc).unwrap_or(f64::NAN)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn python_docstring_examples() {
let rows = reliability_table(
&[0.05, 0.15, 0.15, 0.95],
&[0.0, 0.0, 1.0, 1.0],
10,
BinStrategy::Uniform,
None,
)
.unwrap();
let got: Vec<(usize, f64)> = rows
.iter()
.map(|r| (r.count, r.observed_frequency))
.collect();
assert_eq!(got, vec![(1, 0.0), (2, 0.5), (1, 1.0)]);
assert_eq!(rows[1].bin_lower, 0.1);
assert_eq!(rows[1].bin_upper, 0.2);
assert_eq!(
ece(
&[0.25; 4],
&[1.0, 0.0, 0.0, 0.0],
10,
BinStrategy::Uniform,
None
)
.unwrap(),
0.0
);
assert_eq!(micro_f1(3.0, 1.0, 1.0, 0.0).unwrap(), 0.75);
}
#[test]
fn bins_are_right_closed() {
let rows = reliability_table(
&[0.0, 0.1, 0.1000001],
&[0.0; 3],
10,
BinStrategy::Uniform,
None,
)
.unwrap();
assert_eq!(rows.len(), 2);
assert_eq!(rows[0].count, 2);
assert_eq!(rows[1].count, 1);
let rows = reliability_table(&[1.0], &[1.0], 10, BinStrategy::Uniform, None).unwrap();
assert_eq!((rows[0].bin_lower, rows[0].bin_upper), (0.9, 1.0));
}
#[test]
fn quantile_bins() {
let rows = reliability_table(
&[0.4, 0.1, 0.3, 0.2],
&[1.0, 0.0, 1.0, 0.0],
2,
BinStrategy::Quantile,
None,
)
.unwrap();
assert_eq!(rows.len(), 2);
assert_eq!(rows[0].count, 2);
assert!((rows[0].bin_upper - 0.25).abs() < 1e-15);
let rows = reliability_table(
&[0.3; 5],
&[1.0, 0.0, 0.0, 0.0, 0.0],
10,
BinStrategy::Quantile,
None,
)
.unwrap();
assert_eq!(rows.len(), 1);
assert_eq!((rows[0].bin_lower, rows[0].bin_upper), (0.3, 0.3));
let e = ece(
&[0.3; 5],
&[1.0, 0.0, 0.0, 0.0, 0.0],
10,
BinStrategy::Quantile,
None,
)
.unwrap();
assert!((e - 0.1).abs() < 1e-15);
}
#[test]
fn np_quantile_linear() {
let s = [1.0, 2.0, 4.0, 8.0];
assert_eq!(np_quantile_sorted(&s, 0.0), 1.0);
assert_eq!(np_quantile_sorted(&s, 1.0), 8.0);
assert_eq!(np_quantile_sorted(&s, 0.5), 3.0);
assert!((np_quantile_sorted(&s, 0.25) - 1.75).abs() < 1e-15);
assert_eq!(np_quantile_sorted(&[5.0], 0.3), 5.0);
}
#[test]
fn weights() {
let p = [0.2, 0.2, 0.8];
let y = [0.0, 1.0, 1.0];
let a = ece(&p, &y, 10, BinStrategy::Uniform, Some(&[1.0, 0.0, 1.0])).unwrap();
let b = ece(&[0.2, 0.8], &[0.0, 1.0], 10, BinStrategy::Uniform, None).unwrap();
assert_eq!(a, b);
let c = brier_score(&p, &y, Some(&[2.0, 1.0, 1.0])).unwrap();
let d = brier_score(&[0.2, 0.2, 0.2, 0.8], &[0.0, 0.0, 1.0, 1.0], None).unwrap();
assert!((c - d).abs() < 1e-15);
assert!(ece(&p, &y, 10, BinStrategy::Uniform, Some(&[0.0; 3])).is_err());
assert!(ece(&p, &y, 10, BinStrategy::Uniform, Some(&[1.0, -1.0, 1.0])).is_err());
assert!(ece(&p, &y, 10, BinStrategy::Uniform, Some(&[1.0])).is_err());
}
#[test]
fn input_errors() {
let u = BinStrategy::Uniform;
assert!(ece(&[], &[], 10, u, None).is_err());
assert!(ece(&[0.5], &[1.0, 0.0], 10, u, None).is_err());
assert!(ece(&[1.5], &[1.0], 10, u, None).is_err());
assert!(ece(&[0.5], &[2.0], 10, u, None).is_err());
assert!(ece(&[f64::NAN], &[1.0], 10, u, None).is_err());
assert!(ece(&[0.5], &[1.0], 0, u, None).is_err());
assert!(brier_score(&[0.5], &[f64::INFINITY], None).is_err());
assert!("deciles".parse::<BinStrategy>().is_err());
assert_eq!(
"quantile".parse::<BinStrategy>().unwrap(),
BinStrategy::Quantile
);
assert_eq!(BinStrategy::Uniform.to_string(), "uniform");
}
#[test]
fn micro_f1_and_routed() {
assert_eq!(micro_f1(0.0, 0.0, 0.0, 0.0).unwrap(), 0.0);
assert!(micro_f1(-1.0, 0.0, 0.0, 0.0).is_err());
assert!(micro_f1(f64::NAN, 0.0, 0.0, 0.0).is_err());
let small = [[1.0, 0.0, 1.0], [2.0, 0.0, 0.0]];
let large = [[2.0, 0.0, 0.0], [2.0, 0.0, 0.0]];
let f1 = RoutedMicroF1::new(&small, &large, 0.0).unwrap();
assert_eq!(f1.len(), 2);
assert!(!f1.is_empty());
assert_eq!(f1.score(&[true, true]).unwrap(), 1.0);
assert!(f1.score(&[true]).is_err());
let mut m = f1.metric();
assert!(m(&[true]).is_nan());
assert!(RoutedMicroF1::new(&[], &[], 0.0).is_err());
assert!(RoutedMicroF1::new(&small, &large[..1], 0.0).is_err());
assert!(RoutedMicroF1::new(&[[1.0, -1.0, 0.0]], &[[1.0, 0.0, 0.0]], 0.0).is_err());
}
}