use flodl::Variable;
use flodl::tensor::Result;
pub fn r2_score(pred: &Variable, y: &Variable) -> Result<f32> {
let t = y.data().to_f32_vec()?;
let p = pred.data().to_f32_vec()?;
let n = t.len() as f32;
if n == 0.0 {
return Ok(0.0);
}
let mean_t = t.iter().sum::<f32>() / n;
let ss_tot: f32 = t.iter().map(|&v| (v - mean_t).powi(2)).sum();
let ss_res: f32 = t.iter().zip(&p).map(|(&a, &b)| (a - b).powi(2)).sum();
if ss_tot.abs() < f32::EPSILON {
return Ok(if ss_res.abs() < f32::EPSILON {
1.0
} else {
0.0
});
}
Ok(1.0 - ss_res / ss_tot)
}
pub fn argmax_classes(pred: &Variable, y: &Variable) -> Result<(Vec<i64>, Vec<i64>)> {
let pa = pred.data().argmax(1, false)?.to_i64_vec()?;
let ta = y.data().argmax(1, false)?.to_i64_vec()?;
Ok((pa, ta))
}
pub fn accuracy_score(pred: &Variable, y: &Variable) -> Result<f32> {
let (pa, ta) = argmax_classes(pred, y)?;
let n = pa.len() as f32;
if n == 0.0 {
return Ok(0.0);
}
Ok(pa.iter().zip(&ta).filter(|(a, b)| a == b).count() as f32 / n)
}
pub fn f1_score(pred: &Variable, y: &Variable) -> Result<f32> {
let (pa, ta) = argmax_classes(pred, y)?;
let classes = y.data().shape()[1] as usize;
Ok(f1_from_vecs(&pa, &ta, classes))
}
pub fn f1_from_vecs(pa: &[i64], ta: &[i64], classes: usize) -> f32 {
if classes == 0 || pa.is_empty() {
return 0.0;
}
let mut sum = 0.0f32;
for c in 0..classes as i64 {
let mut tp = 0usize;
let mut fp = 0usize;
let mut fn_ = 0usize;
for (p, t) in pa.iter().zip(ta.iter()) {
match (*p == c, *t == c) {
(true, true) => tp += 1,
(true, false) => fp += 1,
(false, true) => fn_ += 1,
(false, false) => {}
}
}
let denom = (tp + fp + tp + fn_) as f32;
if denom > 0.0 {
sum += 2.0 * tp as f32 / denom;
}
}
sum / classes as f32
}
pub fn precision_score(pred: &Variable, y: &Variable) -> Result<f32> {
let (pa, ta) = argmax_classes(pred, y)?;
let classes = y.data().shape()[1] as usize;
Ok(precision_from_vecs(&pa, &ta, classes))
}
pub fn recall_from_vecs(pa: &[i64], ta: &[i64], classes: usize) -> f32 {
if classes == 0 || pa.is_empty() {
return 0.0;
}
let mut sum = 0.0f32;
let mut present = 0usize;
for c in 0..classes as i64 {
let (mut tp, mut fn_) = (0usize, 0usize);
for (p, t) in pa.iter().zip(ta.iter()) {
if *t == c {
if *p == c {
tp += 1;
} else {
fn_ += 1;
}
}
}
if tp + fn_ > 0 {
sum += tp as f32 / (tp + fn_) as f32;
present += 1;
}
}
if present == 0 {
0.0
} else {
sum / present as f32
}
}
pub fn recall_score(pred: &Variable, y: &Variable) -> Result<f32> {
let (pa, ta) = argmax_classes(pred, y)?;
let classes = y.data().shape()[1] as usize;
Ok(recall_from_vecs(&pa, &ta, classes))
}
pub fn balanced_accuracy_score(pred: &Variable, y: &Variable) -> Result<f32> {
let (pa, ta) = argmax_classes(pred, y)?;
let classes = y.data().shape()[1] as usize;
if classes == 0 || pa.is_empty() {
return Ok(0.0);
}
let n = pa.len() as f32;
let mut sum = 0.0f32;
let mut present = 0usize;
for c in 0..classes as i64 {
let mut present_in_target = false;
let mut correct = 0usize;
for (p, t) in pa.iter().zip(ta.iter()) {
if (*p == c) == (*t == c) {
correct += 1;
}
if *t == c {
present_in_target = true;
}
}
if present_in_target {
sum += correct as f32 / n;
present += 1;
}
}
Ok(if present == 0 {
0.0
} else {
sum / present as f32
})
}
pub fn top_k_accuracy_score(pred: &Variable, y: &Variable, k: usize) -> Result<f32> {
let shape = pred.data().shape();
let rows = shape[0] as usize;
let classes = shape[1] as usize;
if rows == 0 || classes == 0 {
return Ok(0.0);
}
let k = k.clamp(1, classes);
let (_, ta) = argmax_classes(pred, y)?;
let p = pred.data().to_f32_vec()?;
let mut hits = 0usize;
for r in 0..rows {
let truth = ta.get(r).copied().unwrap_or(-1);
if truth < 0 || truth as usize >= classes {
continue;
}
let row = &p[r * classes..(r + 1) * classes];
let target = row[truth as usize];
let strictly_better = row.iter().filter(|v| **v > target).count();
if strictly_better < k {
hits += 1;
}
}
Ok(hits as f32 / rows as f32)
}
pub fn auroc_score(pred: &Variable, y: &Variable) -> Result<f32> {
let p_shape = pred.data().shape().to_vec();
let t_shape = y.data().shape().to_vec();
let rows = p_shape[0] as usize;
if rows == 0 {
return Ok(0.5);
}
let p_cols = *p_shape.get(1).unwrap_or(&1) as usize;
let t_cols = *t_shape.get(1).unwrap_or(&1) as usize;
let p = pred.data().to_f32_vec()?;
let t = y.data().to_f32_vec()?;
let mut pairs: Vec<(f32, bool)> = (0..rows)
.map(|r| {
let score = if p_cols >= 2 {
p[r * p_cols + 1]
} else {
p[r * p_cols]
};
let positive = if t_cols >= 2 {
t[r * t_cols + 1] > 0.5
} else {
t[r * t_cols] > 0.5
};
(score, positive)
})
.collect();
let n_pos = pairs.iter().filter(|(_, pos)| *pos).count();
let n_neg = rows - n_pos;
if n_pos == 0 || n_neg == 0 {
return Ok(0.5);
}
pairs.sort_by(|a, b| a.0.total_cmp(&b.0));
let mut rank_sum_pos = 0.0f64;
let mut i = 0usize;
while i < pairs.len() {
let mut j = i + 1;
while j < pairs.len() && pairs[j].0.total_cmp(&pairs[i].0).is_eq() {
j += 1;
}
let rank = ((i + 1) as f64 + j as f64) / 2.0;
for (_, positive) in &pairs[i..j] {
if *positive {
rank_sum_pos += rank;
}
}
i = j;
}
let u = rank_sum_pos - (n_pos as f64 * (n_pos as f64 + 1.0)) / 2.0;
Ok((u / (n_pos as f64 * n_neg as f64)) as f32)
}
pub fn mape_score(pred: &Variable, y: &Variable) -> Result<f32> {
let t = y.data().to_f32_vec()?;
let p = pred.data().to_f32_vec()?;
let mut sum = 0.0f32;
let mut n = 0usize;
for (&a, &b) in t.iter().zip(&p) {
if a.abs() > f32::EPSILON {
sum += ((a - b) / a).abs();
n += 1;
}
}
Ok(if n == 0 { 0.0 } else { sum / n as f32 })
}
pub fn explained_variance_score(pred: &Variable, y: &Variable) -> Result<f32> {
let t = y.data().to_f32_vec()?;
let p = pred.data().to_f32_vec()?;
let n = t.len() as f32;
if n == 0.0 {
return Ok(0.0);
}
let mean_t = t.iter().sum::<f32>() / n;
let residual: Vec<f32> = t.iter().zip(&p).map(|(&a, &b)| a - b).collect();
let mean_r = residual.iter().sum::<f32>() / n;
let var_t: f32 = t.iter().map(|&v| (v - mean_t).powi(2)).sum::<f32>() / n;
let var_r: f32 = residual.iter().map(|&v| (v - mean_r).powi(2)).sum::<f32>() / n;
if var_t.abs() < f32::EPSILON {
return Ok(if var_r.abs() < f32::EPSILON { 1.0 } else { 0.0 });
}
Ok(1.0 - var_r / var_t)
}
pub fn mse_loss_score(pred: &Variable, y: &Variable) -> Result<f32> {
Ok(flodl::nn::loss::mse_loss(pred, y)?.item()? as f32)
}
pub fn l1_loss_score(pred: &Variable, y: &Variable) -> Result<f32> {
Ok(flodl::nn::loss::l1_loss(pred, y)?.item()? as f32)
}
pub fn rmse_score(pred: &Variable, y: &Variable) -> Result<f32> {
Ok(mse_loss_score(pred, y)?.sqrt())
}
pub fn precision_from_vecs(pa: &[i64], ta: &[i64], classes: usize) -> f32 {
if classes == 0 || pa.is_empty() {
return 0.0;
}
let mut sum = 0.0f32;
for c in 0..classes as i64 {
let mut tp = 0usize;
let mut fp = 0usize;
for (p, t) in pa.iter().zip(ta.iter()) {
if *p == c {
if *t == c {
tp += 1;
} else {
fp += 1;
}
}
}
let denom = tp + fp;
if denom > 0 {
sum += tp as f32 / denom as f32;
}
}
sum / classes as f32
}
pub fn cross_entropy_onehot(pred: &Variable, y: &Variable) -> Result<f32> {
let n = y.data().shape()[0] as f32;
if n == 0.0 {
return Ok(0.0);
}
let ls = pred.data().log_softmax(1)?;
let masked = ls.mul(&y.data())?;
Ok(-masked.sum()?.item()? as f32 / n)
}
pub fn label_smoothing_cross_entropy_loss(
pred: &Variable,
y: &Variable,
smoothing: f32,
) -> Result<Variable> {
let n_classes = y.data().shape()[1] as f32;
let scale = flodl::Tensor::from_f32(&[1.0 - smoothing], &[1], y.data().device())?;
let off = flodl::Tensor::from_f32(&[smoothing / n_classes], &[1], y.data().device())?;
let smoothed_y = y.data().mul(&scale)?.add(&off)?;
cross_entropy_onehot_loss(pred, &Variable::new(smoothed_y, false))
}
pub fn focal_loss_onehot(pred: &Variable, y: &Variable, gamma: f32) -> Result<Variable> {
let log_p = pred.data().log_softmax(1)?;
let log_pt = log_p.mul(&y.data())?.sum_dims(&[1], false)?;
let pt = log_pt.exp()?;
let weight = pt.neg()?.add_scalar(1.0)?.pow_scalar(gamma as f64)?;
let per_row = weight.mul(&log_pt)?.neg()?;
Ok(Variable::new(per_row.mean()?, false))
}
pub fn cross_entropy_onehot_loss(pred: &Variable, y: &Variable) -> Result<Variable> {
let ls = pred.data().log_softmax(1)?;
let masked = ls.mul(&y.data())?;
let neg = flodl::Tensor::from_f32(&[-1.0], &[1], masked.device())?;
let n = flodl::Tensor::from_f32(&[y.data().shape()[0] as f32], &[1], y.data().device())?;
Ok(Variable::new(masked.sum()?.mul(&neg)?.div(&n)?, false))
}
#[cfg(test)]
mod tests {
use super::*;
use flodl::{Device, Tensor};
fn var(xs: &[f32], shape: &[i64]) -> Variable {
Variable::new(Tensor::from_f32(xs, shape, Device::CPU).unwrap(), false)
}
fn three_class_case() -> (Variable, Variable) {
(
var(
&[
3.0, 0.0, 0.0, 0.0, 3.0, 0.5, 0.0, 3.0, 1.0, ],
&[3, 3],
),
var(&[1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0], &[3, 3]),
)
}
#[test]
fn top_k_is_accuracy_at_one_and_inclusive_at_the_class_count() {
let (pred, y) = three_class_case();
let one = top_k_accuracy_score(&pred, &y, 1).unwrap();
let acc = accuracy_score(&pred, &y).unwrap();
assert!(
(one - acc).abs() < 1e-6,
"k=1 must BE accuracy: {one} vs {acc}"
);
assert!((one - 2.0 / 3.0).abs() < 1e-6, "k=1 = {one}");
assert!((top_k_accuracy_score(&pred, &y, 2).unwrap() - 1.0).abs() < 1e-6);
assert!((top_k_accuracy_score(&pred, &y, 99).unwrap() - 1.0).abs() < 1e-6);
assert!((top_k_accuracy_score(&pred, &y, 0).unwrap() - one).abs() < 1e-6);
}
fn binary_case(positive_scores: &[f32; 4]) -> (Variable, Variable) {
let mut xs = Vec::new();
for s in positive_scores {
xs.push(1.0 - s);
xs.push(*s);
}
(
var(&xs, &[4, 2]),
var(&[0.0, 1.0, 0.0, 1.0, 1.0, 0.0, 1.0, 0.0], &[4, 2]),
)
}
#[test]
fn auroc_is_one_perfect_zero_reversed_half_tied() {
let perfect = auroc_score(
&binary_case(&[0.9, 0.8, 0.2, 0.1]).0,
&binary_case(&[0.9, 0.8, 0.2, 0.1]).1,
)
.unwrap();
assert!(
(perfect - 1.0).abs() < 1e-6,
"perfect separation = {perfect}"
);
let reversed = auroc_score(
&binary_case(&[0.1, 0.2, 0.8, 0.9]).0,
&binary_case(&[0.1, 0.2, 0.8, 0.9]).1,
)
.unwrap();
assert!(reversed.abs() < 1e-6, "reversed separation = {reversed}");
let tied = auroc_score(
&binary_case(&[0.5, 0.5, 0.5, 0.5]).0,
&binary_case(&[0.5, 0.5, 0.5, 0.5]).1,
)
.unwrap();
assert!(
(tied - 0.5).abs() < 1e-6,
"all tied = {tied}, expected a coin flip"
);
}
#[test]
fn auroc_on_one_class_says_the_exam_cannot_tell() {
let pred = var(&[0.9, 0.8, 0.7, 0.6], &[4, 1]);
let y = var(&[1.0, 1.0, 1.0, 1.0], &[4, 1]);
let got = auroc_score(&pred, &y).unwrap();
assert!((got - 0.5).abs() < 1e-6, "single-class AUROC = {got}");
assert!(got.is_finite());
}
}