use crate::trainer::BoxedLossFn;
use crate::utils::error::EngineError;
use crate::utils::fitness::{Fitness, Metric};
use flodl::Variable;
pub const DEFAULT_LABEL_SMOOTHING: f32 = 0.1;
pub const DEFAULT_HUBER_BETA: f64 = 1.0;
pub const DEFAULT_FOCAL_GAMMA: f32 = 2.0;
pub const DEFAULT_TOP_K: usize = 3;
fn parse_top_k(name: &str) -> Option<(usize, String)> {
let rest = name.trim().strip_prefix("top_k_accuracy")?;
let k = if rest.is_empty() {
DEFAULT_TOP_K
} else {
rest.strip_prefix('(')?
.strip_suffix(')')?
.trim()
.parse()
.ok()?
};
if k == 0 {
return None;
}
let label = if k == DEFAULT_TOP_K {
"top_k_accuracy".to_string()
} else {
format!("top_k_accuracy({k})")
};
Some((k, label))
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum TargetKind {
OneHot,
Binary,
Regression,
}
pub const METRIC_NAMES: &[&str] = &[
"accuracy",
"balanced_accuracy",
"top_k_accuracy",
"auroc",
"f1",
"precision",
"recall",
"cross_entropy",
"mse",
"mae",
"rmse",
"mape",
"r2",
"explained_variance",
];
pub const FITNESS_NAMES: &[&str] = &[
"cross_entropy",
"accuracy",
"balanced_accuracy",
"top_k_accuracy",
"auroc",
"f1",
"precision",
"recall",
"mse",
"mae",
"rmse",
"mape",
"r2",
"explained_variance",
];
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum Loss {
CrossEntropyOnehot,
LabelSmoothingCe { smoothing: f32 },
Mse,
L1,
SmoothL1 { beta: f64 },
BceWithLogits,
Focal { gamma: f32 },
KlDiv,
PoissonNll,
}
impl Loss {
pub const ALL: &'static [Loss] = &[
Loss::CrossEntropyOnehot,
Loss::LabelSmoothingCe {
smoothing: DEFAULT_LABEL_SMOOTHING,
},
Loss::Mse,
Loss::L1,
Loss::SmoothL1 {
beta: DEFAULT_HUBER_BETA,
},
Loss::BceWithLogits,
Loss::Focal {
gamma: DEFAULT_FOCAL_GAMMA,
},
Loss::KlDiv,
Loss::PoissonNll,
];
pub const NAMES: &'static [&'static str] = &[
"cross_entropy",
"label_smoothing_ce",
"mse",
"l1",
"smooth_l1",
"bce_with_logits",
"focal",
"kl_div",
"poisson_nll",
];
pub fn parse(name: &str) -> Option<Self> {
let name = name.trim();
if let Some((head, rest)) = name.split_once('(') {
let arg = rest.strip_suffix(')')?.trim();
return match head.trim() {
"label_smoothing_ce" => Some(Self::LabelSmoothingCe {
smoothing: arg.parse().ok()?,
}),
"smooth_l1" => Some(Self::SmoothL1 {
beta: arg.parse().ok()?,
}),
"focal" => Some(Self::Focal {
gamma: arg.parse().ok()?,
}),
_ => None,
};
}
match name {
"cross_entropy" | "cross_entropy_onehot" | "ce" => Some(Self::CrossEntropyOnehot),
"label_smoothing_ce" => Some(Self::LabelSmoothingCe {
smoothing: DEFAULT_LABEL_SMOOTHING,
}),
"mse" => Some(Self::Mse),
"l1" | "mae" => Some(Self::L1),
"smooth_l1" | "huber" => Some(Self::SmoothL1 {
beta: DEFAULT_HUBER_BETA,
}),
"bce_with_logits" | "bce" => Some(Self::BceWithLogits),
"focal" | "focal_loss" => Some(Self::Focal {
gamma: DEFAULT_FOCAL_GAMMA,
}),
"kl_div" | "kl" | "kldiv" => Some(Self::KlDiv),
"poisson_nll" | "poisson" => Some(Self::PoissonNll),
_ => None,
}
}
pub fn label(&self) -> String {
match self {
Self::CrossEntropyOnehot => "cross_entropy".to_string(),
Self::LabelSmoothingCe { smoothing } => format!("label_smoothing_ce({smoothing})"),
Self::Mse => "mse".to_string(),
Self::L1 => "l1".to_string(),
Self::SmoothL1 { beta } => format!("smooth_l1({beta})"),
Self::BceWithLogits => "bce_with_logits".to_string(),
Self::Focal { gamma } => format!("focal({gamma})"),
Self::KlDiv => "kl_div".to_string(),
Self::PoissonNll => "poisson_nll".to_string(),
}
}
pub fn build(&self) -> BoxedLossFn {
match *self {
Self::CrossEntropyOnehot => Box::new(crate::utils::score::cross_entropy_onehot_loss),
Self::LabelSmoothingCe { smoothing } => {
Box::new(move |pred: &Variable, y: &Variable| {
crate::utils::score::label_smoothing_cross_entropy_loss(pred, y, smoothing)
})
}
Self::Mse => Box::new(flodl::nn::loss::mse_loss),
Self::L1 => Box::new(flodl::nn::loss::l1_loss),
Self::SmoothL1 { beta } => Box::new(move |pred: &Variable, y: &Variable| {
flodl::nn::loss::smooth_l1_loss(pred, y, beta)
}),
Self::BceWithLogits => Box::new(flodl::nn::loss::bce_with_logits_loss),
Self::Focal { gamma } => Box::new(move |pred: &Variable, y: &Variable| {
crate::utils::score::focal_loss_onehot(pred, y, gamma)
}),
Self::KlDiv => Box::new(|pred: &Variable, y: &Variable| {
let log_p = pred.data().log_softmax(1)?;
flodl::nn::loss::kl_div_loss(&Variable::new(log_p, false), y)
}),
Self::PoissonNll => Box::new(|pred: &Variable, y: &Variable| {
flodl::nn::loss::poisson_nll_loss(pred, y, true)
}),
}
}
pub fn target_kind(&self) -> TargetKind {
match self {
Self::CrossEntropyOnehot
| Self::LabelSmoothingCe { .. }
| Self::Focal { .. }
| Self::KlDiv => TargetKind::OneHot,
Self::BceWithLogits => TargetKind::Binary,
Self::Mse | Self::L1 | Self::SmoothL1 { .. } | Self::PoissonNll => {
TargetKind::Regression
}
}
}
pub fn natural_fitness(&self) -> Fitness {
match self {
Self::CrossEntropyOnehot
| Self::LabelSmoothingCe { .. }
| Self::Focal { .. }
| Self::KlDiv => {
Fitness::minimize("cross_entropy", crate::utils::score::cross_entropy_onehot)
}
Self::BceWithLogits => {
Fitness::maximize("accuracy", crate::utils::score::accuracy_score)
}
Self::Mse => Fitness::minimize("mse", crate::utils::score::mse_loss_score),
Self::L1 | Self::SmoothL1 { .. } => {
Fitness::minimize("mae", crate::utils::score::l1_loss_score)
}
Self::PoissonNll => Fitness::minimize("mse", crate::utils::score::mse_loss_score),
}
}
}
impl std::fmt::Display for Loss {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.label())
}
}
impl std::str::FromStr for Loss {
type Err = EngineError;
fn from_str(s: &str) -> Result<Self, Self::Err> {
Self::parse(s).ok_or_else(|| {
EngineError::InvalidOptions(format!(
"unknown loss {s:?} — expected one of {}",
Loss::NAMES.join(", ")
))
})
}
}
pub fn metric_for(name: &str) -> Option<Metric> {
type ScoreBox = Box<dyn Fn(&Variable, &Variable) -> flodl::tensor::Result<f32> + Send + Sync>;
if let Some((k, label)) = parse_top_k(name) {
return Some(Metric::custom(
label,
move |pred: &Variable, y: &Variable| {
crate::utils::score::top_k_accuracy_score(pred, y, k)
},
));
}
let (label, f): (&'static str, ScoreBox) = match name.trim() {
"accuracy" => ("accuracy", Box::new(crate::utils::score::accuracy_score)),
"balanced_accuracy" => (
"balanced_accuracy",
Box::new(crate::utils::score::balanced_accuracy_score),
),
"auroc" => ("auroc", Box::new(crate::utils::score::auroc_score)),
"f1" => ("f1", Box::new(crate::utils::score::f1_score)),
"precision" => ("precision", Box::new(crate::utils::score::precision_score)),
"recall" => ("recall", Box::new(crate::utils::score::recall_score)),
"cross_entropy" => (
"cross_entropy",
Box::new(crate::utils::score::cross_entropy_onehot),
),
"mse" => ("mse", Box::new(crate::utils::score::mse_loss_score)),
"mae" | "l1" => ("mae", Box::new(crate::utils::score::l1_loss_score)),
"rmse" => ("rmse", Box::new(crate::utils::score::rmse_score)),
"mape" => ("mape", Box::new(crate::utils::score::mape_score)),
"r2" => ("r2", Box::new(crate::utils::score::r2_score)),
"explained_variance" => (
"explained_variance",
Box::new(crate::utils::score::explained_variance_score),
),
_ => return None,
};
Some(Metric::custom(label, f))
}
pub fn metrics_for(names: &[&str]) -> Result<Vec<Metric>, EngineError> {
names
.iter()
.map(|name| {
metric_for(name).ok_or_else(|| {
EngineError::InvalidOptions(format!(
"unknown metric {name:?} — expected one of {}",
METRIC_NAMES.join(", ")
))
})
})
.collect()
}
pub fn fitness_for(name: &str) -> Option<Fitness> {
if let Some((k, label)) = parse_top_k(name) {
return Some(Fitness::maximize(
&label,
move |pred: &Variable, y: &Variable| {
crate::utils::score::top_k_accuracy_score(pred, y, k)
},
));
}
Some(match name.trim() {
"cross_entropy" => {
Fitness::minimize("cross_entropy", crate::utils::score::cross_entropy_onehot)
}
"accuracy" => Fitness::maximize("accuracy", crate::utils::score::accuracy_score),
"balanced_accuracy" => Fitness::maximize(
"balanced_accuracy",
crate::utils::score::balanced_accuracy_score,
),
"auroc" => Fitness::maximize("auroc", crate::utils::score::auroc_score),
"f1" => Fitness::maximize("f1", crate::utils::score::f1_score),
"precision" => Fitness::maximize("precision", crate::utils::score::precision_score),
"recall" => Fitness::maximize("recall", crate::utils::score::recall_score),
"mse" => Fitness::minimize("mse", crate::utils::score::mse_loss_score),
"mae" | "l1" => Fitness::minimize("mae", crate::utils::score::l1_loss_score),
"rmse" => Fitness::minimize("rmse", crate::utils::score::rmse_score),
"mape" => Fitness::minimize("mape", crate::utils::score::mape_score),
"r2" => Fitness::maximize("r2", crate::utils::score::r2_score),
"explained_variance" => Fitness::maximize(
"explained_variance",
crate::utils::score::explained_variance_score,
),
_ => return None,
})
}
#[cfg(test)]
mod tests {
use super::*;
use flodl::{Device, Tensor};
fn var(xs: &[f32], shape: &[i64]) -> Variable {
Variable::new(Tensor::from_f32(xs, shape, Device::CPU).unwrap(), false)
}
fn sample_pair(loss: &Loss) -> (Variable, Variable) {
match loss.target_kind() {
TargetKind::OneHot => (
var(&[1.0, -2.0, 0.5, -1.0, 2.0, 0.0], &[2, 3]),
var(&[0.0, 1.0, 0.0, 0.0, 0.0, 1.0], &[2, 3]),
),
TargetKind::Binary => (var(&[0.4, -0.7], &[2, 1]), var(&[1.0, 0.0], &[2, 1])),
TargetKind::Regression => (var(&[1.5, -2.0], &[2, 1]), var(&[1.0, -1.0], &[2, 1])),
}
}
#[test]
fn every_bare_name_parses() {
for name in Loss::NAMES {
assert!(Loss::parse(name).is_some(), "{name} should parse");
}
}
#[test]
fn label_round_trips_including_parameters() {
for loss in Loss::ALL {
assert_eq!(
Loss::parse(&loss.label()).as_ref(),
Some(loss),
"{} did not round-trip",
loss.label()
);
}
let smoothed = Loss::parse("label_smoothing_ce(0.25)").unwrap();
assert_eq!(smoothed.label(), "label_smoothing_ce(0.25)");
assert_eq!(Loss::parse(&smoothed.label()), Some(smoothed));
let huber = Loss::parse("smooth_l1(0.5)").unwrap();
assert_eq!(Loss::parse(&huber.label()), Some(huber));
}
#[test]
fn aliases_normalise_to_the_canonical_label() {
assert_eq!(Loss::parse("ce").unwrap().label(), "cross_entropy");
assert_eq!(
Loss::parse("cross_entropy_onehot").unwrap().label(),
"cross_entropy"
);
assert_eq!(Loss::parse("huber").unwrap().label(), "smooth_l1(1)");
assert_eq!(Loss::parse("mae").unwrap().label(), "l1");
assert_eq!(Loss::parse("kl").unwrap().label(), "kl_div");
assert_eq!(Loss::parse("poisson").unwrap().label(), "poisson_nll");
assert_eq!(Loss::parse("focal_loss").unwrap().label(), "focal(2)");
let focal = Loss::parse("focal(1)").unwrap();
assert_eq!(focal.label(), "focal(1)");
assert_eq!(Loss::parse(&focal.label()), Some(focal));
assert!(Loss::parse("focal(0.25,2)").is_none());
}
#[test]
fn junk_is_rejected_not_guessed() {
assert!(Loss::parse("").is_none());
assert!(Loss::parse("cross-entropy").is_none());
assert!(Loss::parse("label_smoothing_ce(nope)").is_none());
assert!(Loss::parse("smooth_l1(").is_none());
assert!(Loss::parse("ctc").is_none());
assert!(Loss::parse("nll").is_none());
assert!("mse".parse::<Loss>().is_ok());
assert!("nope".parse::<Loss>().is_err());
}
#[test]
fn every_loss_builds_and_evaluates_on_its_own_target_kind() {
for loss in Loss::ALL {
let (pred, y) = sample_pair(loss);
let value = loss.build()(&pred, &y)
.unwrap_or_else(|e| panic!("{} failed to evaluate: {e}", loss.label()));
let x = value.item().unwrap() as f32;
assert!(x.is_finite(), "{} produced {x}", loss.label());
}
}
#[test]
fn focal_and_kl_reduce_to_cross_entropy_at_their_degenerate_settings() {
let pred = var(&[1.0, -2.0, 0.5, -1.0, 2.0, 0.0], &[2, 3]);
let y = var(&[1.0, 0.0, 0.0, 0.0, 0.0, 1.0], &[2, 3]);
let value = |loss: &Loss| loss.build()(&pred, &y).unwrap().item().unwrap() as f32;
let ce = value(&Loss::CrossEntropyOnehot);
let focal = value(&Loss::Focal { gamma: 0.0 });
assert!((ce - focal).abs() < 1e-5, "focal {focal} vs ce {ce}");
let kl = value(&Loss::KlDiv);
assert!((ce - kl).abs() < 1e-5, "kl_div {kl} vs ce {ce}");
let focused = value(&Loss::Focal { gamma: 2.0 });
assert!(focused < ce, "focal(2) {focused} should be below ce {ce}");
}
#[test]
fn poisson_stays_finite_for_a_net_that_predicts_negative_log_rates() {
let pred = var(&[-3.0, 4.0, 0.0], &[3, 1]);
let y = var(&[0.0, 9.0, 2.0], &[3, 1]);
let x = Loss::PoissonNll.build()(&pred, &y).unwrap().item().unwrap() as f32;
assert!(x.is_finite() && x > 0.0, "poisson_nll produced {x}");
}
#[test]
fn natural_fitness_follows_the_loss_family() {
assert_eq!(
Loss::CrossEntropyOnehot.natural_fitness().label(),
"cross_entropy"
);
assert_eq!(
Loss::LabelSmoothingCe { smoothing: 0.1 }
.natural_fitness()
.label(),
"cross_entropy"
);
assert_eq!(Loss::BceWithLogits.natural_fitness().label(), "accuracy");
assert_eq!(Loss::Mse.natural_fitness().label(), "mse");
assert_eq!(
Loss::Focal { gamma: 2.0 }.natural_fitness().label(),
"cross_entropy"
);
assert_eq!(Loss::KlDiv.natural_fitness().label(), "cross_entropy");
assert_eq!(Loss::PoissonNll.natural_fitness().label(), "mse");
assert_eq!(Loss::L1.natural_fitness().label(), "mae");
assert_eq!(
Loss::SmoothL1 { beta: 1.0 }.natural_fitness().label(),
"mae"
);
for loss in Loss::ALL {
assert!(loss.natural_fitness().is_computed(), "{}", loss.label());
}
}
#[test]
fn every_metric_and_fitness_name_resolves() {
for name in METRIC_NAMES {
assert!(metric_for(name).is_some(), "metric {name}");
}
for name in FITNESS_NAMES {
assert!(fitness_for(name).is_some(), "fitness {name}");
}
assert!(metric_for("nope").is_none());
assert!(fitness_for("nope").is_none());
assert_eq!(metric_for("l1").unwrap().label(), "mae");
}
#[test]
fn top_k_carries_its_k_and_normalises_the_default_spelling() {
assert_eq!(
metric_for("top_k_accuracy").unwrap().label(),
"top_k_accuracy"
);
assert_eq!(
metric_for(&format!("top_k_accuracy({DEFAULT_TOP_K})"))
.unwrap()
.label(),
"top_k_accuracy"
);
assert_eq!(
metric_for("top_k_accuracy(5)").unwrap().label(),
"top_k_accuracy(5)"
);
assert_eq!(
fitness_for("top_k_accuracy(5)").unwrap().label(),
"top_k_accuracy(5)"
);
assert!(metric_for("top_k_accuracy(0)").is_none());
assert!(fitness_for("top_k_accuracy(0)").is_none());
assert!(metric_for("top_k_accuracy(x)").is_none());
}
#[test]
fn metrics_for_names_the_offender_and_lists_the_set() {
assert_eq!(metrics_for(&["accuracy", "f1"]).unwrap().len(), 2);
assert!(metrics_for(&[]).unwrap().is_empty());
let err = metrics_for(&["accuracy", "nope"]).unwrap_err().to_string();
assert!(err.contains("nope"), "{err}");
assert!(
err.contains("accuracy"),
"the valid set should be listed: {err}"
);
}
}