use flodl::Variable;
use flodl::tensor::Result;
use serde::{Deserialize, Serialize};
type ScoreFn = dyn Fn(&Variable, &Variable) -> Result<f32> + Send + Sync;
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)]
pub enum Direction {
#[default]
Minimize,
Maximize,
}
impl Direction {
pub fn is_better(&self, new: f32, current: f32) -> bool {
match self {
Direction::Minimize => new < current,
Direction::Maximize => new > current,
}
}
pub fn arrow(&self) -> &'static str {
match self {
Direction::Minimize => "↓",
Direction::Maximize => "↑",
}
}
pub fn cmp(&self, a: f32, b: f32) -> std::cmp::Ordering {
match self {
Direction::Minimize => b.partial_cmp(&a).unwrap_or(std::cmp::Ordering::Equal),
Direction::Maximize => a.partial_cmp(&b).unwrap_or(std::cmp::Ordering::Equal),
}
}
}
#[derive(Clone)]
pub enum Fitness {
Computed {
score_fn: std::sync::Arc<ScoreFn>,
direction: Direction,
label: String,
},
Reported {
direction: Direction,
label: String,
},
}
impl Fitness {
pub fn new<F>(score_fn: F, direction: Direction, label: &str) -> Self
where
F: Fn(&Variable, &Variable) -> Result<f32> + Send + Sync + 'static,
{
Fitness::Computed {
score_fn: std::sync::Arc::new(score_fn) as std::sync::Arc<ScoreFn>,
direction,
label: label.to_string(),
}
}
pub fn reported(direction: Direction, label: &str) -> Self {
Fitness::Reported {
direction,
label: label.to_string(),
}
}
pub fn minimize<F>(label: &str, score_fn: F) -> Self
where
F: Fn(&Variable, &Variable) -> Result<f32> + Send + Sync + 'static,
{
Fitness::new(score_fn, Direction::Minimize, label)
}
pub fn maximize<F>(label: &str, score_fn: F) -> Self
where
F: Fn(&Variable, &Variable) -> Result<f32> + Send + Sync + 'static,
{
Fitness::new(score_fn, Direction::Maximize, label)
}
pub fn score(&self, pred: &Variable, target: &Variable) -> Result<f32> {
match self {
Fitness::Computed { score_fn, .. } => score_fn(pred, target),
Fitness::Reported { label, .. } => Err(flodl::tensor::TensorError::new(&format!(
"fitness `{label}` is Reported — the trainer supplies the value in StepReport; there is no (pred, target) scorer to call"
))),
}
}
pub fn direction(&self) -> Direction {
match self {
Fitness::Computed { direction, .. } => *direction,
Fitness::Reported { direction, .. } => *direction,
}
}
pub fn label(&self) -> &str {
match self {
Fitness::Computed { label, .. } => label,
Fitness::Reported { label, .. } => label,
}
}
pub fn is_computed(&self) -> bool {
matches!(self, Fitness::Computed { .. })
}
pub fn is_reported(&self) -> bool {
matches!(self, Fitness::Reported { .. })
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct FitnessLabel(pub String);
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct SelectionMetric {
pub label: String,
pub weight: f32,
}
#[derive(Clone)]
pub struct Metric {
pub label: String,
score_fn: std::sync::Arc<ScoreFn>,
}
impl Metric {
pub fn custom<F>(label: impl Into<String>, score_fn: F) -> Self
where
F: Fn(&Variable, &Variable) -> Result<f32> + Send + Sync + 'static,
{
Metric {
label: label.into(),
score_fn: std::sync::Arc::new(score_fn),
}
}
pub fn label(&self) -> &str {
&self.label
}
pub fn score(&self, pred: &Variable, y: &Variable) -> Result<f32> {
(self.score_fn)(pred, y)
}
}
impl PartialEq for Metric {
fn eq(&self, other: &Self) -> bool {
self.label == other.label }
}
impl std::fmt::Debug for Metric {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Metric")
.field("label", &self.label)
.finish()
}
}
impl Default for FitnessLabel {
fn default() -> Self {
FitnessLabel("loss".to_string())
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::graph::network::Network;
use crate::graph::node::Node;
use crate::graph::topology::Topology;
use crate::utils::score::*;
use flodl::nn::Module;
use proptest::prelude::*;
fn tiny_net() -> Network {
let mut graph = Topology::new(0, None);
graph.nodes.push(Node::new_input(0, 1));
graph.nodes.push(Node::new_output(1, 1, 1));
graph.finalize();
Network::build(&graph, flodl::Device::CPU).unwrap()
}
fn input(xs: &[f32]) -> Variable {
Variable::new(
flodl::Tensor::from_f32(xs, &[xs.len() as i64, 1], flodl::Device::CPU).unwrap(),
false,
)
}
#[test]
fn fitness_clone_shares_the_scorer() {
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
let calls = Arc::new(AtomicUsize::new(0));
let counter = Arc::clone(&calls);
let f = Fitness::maximize("counter", move |_p, _t| {
counter.fetch_add(1, Ordering::SeqCst);
Ok(1.0)
});
let g = f.clone();
assert!(f.is_computed() && g.is_computed());
assert_eq!(g.label(), "counter");
assert_eq!(g.direction(), Direction::Maximize);
let x = input(&[3.5]);
assert_eq!(f.score(&x, &x).unwrap(), 1.0);
assert_eq!(g.score(&x, &x).unwrap(), 1.0);
assert_eq!(calls.load(Ordering::SeqCst), 2);
}
#[test]
fn test_direction_semantics() {
assert!(Direction::Minimize.is_better(0.5, 1.0));
assert!(!Direction::Minimize.is_better(1.0, 0.5));
assert!(Direction::Maximize.is_better(1.0, 0.5));
assert!(!Direction::Maximize.is_better(0.5, 1.0));
}
#[test]
fn test_reported_fitness_contract() {
let f = Fitness::reported(Direction::Maximize, "episode_reward");
assert!(f.is_reported());
assert!(!f.is_computed());
assert_eq!(f.direction(), Direction::Maximize);
assert_eq!(f.label(), "episode_reward");
let net = tiny_net();
let x = input(&[1.0, 2.0, 3.0]);
let pred = net.forward(&x).unwrap();
assert!(f.score(&pred, &pred).is_err());
}
#[test]
fn test_computed_fitness_flags() {
let f = Fitness::new(mse_loss_score, Direction::Minimize, "mse");
assert!(f.is_computed());
assert!(!f.is_reported());
}
#[test]
fn test_fitness_minimize_maximize_shorthand() {
let lo = Fitness::minimize("mse", mse_loss_score);
assert!(lo.is_computed());
assert_eq!(lo.direction(), Direction::Minimize);
assert_eq!(lo.label(), "mse");
let hi = Fitness::maximize("f1", f1_score);
assert!(hi.is_computed());
assert_eq!(hi.direction(), Direction::Maximize);
assert_eq!(hi.label(), "f1");
let net = tiny_net();
let x = input(&[1.0, 2.0, 3.0]);
let y = input(&[1.0, 2.0, 3.0]);
let pred = net.forward(&x).unwrap();
let short = Fitness::minimize("mse", mse_loss_score)
.score(&pred, &y)
.unwrap();
let long = Fitness::new(mse_loss_score, Direction::Minimize, "mse")
.score(&pred, &y)
.unwrap();
assert!((short - long).abs() < 1e-9);
}
#[test]
fn test_fitness_new() {
let f = Fitness::new(
|pred, y| {
let diff = pred.data().sub(&y.data())?;
let sq = diff.mul(&diff)?;
Ok(sq.mean()?.item()? as f32)
},
Direction::Minimize,
"mse",
);
let net = tiny_net();
let x = input(&[1.0, 2.0, 3.0]);
let y = input(&[1.0, 2.0, 3.0]);
let pred = net.forward(&x).unwrap();
let score = f.score(&pred, &y).unwrap();
assert!(score.is_finite());
assert!(score >= 0.0);
assert_eq!(f.direction(), Direction::Minimize);
assert_eq!(f.label(), "mse");
}
#[test]
fn test_categorical_helpers() {
let mut graph = Topology::new(0, None);
graph.options.input_dim = Some(2);
graph.nodes.push(Node::new_input(0, 2));
graph.nodes.push(Node::new_hidden(1, 2, 2));
graph.nodes.push(Node::new_output(2, 2, 2));
graph.nodes[2].hidden_dim = Some(2);
graph.finalize();
let net = Network::build(&graph, flodl::Device::CPU).unwrap();
let x = Variable::new(
flodl::Tensor::from_f32(&[2.0, 0.5, 0.1, 0.9, 1.2, 0.3], &[3, 2], flodl::Device::CPU)
.unwrap(),
false,
);
let pred = net.forward(&x).unwrap();
let pa = pred.data().argmax(1, false).unwrap().to_i64_vec().unwrap();
let y = Variable::new(
crate::utils::tabular_data::one_hot(
&pa.iter().map(|&v| v as usize).collect::<Vec<_>>(),
2,
flodl::Device::CPU,
)
.unwrap(),
false,
);
let acc = accuracy_score(&pred, &y).unwrap();
let ce = cross_entropy_onehot(&pred, &y).unwrap();
assert!((acc - 1.0).abs() < 1e-6, "acc = {acc}");
assert!(ce > 0.0 && ce.is_finite(), "ce = {ce}");
}
#[test]
fn test_f1_and_precision_from_vecs() {
assert_eq!(f1_from_vecs(&[0, 1, 0, 1], &[0, 1, 0, 1], 2), 1.0);
assert_eq!(precision_from_vecs(&[0, 1, 0, 1], &[0, 1, 0, 1], 2), 1.0);
let pa = [0i64, 0, 0, 0];
let ta = [0i64, 1, 0, 1];
assert!((precision_from_vecs(&pa, &ta, 2) - 0.25).abs() < 1e-9);
assert!((f1_from_vecs(&pa, &ta, 2) - 1.0 / 3.0).abs() < 1e-9);
assert_eq!(f1_from_vecs(&[], &[], 2), 0.0);
assert_eq!(precision_from_vecs(&[0], &[0], 0), 0.0);
}
#[test]
fn test_all_scoring_helpers() {
let net = tiny_net();
let x = input(&[1.0, 2.0, 3.0]);
let pred = net.forward(&x).unwrap();
let y = Variable::new(pred.data().clone(), false);
let mse = mse_loss_score(&pred, &y).unwrap();
assert!(mse < 1e-6, "mse_score = {mse}");
let l1 = l1_loss_score(&pred, &y).unwrap();
assert!(l1 < 1e-6, "l1_score = {l1}");
let rmse = rmse_score(&pred, &y).unwrap();
assert!(rmse < 1e-3, "rmse_score = {rmse}");
let r2 = r2_score(&pred, &y).unwrap();
assert!((r2 - 1.0).abs() < 1e-3, "r2_score = {r2}");
let mut graph2 = Topology::new(0, None);
graph2.options.input_dim = Some(2);
graph2.nodes.push(Node::new_input(0, 2));
graph2.nodes.push(Node::new_hidden(1, 2, 2));
graph2.nodes.push(Node::new_output(2, 2, 3));
graph2.nodes[2].hidden_dim = Some(3);
graph2.finalize();
let cat_net = Network::build(&graph2, flodl::Device::CPU).unwrap();
let cx = Variable::new(
flodl::Tensor::from_f32(&[1.0, 0.5, 0.1, 0.9], &[2, 2], flodl::Device::CPU).unwrap(),
false,
);
let cpred = cat_net.forward(&cx).unwrap();
let ct = Variable::new(
flodl::Tensor::from_f32(&[1.0, 0.0, 0.0, 0.0, 1.0, 0.0], &[2, 3], flodl::Device::CPU)
.unwrap(),
false,
);
let ce = cross_entropy_onehot(&cpred, &ct).unwrap();
assert!(ce > 0.0 && ce.is_finite(), "cross_entropy = {ce}");
let acc = accuracy_score(&cpred, &ct).unwrap();
assert!((0.0..=1.0).contains(&acc), "accuracy = {acc}");
let f1 = f1_score(&cpred, &ct).unwrap();
assert!((0.0..=1.0).contains(&f1), "f1 = {f1}");
let prec = precision_score(&cpred, &ct).unwrap();
assert!((0.0..=1.0).contains(&prec), "precision = {prec}");
let (pa, ta) = argmax_classes(&cpred, &ct).unwrap();
assert_eq!(pa.len(), 2);
assert_eq!(ta.len(), 2);
}
proptest! {
#[test]
fn prop_direction_is_better_is_antisymmetric(a in -100.0f32..100.0, b in -100.0f32..100.0) {
if a != b {
if Direction::Minimize.is_better(a, b) {
prop_assert!(!Direction::Minimize.is_better(b, a));
}
if Direction::Maximize.is_better(a, b) {
prop_assert!(!Direction::Maximize.is_better(b, a));
}
}
}
}
}