use super::state::AccountBody;
use crate::matrix::Matrix;
pub fn homeostatic_reward(
prev: &AccountBody,
next: &AccountBody,
pnl: f32,
liquidated: bool,
drive_weights: [f32; 4],
pnl_lambda: f32,
terminal_penalty: f32,
) -> f32 {
let relief = prev.drive(drive_weights) - next.drive(drive_weights);
relief + pnl_lambda * pnl - if liquidated { terminal_penalty } else { 0.0 }
}
pub fn bellman_targets(
rewards: &[f32],
dones: &[f32],
next_q: &Matrix,
gamma: f32,
out: &mut [f32],
) {
for (i, y) in out.iter_mut().enumerate() {
let best = next_q
.row(i)
.iter()
.copied()
.fold(f32::NEG_INFINITY, f32::max);
*y = rewards[i] + gamma * (1.0 - dones[i]) * best;
}
}
pub fn double_bellman_targets(
rewards: &[f32],
dones: &[f32],
next_q_online: &Matrix,
next_q_target: &Matrix,
gamma: f32,
out: &mut [f32],
) {
for (i, y) in out.iter_mut().enumerate() {
let row = next_q_online.row(i);
let mut best = 0;
for (a, v) in row.iter().enumerate() {
if *v > row[best] {
best = a;
}
}
*y = rewards[i] + gamma * (1.0 - dones[i]) * next_q_target.row(i)[best];
}
}
pub fn td_huber_loss(q: &Matrix, actions: &[usize], targets: &[f32], grad: &mut Matrix) -> f32 {
let mut errors = vec![0.0; actions.len()];
weighted_td_huber_loss(q, actions, targets, &[], grad, &mut errors)
}
pub fn weighted_td_huber_loss(
q: &Matrix,
actions: &[usize],
targets: &[f32],
weights: &[f32],
grad: &mut Matrix,
errors: &mut [f32],
) -> f32 {
grad.zeros();
let n = q.rows as f32;
let mut loss = 0.0;
for (i, (&a, &y)) in actions.iter().zip(targets).enumerate() {
let weight = weights.get(i).copied().unwrap_or(1.0);
let error = q.row(i)[a] - y;
errors[i] = error;
loss += weight
* if error.abs() <= 1.0 {
0.5 * error * error
} else {
error.abs() - 0.5
};
grad.row_mut(i)[a] = weight * error.clamp(-1.0, 1.0) / n;
}
loss / n
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn relief_pays_and_liquidation_costs() {
let stressed = AccountBody::from_array([0.9, 0.5, 0.05, 0.0]);
let calm = AccountBody::default();
let w = [1.0; 4];
assert!(homeostatic_reward(&stressed, &calm, 0.0, false, w, 1.0, 1.0) > 0.5);
assert!(homeostatic_reward(&calm, &calm, 0.0, true, w, 1.0, 1.0) == -1.0);
}
#[test]
fn td_loss_touches_only_the_chosen_action_and_done_cuts_the_bootstrap() {
let next_q = Matrix::from_vec(2, 2, vec![1.0, 3.0, 5.0, 2.0]);
let mut y = [0.0; 2];
bellman_targets(&[1.0, 1.0], &[0.0, 1.0], &next_q, 0.5, &mut y);
assert_eq!(y, [2.5, 1.0]);
let online = Matrix::from_vec(2, 2, vec![4.0, 0.0, 0.0, 1.0]);
let mut y2 = [0.0; 2];
double_bellman_targets(&[1.0, 1.0], &[0.0, 1.0], &online, &next_q, 0.5, &mut y2);
assert_eq!(y2, [1.5, 1.0]);
let q = Matrix::from_vec(2, 2, vec![0.0, 2.0, 1.0, 9.0]);
let mut grad = Matrix::new(2, 2);
let loss = td_huber_loss(&q, &[1, 0], &y, &mut grad);
assert!((loss - 0.125 / 2.0).abs() < 1e-6);
assert_eq!(grad.data, vec![0.0, -0.25, 0.0, 0.0]);
}
#[test]
fn weights_scale_each_row_of_the_td_loss() {
let q = Matrix::from_vec(2, 2, vec![1.0, 0.0, 0.0, 3.0]);
let (actions, targets) = ([0, 1], [0.5, 0.0]);
let mut plain = Matrix::new(2, 2);
let mut weighted = Matrix::new(2, 2);
let mut errors = [0.0; 2];
let loss = td_huber_loss(&q, &actions, &targets, &mut plain);
let ones = weighted_td_huber_loss(&q, &actions, &targets, &[], &mut weighted, &mut errors);
assert_eq!((loss, &plain.data), (ones, &weighted.data));
assert_eq!(errors, [0.5, 3.0]);
let scaled = weighted_td_huber_loss(
&q,
&actions,
&targets,
&[0.5, 2.0],
&mut weighted,
&mut errors,
);
assert!((scaled - (0.5 * 0.125 + 2.0 * 2.5) / 2.0).abs() < 1e-6);
assert_eq!(weighted.data, [0.5 * 0.5 / 2.0, 0.0, 0.0, 2.0 * 1.0 / 2.0]);
}
}