use crate::activations::softmax;
use crate::batch::TokenBatch;
use crate::matrix::Matrix;
use crate::network::NetworkError;
use rayon::prelude::*;
#[derive(Clone, Debug, PartialEq)]
pub struct CausalLmLoss {
pub loss: f32,
pub grad_logits: Matrix,
pub predicted: usize,
}
impl CausalLmLoss {
pub fn perplexity(&self) -> f32 {
self.loss.exp()
}
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct TotalLoss {
pub lm_loss: f32,
pub auxiliary_loss: f32,
}
impl TotalLoss {
pub fn total(&self) -> f32 {
self.lm_loss + self.auxiliary_loss
}
pub fn perplexity(&self) -> f32 {
self.lm_loss.exp()
}
}
pub fn causal_lm_loss(logits: &Matrix, ids: &[u32]) -> Result<CausalLmLoss, NetworkError> {
causal_lm_loss_batch(logits, &TokenBatch::new(&[ids])?)
}
pub fn causal_lm_loss_batch(
logits: &Matrix,
batch: &TokenBatch,
) -> Result<CausalLmLoss, NetworkError> {
if logits.rows != batch.rows() {
return Err(NetworkError::InvalidTarget {
expected: logits.rows,
actual: batch.rows(),
});
}
let ids = batch.ids();
let predicted = batch.predicted();
if predicted == 0 {
return Err(NetworkError::InvalidConfig(
"a causal language-modelling loss needs at least two tokens".into(),
));
}
cross_entropy_rows(logits, predicted, |row| {
batch.predicts(row).then(|| ids[row + 1])
})
}
pub(crate) fn cross_entropy_rows(
logits: &Matrix,
predicted: usize,
target: impl Fn(usize) -> Option<u32> + Sync,
) -> Result<CausalLmLoss, NetworkError> {
let vocab_size = logits.cols;
let mut grad_logits = Matrix::new(logits.rows, vocab_size);
let inverse_predicted = 1.0 / predicted as f32;
let per_row: Vec<f32> = grad_logits
.data
.par_chunks_mut(vocab_size)
.enumerate()
.map(|(row_index, row)| {
let Some(id) = target(row_index) else {
return Ok(0.0);
};
let target = id as usize;
if target >= vocab_size {
return Err(NetworkError::TokenOutOfRange { id, vocab_size });
}
row.copy_from_slice(logits.row(row_index));
softmax(row);
let contribution = -row[target].max(f32::MIN_POSITIVE).ln();
for value in row.iter_mut() {
*value *= inverse_predicted;
}
row[target] -= inverse_predicted;
Ok(contribution)
})
.collect::<Result<Vec<f32>, NetworkError>>()?;
let loss: f32 = per_row.iter().sum();
Ok(CausalLmLoss {
loss: loss / predicted as f32,
grad_logits,
predicted,
})
}
pub(crate) fn add_z_loss(
logits: &Matrix,
batch: &TokenBatch,
weight: f32,
grad_logits: &mut Matrix,
) -> f32 {
if weight == 0.0 {
return 0.0;
}
let inverse_predicted = 1.0 / batch.predicted() as f32;
let vocab_size = logits.cols;
let per_row: Vec<f32> = grad_logits
.data
.par_chunks_mut(vocab_size)
.enumerate()
.map(|(row_index, grad)| {
if !batch.predicts(row_index) {
return 0.0;
}
let row = logits.row(row_index);
let max = row.iter().copied().fold(f32::NEG_INFINITY, f32::max);
let sum: f32 = row.iter().map(|&value| (value - max).exp()).sum();
let lse = max + sum.ln();
let scale = 2.0 * weight * lse * inverse_predicted / sum;
for (g, &value) in grad.iter_mut().zip(row) {
*g += (value - max).exp() * scale;
}
weight * lse * lse
})
.collect();
per_row.iter().sum::<f32>() * inverse_predicted
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn uniform_logits_give_the_log_of_the_vocabulary() {
let logits = Matrix::new(3, 4);
let result = causal_lm_loss(&logits, &[0, 1, 2]).unwrap();
assert_eq!(result.predicted, 2);
assert!((result.loss - 4.0f32.ln()).abs() < 1e-5);
}
#[test]
fn the_last_position_has_no_target() {
let logits = Matrix::from_vec(2, 2, vec![0.5, -0.5, 2.0, -2.0]);
let result = causal_lm_loss(&logits, &[0, 1]).unwrap();
assert_eq!(result.grad_logits.row(1), &[0.0, 0.0]);
assert!(result.grad_logits.row(0).iter().any(|&g| g != 0.0));
}
#[test]
fn gradient_matches_finite_differences() {
let logits = Matrix::from_vec(3, 3, vec![0.2, -1.0, 0.5, 1.5, 0.3, -0.7, 0.1, 0.9, -0.2]);
let ids = [2u32, 0, 1];
let analytic = causal_lm_loss(&logits, &ids).unwrap().grad_logits;
let epsilon = 1e-3;
for index in 0..logits.data.len() {
let mut bumped = logits.clone();
bumped.data[index] += epsilon;
let high = causal_lm_loss(&bumped, &ids).unwrap().loss;
bumped.data[index] -= 2.0 * epsilon;
let low = causal_lm_loss(&bumped, &ids).unwrap().loss;
let numeric = (high - low) / (2.0 * epsilon);
assert!((analytic.data[index] - numeric).abs() < 1e-3);
}
}
#[test]
fn z_loss_gradient_matches_finite_differences() {
let logits = Matrix::from_vec(3, 3, vec![2.2, -1.0, 0.5, 1.5, 3.3, -0.7, 0.1, 0.9, -0.2]);
let batch = TokenBatch::new(&[[2u32, 0, 1]]).unwrap();
let z = |logits: &Matrix| add_z_loss(logits, &batch, 0.1, &mut Matrix::new(3, 3));
let mut analytic = Matrix::new(3, 3);
let value = add_z_loss(&logits, &batch, 0.1, &mut analytic);
assert!(value > 0.0);
assert_eq!(analytic.row(2), &[0.0, 0.0, 0.0]);
let epsilon = 1e-3;
for index in 0..logits.data.len() {
let mut bumped = logits.clone();
bumped.data[index] += epsilon;
let high = z(&bumped);
bumped.data[index] -= 2.0 * epsilon;
let low = z(&bumped);
let numeric = (high - low) / (2.0 * epsilon);
assert!((analytic.data[index] - numeric).abs() < 1e-3);
}
}
#[test]
fn a_loss_mask_drops_a_position_to_exactly_zero() {
let logits = Matrix::from_vec(3, 3, vec![0.2, -1.0, 0.5, 1.5, 0.3, -0.7, 0.1, 0.9, -0.2]);
let ids = [0u32, 1, 2];
let full = causal_lm_loss(&logits, &ids).unwrap();
let masked = causal_lm_loss_batch(
&logits,
&TokenBatch::new(&[ids])
.unwrap()
.with_loss_mask(&[false, false, true])
.unwrap(),
)
.unwrap();
assert_eq!(masked.predicted, 1);
assert_eq!(masked.grad_logits.row(0), &[0.0, 0.0, 0.0]);
assert_eq!(masked.grad_logits.row(2), &[0.0, 0.0, 0.0]);
let surviving = -softmax_probability(logits.row(1), 2).ln();
assert!((masked.loss - surviving).abs() < 1e-5);
let dropped = -softmax_probability(logits.row(0), 1).ln();
assert!((full.loss * 2.0 - masked.loss - dropped).abs() < 1e-5);
for (full_grad, masked_grad) in full
.grad_logits
.row(1)
.iter()
.zip(masked.grad_logits.row(1))
{
assert!((masked_grad - 2.0 * full_grad).abs() < 1e-6);
}
}
#[test]
fn a_mask_that_flags_everything_changes_nothing() {
let logits = Matrix::from_vec(3, 3, vec![0.2, -1.0, 0.5, 1.5, 0.3, -0.7, 0.1, 0.9, -0.2]);
let ids = [2u32, 0, 1];
let plain = causal_lm_loss(&logits, &ids).unwrap();
let flagged = causal_lm_loss_batch(
&logits,
&TokenBatch::new(&[ids])
.unwrap()
.with_loss_mask(&[true; 3])
.unwrap(),
)
.unwrap();
assert_eq!(plain, flagged);
}
fn softmax_probability(row: &[f32], index: usize) -> f32 {
let mut row = row.to_vec();
softmax(&mut row);
row[index]
}
#[test]
fn confident_and_correct_beats_confident_and_wrong() {
let right = Matrix::from_vec(2, 2, vec![-5.0, 5.0, 0.0, 0.0]);
let wrong = Matrix::from_vec(2, 2, vec![5.0, -5.0, 0.0, 0.0]);
let right = causal_lm_loss(&right, &[0, 1]).unwrap().loss;
let wrong = causal_lm_loss(&wrong, &[0, 1]).unwrap().loss;
assert!(right < wrong);
}
#[test]
fn a_single_token_has_nothing_to_predict() {
assert!(causal_lm_loss(&Matrix::new(1, 4), &[0]).is_err());
}
#[test]
fn total_loss_adds_the_auxiliary_terms() {
let total = TotalLoss {
lm_loss: 2.0,
auxiliary_loss: 0.25,
};
assert!((total.total() - 2.25).abs() < 1e-6);
}
}