use crate::matrix::Matrix;
use crate::optimizers::Optimizer;
use rand::{Rng, rngs::StdRng};
use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(from = "Matrix", into = "Matrix")]
pub struct Param {
pub value: Matrix,
pub grad: Matrix,
moment1: Matrix,
moment2: Matrix,
#[cfg(feature = "cuda")]
pub(crate) device: Option<crate::gpu_transformer::DeviceParam>,
}
impl From<Matrix> for Param {
fn from(value: Matrix) -> Self {
Self::new(value)
}
}
impl From<Param> for Matrix {
fn from(param: Param) -> Self {
param.value
}
}
impl PartialEq for Param {
fn eq(&self, other: &Self) -> bool {
self.value == other.value
}
}
impl Param {
pub fn new(value: Matrix) -> Self {
let (rows, cols) = (value.rows, value.cols);
Self {
value,
grad: Matrix::new(rows, cols),
moment1: Matrix::new(rows, cols),
moment2: Matrix::new(rows, cols),
#[cfg(feature = "cuda")]
device: None,
}
}
pub fn zeros(rows: usize, cols: usize) -> Self {
Self::new(Matrix::new(rows, cols))
}
pub fn filled(rows: usize, cols: usize, value: f32) -> Self {
Self::new(Matrix::from_vec(rows, cols, vec![value; rows * cols]))
}
pub fn he_uniform(rows: usize, cols: usize, fan_in: usize, rng: &mut StdRng) -> Self {
let scale = (2.0 / fan_in as f32).sqrt();
let data = (0..rows * cols)
.map(|_| rng.gen_range(-scale..scale))
.collect();
Self::new(Matrix::from_vec(rows, cols, data))
}
pub fn len(&self) -> usize {
self.value.data.len()
}
pub fn is_empty(&self) -> bool {
self.value.data.is_empty()
}
pub fn zero_grad(&mut self) {
#[cfg(feature = "cuda")]
if let Some(device) = &mut self.device {
device.zero_grad();
return;
}
self.grad.zeros();
}
#[cfg(feature = "cuda")]
pub(crate) fn move_to_cuda(
&mut self,
context: &std::sync::Arc<crate::gpu_transformer::GpuContext>,
) -> Result<(), crate::network::NetworkError> {
if self.device.is_none() {
self.device = Some(crate::gpu_transformer::DeviceParam::new(
context.clone(),
&self.value,
)?);
}
Ok(())
}
#[cfg(feature = "cuda")]
pub(crate) fn move_to_cpu(&mut self) -> Result<(), crate::network::NetworkError> {
if let Some(device) = self.device.take() {
device.download_value(&mut self.value)?;
device.download_grad(&mut self.grad)?;
}
Ok(())
}
#[cfg(feature = "cuda")]
pub(crate) fn sync_from_device(&mut self) -> Result<(), crate::network::NetworkError> {
if let Some(device) = &self.device {
device.download_value(&mut self.value)?;
device.download_grad(&mut self.grad)?;
}
Ok(())
}
pub fn moments(&mut self) -> Result<(&Matrix, &Matrix), crate::network::NetworkError> {
#[cfg(feature = "cuda")]
if let Some(device) = &self.device {
device.download_moments(&mut self.moment1, &mut self.moment2)?;
}
Ok((&self.moment1, &self.moment2))
}
pub fn set_moments(
&mut self,
first: Matrix,
second: Matrix,
) -> Result<(), crate::network::NetworkError> {
self.moment1 = first;
self.moment2 = second;
#[cfg(feature = "cuda")]
if let Some(device) = &mut self.device {
device.upload_moments(&self.moment1, &self.moment2)?;
}
Ok(())
}
pub fn step(&mut self, optimizer: &Optimizer, step: usize, scale: f32) {
#[cfg(feature = "cuda")]
if let Some(device) = &mut self.device {
device.step(optimizer, step, scale);
return;
}
match *optimizer {
Optimizer::Sgd { learning_rate } => {
for (value, grad) in self.value.data.iter_mut().zip(&self.grad.data) {
*value -= learning_rate * grad * scale;
}
}
Optimizer::Adam {
learning_rate,
beta1,
beta2,
epsilon,
weight_decay,
} => {
let bias_correction1 = 1.0 - beta1.powi(step as i32);
let bias_correction2 = 1.0 - beta2.powi(step as i32);
for (((value, grad), m), v) in self
.value
.data
.iter_mut()
.zip(&self.grad.data)
.zip(&mut self.moment1.data)
.zip(&mut self.moment2.data)
{
let grad = grad * scale;
*m = beta1 * *m + (1.0 - beta1) * grad;
*v = beta2 * *v + (1.0 - beta2) * grad * grad;
let m_hat = *m / bias_correction1;
let v_hat = *v / bias_correction2;
*value -= learning_rate * weight_decay * *value;
*value -= learning_rate * m_hat / (v_hat.sqrt() + epsilon);
}
}
}
self.zero_grad();
}
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)]
pub struct Linear {
pub weight: Param,
}
impl Linear {
pub fn new(in_features: usize, out_features: usize, rng: &mut StdRng) -> Self {
Self {
weight: Param::he_uniform(out_features, in_features, in_features, rng),
}
}
pub fn in_features(&self) -> usize {
self.weight.value.cols
}
pub fn out_features(&self) -> usize {
self.weight.value.rows
}
pub fn forward(&self, input: &Matrix) -> Matrix {
debug_assert_eq!(input.cols, self.in_features());
#[cfg(feature = "cuda")]
if let Some(device) = &self.weight.device {
return device.matmul_rhs_transposed(input);
}
let mut output = Matrix::new(input.rows, self.out_features());
input.dot_rhs_transposed(&self.weight.value, &mut output);
output
}
pub fn backward(&mut self, input: &Matrix, grad_output: &Matrix) -> Matrix {
debug_assert_eq!(grad_output.cols, self.out_features());
#[cfg(feature = "cuda")]
if let Some(device) = &mut self.weight.device {
device.accumulate_grad(grad_output, input);
return device.matmul(grad_output);
}
grad_output.dot_self_transposed_accumulate(input, &mut self.weight.grad);
let mut grad_input = Matrix::new(grad_output.rows, self.in_features());
grad_output.dot(&self.weight.value, &mut grad_input);
grad_input
}
pub fn params_mut(&mut self) -> Vec<&mut Param> {
vec![&mut self.weight]
}
}
#[cfg(test)]
mod tests {
use super::*;
use rand::SeedableRng;
#[test]
fn linear_backward_matches_finite_differences() {
let mut rng = StdRng::seed_from_u64(7);
let mut linear = Linear::new(3, 2, &mut rng);
let input = Matrix::from_vec(2, 3, vec![0.5, -1.0, 2.0, 0.25, 0.75, -0.5]);
let grad_output = Matrix::from_vec(2, 2, vec![1.0; 4]);
let grad_input = linear.backward(&input, &grad_output);
let epsilon = 1e-3;
for index in 0..input.data.len() {
let mut bumped = input.clone();
bumped.data[index] += epsilon;
let high: f32 = linear.forward(&bumped).data.iter().sum();
bumped.data[index] -= 2.0 * epsilon;
let low: f32 = linear.forward(&bumped).data.iter().sum();
let numeric = (high - low) / (2.0 * epsilon);
assert!((grad_input.data[index] - numeric).abs() < 1e-2);
}
}
#[test]
fn param_round_trips_as_a_bare_matrix() {
let param = Param::new(Matrix::from_vec(1, 3, vec![1.0, 2.0, 3.0]));
let json = serde_json::to_string(¶m).unwrap();
let restored: Param = serde_json::from_str(&json).unwrap();
assert_eq!(restored.value, param.value);
assert!(restored.grad.data.iter().all(|&g| g == 0.0));
}
}