use crate::tensor::Tensor;
use std::cell::RefCell;
macro_rules! assert_shape {
($a:expr, $b:expr) => {
assert_eq!(
$a.shape,
$b.shape,
"shape mismatch: lhs {:?} vs rhs {:?}",
$a.shape,
$b.shape
);
};
}
macro_rules! assert_matmul {
($a:expr, $b:expr) => {
assert_eq!(
$a.shape[1],
$b.shape[0],
"matmul dimension mismatch: lhs cols {} vs rhs rows {}",
$a.shape[1],
$b.shape[0]
);
};
}
thread_local! {
static GRAPH: RefCell<Vec<GraphEntry>> = const { RefCell::new(Vec::new()) };
static GRAD_REGISTRY: RefCell<Vec<Option<Tensor>>> = RefCell::new(Vec::new());
}
enum GraphEntry {
Tensor(Tensor),
Op { inputs: Vec<usize>, backward_fn: usize },
}
pub fn clear_graph() {
GRAPH.with(|g| g.borrow_mut().clear());
GRAD_REGISTRY.with(|g| g.borrow_mut().clear());
}
#[derive(Clone)]
pub struct GradTensor {
pub tensor: Tensor,
pub grad: Option<Tensor>,
node_id: usize,
}
impl GradTensor {
pub fn new(tensor: Tensor) -> Self {
let id = GRAPH.with(|g| {
let mut g = g.borrow_mut();
let id = g.len();
g.push(GraphEntry::Tensor(tensor.clone()));
id
});
GRAD_REGISTRY.with(|g| {
let mut g = g.borrow_mut();
if g.len() <= id {
g.resize_with(id + 1, || None);
}
});
Self { tensor, grad: None, node_id: id }
}
pub fn from_data(shape: &[usize], data: Vec<f64>) -> Self {
let t = Tensor::from_vec(shape, data);
assert!(t.is_ok(), "from_vec failed for shape {:?}", shape);
Self::new(t.expect("shape check above"))
}
pub fn zero_grad(&mut self) { self.grad = None; }
}
pub fn add(a: &GradTensor, b: &GradTensor) -> GradTensor {
assert_shape!(a.tensor, b.tensor);
let out = a.tensor.add(&b.tensor).unwrap();
let out_id = GRAPH.with(|g| {
let mut g = g.borrow_mut();
let id = g.len();
g.push(GraphEntry::Op { inputs: vec![a.node_id, b.node_id], backward_fn: 0 });
id
});
GRAD_REGISTRY.with(|g| {
let mut g = g.borrow_mut();
if g.len() <= out_id {
g.resize_with(out_id + 1, || None);
}
});
GradTensor { tensor: out, grad: None, node_id: out_id }
}
pub fn mul(a: &GradTensor, b: &GradTensor) -> GradTensor {
assert_shape!(a.tensor, b.tensor);
let out = a.tensor.mul(&b.tensor).unwrap();
let out_id = GRAPH.with(|g| {
let mut g = g.borrow_mut();
let id = g.len();
g.push(GraphEntry::Op { inputs: vec![a.node_id, b.node_id], backward_fn: 1 });
id
});
GRAD_REGISTRY.with(|g| {
let mut g = g.borrow_mut();
if g.len() <= out_id {
g.resize_with(out_id + 1, || None);
}
});
GradTensor { tensor: out, grad: None, node_id: out_id }
}
pub fn matmul(a: &GradTensor, b: &GradTensor) -> GradTensor {
assert_matmul!(a.tensor, b.tensor);
let out = a.tensor.matmul(&b.tensor).unwrap();
let out_id = GRAPH.with(|g| {
let mut g = g.borrow_mut();
let id = g.len();
g.push(GraphEntry::Op { inputs: vec![a.node_id, b.node_id], backward_fn: 2 });
id
});
GRAD_REGISTRY.with(|g| {
let mut g = g.borrow_mut();
if g.len() <= out_id {
g.resize_with(out_id + 1, || None);
}
});
GradTensor { tensor: out, grad: None, node_id: out_id }
}
pub fn relu_op(a: &GradTensor) -> GradTensor {
let out = crate::activations::relu(&a.tensor);
let out_id = GRAPH.with(|g| {
let mut g = g.borrow_mut();
let id = g.len();
g.push(GraphEntry::Op { inputs: vec![a.node_id], backward_fn: 3 });
id
});
GRAD_REGISTRY.with(|g| {
let mut g = g.borrow_mut();
if g.len() <= out_id {
g.resize_with(out_id + 1, || None);
}
});
GradTensor { tensor: out, grad: None, node_id: out_id }
}
pub fn sum(a: &GradTensor) -> GradTensor {
let val = a.tensor.sum();
let out = Tensor::scalar(val);
let out_id = GRAPH.with(|g| {
let mut g = g.borrow_mut();
let id = g.len();
g.push(GraphEntry::Op { inputs: vec![a.node_id], backward_fn: 4 });
id
});
GRAD_REGISTRY.with(|g| {
let mut g = g.borrow_mut();
if g.len() <= out_id {
g.resize_with(out_id + 1, || None);
}
});
GradTensor { tensor: out, grad: None, node_id: out_id }
}
pub fn mse_loss(pred: &GradTensor, target: &GradTensor) -> GradTensor {
assert_shape!(pred.tensor, target.tensor);
let val = crate::losses::mse(&pred.tensor, &target.tensor).unwrap();
let out = Tensor::scalar(val);
let out_id = GRAPH.with(|g| {
let mut g = g.borrow_mut();
let id = g.len();
g.push(GraphEntry::Op { inputs: vec![pred.node_id, target.node_id], backward_fn: 5 });
id
});
GRAD_REGISTRY.with(|g| {
let mut g = g.borrow_mut();
if g.len() <= out_id {
g.resize_with(out_id + 1, || None);
}
});
GradTensor { tensor: out, grad: None, node_id: out_id }
}
pub fn backward(loss: &mut GradTensor, scale: f64) {
GRAPH.with(|g| {
let g = g.borrow();
let n = g.len();
let mut grads: Vec<Option<Tensor>> = vec![None; n];
grads[loss.node_id] = Some(Tensor::scalar(scale));
for i in (0..n).rev() {
let grad_out = match &grads[i] {
Some(g) => g.clone(),
None => continue,
};
match &g[i] {
GraphEntry::Tensor(_) => {}
GraphEntry::Op { inputs, backward_fn } => {
let input_grads = match backward_fn {
0 => vec![grad_out.clone(), grad_out.clone()],
1 => {
let a = match &g[inputs[1]] {
GraphEntry::Tensor(t) => t.clone(),
_ => Tensor::zeros(&[1]),
};
let b = match &g[inputs[0]] {
GraphEntry::Tensor(t) => t.clone(),
_ => Tensor::zeros(&[1]),
};
assert_shape!(grad_out, a);
assert_shape!(grad_out, b);
vec![grad_out.mul(&a).unwrap(), grad_out.mul(&b).unwrap()]
}
2 => {
let a = match &g[inputs[0]] {
GraphEntry::Tensor(t) => t.clone(),
_ => Tensor::zeros(&[1]),
};
let b = match &g[inputs[1]] {
GraphEntry::Tensor(t) => t.clone(),
_ => Tensor::zeros(&[1]),
};
let bt = b.transpose().unwrap();
let at = a.transpose().unwrap();
assert_matmul!(grad_out, bt);
assert_matmul!(at, grad_out);
vec![grad_out.matmul(&bt).unwrap(), at.matmul(&grad_out).unwrap()]
}
3 => {
let input = match &g[inputs[0]] {
GraphEntry::Tensor(t) => t.clone(),
_ => Tensor::zeros(&[1]),
};
let mask = crate::activations::relu_grad(&input);
assert_shape!(grad_out, mask);
vec![grad_out.mul(&mask).unwrap()]
}
4 => {
let input = match &g[inputs[0]] {
GraphEntry::Tensor(t) => t.clone(),
_ => Tensor::zeros(&[1]),
};
vec![Tensor::full(&input.shape, grad_out.data[0])]
}
5 => {
let pred = match &g[inputs[0]] {
GraphEntry::Tensor(t) => t.clone(),
_ => Tensor::zeros(&[1]),
};
let target = match &g[inputs[1]] {
GraphEntry::Tensor(t) => t.clone(),
_ => Tensor::zeros(&[1]),
};
assert_shape!(pred, target);
let g = crate::losses::mse_grad(&pred, &target).unwrap();
vec![g, Tensor::zeros(&target.shape)]
}
_ => vec![],
};
for (j, &input_id) in inputs.iter().enumerate() {
if j < input_grads.len() {
grads[input_id] = Some(match &grads[input_id] {
Some(existing) => {
assert_shape!(existing, &input_grads[j]);
existing.add(&input_grads[j]).unwrap()
}
None => input_grads[j].clone(),
});
}
}
}
}
}
GRAD_REGISTRY.with(|reg| {
let mut reg = reg.borrow_mut();
for (id, grad) in grads.into_iter().enumerate() {
if let Some(g) = grad {
if id < reg.len() {
reg[id] = Some(g);
}
}
}
});
});
loss.grad = Some(Tensor::scalar(scale));
}
pub fn get_grad(node_id: usize) -> Option<Tensor> {
GRAD_REGISTRY.with(|reg| {
let reg = reg.borrow();
reg.get(node_id).cloned().flatten()
})
}
pub fn node_id(gt: &GradTensor) -> usize {
gt.node_id
}