use rayon::prelude::*;
use std::cell::RefCell;
use std::collections::HashSet;
use std::fmt;
use std::ops::{Add, Div, Mul, Sub};
use std::rc::Rc;
use std::sync::atomic::{AtomicU8, AtomicUsize, Ordering};
pub mod act_quant;
pub mod advanced_quant;
pub mod amp;
pub mod analyze;
pub mod anomaly;
pub mod attention;
pub mod calibration;
pub mod kv_quant;
pub mod checkpoint;
#[cfg(feature = "training")]
pub mod config;
#[cfg(feature = "training")]
pub mod corpus;
#[cfg(feature = "training")]
pub mod data_loader;
#[cfg(feature = "training")]
pub mod data_parallel;
pub mod generate;
pub mod gguf;
#[cfg(feature = "gpu")]
pub mod gpu;
#[cfg(not(feature = "gpu"))]
#[path = "gpu_stub.rs"]
pub mod gpu;
pub mod grad;
pub mod initialization;
pub mod inspect;
pub mod layers;
pub mod log;
pub mod memory;
pub mod model;
pub mod norm;
pub mod optimizer;
pub mod peft;
pub mod perf_regression;
pub mod profile;
pub mod ptq;
pub mod qat;
pub mod quantize;
pub mod quality;
pub mod weight_quant;
pub mod rope;
pub mod state_dict;
pub mod tokenizer;
pub mod tokenizer_quality;
pub mod trainer;
pub mod transformer;
pub use grad::{
enable_grad, is_grad_enabled, no_grad, set_grad_enabled, with_no_grad, GradModeGuard,
};
pub fn checkpoint<F>(input: &Tensor, function: F) -> Tensor
where
F: Fn(&Tensor) -> Tensor + 'static,
{
if !is_grad_enabled() {
return function(input);
}
let recompute: Rc<dyn Fn(&Tensor) -> Tensor> = Rc::new(function);
let output = with_no_grad(|| recompute(input));
wrap_checkpoint_output(output, input.clone(), CheckpointRecompute::Cpu(recompute))
}
pub fn checkpoint_gpu<F>(
input: &Tensor,
gpu: &gpu::GpuContext,
function: F,
) -> Result<Tensor, gpu::GpuError>
where
F: Fn(&Tensor, &gpu::GpuContext) -> Result<Tensor, gpu::GpuError> + 'static,
{
if !is_grad_enabled() {
return function(input, gpu);
}
let recompute: Rc<dyn Fn(&Tensor, &gpu::GpuContext) -> Result<Tensor, gpu::GpuError>> =
Rc::new(function);
let output = with_no_grad(|| recompute(input, gpu))?;
Ok(wrap_checkpoint_output(
output,
input.clone(),
CheckpointRecompute::Gpu(recompute),
))
}
fn wrap_checkpoint_output(
output: Tensor,
input: Tensor,
recompute: CheckpointRecompute,
) -> Tensor {
let op = Op::Checkpoint { input, recompute };
let node = output.inner.borrow();
match node.device {
Device::Cpu => Tensor::from_shared_view(
node.data.clone(),
node.offset,
node.shape.clone(),
node.strides.clone(),
true,
op,
),
Device::Gpu => {
let storage = node
.gpu
.clone()
.expect("GPU checkpoint output missing storage");
drop(node);
Tensor::from_gpu_operation(storage, true, op)
}
}
}
pub(crate) fn detach_for_checkpoint(input: &Tensor) -> Tensor {
let node = input.inner.borrow();
match node.device {
Device::Cpu => Tensor::from_shared_view(
node.data.clone(),
node.offset,
node.shape.clone(),
node.strides.clone(),
true,
Op::Leaf,
),
Device::Gpu => {
let storage = node
.gpu
.clone()
.expect("GPU checkpoint input missing storage");
drop(node);
Tensor::from_gpu_operation(storage, true, Op::Leaf)
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Device {
Cpu,
Gpu,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct TensorError(pub String);
impl TensorError {
pub fn new(message: impl Into<String>) -> Self {
Self(message.into())
}
}
impl fmt::Display for TensorError {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(formatter, "tensor error: {}", self.0)
}
}
impl std::error::Error for TensorError {}
fn panic_tensor<T>(error: TensorError) -> T {
panic!("{error}")
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
#[repr(u8)]
pub enum CpuMatmulBackend {
Reference = 0,
#[default]
BlockedParallel = 1,
}
static CPU_MATMUL_BACKEND: AtomicU8 = AtomicU8::new(CpuMatmulBackend::BlockedParallel as u8);
static CPU_LIVE_TENSOR_BYTES: AtomicUsize = AtomicUsize::new(0);
static CPU_PEAK_TENSOR_BYTES: AtomicUsize = AtomicUsize::new(0);
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct CpuMemoryStats {
pub live_tensor_bytes: usize,
pub peak_tensor_bytes: usize,
}
pub fn cpu_memory_stats() -> CpuMemoryStats {
CpuMemoryStats {
live_tensor_bytes: CPU_LIVE_TENSOR_BYTES.load(Ordering::Relaxed),
peak_tensor_bytes: CPU_PEAK_TENSOR_BYTES.load(Ordering::Relaxed),
}
}
pub fn reset_cpu_peak_memory() {
CPU_PEAK_TENSOR_BYTES.store(
CPU_LIVE_TENSOR_BYTES.load(Ordering::Relaxed),
Ordering::Relaxed,
)
}
fn track_cpu_tensor_allocation(bytes: usize) {
if bytes == 0 {
return;
}
let live = CPU_LIVE_TENSOR_BYTES.fetch_add(bytes, Ordering::Relaxed) + bytes;
CPU_PEAK_TENSOR_BYTES.fetch_max(live, Ordering::Relaxed);
}
fn track_cpu_tensor_deallocation(bytes: usize) {
if bytes > 0 {
CPU_LIVE_TENSOR_BYTES.fetch_sub(bytes, Ordering::Relaxed);
}
}
#[derive(Clone)]
struct SharedCpuStorage {
data: Rc<Vec<f32>>,
}
impl SharedCpuStorage {
fn new(data: Vec<f32>) -> Self {
track_cpu_tensor_allocation(data.len() * std::mem::size_of::<f32>());
Self {
data: Rc::new(data),
}
}
fn empty() -> Self {
Self {
data: Rc::new(Vec::new()),
}
}
fn rc(&self) -> Rc<Vec<f32>> {
Rc::clone(&self.data)
}
fn is_empty(&self) -> bool {
self.data.is_empty()
}
fn make_mut(&mut self) -> &mut Vec<f32> {
if Rc::strong_count(&self.data) > 1 {
let cloned = (*self.data).clone();
track_cpu_tensor_allocation(cloned.len() * std::mem::size_of::<f32>());
self.data = Rc::new(cloned);
}
Rc::make_mut(&mut self.data)
}
fn set(&mut self, data: Vec<f32>) {
*self = Self::new(data);
}
fn clear(&mut self) {
*self = Self::empty();
}
fn ptr_eq(&self, other: &Self) -> bool {
Rc::ptr_eq(&self.data, &other.data)
}
}
impl Drop for SharedCpuStorage {
fn drop(&mut self) {
if Rc::strong_count(&self.data) == 1 {
track_cpu_tensor_deallocation(self.data.len() * std::mem::size_of::<f32>());
}
}
}
impl std::ops::Deref for SharedCpuStorage {
type Target = [f32];
fn deref(&self) -> &[f32] {
self.data.as_slice()
}
}
impl fmt::Debug for SharedCpuStorage {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
self.data.fmt(formatter)
}
}
#[derive(Clone)]
struct SharedIndexStorage {
data: Rc<Vec<usize>>,
}
impl SharedIndexStorage {
fn new(data: Vec<usize>) -> Self {
track_cpu_tensor_allocation(data.len() * std::mem::size_of::<usize>());
Self {
data: Rc::new(data),
}
}
fn to_vec(&self) -> Vec<usize> {
(*self.data).clone()
}
}
impl Drop for SharedIndexStorage {
fn drop(&mut self) {
if Rc::strong_count(&self.data) == 1 {
track_cpu_tensor_deallocation(self.data.len() * std::mem::size_of::<usize>());
}
}
}
impl std::ops::Deref for SharedIndexStorage {
type Target = [usize];
fn deref(&self) -> &[usize] {
self.data.as_slice()
}
}
pub fn set_cpu_matmul_backend(backend: CpuMatmulBackend) {
CPU_MATMUL_BACKEND.store(backend as u8, Ordering::Relaxed);
}
pub fn cpu_matmul_backend() -> CpuMatmulBackend {
match CPU_MATMUL_BACKEND.load(Ordering::Relaxed) {
0 => CpuMatmulBackend::Reference,
_ => CpuMatmulBackend::BlockedParallel,
}
}
#[derive(Clone)]
pub struct Tensor {
inner: Rc<RefCell<TensorData>>,
}
#[derive(Clone)]
pub struct IndexTensor {
inner: Rc<RefCell<IndexTensorData>>,
}
struct IndexTensorData {
data: SharedIndexStorage,
gpu: Option<gpu::GpuIndexTensor>,
shape: Vec<usize>,
}
struct TensorData {
data: SharedCpuStorage,
offset: usize,
gpu: Option<gpu::GpuTensor>,
gpu_grad: Option<gpu::GpuTensor>,
shape: Vec<usize>,
strides: Vec<usize>,
grad: Option<Vec<f32>>,
requires_grad: bool,
device: Device,
op: Op,
}
impl TensorData {
fn numel(&self) -> usize {
element_count(&self.shape)
}
fn is_contiguous(&self) -> bool {
is_contiguous(&self.shape, &self.strides)
}
fn contig_slice(&self) -> &[f32] {
debug_assert!(self.is_contiguous());
let numel = self.numel();
&self.data[self.offset..self.offset + numel]
}
}
#[derive(Clone)]
enum Op {
Leaf,
Transfer(Tensor),
Add(Tensor, Tensor),
Sub(Tensor, Tensor),
Mul(Tensor, Tensor),
Div(Tensor, Tensor),
Pow(Tensor, f32),
MatMul(Tensor, Tensor),
Reshape(Tensor),
Contiguous(Tensor),
Permute {
input: Tensor,
axes: Vec<usize>,
},
Narrow {
input: Tensor,
dim: usize,
start: usize,
length: usize,
},
Embedding {
weight: Tensor,
indices: IndexTensor,
},
Rope {
input: Tensor,
sequence_dim: usize,
theta: f32,
position_offset: usize,
scaling: Option<rope::RopeScaling>,
},
SoftmaxLast(Tensor),
CausalMask {
input: Tensor,
past_length: usize,
},
Gelu(Tensor),
Silu(Tensor),
RmsNormGpu {
input: Tensor,
weight: Tensor,
epsilon: f32,
},
ResidualRmsNormGpu {
left: Tensor,
right: Tensor,
weight: Tensor,
epsilon: f32,
},
SwiGluGpu {
gate: Tensor,
up: Tensor,
},
MaskedSoftmaxGpu {
input: Tensor,
past_length: usize,
},
TiledAttention {
query: Tensor,
key: Tensor,
value: Tensor,
attention_mask: Option<IndexTensor>,
logsumexp: Tensor,
scale: f32,
past_length: usize,
dropout_probability: f32,
dropout_seed: u32,
},
CrossEntropy {
logits: Tensor,
targets: IndexTensor,
ignore_index: Option<usize>,
},
Relu(Tensor),
Sigmoid(Tensor),
FakeQuant(Tensor),
Sum(Tensor),
SumDim {
input: Tensor,
dim: usize,
keepdim: bool,
},
Mean(Tensor),
MeanDim {
input: Tensor,
dim: usize,
keepdim: bool,
},
Checkpoint {
input: Tensor,
recompute: CheckpointRecompute,
},
Cat {
inputs: Vec<Tensor>,
dim: usize,
},
}
#[derive(Clone)]
pub(crate) enum CheckpointRecompute {
Cpu(Rc<dyn Fn(&Tensor) -> Tensor>),
Gpu(Rc<dyn Fn(&Tensor, &gpu::GpuContext) -> Result<Tensor, gpu::GpuError>>),
}
impl Op {
fn name(&self) -> &'static str {
match self {
Op::Leaf => "leaf",
Op::Transfer(_) => "transfer",
Op::Add(_, _) => "add",
Op::Sub(_, _) => "sub",
Op::Mul(_, _) => "mul",
Op::Div(_, _) => "div",
Op::Pow(_, _) => "pow",
Op::MatMul(_, _) => "matmul",
Op::Reshape(_) => "reshape",
Op::Contiguous(_) => "contiguous",
Op::Permute { .. } => "permute",
Op::Narrow { .. } => "narrow",
Op::Embedding { .. } => "embedding",
Op::Rope { .. } => "rope",
Op::SoftmaxLast(_) => "softmax",
Op::CausalMask { .. } => "causal_mask",
Op::Gelu(_) => "gelu",
Op::Silu(_) => "silu",
Op::RmsNormGpu { .. } => "rms_norm",
Op::ResidualRmsNormGpu { .. } => "residual_rms_norm",
Op::SwiGluGpu { .. } => "swiglu",
Op::MaskedSoftmaxGpu { .. } => "masked_softmax",
Op::TiledAttention { .. } => "tiled_attention",
Op::CrossEntropy { .. } => "cross_entropy",
Op::Relu(_) => "relu",
Op::Sigmoid(_) => "sigmoid",
Op::FakeQuant(_) => "fake_quant",
Op::Sum(_) => "sum",
Op::SumDim { .. } => "sum_dim",
Op::Mean(_) => "mean",
Op::MeanDim { .. } => "mean_dim",
Op::Checkpoint { .. } => "checkpoint",
Op::Cat { .. } => "cat",
}
}
fn produces_values(&self) -> bool {
!matches!(
self,
Op::Leaf
| Op::Transfer(_)
| Op::Reshape(_)
| Op::Contiguous(_)
| Op::Permute { .. }
| Op::Narrow { .. }
)
}
}
#[derive(Clone, Copy)]
enum BinaryOp {
Add,
Sub,
Mul,
Div,
}
fn resolve_grad_tracking(requires_grad: bool, op: Op) -> (bool, Op) {
match op {
Op::Leaf => (requires_grad, Op::Leaf),
other if requires_grad && is_grad_enabled() => (true, other),
_ => (false, Op::Leaf),
}
}
impl IndexTensor {
pub fn try_new(data: Vec<usize>, shape: Vec<usize>) -> Result<Self, TensorError> {
if shape.iter().any(|dimension| *dimension == 0) {
return Err(TensorError::new("zero-sized dimensions are not supported"));
}
if data.len() != element_count(&shape) {
return Err(TensorError::new("index data length does not match shape"));
}
Ok(Self {
inner: Rc::new(RefCell::new(IndexTensorData {
data: SharedIndexStorage::new(data),
gpu: None,
shape,
})),
})
}
pub fn new(data: Vec<usize>, shape: Vec<usize>) -> Self {
Self::try_new(data, shape).unwrap_or_else(panic_tensor)
}
pub fn shape(&self) -> Vec<usize> {
self.inner.borrow().shape.clone()
}
pub fn numel(&self) -> usize {
self.inner.borrow().data.len()
}
pub fn data(&self) -> Vec<usize> {
self.inner.borrow().data.to_vec()
}
pub fn try_validate_upper_bound(
&self,
upper_bound: usize,
label: &str,
) -> Result<(), TensorError> {
if self
.inner
.borrow()
.data
.iter()
.any(|index| *index >= upper_bound)
{
return Err(TensorError::new(format!("{label} is out of range")));
}
Ok(())
}
pub(crate) fn validate_upper_bound(&self, upper_bound: usize, label: &str) {
self.try_validate_upper_bound(upper_bound, label)
.unwrap_or_else(panic_tensor)
}
pub(crate) fn cached_gpu(&self) -> Option<gpu::GpuIndexTensor> {
self.inner.borrow().gpu.clone()
}
pub(crate) fn set_cached_gpu(&self, storage: gpu::GpuIndexTensor) {
self.inner.borrow_mut().gpu = Some(storage);
}
}
impl Tensor {
pub fn try_new(
data: Vec<f32>,
shape: Vec<usize>,
requires_grad: bool,
) -> Result<Self, TensorError> {
validate_shape(&data, &shape)?;
Ok(Self::from_operation(data, shape, requires_grad, Op::Leaf))
}
pub fn new(data: Vec<f32>, shape: Vec<usize>, requires_grad: bool) -> Self {
Self::try_new(data, shape, requires_grad).unwrap_or_else(panic_tensor)
}
pub fn scalar(value: f32, requires_grad: bool) -> Self {
Self::new(vec![value], vec![], requires_grad)
}
pub fn try_zeros(shape: Vec<usize>, requires_grad: bool) -> Result<Self, TensorError> {
let length = element_count(&shape);
Self::try_new(vec![0.0; length], shape, requires_grad)
}
pub fn zeros(shape: Vec<usize>, requires_grad: bool) -> Self {
Self::try_zeros(shape, requires_grad).unwrap_or_else(panic_tensor)
}
pub fn try_ones(shape: Vec<usize>, requires_grad: bool) -> Result<Self, TensorError> {
let length = element_count(&shape);
Self::try_new(vec![1.0; length], shape, requires_grad)
}
pub fn ones(shape: Vec<usize>, requires_grad: bool) -> Self {
Self::try_ones(shape, requires_grad).unwrap_or_else(panic_tensor)
}
fn from_operation(data: Vec<f32>, shape: Vec<usize>, requires_grad: bool, op: Op) -> Self {
Self::from_shared_operation(SharedCpuStorage::new(data), shape, requires_grad, op)
}
fn from_shared_operation(
data: SharedCpuStorage,
shape: Vec<usize>,
requires_grad: bool,
op: Op,
) -> Self {
let strides = compute_strides(&shape);
Self::from_shared_view(data, 0, shape, strides, requires_grad, op)
}
fn from_shared_view(
data: SharedCpuStorage,
offset: usize,
shape: Vec<usize>,
strides: Vec<usize>,
requires_grad: bool,
op: Op,
) -> Self {
if anomaly::detect_anomaly_enabled() && op.produces_values() {
let numel = element_count(&shape);
if offset == 0 && strides == compute_strides(&shape) && data.len() == numel {
anomaly::check_finite(op.name(), &shape, &data[..]);
} else {
let logical_strides = compute_strides(&shape);
let mut gathered = vec![0.0; numel];
for flat in 0..numel {
let mut index = offset;
for dim in 0..shape.len() {
let coordinate = (flat / logical_strides[dim]) % shape[dim];
index += coordinate * strides[dim];
}
gathered[flat] = data[index];
}
anomaly::check_finite(op.name(), &shape, &gathered);
}
}
let (requires_grad, op) = resolve_grad_tracking(requires_grad, op);
Self {
inner: Rc::new(RefCell::new(TensorData {
data,
offset,
gpu: None,
gpu_grad: None,
shape,
strides,
grad: None,
requires_grad,
device: Device::Cpu,
op,
})),
}
}
pub(crate) fn from_gpu_operation(gpu: gpu::GpuTensor, requires_grad: bool, op: Op) -> Self {
let shape = gpu.shape().to_vec();
let strides = compute_strides(&shape);
if anomaly::detect_anomaly_enabled() && op.produces_values() {
let values = gpu
.to_vec()
.unwrap_or_else(|error| panic!("anomaly: failed to read GPU tensor: {error}"));
anomaly::check_finite(op.name(), &shape, &values);
}
let (requires_grad, op) = resolve_grad_tracking(requires_grad, op);
Self {
inner: Rc::new(RefCell::new(TensorData {
data: SharedCpuStorage::empty(),
offset: 0,
gpu: Some(gpu),
gpu_grad: None,
shape,
strides,
grad: None,
requires_grad,
device: Device::Gpu,
op,
})),
}
}
pub fn shape(&self) -> Vec<usize> {
self.inner.borrow().shape.clone()
}
pub fn strides(&self) -> Vec<usize> {
self.inner.borrow().strides.clone()
}
pub fn numel(&self) -> usize {
self.inner.borrow().numel()
}
pub fn requires_grad(&self) -> bool {
self.inner.borrow().requires_grad
}
pub fn set_requires_grad(&self, requires_grad: bool) {
self.inner.borrow_mut().requires_grad = requires_grad;
}
pub fn is_leaf(&self) -> bool {
matches!(self.inner.borrow().op, Op::Leaf)
}
pub fn device(&self) -> Device {
self.inner.borrow().device
}
pub fn is_contiguous(&self) -> bool {
self.inner.borrow().is_contiguous()
}
pub fn shares_storage_with(&self, other: &Tensor) -> bool {
self.inner.borrow().data.ptr_eq(&other.inner.borrow().data)
}
fn to_contiguous_vec(&self) -> Vec<f32> {
let node = self.inner.borrow();
if node.is_contiguous() {
return node.contig_slice().to_vec();
}
let numel = node.numel();
let logical_strides = compute_strides(&node.shape);
let mut output = vec![0.0; numel];
for flat in 0..numel {
let mut index = node.offset;
for dim in 0..node.shape.len() {
let coordinate = (flat / logical_strides[dim]) % node.shape[dim];
index += coordinate * node.strides[dim];
}
output[flat] = node.data[index];
}
output
}
pub fn contiguous(&self) -> Tensor {
self.assert_cpu("contiguous()");
if self.is_contiguous() {
return self.clone();
}
Tensor::from_operation(
self.to_contiguous_vec(),
self.shape(),
self.requires_grad(),
Op::Contiguous(self.clone()),
)
}
pub fn shared_data(&self) -> Rc<Vec<f32>> {
self.assert_cpu("shared_data()");
{
let node = self.inner.borrow();
if node.is_contiguous() && node.offset == 0 && node.data.len() == node.numel() {
return node.data.rc();
}
}
Rc::new(self.to_contiguous_vec())
}
pub fn data(&self) -> Vec<f32> {
self.assert_cpu("data()");
self.to_contiguous_vec()
}
fn try_assert_cpu(&self, operation: &str) -> Result<(), TensorError> {
if self.device() != Device::Cpu {
return Err(TensorError::new(format!(
"{operation} requires CPU storage; call to_cpu() first"
)));
}
Ok(())
}
fn assert_cpu(&self, operation: &str) {
self.try_assert_cpu(operation).unwrap_or_else(panic_tensor)
}
fn ensure_cpu_data(&self) {
let gpu = {
let node = self.inner.borrow();
if !node.data.is_empty() {
return;
}
node.gpu
.clone()
.expect("tensor has neither CPU nor GPU storage")
};
let values = gpu.to_vec().expect("GPU tensor readback failed");
let mut node = self.inner.borrow_mut();
if node.data.is_empty() {
node.data.set(values);
node.offset = 0;
}
}
pub(crate) fn cached_gpu(&self) -> Option<gpu::GpuTensor> {
self.inner.borrow().gpu.clone()
}
pub(crate) fn cached_cpu_data(&self) -> Option<Vec<f32>> {
let node = self.inner.borrow();
if node.data.is_empty() {
None
} else {
drop(node);
Some(self.to_contiguous_vec())
}
}
pub(crate) fn set_cached_gpu(&self, storage: gpu::GpuTensor) {
self.inner.borrow_mut().gpu = Some(storage);
}
pub(crate) fn mark_gpu_authoritative(&self) {
let mut node = self.inner.borrow_mut();
assert!(
node.gpu.is_some(),
"GPU-authoritative tensor needs GPU storage"
);
node.data.clear();
node.offset = 0;
node.device = Device::Gpu;
}
pub fn offload_gpu_cache(&self) {
self.ensure_cpu_data();
let mut node = self.inner.borrow_mut();
node.gpu = None;
node.gpu_grad = None;
node.device = Device::Cpu;
}
pub(crate) fn copy_to_gpu_storage(&self, storage: gpu::GpuTensor) -> Self {
Tensor::from_gpu_operation(storage, self.requires_grad(), Op::Transfer(self.clone()))
}
pub(crate) fn copy_to_cpu_storage(&self, data: Vec<f32>) -> Self {
Tensor::from_operation(
data,
self.shape(),
self.requires_grad(),
Op::Transfer(self.clone()),
)
}
pub fn grad(&self) -> Option<Vec<f32>> {
self.inner.borrow().grad.clone()
}
pub fn set_grad(&self, gradient: Option<Vec<f32>>) {
let mut node = self.inner.borrow_mut();
if let Some(values) = gradient.as_ref() {
assert_eq!(
values.len(),
node.numel(),
"set_grad() length does not match tensor numel"
);
}
node.grad = gradient;
node.gpu_grad = None;
}
pub fn add_grad(&self, contribution: &[f32]) {
accumulate_gradient(self, contribution);
}
pub(crate) fn gpu_grad(&self) -> Option<gpu::GpuTensor> {
self.inner.borrow().gpu_grad.clone()
}
pub(crate) fn set_gpu_grad(&self, gradient: Option<gpu::GpuTensor>) {
self.inner.borrow_mut().gpu_grad = gradient;
}
pub fn scale_grad(&self, scale: f32) {
if let Some(gradient) = self.inner.borrow_mut().grad.as_mut() {
for value in gradient {
*value *= scale;
}
}
}
pub fn try_item(&self) -> Result<f32, TensorError> {
self.try_assert_cpu("item()")?;
if self.numel() != 1 {
return Err(TensorError::new("item() requires a one-element tensor"));
}
let node = self.inner.borrow();
if node.is_contiguous() {
Ok(node.data[node.offset])
} else {
drop(node);
Ok(self.to_contiguous_vec()[0])
}
}
pub fn item(&self) -> f32 {
self.try_item().unwrap_or_else(panic_tensor)
}
pub fn try_set_data(&self, values: Vec<f32>) -> Result<(), TensorError> {
let mut node = self.inner.borrow_mut();
if element_count(&node.shape) != values.len() {
return Err(TensorError::new("replacement data has the wrong length"));
}
node.data.set(values);
node.offset = 0;
node.strides = compute_strides(&node.shape);
node.gpu = None;
node.gpu_grad = None;
node.device = Device::Cpu;
Ok(())
}
pub fn set_data(&self, values: Vec<f32>) {
self.try_set_data(values).unwrap_or_else(panic_tensor)
}
pub fn set_data_keep_grad(&self, values: Vec<f32>) {
self.try_set_data_keep_grad(values)
.unwrap_or_else(panic_tensor)
}
pub fn try_set_data_keep_grad(&self, values: Vec<f32>) -> Result<(), TensorError> {
let mut node = self.inner.borrow_mut();
if element_count(&node.shape) != values.len() {
return Err(TensorError::new("replacement data has the wrong length"));
}
let grad = node.grad.take();
let gpu_grad = node.gpu_grad.take();
node.data.set(values);
node.offset = 0;
node.strides = compute_strides(&node.shape);
node.gpu = None;
node.device = Device::Cpu;
node.grad = grad;
node.gpu_grad = gpu_grad;
Ok(())
}
pub fn try_add(&self, other: &Tensor) -> Result<Tensor, TensorError> {
self.try_elementwise(other, BinaryOp::Add)
}
pub fn add(&self, other: &Tensor) -> Tensor {
self.try_add(other).unwrap_or_else(panic_tensor)
}
pub fn try_sub(&self, other: &Tensor) -> Result<Tensor, TensorError> {
self.try_elementwise(other, BinaryOp::Sub)
}
pub fn sub(&self, other: &Tensor) -> Tensor {
self.try_sub(other).unwrap_or_else(panic_tensor)
}
pub fn try_mul(&self, other: &Tensor) -> Result<Tensor, TensorError> {
self.try_elementwise(other, BinaryOp::Mul)
}
pub fn mul(&self, other: &Tensor) -> Tensor {
self.try_mul(other).unwrap_or_else(panic_tensor)
}
pub fn try_div(&self, other: &Tensor) -> Result<Tensor, TensorError> {
self.try_elementwise(other, BinaryOp::Div)
}
pub fn div(&self, other: &Tensor) -> Tensor {
self.try_div(other).unwrap_or_else(panic_tensor)
}
pub fn try_dropout(&self, probability: f32, seed: u64) -> Result<Tensor, TensorError> {
try_validate_dropout_probability(probability)?;
if probability == 0.0 {
return Ok(self.clone());
}
let mask = Tensor::try_new(
dropout_mask_values(self.numel(), probability, seed),
self.shape(),
false,
)?;
self.try_mul(&mask)
}
pub fn dropout(&self, probability: f32, seed: u64) -> Tensor {
self.try_dropout(probability, seed)
.unwrap_or_else(panic_tensor)
}
fn try_elementwise(&self, other: &Tensor, operation: BinaryOp) -> Result<Tensor, TensorError> {
self.try_assert_cpu("elementwise operation")?;
other.try_assert_cpu("elementwise operation")?;
let left_tensor = self.contiguous();
let right_tensor = other.contiguous();
let (output, output_shape) = {
let left = left_tensor.inner.borrow();
let right = right_tensor.inner.borrow();
let output_shape = try_broadcast_shape(&left.shape, &right.shape)?;
let output_strides = compute_strides(&output_shape);
let mut output = Vec::with_capacity(element_count(&output_shape));
for flat_index in 0..element_count(&output_shape) {
let left_index = broadcast_offset(
flat_index,
&output_shape,
&output_strides,
&left.shape,
&left.strides,
);
let right_index = broadcast_offset(
flat_index,
&output_shape,
&output_strides,
&right.shape,
&right.strides,
);
let left_value = left.data[left.offset + left_index];
let right_value = right.data[right.offset + right_index];
output.push(match operation {
BinaryOp::Add => left_value + right_value,
BinaryOp::Sub => left_value - right_value,
BinaryOp::Mul => left_value * right_value,
BinaryOp::Div => left_value / right_value,
});
}
(output, output_shape)
};
let requires_grad = left_tensor.requires_grad() || right_tensor.requires_grad();
let op = match operation {
BinaryOp::Add => Op::Add(left_tensor, right_tensor),
BinaryOp::Sub => Op::Sub(left_tensor, right_tensor),
BinaryOp::Mul => Op::Mul(left_tensor, right_tensor),
BinaryOp::Div => Op::Div(left_tensor, right_tensor),
};
Ok(Tensor::from_operation(
output,
output_shape,
requires_grad,
op,
))
}
pub fn powf(&self, exponent: f32) -> Tensor {
self.assert_cpu("powf()");
let source = self.contiguous();
let output = source
.inner
.borrow()
.contig_slice()
.iter()
.map(|value| value.powf(exponent))
.collect();
Tensor::from_operation(
output,
source.shape(),
source.requires_grad(),
Op::Pow(source, exponent),
)
}
pub fn try_reshape(&self, shape: Vec<usize>) -> Result<Tensor, TensorError> {
if shape.iter().any(|dimension| *dimension == 0) {
return Err(TensorError::new("zero-sized dimensions are not supported"));
}
if self.numel() != element_count(&shape) {
return Err(TensorError::new("reshape changes the number of elements"));
}
if self.device() == Device::Gpu {
if let Some(storage) = self.cached_gpu() {
return Ok(Tensor::from_gpu_operation(
storage.reshape(shape),
self.requires_grad(),
Op::Reshape(self.clone()),
));
}
}
let source = if self.is_contiguous() {
self.clone()
} else {
self.contiguous()
};
let (data, offset) = {
let node = source.inner.borrow();
if shape == node.shape {
drop(node);
return Ok(source);
}
(node.data.clone(), node.offset)
};
Ok(Tensor::from_shared_view(
data,
offset,
shape.clone(),
compute_strides(&shape),
source.requires_grad(),
Op::Reshape(source),
))
}
pub fn reshape(&self, shape: Vec<usize>) -> Tensor {
self.try_reshape(shape).unwrap_or_else(panic_tensor)
}
pub fn try_permute(&self, axes: Vec<usize>) -> Result<Tensor, TensorError> {
self.try_assert_cpu("permute()")?;
let (data, offset, output_shape, output_strides) = {
let input = self.inner.borrow();
if axes.len() != input.shape.len() {
return Err(TensorError::new("permute rank mismatch"));
}
let mut seen = vec![false; axes.len()];
for axis in &axes {
if *axis >= axes.len() {
return Err(TensorError::new("permute axis out of range"));
}
if seen[*axis] {
return Err(TensorError::new("permute axes must be unique"));
}
seen[*axis] = true;
}
if axes.iter().copied().eq(0..axes.len()) {
return Ok(self.clone());
}
let output_shape: Vec<usize> = axes.iter().map(|axis| input.shape[*axis]).collect();
let output_strides: Vec<usize> = axes.iter().map(|axis| input.strides[*axis]).collect();
(
input.data.clone(),
input.offset,
output_shape,
output_strides,
)
};
Ok(Tensor::from_shared_view(
data,
offset,
output_shape,
output_strides,
self.requires_grad(),
Op::Permute {
input: self.clone(),
axes,
},
))
}
pub fn permute(&self, axes: Vec<usize>) -> Tensor {
self.try_permute(axes).unwrap_or_else(panic_tensor)
}
pub fn try_transpose(&self, dim0: usize, dim1: usize) -> Result<Tensor, TensorError> {
let rank = self.shape().len();
if dim0 >= rank || dim1 >= rank {
return Err(TensorError::new("transpose dimension out of range"));
}
if dim0 == dim1 {
return Ok(self.clone());
}
let mut axes: Vec<usize> = (0..rank).collect();
axes.swap(dim0, dim1);
self.try_permute(axes)
}
pub fn transpose(&self, dim0: usize, dim1: usize) -> Tensor {
self.try_transpose(dim0, dim1).unwrap_or_else(panic_tensor)
}
pub fn try_narrow(
&self,
dim: usize,
start: usize,
length: usize,
) -> Result<Tensor, TensorError> {
self.try_assert_cpu("narrow()")?;
let (data, offset, shape, strides) = {
let input = self.inner.borrow();
if dim >= input.shape.len() {
return Err(TensorError::new("narrow dimension out of range"));
}
if length == 0 {
return Err(TensorError::new("narrow length must be positive"));
}
if start + length > input.shape[dim] {
return Err(TensorError::new("narrow range exceeds dimension size"));
}
if start == 0 && length == input.shape[dim] {
return Ok(self.clone());
}
let mut shape = input.shape.clone();
shape[dim] = length;
let offset = input.offset + start * input.strides[dim];
(input.data.clone(), offset, shape, input.strides.clone())
};
Ok(Tensor::from_shared_view(
data,
offset,
shape,
strides,
self.requires_grad(),
Op::Narrow {
input: self.clone(),
dim,
start,
length,
},
))
}
pub fn narrow(&self, dim: usize, start: usize, length: usize) -> Tensor {
self.try_narrow(dim, start, length)
.unwrap_or_else(panic_tensor)
}
pub fn try_slice(&self, dim: usize, start: usize, end: usize) -> Result<Tensor, TensorError> {
if end < start {
return Err(TensorError::new("slice end must be >= start"));
}
self.try_narrow(dim, start, end - start)
}
pub fn slice(&self, dim: usize, start: usize, end: usize) -> Tensor {
self.try_slice(dim, start, end).unwrap_or_else(panic_tensor)
}
pub fn try_embedding(&self, indices: &IndexTensor) -> Result<Tensor, TensorError> {
self.try_assert_cpu("embedding()")?;
let weight = self.contiguous();
indices.try_validate_upper_bound(weight.shape()[0], "embedding token ID")?;
let (output, output_shape) = {
let weight_node = weight.inner.borrow();
let index_node = indices.inner.borrow();
if weight_node.shape.len() != 2 {
return Err(TensorError::new("embedding weight must be 2D"));
}
let width = weight_node.shape[1];
let mut output_shape = index_node.shape.clone();
output_shape.push(width);
let mut output = Vec::with_capacity(index_node.data.len() * width);
let values = weight_node.contig_slice();
for token_id in index_node.data.iter() {
let start = token_id * width;
output.extend_from_slice(&values[start..start + width]);
}
(output, output_shape)
};
Ok(Tensor::from_operation(
output,
output_shape,
weight.requires_grad(),
Op::Embedding {
weight,
indices: indices.clone(),
},
))
}
pub fn embedding(&self, indices: &IndexTensor) -> Tensor {
self.try_embedding(indices).unwrap_or_else(panic_tensor)
}
pub fn try_rope(&self, sequence_dim: usize, theta: f32) -> Result<Tensor, TensorError> {
self.try_rope_with_offset(sequence_dim, theta, 0)
}
pub fn rope(&self, sequence_dim: usize, theta: f32) -> Tensor {
self.try_rope(sequence_dim, theta)
.unwrap_or_else(panic_tensor)
}
pub fn try_rope_with_offset(
&self,
sequence_dim: usize,
theta: f32,
position_offset: usize,
) -> Result<Tensor, TensorError> {
self.try_rope_with_offset_and_scaling(sequence_dim, theta, position_offset, None)
}
pub fn rope_with_offset(
&self,
sequence_dim: usize,
theta: f32,
position_offset: usize,
) -> Tensor {
self.try_rope_with_offset(sequence_dim, theta, position_offset)
.unwrap_or_else(panic_tensor)
}
pub fn try_rope_with_offset_and_scaling(
&self,
sequence_dim: usize,
theta: f32,
position_offset: usize,
scaling: Option<&rope::RopeScaling>,
) -> Result<Tensor, TensorError> {
self.try_assert_cpu("rope_with_offset()")?;
let source = self.contiguous();
let output = {
let input = source.inner.borrow();
if !(theta > 0.0) {
return Err(TensorError::new("RoPE theta must be positive"));
}
if sequence_dim + 1 >= input.shape.len() {
return Err(TensorError::new(
"RoPE sequence dimension must precede the feature dimension",
));
}
let width = *input.shape.last().unwrap();
if width % 2 != 0 {
return Err(TensorError::new("RoPE feature width must be even"));
}
let values = input.contig_slice();
let mut output = values.to_vec();
for vector_start in (0..values.len()).step_by(width) {
let position = position_offset
+ (vector_start / input.strides[sequence_dim]) % input.shape[sequence_dim];
for even in (0..width).step_by(2) {
let frequency = scaling.map_or_else(
|| theta.powf(-(even as f32) / width as f32),
|scaling| scaling.frequency(even, width, theta),
);
let angle = position as f32 * frequency;
let attention_factor = scaling.map_or(1.0, rope::RopeScaling::attention_factor);
let (cosine, sine) = (
angle.cos() * attention_factor,
angle.sin() * attention_factor,
);
let left = values[vector_start + even];
let right = values[vector_start + even + 1];
output[vector_start + even] = left * cosine - right * sine;
output[vector_start + even + 1] = left * sine + right * cosine;
}
}
output
};
Ok(Tensor::from_operation(
output,
source.shape(),
source.requires_grad(),
Op::Rope {
input: source,
sequence_dim,
theta,
position_offset,
scaling: scaling.cloned(),
},
))
}
pub fn rope_with_offset_and_scaling(
&self,
sequence_dim: usize,
theta: f32,
position_offset: usize,
scaling: Option<&rope::RopeScaling>,
) -> Tensor {
self.try_rope_with_offset_and_scaling(sequence_dim, theta, position_offset, scaling)
.unwrap_or_else(panic_tensor)
}
pub fn try_softmax_last_dim(&self) -> Result<Tensor, TensorError> {
self.try_assert_cpu("softmax_last_dim()")?;
let source = self.contiguous();
let output = {
let input = source.inner.borrow();
if input.shape.is_empty() {
return Err(TensorError::new("softmax requires at least one dimension"));
}
let width = *input.shape.last().unwrap();
let values = input.contig_slice();
let mut output = vec![0.0; values.len()];
for (input_row, output_row) in values
.chunks_exact(width)
.zip(output.chunks_exact_mut(width))
{
let maximum = input_row.iter().copied().fold(f32::NEG_INFINITY, f32::max);
let mut total = 0.0;
for (target, value) in output_row.iter_mut().zip(input_row.iter()) {
*target = (*value - maximum).exp();
total += *target;
}
for value in output_row {
*value /= total;
}
}
output
};
Ok(Tensor::from_operation(
output,
source.shape(),
source.requires_grad(),
Op::SoftmaxLast(source),
))
}
pub fn softmax_last_dim(&self) -> Tensor {
self.try_softmax_last_dim().unwrap_or_else(panic_tensor)
}
pub fn try_masked_softmax_last_dim(&self, past_length: usize) -> Result<Tensor, TensorError> {
self.try_assert_cpu("masked_softmax_last_dim()")?;
let source = self.contiguous();
let output = {
let input = source.inner.borrow();
if input.shape.len() < 2 {
return Err(TensorError::new(
"masked softmax requires at least 2 dimensions",
));
}
let key_length = *input.shape.last().unwrap();
let query_length = input.shape[input.shape.len() - 2];
let values = input.contig_slice();
let mut output = vec![0.0; values.len()];
for (row_index, (input_row, output_row)) in values
.chunks_exact(key_length)
.zip(output.chunks_exact_mut(key_length))
.enumerate()
{
let query = row_index % query_length;
let allowed = (past_length + query + 1).min(key_length);
let maximum = input_row[..allowed]
.iter()
.copied()
.fold(f32::NEG_INFINITY, f32::max);
let mut total = 0.0;
for (key, (target, value)) in
output_row.iter_mut().zip(input_row.iter()).enumerate()
{
if key < allowed {
*target = (*value - maximum).exp();
total += *target;
} else {
*target = 0.0;
}
}
if total > 0.0 {
for value in &mut output_row[..allowed] {
*value /= total;
}
}
}
output
};
Ok(Tensor::from_operation(
output,
source.shape(),
source.requires_grad(),
Op::MaskedSoftmaxGpu {
input: source,
past_length,
},
))
}
pub fn masked_softmax_last_dim(&self, past_length: usize) -> Tensor {
self.try_masked_softmax_last_dim(past_length)
.unwrap_or_else(panic_tensor)
}
pub fn try_tiled_causal_attention(
&self,
key: &Tensor,
value: &Tensor,
scale: f32,
past_length: usize,
) -> Result<Tensor, TensorError> {
self.try_tiled_causal_attention_with_optional_mask(
key,
value,
scale,
past_length,
None,
0.0,
0,
)
}
pub fn tiled_causal_attention(
&self,
key: &Tensor,
value: &Tensor,
scale: f32,
past_length: usize,
) -> Tensor {
self.try_tiled_causal_attention(key, value, scale, past_length)
.unwrap_or_else(panic_tensor)
}
pub fn try_tiled_causal_attention_with_mask(
&self,
key: &Tensor,
value: &Tensor,
scale: f32,
past_length: usize,
attention_mask: &IndexTensor,
) -> Result<Tensor, TensorError> {
self.try_tiled_causal_attention_with_optional_mask(
key,
value,
scale,
past_length,
Some(attention_mask),
0.0,
0,
)
}
pub fn tiled_causal_attention_with_mask(
&self,
key: &Tensor,
value: &Tensor,
scale: f32,
past_length: usize,
attention_mask: &IndexTensor,
) -> Tensor {
self.try_tiled_causal_attention_with_mask(key, value, scale, past_length, attention_mask)
.unwrap_or_else(panic_tensor)
}
pub(crate) fn try_tiled_causal_attention_with_optional_mask(
&self,
key: &Tensor,
value: &Tensor,
scale: f32,
past_length: usize,
attention_mask: Option<&IndexTensor>,
dropout_probability: f32,
dropout_seed: u32,
) -> Result<Tensor, TensorError> {
self.try_assert_cpu("tiled_causal_attention()")?;
key.try_assert_cpu("tiled_causal_attention()")?;
value.try_assert_cpu("tiled_causal_attention()")?;
let query = self.contiguous();
let key = key.contiguous();
let value = value.contiguous();
let query_shape = query.shape();
let key_shape = key.shape();
if query_shape.len() != 4 {
return Err(TensorError::new("tiled attention requires [B, H, Q, D]"));
}
if key_shape.len() != 4 {
return Err(TensorError::new("tiled attention requires [B, H, K, D]"));
}
if value.shape() != key_shape {
return Err(TensorError::new("attention K/V shapes differ"));
}
if query_shape[0] != key_shape[0] {
return Err(TensorError::new("attention batch sizes differ"));
}
if key_shape[1] == 0 || query_shape[1] % key_shape[1] != 0 {
return Err(TensorError::new(
"query heads must be divisible by key/value heads",
));
}
if query_shape[3] != key_shape[3] {
return Err(TensorError::new("attention head widths differ"));
}
if past_length + query_shape[2] != key_shape[2] {
return Err(TensorError::new(
"cached attention dimensions are inconsistent",
));
}
if !(scale.is_finite() && scale > 0.0) {
return Err(TensorError::new("attention scale must be positive"));
}
try_validate_dropout_probability(dropout_probability)?;
let attention_mask_data = match attention_mask {
Some(mask) => {
if mask.shape() != vec![query_shape[0], key_shape[2]] {
return Err(TensorError::new("attention mask must have shape [B, K]"));
}
let data = mask.data();
if data.iter().any(|value| *value > 1) {
return Err(TensorError::new(
"attention mask values must be zero or one",
));
}
Some(data)
}
None => None,
};
let query_data = query.shared_data();
let key_data = key.shared_data();
let value_data = value.shared_data();
let (output, logsumexp) = tiled_attention_forward(
&query_data,
&key_data,
&value_data,
attention_mask_data.as_deref(),
&query_shape,
key_shape[1],
key_shape[2],
scale,
past_length,
dropout_probability,
dropout_seed,
);
let logsumexp = Tensor::new(
logsumexp,
vec![query_shape[0], query_shape[1], query_shape[2]],
false,
);
Ok(Tensor::from_operation(
output,
query_shape,
query.requires_grad() || key.requires_grad() || value.requires_grad(),
Op::TiledAttention {
query,
key,
value,
attention_mask: attention_mask.cloned(),
logsumexp,
scale,
past_length,
dropout_probability,
dropout_seed,
},
))
}
pub(crate) fn tiled_causal_attention_with_optional_mask(
&self,
key: &Tensor,
value: &Tensor,
scale: f32,
past_length: usize,
attention_mask: Option<&IndexTensor>,
dropout_probability: f32,
dropout_seed: u32,
) -> Tensor {
self.try_tiled_causal_attention_with_optional_mask(
key,
value,
scale,
past_length,
attention_mask,
dropout_probability,
dropout_seed,
)
.unwrap_or_else(panic_tensor)
}
pub fn try_cross_entropy(&self, targets: &IndexTensor) -> Result<Tensor, TensorError> {
self.try_cross_entropy_with_ignore_index(targets, None)
}
pub fn cross_entropy(&self, targets: &IndexTensor) -> Tensor {
self.try_cross_entropy(targets).unwrap_or_else(panic_tensor)
}
pub fn try_cross_entropy_ignore_index(
&self,
targets: &IndexTensor,
ignore_index: usize,
) -> Result<Tensor, TensorError> {
self.try_cross_entropy_with_ignore_index(targets, Some(ignore_index))
}
pub fn cross_entropy_ignore_index(&self, targets: &IndexTensor, ignore_index: usize) -> Tensor {
self.try_cross_entropy_ignore_index(targets, ignore_index)
.unwrap_or_else(panic_tensor)
}
fn try_cross_entropy_with_ignore_index(
&self,
targets: &IndexTensor,
ignore_index: Option<usize>,
) -> Result<Tensor, TensorError> {
self.try_assert_cpu("cross_entropy()")?;
let logits_tensor = self.contiguous();
let loss = {
let logits = logits_tensor.inner.borrow();
let target_node = targets.inner.borrow();
if logits.shape.is_empty() {
return Err(TensorError::new("cross entropy logits need a class axis"));
}
if target_node.shape != logits.shape[..logits.shape.len() - 1] {
return Err(TensorError::new("cross entropy target shape mismatch"));
}
let classes = *logits.shape.last().unwrap();
if target_node
.data
.iter()
.any(|target| Some(*target) != ignore_index && *target >= classes)
{
return Err(TensorError::new("cross entropy target is out of range"));
}
let valid_rows = target_node
.data
.iter()
.filter(|target| Some(**target) != ignore_index)
.count();
if valid_rows == 0 {
return Err(TensorError::new(
"cross entropy needs at least one non-ignored target",
));
}
let values = logits.contig_slice();
let mut loss = 0.0;
for (row_index, row) in values.chunks_exact(classes).enumerate() {
let target = target_node.data[row_index];
if Some(target) == ignore_index {
continue;
}
let maximum = row.iter().copied().fold(f32::NEG_INFINITY, f32::max);
let exp_sum: f32 = row.iter().map(|value| (*value - maximum).exp()).sum();
loss += maximum + exp_sum.ln() - row[target];
}
loss / valid_rows as f32
};
Ok(Tensor::from_operation(
vec![loss],
vec![],
logits_tensor.requires_grad(),
Op::CrossEntropy {
logits: logits_tensor,
targets: targets.clone(),
ignore_index,
},
))
}
pub fn try_matmul(&self, other: &Tensor) -> Result<Tensor, TensorError> {
self.try_matmul_with_backend(other, cpu_matmul_backend())
}
pub fn matmul(&self, other: &Tensor) -> Tensor {
self.try_matmul(other).unwrap_or_else(panic_tensor)
}
pub fn try_matmul_with_backend(
&self,
other: &Tensor,
backend: CpuMatmulBackend,
) -> Result<Tensor, TensorError> {
self.try_assert_cpu("matmul()")?;
other.try_assert_cpu("matmul()")?;
let left_tensor = self.contiguous();
let right_tensor = other.contiguous();
let (output, output_shape) = {
let left = left_tensor.inner.borrow();
let right = right_tensor.inner.borrow();
let dimensions = try_matmul_dimensions(&left.shape, &right.shape)?;
let left_values = left.contig_slice();
let right_values = right.contig_slice();
let output = match backend {
CpuMatmulBackend::Reference => {
matmul_reference(left_values, right_values, dimensions)
}
CpuMatmulBackend::BlockedParallel => {
matmul_blocked_parallel(left_values, right_values, dimensions)
}
};
let mut output_shape = left.shape[..dimensions.rank - 2].to_vec();
output_shape.extend([dimensions.rows, dimensions.columns]);
(output, output_shape)
};
Ok(Tensor::from_operation(
output,
output_shape,
left_tensor.requires_grad() || right_tensor.requires_grad(),
Op::MatMul(left_tensor, right_tensor),
))
}
pub fn matmul_with_backend(&self, other: &Tensor, backend: CpuMatmulBackend) -> Tensor {
self.try_matmul_with_backend(other, backend)
.unwrap_or_else(panic_tensor)
}
pub fn relu(&self) -> Tensor {
self.assert_cpu("relu()");
let source = self.contiguous();
let output = source
.inner
.borrow()
.contig_slice()
.iter()
.map(|value| value.max(0.0))
.collect();
Tensor::from_operation(
output,
source.shape(),
source.requires_grad(),
Op::Relu(source),
)
}
pub fn sigmoid(&self) -> Tensor {
self.assert_cpu("sigmoid()");
let source = self.contiguous();
let output = source
.inner
.borrow()
.contig_slice()
.iter()
.map(|value| 1.0 / (1.0 + (-value).exp()))
.collect();
Tensor::from_operation(
output,
source.shape(),
source.requires_grad(),
Op::Sigmoid(source),
)
}
pub fn ste_replace(&self, values: Vec<f32>) -> Tensor {
self.assert_cpu("ste_replace()");
assert_eq!(
values.len(),
self.numel(),
"ste_replace length must match the tensor"
);
let source = self.contiguous();
Tensor::from_operation(
values,
source.shape(),
source.requires_grad(),
Op::FakeQuant(source),
)
}
pub fn gelu(&self) -> Tensor {
self.assert_cpu("gelu()");
let source = self.contiguous();
let output = source
.inner
.borrow()
.contig_slice()
.iter()
.map(|value| gelu_value(*value))
.collect();
Tensor::from_operation(
output,
source.shape(),
source.requires_grad(),
Op::Gelu(source),
)
}
pub fn silu(&self) -> Tensor {
self.assert_cpu("silu()");
let source = self.contiguous();
let output = source
.inner
.borrow()
.contig_slice()
.iter()
.map(|value| {
let sigmoid = 1.0 / (1.0 + (-value).exp());
value * sigmoid
})
.collect();
Tensor::from_operation(
output,
source.shape(),
source.requires_grad(),
Op::Silu(source),
)
}
pub fn sum(&self) -> Tensor {
self.assert_cpu("sum()");
let source = self.contiguous();
let value = source.inner.borrow().contig_slice().iter().sum();
Tensor::from_operation(vec![value], vec![], source.requires_grad(), Op::Sum(source))
}
pub fn try_sum_dim(&self, dim: usize, keepdim: bool) -> Result<Tensor, TensorError> {
self.try_reduce_dim(dim, keepdim, false)
}
pub fn sum_dim(&self, dim: usize, keepdim: bool) -> Tensor {
self.try_sum_dim(dim, keepdim).unwrap_or_else(panic_tensor)
}
pub fn mean(&self) -> Tensor {
self.assert_cpu("mean()");
let source = self.contiguous();
let node = source.inner.borrow();
let values = node.contig_slice();
let value = values.iter().sum::<f32>() / values.len() as f32;
drop(node);
Tensor::from_operation(
vec![value],
vec![],
source.requires_grad(),
Op::Mean(source),
)
}
pub fn try_mean_dim(&self, dim: usize, keepdim: bool) -> Result<Tensor, TensorError> {
self.try_reduce_dim(dim, keepdim, true)
}
pub fn mean_dim(&self, dim: usize, keepdim: bool) -> Tensor {
self.try_mean_dim(dim, keepdim).unwrap_or_else(panic_tensor)
}
fn try_reduce_dim(
&self,
dim: usize,
keepdim: bool,
average: bool,
) -> Result<Tensor, TensorError> {
self.try_assert_cpu("reduction")?;
let source = self.contiguous();
let (output, output_shape) = {
let input = source.inner.borrow();
if dim >= input.shape.len() {
return Err(TensorError::new("reduction dimension out of range"));
}
let mut output_shape = input.shape.clone();
if keepdim {
output_shape[dim] = 1;
} else {
output_shape.remove(dim);
}
let output_strides = compute_strides(&output_shape);
let values = input.contig_slice();
let mut output = vec![0.0; element_count(&output_shape)];
for input_index in 0..values.len() {
let output_index = reduction_offset(
input_index,
&input.shape,
&input.strides,
dim,
keepdim,
&output_strides,
);
output[output_index] += values[input_index];
}
if average {
let divisor = input.shape[dim] as f32;
for value in &mut output {
*value /= divisor;
}
}
(output, output_shape)
};
let op = if average {
Op::MeanDim {
input: source.clone(),
dim,
keepdim,
}
} else {
Op::SumDim {
input: source.clone(),
dim,
keepdim,
}
};
Ok(Tensor::from_operation(
output,
output_shape,
source.requires_grad(),
op,
))
}
pub fn try_backward(&self) -> Result<(), TensorError> {
if self.numel() != 1 {
return Err(TensorError::new("backward() requires a scalar tensor"));
}
if !self.requires_grad() {
return Err(TensorError::new(
"backward() requires a tensor that tracks gradients",
));
}
self.backward_unchecked();
Ok(())
}
pub fn backward(&self) {
self.try_backward().unwrap_or_else(panic_tensor)
}
fn backward_unchecked(&self) {
self.backward_with_gradient(&[1.0]);
}
fn backward_with_gradient(&self, root_gradient: &[f32]) {
let topology = self.topological_order();
for tensor in &topology {
tensor.ensure_cpu_data();
}
for tensor in &topology {
let mut node = tensor.inner.borrow_mut();
if !matches!(node.op, Op::Leaf) {
node.grad = None;
}
}
if self.is_leaf() {
accumulate_gradient(self, root_gradient);
} else {
self.inner.borrow_mut().grad = Some(root_gradient.to_vec());
}
for tensor in topology.iter().rev() {
let (operation, output_gradient, output_shape) = {
let node = tensor.inner.borrow();
(node.op.clone(), node.grad.clone(), node.shape.clone())
};
if let Some(output_gradient) = output_gradient {
match operation {
Op::Leaf => {}
Op::Checkpoint { input, recompute } => {
checkpoint_cpu_backward(&input, &recompute, &output_gradient);
}
Op::Cat { inputs, dim } => {
cat_backward(&inputs, dim, &output_gradient, &output_shape);
}
Op::Transfer(input) => accumulate_gradient(&input, &output_gradient),
Op::Add(left, right) => binary_backward(
&left,
&right,
&output_gradient,
&output_shape,
BinaryOp::Add,
),
Op::Sub(left, right) => binary_backward(
&left,
&right,
&output_gradient,
&output_shape,
BinaryOp::Sub,
),
Op::Mul(left, right) => binary_backward(
&left,
&right,
&output_gradient,
&output_shape,
BinaryOp::Mul,
),
Op::Div(left, right) => binary_backward(
&left,
&right,
&output_gradient,
&output_shape,
BinaryOp::Div,
),
Op::Pow(input, exponent) => {
let input_node = input.inner.borrow();
let gradient: Vec<f32> = input_node
.contig_slice()
.iter()
.zip(output_gradient.iter())
.map(|(value, gradient)| {
gradient * exponent * value.powf(exponent - 1.0)
})
.collect();
drop(input_node);
accumulate_gradient(&input, &gradient);
}
Op::MatMul(left, right) => {
matmul_backward(&left, &right, &output_gradient);
}
Op::Reshape(input) | Op::Contiguous(input) | Op::FakeQuant(input) => {
accumulate_gradient(&input, &output_gradient);
}
Op::Permute { input, axes } => {
permute_backward(&input, &output_gradient, &output_shape, &axes);
}
Op::Narrow {
input,
dim,
start,
length,
} => narrow_backward(&input, &output_gradient, dim, start, length),
Op::Embedding { weight, indices } => {
embedding_backward(&weight, &indices, &output_gradient);
}
Op::Rope {
input,
sequence_dim,
theta,
position_offset,
scaling,
} => rope_backward(
&input,
&output_gradient,
sequence_dim,
theta,
position_offset,
scaling.as_ref(),
),
Op::SoftmaxLast(input) => {
let output_data = tensor.inner.borrow();
softmax_backward(&input, output_data.contig_slice(), &output_gradient);
}
Op::CausalMask { input, past_length } => {
causal_mask_backward(&input, &output_gradient, past_length)
}
Op::Gelu(input) => {
let input_node = input.inner.borrow();
let gradient: Vec<f32> = input_node
.contig_slice()
.iter()
.zip(output_gradient.iter())
.map(|(value, gradient)| gradient * gelu_derivative(*value))
.collect();
drop(input_node);
accumulate_gradient(&input, &gradient);
}
Op::Silu(input) => {
let input_node = input.inner.borrow();
let gradient: Vec<f32> = input_node
.contig_slice()
.iter()
.zip(output_gradient.iter())
.map(|(value, gradient)| {
let sigmoid = 1.0 / (1.0 + (-value).exp());
gradient * (sigmoid + value * sigmoid * (1.0 - sigmoid))
})
.collect();
drop(input_node);
accumulate_gradient(&input, &gradient);
}
Op::RmsNormGpu {
input,
weight,
epsilon,
} => rms_norm_backward(&input, &weight, &output_gradient, epsilon),
Op::ResidualRmsNormGpu {
left,
right,
weight,
epsilon,
} => {
residual_rms_norm_backward(&left, &right, &weight, &output_gradient, epsilon)
}
Op::SwiGluGpu { gate, up } => swiglu_backward(&gate, &up, &output_gradient),
Op::MaskedSoftmaxGpu { input, past_length } => {
masked_softmax_backward(&input, tensor, &output_gradient, past_length)
}
Op::TiledAttention {
query,
key,
value,
attention_mask,
logsumexp,
scale,
past_length,
dropout_probability,
dropout_seed,
} => tiled_attention_backward(
&query,
&key,
&value,
tensor,
&logsumexp,
&output_gradient,
scale,
past_length,
dropout_probability,
dropout_seed,
attention_mask.as_ref(),
),
Op::CrossEntropy {
logits,
targets,
ignore_index,
} => {
cross_entropy_backward(&logits, &targets, output_gradient[0], ignore_index);
}
Op::Relu(input) => {
let input_node = input.inner.borrow();
let gradient: Vec<f32> = input_node
.contig_slice()
.iter()
.zip(output_gradient.iter())
.map(|(value, gradient)| if *value > 0.0 { *gradient } else { 0.0 })
.collect();
drop(input_node);
accumulate_gradient(&input, &gradient);
}
Op::Sigmoid(input) => {
let input_node = input.inner.borrow();
let gradient: Vec<f32> = input_node
.contig_slice()
.iter()
.zip(output_gradient.iter())
.map(|(value, gradient)| {
let sigmoid = 1.0 / (1.0 + (-value).exp());
gradient * sigmoid * (1.0 - sigmoid)
})
.collect();
drop(input_node);
accumulate_gradient(&input, &gradient);
}
Op::Sum(input) => {
let gradient = vec![output_gradient[0]; input.numel()];
accumulate_gradient(&input, &gradient);
}
Op::SumDim {
input,
dim,
keepdim,
} => reduction_backward(&input, &output_gradient, dim, keepdim, 1.0),
Op::Mean(input) => {
let scale = output_gradient[0] / input.numel() as f32;
let gradient = vec![scale; input.numel()];
accumulate_gradient(&input, &gradient);
}
Op::MeanDim {
input,
dim,
keepdim,
} => {
let divisor = input.inner.borrow().shape[dim] as f32;
reduction_backward(&input, &output_gradient, dim, keepdim, 1.0 / divisor);
}
}
}
tensor.release_finished_autograd_storage();
}
}
pub(crate) fn release_finished_autograd_storage(&self) {
{
let mut node = self.inner.borrow_mut();
if matches!(node.op, Op::Leaf) {
return;
}
node.grad = None;
node.gpu_grad = None;
node.op = Op::Leaf;
}
if Rc::strong_count(&self.inner) == 1 {
let mut node = self.inner.borrow_mut();
node.data.clear();
node.gpu = None;
node.offset = 0;
}
}
pub fn zero_grad(&self) {
for tensor in self.topological_order() {
let mut node = tensor.inner.borrow_mut();
node.grad = None;
node.gpu_grad = None;
}
}
pub fn with_mut_data(&self, update: impl FnOnce(&mut [f32])) {
self.assert_cpu("with_mut_data()");
let mut node = self.inner.borrow_mut();
assert!(
node.is_contiguous(),
"with_mut_data() requires a contiguous tensor"
);
let offset = node.offset;
let numel = node.numel();
let values = node.data.make_mut();
update(&mut values[offset..offset + numel]);
node.gpu = None;
node.gpu_grad = None;
node.device = Device::Cpu;
}
pub fn sgd_step(&self, learning_rate: f32) {
let gradient = self.inner.borrow().grad.clone();
if let Some(gradient) = gradient {
self.with_mut_data(|values| {
for (value, gradient) in values.iter_mut().zip(gradient.iter()) {
*value -= learning_rate * gradient;
}
});
}
}
fn topological_order(&self) -> Vec<Tensor> {
let mut ordered = Vec::new();
let mut seen = HashSet::new();
let mut stack = vec![(self.clone(), false)];
while let Some((tensor, expanded)) = stack.pop() {
let identity = Rc::as_ptr(&tensor.inner) as usize;
if expanded {
ordered.push(tensor);
continue;
}
if !seen.insert(identity) {
continue;
}
stack.push((tensor.clone(), true));
let parents = tensor.parents();
for parent in parents.into_iter().rev() {
stack.push((parent, false));
}
}
ordered
}
fn parents(&self) -> Vec<Tensor> {
match &self.inner.borrow().op {
Op::Leaf => vec![],
Op::Transfer(input) => vec![input.clone()],
Op::Add(left, right)
| Op::Sub(left, right)
| Op::Mul(left, right)
| Op::Div(left, right)
| Op::MatMul(left, right) => vec![left.clone(), right.clone()],
Op::Pow(input, _)
| Op::Reshape(input)
| Op::Contiguous(input)
| Op::Relu(input)
| Op::Sigmoid(input)
| Op::FakeQuant(input)
| Op::SoftmaxLast(input)
| Op::Gelu(input)
| Op::Silu(input)
| Op::Sum(input)
| Op::Mean(input) => vec![input.clone()],
Op::Permute { input, .. }
| Op::Narrow { input, .. }
| Op::Rope { input, .. }
| Op::CausalMask { input, .. }
| Op::SumDim { input, .. }
| Op::MeanDim { input, .. } => vec![input.clone()],
Op::Embedding { weight, .. } => vec![weight.clone()],
Op::RmsNormGpu { input, weight, .. } => vec![input.clone(), weight.clone()],
Op::ResidualRmsNormGpu {
left,
right,
weight,
..
} => vec![left.clone(), right.clone(), weight.clone()],
Op::SwiGluGpu { gate, up } => vec![gate.clone(), up.clone()],
Op::MaskedSoftmaxGpu { input, .. } => vec![input.clone()],
Op::TiledAttention {
query, key, value, ..
} => vec![query.clone(), key.clone(), value.clone()],
Op::CrossEntropy { logits, .. } => vec![logits.clone()],
Op::Checkpoint { input, .. } => vec![input.clone()],
Op::Cat { inputs, .. } => inputs.clone(),
}
}
pub fn cat(tensors: &[Tensor], dim: usize) -> Tensor {
Self::try_cat(tensors, dim).unwrap_or_else(panic_tensor)
}
pub fn try_cat(tensors: &[Tensor], dim: usize) -> Result<Tensor, TensorError> {
if tensors.is_empty() {
return Err(TensorError::new("cat requires at least one tensor"));
}
let first_shape = tensors[0].shape();
if dim >= first_shape.len() {
return Err(TensorError::new("cat dim is out of range"));
}
let mut out_shape = first_shape.clone();
out_shape[dim] = 0;
let mut requires_grad = false;
for tensor in tensors {
tensor.try_assert_cpu("cat()")?;
let shape = tensor.shape();
if shape.len() != first_shape.len() {
return Err(TensorError::new("cat inputs must have the same rank"));
}
for (axis, (left, right)) in first_shape.iter().zip(shape.iter()).enumerate() {
if axis != dim && left != right {
return Err(TensorError::new(
"cat inputs must match on all non-concat dimensions",
));
}
}
out_shape[dim] = out_shape[dim].saturating_add(shape[dim]);
requires_grad = requires_grad || tensor.requires_grad();
}
let mut data = Vec::with_capacity(element_count(&out_shape));
let outer = first_shape[..dim].iter().product::<usize>().max(1);
let inner = first_shape[dim + 1..].iter().product::<usize>().max(1);
for outer_index in 0..outer {
for tensor in tensors {
let contiguous = tensor.contiguous();
let values = contiguous.shared_data();
let concat_len = tensor.shape()[dim];
let block = concat_len * inner;
let start = outer_index * block;
data.extend_from_slice(&values[start..start + block]);
}
}
Ok(Tensor::from_operation(
data,
out_shape,
requires_grad,
Op::Cat {
inputs: tensors.to_vec(),
dim,
},
))
}
}
fn cat_backward(inputs: &[Tensor], dim: usize, output_gradient: &[f32], output_shape: &[usize]) {
let outer = output_shape[..dim].iter().product::<usize>().max(1);
let inner = output_shape[dim + 1..].iter().product::<usize>().max(1);
let mut input_gradients: Vec<Vec<f32>> = inputs
.iter()
.map(|input| vec![0.0; input.numel()])
.collect();
let mut out_offset = 0usize;
for outer_index in 0..outer {
for (input_index, input) in inputs.iter().enumerate() {
let concat_len = input.shape()[dim];
let block = concat_len * inner;
let dst = outer_index * block;
input_gradients[input_index][dst..dst + block]
.copy_from_slice(&output_gradient[out_offset..out_offset + block]);
out_offset += block;
}
}
for (input, gradient) in inputs.iter().zip(input_gradients.iter()) {
accumulate_gradient(input, gradient);
}
}
fn checkpoint_cpu_backward(
input: &Tensor,
recompute: &CheckpointRecompute,
output_gradient: &[f32],
) {
let CheckpointRecompute::Cpu(recompute) = recompute else {
panic!("checkpoint CPU backward requires a CPU recompute closure");
};
let detached = detach_for_checkpoint(input);
let recomputed = {
let _guard = enable_grad();
recompute(&detached)
};
recomputed.backward_with_gradient(output_gradient);
if let Some(gradient) = detached.grad() {
accumulate_gradient(input, &gradient);
}
}
fn validate_shape(data: &[f32], shape: &[usize]) -> Result<(), TensorError> {
if shape.iter().any(|dimension| *dimension == 0) {
return Err(TensorError::new("zero-sized dimensions are not supported"));
}
if data.len() != element_count(shape) {
return Err(TensorError::new("data length does not match shape"));
}
Ok(())
}
fn element_count(shape: &[usize]) -> usize {
shape.iter().product()
}
pub(crate) fn try_validate_dropout_probability(probability: f32) -> Result<(), TensorError> {
if !(probability.is_finite() && (0.0..1.0).contains(&probability)) {
return Err(TensorError::new(
"dropout probability must be finite and in [0, 1)",
));
}
Ok(())
}
pub(crate) fn validate_dropout_probability(probability: f32) {
try_validate_dropout_probability(probability).unwrap_or_else(panic_tensor)
}
pub(crate) fn derive_seed(seed: u64, stream: u64) -> u64 {
splitmix64(seed ^ stream.wrapping_mul(0x9e37_79b9_7f4a_7c15))
}
pub(crate) fn dropout_mask_values(length: usize, probability: f32, seed: u64) -> Vec<f32> {
validate_dropout_probability(probability);
if probability == 0.0 {
return vec![1.0; length];
}
let scale = 1.0 / (1.0 - probability);
(0..length)
.map(|index| {
let random =
splitmix64(seed.wrapping_add(index as u64)) as f64 / (u64::MAX as f64 + 1.0);
if random >= probability as f64 {
scale
} else {
0.0
}
})
.collect()
}
#[allow(clippy::too_many_arguments)]
fn attention_dropout_scale(
batch: usize,
head: usize,
query: usize,
key: usize,
heads: usize,
query_length: usize,
key_length: usize,
probability: f32,
seed: u32,
) -> f32 {
if probability == 0.0 {
return 1.0;
}
let index = (((batch * heads + head) * query_length + query) * key_length + key) as u32;
let random = hash_u32(index ^ seed) as f32 / 4_294_967_296.0;
if random >= probability {
1.0 / (1.0 - probability)
} else {
0.0
}
}
fn hash_u32(mut value: u32) -> u32 {
value ^= value >> 16;
value = value.wrapping_mul(0x7feb_352d);
value ^= value >> 15;
value = value.wrapping_mul(0x846c_a68b);
value ^ (value >> 16)
}
fn splitmix64(mut value: u64) -> u64 {
value = value.wrapping_add(0x9e37_79b9_7f4a_7c15);
value = (value ^ (value >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9);
value = (value ^ (value >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb);
value ^ (value >> 31)
}
#[derive(Clone, Copy)]
struct MatmulDimensions {
rank: usize,
batches: usize,
rows: usize,
shared: usize,
columns: usize,
}
fn try_matmul_dimensions(left: &[usize], right: &[usize]) -> Result<MatmulDimensions, TensorError> {
if left.len() < 2 {
return Err(TensorError::new("left matmul input must be at least 2D"));
}
if left.len() != right.len() {
return Err(TensorError::new("matmul inputs must have equal rank"));
}
let rank = left.len();
if left[..rank - 2] != right[..rank - 2] {
return Err(TensorError::new("matmul batch dimensions differ"));
}
if left[rank - 1] != right[rank - 2] {
return Err(TensorError::new("matmul inner dimensions differ"));
}
Ok(MatmulDimensions {
rank,
batches: element_count(&left[..rank - 2]),
rows: left[rank - 2],
shared: left[rank - 1],
columns: right[rank - 1],
})
}
fn matmul_reference(left: &[f32], right: &[f32], dimensions: MatmulDimensions) -> Vec<f32> {
let mut output = vec![0.0; dimensions.batches * dimensions.rows * dimensions.columns];
for batch in 0..dimensions.batches {
let left_base = batch * dimensions.rows * dimensions.shared;
let right_base = batch * dimensions.shared * dimensions.columns;
let output_base = batch * dimensions.rows * dimensions.columns;
for row in 0..dimensions.rows {
for shared_index in 0..dimensions.shared {
let left_value = left[left_base + row * dimensions.shared + shared_index];
for column in 0..dimensions.columns {
output[output_base + row * dimensions.columns + column] +=
left_value * right[right_base + shared_index * dimensions.columns + column];
}
}
}
}
output
}
fn matmul_blocked_parallel(left: &[f32], right: &[f32], dimensions: MatmulDimensions) -> Vec<f32> {
let operation_count = dimensions
.batches
.saturating_mul(dimensions.rows)
.saturating_mul(dimensions.shared)
.saturating_mul(dimensions.columns);
let thread_count = rayon::current_num_threads();
let use_threads = thread_count > 1
&& operation_count >= 128 * 1024
&& dimensions.batches * dimensions.rows >= 2;
if !use_threads {
return matmul_reference(left, right, dimensions);
}
let mut output = vec![0.0; dimensions.batches * dimensions.rows * dimensions.columns];
output
.par_chunks_mut(dimensions.columns)
.enumerate()
.for_each(|(output_row, values)| {
matmul_blocked_row(values, output_row, left, right, dimensions);
});
output
}
fn matmul_blocked_row(
output: &mut [f32],
output_row: usize,
left: &[f32],
right: &[f32],
dimensions: MatmulDimensions,
) {
const SHARED_BLOCK: usize = 64;
const COLUMN_BLOCK: usize = 128;
let batch = output_row / dimensions.rows;
let row = output_row % dimensions.rows;
let left_start = (batch * dimensions.rows + row) * dimensions.shared;
let right_start = batch * dimensions.shared * dimensions.columns;
let left_row = &left[left_start..left_start + dimensions.shared];
let right_batch = &right[right_start..right_start + dimensions.shared * dimensions.columns];
for column_start in (0..dimensions.columns).step_by(COLUMN_BLOCK) {
let column_end = (column_start + COLUMN_BLOCK).min(dimensions.columns);
for shared_start in (0..dimensions.shared).step_by(SHARED_BLOCK) {
let shared_end = (shared_start + SHARED_BLOCK).min(dimensions.shared);
for shared_index in shared_start..shared_end {
let left_value = left_row[shared_index];
let right_start = shared_index * dimensions.columns + column_start;
let right_values =
&right_batch[right_start..right_start + column_end - column_start];
for (target, right_value) in output[column_start..column_end]
.iter_mut()
.zip(right_values)
{
*target += left_value * right_value;
}
}
}
}
}
fn gelu_value(value: f32) -> f32 {
const SCALE: f32 = 0.797_884_6;
0.5 * value * (1.0 + (SCALE * (value + 0.044_715 * value.powi(3))).tanh())
}
fn gelu_derivative(value: f32) -> f32 {
const SCALE: f32 = 0.797_884_6;
let inner = SCALE * (value + 0.044_715 * value.powi(3));
let tanh = inner.tanh();
0.5 * (1.0 + tanh)
+ 0.5 * value * (1.0 - tanh * tanh) * SCALE * (1.0 + 3.0 * 0.044_715 * value * value)
}
fn compute_strides(shape: &[usize]) -> Vec<usize> {
let mut strides = vec![1; shape.len()];
for index in (0..shape.len()).rev() {
if index + 1 < shape.len() {
strides[index] = strides[index + 1] * shape[index + 1];
}
}
strides
}
fn is_contiguous(shape: &[usize], strides: &[usize]) -> bool {
if shape.len() != strides.len() {
return false;
}
let mut expected = 1;
for index in (0..shape.len()).rev() {
if strides[index] != expected {
return false;
}
expected *= shape[index];
}
true
}
fn try_broadcast_shape(left: &[usize], right: &[usize]) -> Result<Vec<usize>, TensorError> {
let rank = left.len().max(right.len());
let mut output = vec![1; rank];
for reverse_index in 0..rank {
let left_dim = left
.len()
.checked_sub(reverse_index + 1)
.map(|index| left[index])
.unwrap_or(1);
let right_dim = right
.len()
.checked_sub(reverse_index + 1)
.map(|index| right[index])
.unwrap_or(1);
if !(left_dim == right_dim || left_dim == 1 || right_dim == 1) {
return Err(TensorError::new(format!(
"shapes {left:?} and {right:?} cannot be broadcast"
)));
}
output[rank - reverse_index - 1] = left_dim.max(right_dim);
}
Ok(output)
}
fn broadcast_offset(
output_flat_index: usize,
output_shape: &[usize],
output_strides: &[usize],
input_shape: &[usize],
input_strides: &[usize],
) -> usize {
if input_shape.is_empty() {
return 0;
}
let rank_offset = output_shape.len() - input_shape.len();
let mut offset = 0;
for input_dim in 0..input_shape.len() {
let output_dim = rank_offset + input_dim;
let coordinate =
(output_flat_index / output_strides[output_dim]) % output_shape[output_dim];
if input_shape[input_dim] != 1 {
offset += coordinate * input_strides[input_dim];
}
}
offset
}
fn reduction_offset(
input_flat_index: usize,
input_shape: &[usize],
input_strides: &[usize],
reduced_dim: usize,
keepdim: bool,
output_strides: &[usize],
) -> usize {
let mut output_offset = 0;
let mut output_dim = 0;
for input_dim in 0..input_shape.len() {
let coordinate = (input_flat_index / input_strides[input_dim]) % input_shape[input_dim];
if input_dim == reduced_dim {
if keepdim {
output_dim += 1;
}
continue;
}
output_offset += coordinate * output_strides[output_dim];
output_dim += 1;
}
output_offset
}
fn accumulate_gradient(tensor: &Tensor, contribution: &[f32]) {
if !tensor.requires_grad() {
return;
}
let mut node = tensor.inner.borrow_mut();
let numel = node.numel();
assert_eq!(numel, contribution.len());
let gradient = node.grad.get_or_insert_with(|| vec![0.0; numel]);
for (target, addition) in gradient.iter_mut().zip(contribution.iter()) {
*target += addition;
}
}
fn binary_backward(
left: &Tensor,
right: &Tensor,
output_gradient: &[f32],
output_shape: &[usize],
operation: BinaryOp,
) {
let (left_gradient, right_gradient) = {
let left_node = left.inner.borrow();
let right_node = right.inner.borrow();
let output_strides = compute_strides(output_shape);
let mut left_gradient = vec![0.0; left_node.numel()];
let mut right_gradient = vec![0.0; right_node.numel()];
for (output_index, gradient) in output_gradient.iter().enumerate() {
let left_index = broadcast_offset(
output_index,
output_shape,
&output_strides,
&left_node.shape,
&left_node.strides,
);
let right_index = broadcast_offset(
output_index,
output_shape,
&output_strides,
&right_node.shape,
&right_node.strides,
);
let left_value = left_node.data[left_node.offset + left_index];
let right_value = right_node.data[right_node.offset + right_index];
match operation {
BinaryOp::Add => {
left_gradient[left_index] += gradient;
right_gradient[right_index] += gradient;
}
BinaryOp::Sub => {
left_gradient[left_index] += gradient;
right_gradient[right_index] -= gradient;
}
BinaryOp::Mul => {
left_gradient[left_index] += gradient * right_value;
right_gradient[right_index] += gradient * left_value;
}
BinaryOp::Div => {
left_gradient[left_index] += gradient / right_value;
right_gradient[right_index] -=
gradient * left_value / (right_value * right_value);
}
}
}
(left_gradient, right_gradient)
};
accumulate_gradient(left, &left_gradient);
accumulate_gradient(right, &right_gradient);
}
fn matmul_backward(left: &Tensor, right: &Tensor, output_gradient: &[f32]) {
let (left_gradient, right_gradient) = {
let left_node = left.inner.borrow();
let right_node = right.inner.borrow();
let left_values = left_node.contig_slice();
let right_values = right_node.contig_slice();
let rank = left_node.shape.len();
let (rows, shared, columns) = (
left_node.shape[rank - 2],
left_node.shape[rank - 1],
right_node.shape[rank - 1],
);
let batch_count = element_count(&left_node.shape[..rank - 2]);
let mut left_gradient = vec![0.0; left_node.numel()];
let mut right_gradient = vec![0.0; right_node.numel()];
for batch in 0..batch_count {
let left_base = batch * rows * shared;
let right_base = batch * shared * columns;
let output_base = batch * rows * columns;
for row in 0..rows {
for shared_index in 0..shared {
for column in 0..columns {
let output_grad = output_gradient[output_base + row * columns + column];
left_gradient[left_base + row * shared + shared_index] += output_grad
* right_values[right_base + shared_index * columns + column];
right_gradient[right_base + shared_index * columns + column] +=
left_values[left_base + row * shared + shared_index] * output_grad;
}
}
}
}
(left_gradient, right_gradient)
};
accumulate_gradient(left, &left_gradient);
accumulate_gradient(right, &right_gradient);
}
fn permute_backward(
input: &Tensor,
output_gradient: &[f32],
output_shape: &[usize],
axes: &[usize],
) {
let input_shape = input.shape();
let input_logical_strides = compute_strides(&input_shape);
let output_strides = compute_strides(output_shape);
let mut input_gradient = vec![0.0; input.numel()];
for (output_index, gradient) in output_gradient.iter().enumerate() {
let mut input_index = 0;
for output_dim in 0..axes.len() {
let coordinate = (output_index / output_strides[output_dim]) % output_shape[output_dim];
input_index += coordinate * input_logical_strides[axes[output_dim]];
}
input_gradient[input_index] += gradient;
}
accumulate_gradient(input, &input_gradient);
}
fn narrow_backward(
input: &Tensor,
output_gradient: &[f32],
dim: usize,
start: usize,
length: usize,
) {
let input_shape = input.shape();
let mut output_shape = input_shape.clone();
output_shape[dim] = length;
let input_strides = compute_strides(&input_shape);
let output_strides = compute_strides(&output_shape);
let mut input_gradient = vec![0.0; input.numel()];
for (output_index, gradient) in output_gradient.iter().enumerate() {
let mut input_index = 0;
for dimension in 0..input_shape.len() {
let coordinate = (output_index / output_strides[dimension]) % output_shape[dimension];
let input_coordinate = if dimension == dim {
coordinate + start
} else {
coordinate
};
input_index += input_coordinate * input_strides[dimension];
}
input_gradient[input_index] += gradient;
}
accumulate_gradient(input, &input_gradient);
}
fn embedding_backward(weight: &Tensor, indices: &IndexTensor, output_gradient: &[f32]) {
let gradient = {
let weight_node = weight.inner.borrow();
let index_node = indices.inner.borrow();
let width = weight_node.shape[1];
let mut gradient = vec![0.0; weight_node.numel()];
for (position, token_id) in index_node.data.iter().copied().enumerate() {
let source_start = position * width;
let target_start = token_id * width;
for feature in 0..width {
gradient[target_start + feature] += output_gradient[source_start + feature];
}
}
gradient
};
accumulate_gradient(weight, &gradient);
}
fn rope_backward(
input: &Tensor,
output_gradient: &[f32],
sequence_dim: usize,
theta: f32,
position_offset: usize,
scaling: Option<&rope::RopeScaling>,
) {
let gradient = {
let input_node = input.inner.borrow();
let width = *input_node.shape.last().unwrap();
let numel = input_node.numel();
let mut gradient = vec![0.0; numel];
for vector_start in (0..numel).step_by(width) {
let position = position_offset
+ (vector_start / input_node.strides[sequence_dim])
% input_node.shape[sequence_dim];
for even in (0..width).step_by(2) {
let frequency = scaling.map_or_else(
|| theta.powf(-(even as f32) / width as f32),
|scaling| scaling.frequency(even, width, theta),
);
let angle = position as f32 * frequency;
let attention_factor = scaling.map_or(1.0, rope::RopeScaling::attention_factor);
let (cosine, sine) = (
angle.cos() * attention_factor,
angle.sin() * attention_factor,
);
let left_gradient = output_gradient[vector_start + even];
let right_gradient = output_gradient[vector_start + even + 1];
gradient[vector_start + even] = left_gradient * cosine + right_gradient * sine;
gradient[vector_start + even + 1] = -left_gradient * sine + right_gradient * cosine;
}
}
gradient
};
accumulate_gradient(input, &gradient);
}
fn softmax_backward(input: &Tensor, output: &[f32], output_gradient: &[f32]) {
let width = *input.shape().last().unwrap();
let mut input_gradient = vec![0.0; input.numel()];
for row in 0..input.numel() / width {
let start = row * width;
let dot: f32 = (0..width)
.map(|index| output_gradient[start + index] * output[start + index])
.sum();
for index in 0..width {
input_gradient[start + index] =
output[start + index] * (output_gradient[start + index] - dot);
}
}
accumulate_gradient(input, &input_gradient);
}
fn causal_mask_backward(input: &Tensor, output_gradient: &[f32], past_length: usize) {
let shape = input.shape();
let query_length = shape[shape.len() - 2];
let key_length = shape[shape.len() - 1];
let mut gradient = output_gradient.to_vec();
for (index, value) in gradient.iter_mut().enumerate() {
let key = index % key_length;
let query = (index / key_length) % query_length;
if key > past_length + query {
*value = 0.0;
}
}
accumulate_gradient(input, &gradient);
}
fn cross_entropy_backward(
logits: &Tensor,
targets: &IndexTensor,
upstream_gradient: f32,
ignore_index: Option<usize>,
) {
let gradient = {
let logits_node = logits.inner.borrow();
let target_node = targets.inner.borrow();
let classes = *logits_node.shape.last().unwrap();
let valid_rows = target_node
.data
.iter()
.filter(|target| Some(**target) != ignore_index)
.count();
let scale = upstream_gradient / valid_rows as f32;
let values = logits_node.contig_slice();
let mut gradient = vec![0.0; values.len()];
for (row_index, row) in values.chunks_exact(classes).enumerate() {
let target = target_node.data[row_index];
if Some(target) == ignore_index {
continue;
}
let maximum = row.iter().copied().fold(f32::NEG_INFINITY, f32::max);
let exp_sum: f32 = row.iter().map(|value| (*value - maximum).exp()).sum();
let start = row_index * classes;
for class in 0..classes {
let probability = (row[class] - maximum).exp() / exp_sum;
let target_probability = usize::from(class == target) as f32;
gradient[start + class] = scale * (probability - target_probability);
}
}
gradient
};
accumulate_gradient(logits, &gradient);
}
fn rms_norm_backward(input: &Tensor, weight: &Tensor, output_gradient: &[f32], epsilon: f32) {
let (input_gradient, weight_gradient) = {
let input_node = input.inner.borrow();
let weight_node = weight.inner.borrow();
let width = *input_node.shape.last().unwrap();
let input_values = input_node.contig_slice();
let weight_values = weight_node.contig_slice();
assert_eq!(weight_values.len(), width);
let mut input_gradient = vec![0.0; input_values.len()];
let mut weight_gradient = vec![0.0; width];
for row in 0..input_values.len() / width {
let start = row * width;
let mean_square = input_values[start..start + width]
.iter()
.map(|value| value * value)
.sum::<f32>()
/ width as f32;
let inverse_rms = (mean_square + epsilon).powf(-0.5);
let weighted_dot = (0..width)
.map(|column| {
output_gradient[start + column]
* weight_values[column]
* input_values[start + column]
})
.sum::<f32>();
for column in 0..width {
let index = start + column;
input_gradient[index] =
output_gradient[index] * weight_values[column] * inverse_rms
- input_values[index] * inverse_rms.powi(3) * weighted_dot / width as f32;
weight_gradient[column] +=
output_gradient[index] * input_values[index] * inverse_rms;
}
}
(input_gradient, weight_gradient)
};
accumulate_gradient(input, &input_gradient);
accumulate_gradient(weight, &weight_gradient);
}
fn residual_rms_norm_backward(
left: &Tensor,
right: &Tensor,
weight: &Tensor,
output_gradient: &[f32],
epsilon: f32,
) {
let (residual_gradient, weight_gradient) = {
let left_node = left.inner.borrow();
let right_node = right.inner.borrow();
let weight_node = weight.inner.borrow();
let width = *left_node.shape.last().unwrap();
let left_values = left_node.contig_slice();
let right_values = right_node.contig_slice();
let weight_values = weight_node.contig_slice();
let mut residual_gradient = vec![0.0; left_values.len()];
let mut weight_gradient = vec![0.0; width];
for row in 0..left_values.len() / width {
let start = row * width;
let mean_square = (0..width)
.map(|column| {
let residual = left_values[start + column] + right_values[start + column];
residual * residual
})
.sum::<f32>()
/ width as f32;
let inverse_rms = (mean_square + epsilon).powf(-0.5);
let weighted_dot = (0..width)
.map(|column| {
let index = start + column;
output_gradient[index]
* weight_values[column]
* (left_values[index] + right_values[index])
})
.sum::<f32>();
for column in 0..width {
let index = start + column;
let residual = left_values[index] + right_values[index];
residual_gradient[index] =
output_gradient[index] * weight_values[column] * inverse_rms
- residual * inverse_rms.powi(3) * weighted_dot / width as f32;
weight_gradient[column] += output_gradient[index] * residual * inverse_rms;
}
}
(residual_gradient, weight_gradient)
};
accumulate_gradient(left, &residual_gradient);
accumulate_gradient(right, &residual_gradient);
accumulate_gradient(weight, &weight_gradient);
}
fn swiglu_backward(gate: &Tensor, up: &Tensor, output_gradient: &[f32]) {
let (gate_gradient, up_gradient) = {
let gate_node = gate.inner.borrow();
let up_node = up.inner.borrow();
let gate_values = gate_node.contig_slice();
let up_values = up_node.contig_slice();
let mut gate_gradient = vec![0.0; gate_values.len()];
let mut up_gradient = vec![0.0; up_values.len()];
for index in 0..gate_values.len() {
let value = gate_values[index];
let sigmoid = 1.0 / (1.0 + (-value).exp());
let silu = value * sigmoid;
let silu_derivative = sigmoid + value * sigmoid * (1.0 - sigmoid);
gate_gradient[index] = output_gradient[index] * up_values[index] * silu_derivative;
up_gradient[index] = output_gradient[index] * silu;
}
(gate_gradient, up_gradient)
};
accumulate_gradient(gate, &gate_gradient);
accumulate_gradient(up, &up_gradient);
}
fn tiled_attention_forward(
query: &[f32],
key: &[f32],
value: &[f32],
attention_mask: Option<&[usize]>,
query_shape: &[usize],
key_value_heads: usize,
key_length: usize,
scale: f32,
past_length: usize,
dropout_probability: f32,
dropout_seed: u32,
) -> (Vec<f32>, Vec<f32>) {
const TILE_KEYS: usize = 32;
let query_heads = query_shape[1];
let query_heads_per_key_value_head = query_heads / key_value_heads;
let outer = query_shape[0] * query_heads;
let query_length = query_shape[2];
let head_dim = query_shape[3];
let mut output = vec![0.0; query.len()];
let mut logsumexp = vec![0.0; outer * query_length];
for outer_index in 0..outer {
let batch = outer_index / query_heads;
let query_head = outer_index % query_heads;
let key_value_head = query_head / query_heads_per_key_value_head;
let key_value_outer = batch * key_value_heads + key_value_head;
for query_position in 0..query_length {
let query_start = (outer_index * query_length + query_position) * head_dim;
let output_row = &mut output[query_start..query_start + head_dim];
if attention_mask
.is_some_and(|mask| mask[batch * key_length + past_length + query_position] == 0)
{
continue;
}
let allowed_keys = (past_length + query_position + 1).min(key_length);
let mut running_maximum = f32::NEG_INFINITY;
let mut running_sum = 0.0;
for tile_start in (0..allowed_keys).step_by(TILE_KEYS) {
let tile_count = TILE_KEYS.min(allowed_keys - tile_start);
let mut scores = [0.0; TILE_KEYS];
let mut tile_maximum = f32::NEG_INFINITY;
for tile_key in 0..tile_count {
let key_position = tile_start + tile_key;
if attention_mask
.is_some_and(|mask| mask[batch * key_length + key_position] == 0)
{
scores[tile_key] = f32::NEG_INFINITY;
continue;
}
let key_start = (key_value_outer * key_length + key_position) * head_dim;
let score = query[query_start..query_start + head_dim]
.iter()
.zip(&key[key_start..key_start + head_dim])
.map(|(query_value, key_value)| query_value * key_value)
.sum::<f32>()
* scale;
scores[tile_key] = score;
tile_maximum = tile_maximum.max(score);
}
let new_maximum = running_maximum.max(tile_maximum);
let old_scale = if running_sum == 0.0 {
0.0
} else {
(running_maximum - new_maximum).exp()
};
running_sum *= old_scale;
for output_value in output_row.iter_mut() {
*output_value *= old_scale;
}
for (tile_key, score) in scores[..tile_count].iter().enumerate() {
if !score.is_finite() {
continue;
}
let weight = (*score - new_maximum).exp();
running_sum += weight;
let key_position = tile_start + tile_key;
let dropout_scale = attention_dropout_scale(
batch,
query_head,
query_position,
key_position,
query_heads,
query_length,
key_length,
dropout_probability,
dropout_seed,
);
let value_start = (key_value_outer * key_length + key_position) * head_dim;
for (output_value, value_value) in output_row
.iter_mut()
.zip(&value[value_start..value_start + head_dim])
{
*output_value += weight * dropout_scale * value_value;
}
}
running_maximum = new_maximum;
}
for output_value in output_row {
*output_value /= running_sum;
}
logsumexp[outer_index * query_length + query_position] =
running_maximum + running_sum.ln();
}
}
(output, logsumexp)
}
fn tiled_attention_backward(
query: &Tensor,
key: &Tensor,
value: &Tensor,
output: &Tensor,
logsumexp: &Tensor,
output_gradient: &[f32],
scale: f32,
past_length: usize,
dropout_probability: f32,
dropout_seed: u32,
attention_mask: Option<&IndexTensor>,
) {
let query_shape = query.shape();
let key_shape = key.shape();
let query_heads = query_shape[1];
let key_value_heads = key_shape[1];
let query_heads_per_key_value_head = query_heads / key_value_heads;
let outer = query_shape[0] * query_heads;
let query_length = query_shape[2];
let key_length = key_shape[2];
let head_dim = query_shape[3];
let query_data = query.shared_data();
let key_data = key.shared_data();
let value_data = value.shared_data();
let output_data = output.shared_data();
let logsumexp_data = logsumexp.shared_data();
let attention_mask_data = attention_mask.map(IndexTensor::data);
let mut query_gradient = vec![0.0; query.numel()];
let mut key_gradient = vec![0.0; key.numel()];
let mut value_gradient = vec![0.0; value.numel()];
for outer_index in 0..outer {
let batch = outer_index / query_heads;
let query_head = outer_index % query_heads;
let key_value_head = query_head / query_heads_per_key_value_head;
let key_value_outer = batch * key_value_heads + key_value_head;
for query_position in 0..query_length {
if attention_mask_data
.as_ref()
.is_some_and(|mask| mask[batch * key_length + past_length + query_position] == 0)
{
continue;
}
let query_start = (outer_index * query_length + query_position) * head_dim;
let delta = output_gradient[query_start..query_start + head_dim]
.iter()
.zip(&output_data[query_start..query_start + head_dim])
.map(|(gradient, output_value)| gradient * output_value)
.sum::<f32>();
let row_logsumexp = logsumexp_data[outer_index * query_length + query_position];
let allowed_keys = (past_length + query_position + 1).min(key_length);
for key_position in 0..allowed_keys {
if attention_mask_data
.as_ref()
.is_some_and(|mask| mask[batch * key_length + key_position] == 0)
{
continue;
}
let key_start = (key_value_outer * key_length + key_position) * head_dim;
let score = query_data[query_start..query_start + head_dim]
.iter()
.zip(&key_data[key_start..key_start + head_dim])
.map(|(query_value, key_value)| query_value * key_value)
.sum::<f32>()
* scale;
let probability = (score - row_logsumexp).exp();
let dropout_scale = attention_dropout_scale(
batch,
query_head,
query_position,
key_position,
query_heads,
query_length,
key_length,
dropout_probability,
dropout_seed,
);
let probability_gradient = output_gradient[query_start..query_start + head_dim]
.iter()
.zip(&value_data[key_start..key_start + head_dim])
.map(|(gradient, value_value)| gradient * value_value)
.sum::<f32>()
* dropout_scale;
let score_gradient = probability * (probability_gradient - delta);
for feature in 0..head_dim {
query_gradient[query_start + feature] +=
score_gradient * key_data[key_start + feature] * scale;
key_gradient[key_start + feature] +=
score_gradient * query_data[query_start + feature] * scale;
value_gradient[key_start + feature] +=
probability * dropout_scale * output_gradient[query_start + feature];
}
}
}
}
accumulate_gradient(query, &query_gradient);
accumulate_gradient(key, &key_gradient);
accumulate_gradient(value, &value_gradient);
}
fn masked_softmax_backward(
input: &Tensor,
output: &Tensor,
output_gradient: &[f32],
past_length: usize,
) {
let shape = input.shape();
let query_length = shape[shape.len() - 2];
let key_length = shape[shape.len() - 1];
let output_node = output.inner.borrow();
let output_values = output_node.contig_slice();
let mut input_gradient = vec![0.0; input.numel()];
for row in 0..input.numel() / key_length {
let start = row * key_length;
let query = row % query_length;
let dot = (0..key_length)
.map(|key| output_gradient[start + key] * output_values[start + key])
.sum::<f32>();
for key in 0..key_length {
if key <= past_length + query {
let index = start + key;
input_gradient[index] = output_values[index] * (output_gradient[index] - dot);
}
}
}
drop(output_node);
accumulate_gradient(input, &input_gradient);
}
fn reduction_backward(
input: &Tensor,
output_gradient: &[f32],
dim: usize,
keepdim: bool,
scale: f32,
) {
let gradient = {
let input_node = input.inner.borrow();
let mut output_shape = input_node.shape.clone();
if keepdim {
output_shape[dim] = 1;
} else {
output_shape.remove(dim);
}
let output_strides = compute_strides(&output_shape);
let numel = input_node.numel();
let mut gradient = vec![0.0; numel];
for input_index in 0..numel {
let output_index = reduction_offset(
input_index,
&input_node.shape,
&input_node.strides,
dim,
keepdim,
&output_strides,
);
gradient[input_index] = output_gradient[output_index] * scale;
}
gradient
};
accumulate_gradient(input, &gradient);
}
impl<'b> Add<&'b Tensor> for &Tensor {
type Output = Tensor;
fn add(self, rhs: &'b Tensor) -> Self::Output {
Tensor::add(self, rhs)
}
}
impl<'b> Sub<&'b Tensor> for &Tensor {
type Output = Tensor;
fn sub(self, rhs: &'b Tensor) -> Self::Output {
Tensor::sub(self, rhs)
}
}
impl<'b> Mul<&'b Tensor> for &Tensor {
type Output = Tensor;
fn mul(self, rhs: &'b Tensor) -> Self::Output {
Tensor::mul(self, rhs)
}
}
impl<'b> Div<&'b Tensor> for &Tensor {
type Output = Tensor;
fn div(self, rhs: &'b Tensor) -> Self::Output {
Tensor::div(self, rhs)
}
}
impl fmt::Debug for Tensor {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
let node = self.inner.borrow();
let mut debug = formatter.debug_struct("Tensor");
if node.device == Device::Cpu {
drop(node);
debug.field("data", &self.to_contiguous_vec());
let node = self.inner.borrow();
debug
.field("shape", &node.shape)
.field("strides", &node.strides)
.field("offset", &node.offset)
.field("grad", &node.grad)
.field("requires_grad", &node.requires_grad)
.field("device", &node.device)
.finish()
} else {
debug.field("data", &"<GPU resident>");
debug
.field("shape", &node.shape)
.field("strides", &node.strides)
.field("offset", &node.offset)
.field("grad", &node.grad)
.field("requires_grad", &node.requires_grad)
.field("device", &node.device)
.finish()
}
}
}
impl fmt::Display for Tensor {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
let node = self.inner.borrow();
if node.device == Device::Cpu {
let shape = node.shape.clone();
drop(node);
write!(
formatter,
"Tensor(shape={:?}, data={:?})",
shape,
self.to_contiguous_vec()
)
} else {
write!(formatter, "Tensor(shape={:?}, device=Gpu)", node.shape)
}
}
}
macro_rules! assert_not_impl_any {
($type:ty: $($trait:path),+ $(,)?) => {
const _: fn() = || {
#[allow(dead_code)]
struct Wrapper<T: ?Sized>(core::marker::PhantomData<T>);
trait AmbiguousIfImpl<A> {
fn check() {}
}
impl<T: ?Sized> AmbiguousIfImpl<()> for Wrapper<T> {}
$({
#[allow(dead_code)]
struct OnlyOneImpl;
impl<T: ?Sized + $trait> AmbiguousIfImpl<OnlyOneImpl> for Wrapper<T> {}
})+
let _ = <Wrapper<$type> as AmbiguousIfImpl<_>>::check;
};
};
}
assert_not_impl_any!(Tensor: Send, Sync);
assert_not_impl_any!(IndexTensor: Send, Sync);
#[cfg(test)]
mod tests {
use super::{IndexTensor, Tensor, TensorError};
use crate::rope::{RopeScaling, RopeScalingType};
#[test]
fn shape_errors_are_recoverable() {
assert!(matches!(
Tensor::try_new(vec![1.0], vec![2], false),
Err(TensorError(_))
));
assert!(matches!(
IndexTensor::try_new(vec![1], vec![2]),
Err(TensorError(_))
));
let matrix = Tensor::new(vec![1.0, 2.0, 3.0, 4.0], vec![2, 2], false);
assert!(matrix.try_reshape(vec![3, 1]).is_err());
assert!(matrix
.try_matmul(&Tensor::zeros(vec![3, 2], false))
.is_err());
assert!(matrix.try_permute(vec![0, 0]).is_err());
assert!(matrix.try_narrow(0, 0, 3).is_err());
assert!(matrix.try_add(&Tensor::zeros(vec![3], false)).is_err());
}
fn assert_close(actual: &[f32], expected: &[f32]) {
assert_eq!(actual.len(), expected.len());
for (index, (actual, expected)) in actual.iter().zip(expected.iter()).enumerate() {
assert!(
(actual - expected).abs() < 1e-5,
"value {index} differs: expected {expected}, got {actual}"
);
}
}
#[test]
fn matrix_multiplication_is_numerically_correct() {
let left = Tensor::new(vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0], vec![2, 3], false);
let right = Tensor::new(vec![7.0, 8.0, 9.0, 10.0, 11.0, 12.0], vec![3, 2], false);
let result = left.matmul(&right);
assert_eq!(result.shape(), vec![2, 2]);
assert_close(&result.data(), &[58.0, 64.0, 139.0, 154.0]);
}
#[test]
fn autograd_accumulates_gradients_from_multiple_paths() {
let x = Tensor::new(vec![1.0, 2.0, 3.0], vec![3], true);
let squared = &x * &x;
let branched = &squared + &x;
let loss = branched.sum();
loss.backward();
assert_close(&x.grad().unwrap(), &[3.0, 5.0, 7.0]);
}
#[test]
fn broadcast_bias_gradient_is_reduced_to_bias_shape() {
let matrix = Tensor::new(vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0], vec![2, 3], true);
let bias = Tensor::new(vec![10.0, 20.0, 30.0], vec![3], true);
let loss = (&matrix + &bias).sum();
loss.backward();
assert_close(&matrix.grad().unwrap(), &[1.0; 6]);
assert_close(&bias.grad().unwrap(), &[2.0, 2.0, 2.0]);
}
#[test]
fn matmul_backward_matches_analytical_gradients() {
let left = Tensor::new(vec![1.0, 2.0, 3.0, 4.0], vec![2, 2], true);
let right = Tensor::new(vec![5.0, 6.0, 7.0, 8.0], vec![2, 2], true);
let loss = left.matmul(&right).sum();
loss.backward();
assert_close(&left.grad().unwrap(), &[11.0, 15.0, 11.0, 15.0]);
assert_close(&right.grad().unwrap(), &[4.0, 4.0, 6.0, 6.0]);
}
#[test]
fn cross_entropy_ignores_selected_targets() {
let logits = Tensor::new(vec![2.0, 0.0, -1.0, 100.0, -100.0, 0.0], vec![2, 3], true);
let targets = IndexTensor::new(vec![0, 99], vec![2]);
let loss = logits.cross_entropy_ignore_index(&targets, 99);
let expected = (1.0 + (-2.0f32).exp() + (-3.0f32).exp()).ln();
assert_close(&[loss.item()], &[expected]);
loss.backward();
assert_close(&logits.grad().unwrap()[3..], &[0.0, 0.0, 0.0]);
}
#[test]
fn short_context_rope_remains_unchanged_when_scaling_is_disabled() {
let values: Vec<f32> = (0..16).map(|index| index as f32 / 10.0).collect();
let input = Tensor::new(values.clone(), vec![1, 1, 4, 4], false);
let mut expected = values.clone();
for (position, vector) in values.chunks_exact(4).enumerate() {
for even in (0..4).step_by(2) {
let frequency = 10_000.0f32.powf(-(even as f32) / 4.0);
let angle = position as f32 * frequency;
let (cosine, sine) = (angle.cos(), angle.sin());
let start = position * 4 + even;
expected[start] = vector[even] * cosine - vector[even + 1] * sine;
expected[start + 1] = vector[even] * sine + vector[even + 1] * cosine;
}
}
let output = input.rope_with_offset_and_scaling(2, 10_000.0, 0, None);
assert_close(&output.data(), &expected);
}
#[test]
fn linear_and_yarn_scaling_change_long_context_frequencies() {
let input = Tensor::new(vec![1.0, 0.0, 1.0, 0.0], vec![1, 1, 1, 4], false);
let linear = RopeScaling {
kind: RopeScalingType::Linear,
factor: 4.0,
original_max_position_embeddings: 2_048,
attention_factor: None,
beta_fast: 32.0,
beta_slow: 1.0,
};
let yarn = RopeScaling {
kind: RopeScalingType::Yarn,
..linear.clone()
};
let unscaled = input.rope_with_offset(2, 10_000.0, 4_096).data();
let linear = input
.rope_with_offset_and_scaling(2, 10_000.0, 4_096, Some(&linear))
.data();
let yarn = input
.rope_with_offset_and_scaling(2, 10_000.0, 4_096, Some(&yarn))
.data();
assert_ne!(linear, unscaled);
assert_ne!(yarn, unscaled);
assert_ne!(yarn, linear);
}
#[test]
fn reshape_shares_cpu_storage_without_duplicating_bytes() {
let input = Tensor::new(vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0], vec![2, 3], false);
let reshaped = input.reshape(vec![3, 2]);
assert!(input.shares_storage_with(&reshaped));
assert!(reshaped.is_contiguous());
assert_eq!(reshaped.shape(), vec![3, 2]);
assert_close(&reshaped.data(), &[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]);
}
#[test]
fn mutating_shared_storage_uses_copy_on_write() {
let input = Tensor::new(vec![1.0, 2.0, 3.0, 4.0], vec![4], true);
let view = input.reshape(vec![2, 2]);
assert!(input.shares_storage_with(&view));
input.with_mut_data(|values| values[0] = 9.0);
assert!(!input.shares_storage_with(&view));
assert_eq!(input.data()[0], 9.0);
assert_eq!(view.data()[0], 1.0);
}
#[test]
fn reshape_backward_through_shared_storage() {
let input = Tensor::new(vec![1.0, 2.0, 3.0, 4.0], vec![2, 2], true);
let loss = input.reshape(vec![4]).sum();
loss.backward();
assert_close(&input.grad().unwrap(), &[1.0; 4]);
}
#[test]
fn permute_and_transpose_are_zero_copy_views() {
let input = Tensor::new(vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0], vec![2, 3], false);
let permuted = input.permute(vec![1, 0]);
let transposed = input.transpose(0, 1);
assert!(input.shares_storage_with(&permuted));
assert!(input.shares_storage_with(&transposed));
assert!(!permuted.is_contiguous());
assert_eq!(permuted.shape(), vec![3, 2]);
assert_close(&permuted.data(), &[1.0, 4.0, 2.0, 5.0, 3.0, 6.0]);
assert_close(&transposed.data(), &permuted.data());
let packed = permuted.contiguous();
assert!(packed.is_contiguous());
assert!(!input.shares_storage_with(&packed));
assert_close(&packed.data(), &permuted.data());
}
#[test]
fn narrow_and_slice_are_zero_copy_views() {
let input = Tensor::new(
vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0],
vec![3, 3],
false,
);
let narrowed = input.narrow(0, 1, 2);
let sliced = input.slice(1, 1, 3);
assert!(input.shares_storage_with(&narrowed));
assert!(input.shares_storage_with(&sliced));
assert_eq!(narrowed.shape(), vec![2, 3]);
assert_eq!(sliced.shape(), vec![3, 2]);
assert_close(&narrowed.data(), &[4.0, 5.0, 6.0, 7.0, 8.0, 9.0]);
assert_close(&sliced.data(), &[2.0, 3.0, 5.0, 6.0, 8.0, 9.0]);
}
#[test]
fn view_matmul_materializes_then_matches_dense_path() {
let left = Tensor::new(vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0], vec![2, 3], false);
let right = Tensor::new(vec![7.0, 8.0, 9.0, 10.0, 11.0, 12.0], vec![3, 2], false);
let dense = left.matmul(&right);
let viewed = left.transpose(0, 1).transpose(0, 1).matmul(&right);
assert_close(&viewed.data(), &dense.data());
}
#[test]
fn narrow_and_permute_backward_match_dense_grads() {
let values = vec![0.5, -1.0, 2.0, 0.25, 1.5, -0.5];
let input = Tensor::new(values, vec![2, 3], true);
let loss = input.narrow(1, 1, 2).transpose(0, 1).sum();
loss.backward();
assert_close(&input.grad().unwrap(), &[0.0, 1.0, 1.0, 0.0, 1.0, 1.0]);
}
}