use alloc::vec::Vec;
use burn::{
Tensor,
config::Config,
module::Module,
prelude::{
Int,
TensorData,
},
tensor::{
Device,
FloatDType,
},
};
#[derive(Config, Debug)]
pub struct DeltaNetStateConfig {
pub rows: usize,
pub conv_channels: usize,
pub conv_kernel: usize,
pub heads: usize,
pub key_dim: usize,
pub value_dim: usize,
}
impl DeltaNetStateConfig {
pub fn init(
&self,
device: &Device,
) -> DeltaNetState {
assert!(
self.conv_kernel >= 2,
"a causal convolution of at least 2 taps; got {}",
self.conv_kernel
);
let f32 = (device, FloatDType::F32.into());
DeltaNetState {
conv: Tensor::zeros([self.rows, self.conv_kernel - 1, self.conv_channels], f32),
recurrent: Tensor::zeros([self.rows, self.heads, self.key_dim, self.value_dim], f32),
}
}
}
#[derive(Module, Debug)]
pub struct DeltaNetState {
pub conv: Tensor<3>,
pub recurrent: Tensor<4>,
}
impl DeltaNetState {
pub fn rows(&self) -> usize {
self.conv.dims()[0]
}
pub fn reset(&mut self) {
self.conv = self.conv.zeros_like();
self.recurrent = self.recurrent.zeros_like();
}
pub fn reorder(
&mut self,
sources: &[usize],
) {
let rows = self.rows();
assert_eq!(sources.len(), rows, "one source row per row");
assert!(
sources.iter().all(|&s| s < rows),
"a source row past the rows"
);
if sources.iter().enumerate().all(|(r, &s)| r == s) {
return;
}
let indices: Tensor<1, Int> = Tensor::from_data(
TensorData::new(
sources.iter().map(|&s| s as i64).collect::<Vec<_>>(),
[rows],
),
&self.conv.device(),
);
self.conv = self.conv.clone().select(0, indices.clone());
self.recurrent = self.recurrent.clone().select(0, indices);
}
}
#[derive(Module, Debug, Default)]
pub struct DeltaNetCache {
pub layers: Vec<DeltaNetState>,
pub snapshot: Option<DeltaNetSnapshot>,
}
#[derive(Module, Debug)]
pub struct DeltaNetSnapshot {
pub pos: usize,
pub layers: Vec<DeltaNetState>,
}
impl DeltaNetCache {
pub fn new(layers: Vec<DeltaNetState>) -> Self {
Self {
layers,
snapshot: None,
}
}
pub fn is_empty(&self) -> bool {
self.layers.is_empty()
}
pub fn layer_mut(
&mut self,
slot: usize,
) -> &mut DeltaNetState {
let len = self.layers.len();
self.layers
.get_mut(slot)
.unwrap_or_else(|| panic!("linear-attention slot {slot} of {len}"))
}
pub fn reset(&mut self) {
self.layers.iter_mut().for_each(DeltaNetState::reset);
self.snapshot = None;
}
pub fn take_snapshot(
&mut self,
pos: usize,
) {
self.snapshot = Some(DeltaNetSnapshot {
pos,
layers: self.layers.clone(),
});
}
pub fn snapshot_pos(&self) -> Option<usize> {
self.snapshot.as_ref().map(|s| s.pos)
}
pub fn restore_snapshot(&mut self) {
let snapshot = self.snapshot.as_ref().expect("a snapshot to restore");
self.layers = snapshot.layers.clone();
}
pub fn reorder(
&mut self,
sources: &[usize],
) {
let snapshot = self.snapshot.iter_mut().flat_map(|s| s.layers.iter_mut());
for layer in self.layers.iter_mut().chain(snapshot) {
layer.reorder(sources);
}
}
}
#[cfg(test)]
mod tests {
use burn::tensor::{
DType,
Distribution,
};
use super::*;
use crate::support::testing::{
cpu_device,
seeded_tensor,
};
fn state() -> DeltaNetState {
let device = cpu_device();
let mut state = DeltaNetStateConfig::new(3, 5, 4, 2, 3, 2).init(&device);
state.conv = seeded_tensor::<3>(1, [3, 3, 5], Distribution::Default, &device);
state.recurrent = seeded_tensor::<4>(2, [3, 2, 3, 2], Distribution::Default, &device);
state
}
#[test]
fn test_init() {
let device = cpu_device();
let state = DeltaNetStateConfig::new(2, 10, 4, 3, 5, 6).init(&device);
assert_eq!(state.rows(), 2);
assert_eq!(state.conv.dims(), [2, 3, 10]);
assert_eq!(state.recurrent.dims(), [2, 3, 5, 6]);
for dtype in [state.conv.dtype(), state.recurrent.dtype()] {
assert_eq!(dtype, DType::F32);
}
assert_eq!(state.recurrent.abs().max().into_scalar::<f32>(), 0.0);
}
#[test]
fn test_reorder() {
let before = state();
let mut after = state();
after.reorder(&[2, 0, 0]);
for (r, s) in [(0, 2), (1, 0), (2, 0)] {
assert_eq!(
after.conv.clone().slice_dim(0, r..=r).into_data(),
before.conv.clone().slice_dim(0, s..=s).into_data()
);
assert_eq!(
after.recurrent.clone().slice_dim(0, r..=r).into_data(),
before.recurrent.clone().slice_dim(0, s..=s).into_data()
);
}
let mut same = state();
same.reorder(&[0, 1, 2]);
assert_eq!(same.conv.into_data(), before.conv.into_data());
}
#[test]
fn test_reset() {
let mut state = state();
state.reset();
assert_eq!(state.conv.dims(), [3, 3, 5]);
assert_eq!(state.conv.abs().max().into_scalar::<f32>(), 0.0);
assert_eq!(state.recurrent.abs().max().into_scalar::<f32>(), 0.0);
}
#[test]
fn test_cache() {
let mut cache = DeltaNetCache::new(vec![state(), state()]);
assert!(!cache.is_empty() && DeltaNetCache::default().is_empty());
let before = state();
cache.reorder(&[1, 1, 0]);
for layer in &cache.layers {
assert_eq!(
layer.conv.clone().slice_dim(0, 0..1).into_data(),
before.conv.clone().slice_dim(0, 1..2).into_data()
);
}
cache.reset();
assert_eq!(
cache
.layer_mut(1)
.recurrent
.clone()
.abs()
.max()
.into_scalar::<f32>(),
0.0
);
let mut cache = DeltaNetCache::new(vec![state()]);
cache.take_snapshot(5);
let kept = cache.layer_mut(0).recurrent.clone();
cache.layer_mut(0).reset();
assert_eq!(cache.snapshot_pos(), Some(5));
cache.restore_snapshot();
assert_eq!(
cache.layer_mut(0).recurrent.clone().into_data(),
kept.clone().into_data()
);
cache.reorder(&[2, 1, 0]);
assert_eq!(
cache.snapshot.as_ref().unwrap().layers[0]
.recurrent
.clone()
.slice_dim(0, 0..1)
.into_data(),
kept.slice_dim(0, 2..3).into_data(),
"the snapshot's rows reorder too"
);
cache.reset();
assert_eq!(cache.snapshot_pos(), None);
let refused = std::panic::catch_unwind(core::panic::AssertUnwindSafe(|| {
let _ = cache.layer_mut(2);
}));
assert!(refused.is_err());
}
#[test]
#[should_panic(expected = "a source row past the rows")]
fn test_reorder_past_the_rows() {
state().reorder(&[0, 1, 3]);
}
}