use alloc::vec::Vec;
use burn::{
Tensor,
prelude::TensorData,
tensor::Device,
};
#[cfg(not(feature = "std"))]
#[allow(unused_imports)]
use num_traits::Float;
pub fn rel_position_table(
seq_len: usize,
d_model: usize,
device: &Device,
) -> Tensor<3> {
assert!(seq_len > 0, "rel_position_table: seq_len is zero");
assert!(
d_model.is_multiple_of(2),
"rel_position_table: d_model ({d_model}) is odd"
);
let rows = 2 * seq_len - 1;
let mut data = Vec::with_capacity(rows * d_model);
for r in 0..rows {
let p = (seq_len as f64 - 1.0) - r as f64;
for i in 0..d_model / 2 {
let freq = 1.0f32 / 10000f32.powf((2 * i) as f32 / d_model as f32);
let angle = (p as f32) * freq;
data.push(angle.sin());
data.push(angle.cos());
}
}
Tensor::from_data(TensorData::new(data, [1, rows, d_model]), device)
}
pub fn rel_shift(scores: Tensor<4>) -> Tensor<4> {
let [batch, heads, t, p] = scores.dims();
assert_eq!(p, 2 * t - 1, "rel_shift: positions ({p}) != 2 * {t} - 1");
let padded = Tensor::cat(
vec![
Tensor::zeros([batch, heads, t, 1], &scores.device()),
scores,
],
3,
);
padded
.reshape([batch, heads, p + 1, t])
.slice_dim(2, 1..)
.reshape([batch, heads, t, p])
.slice_dim(3, 0..t)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::support::testing::cpu_device;
#[test]
fn test_rel_position_table() {
let device = cpu_device();
let table = rel_position_table(3, 4, &device);
assert_eq!(table.dims(), [1, 5, 4]);
let values: Vec<f32> = table.into_data().try_into_vec_as::<f32>().unwrap();
assert_eq!(&values[8..12], &[0.0, 1.0, 0.0, 1.0]);
let expected = [2f32.sin(), 2f32.cos(), 0.02f32.sin(), 0.02f32.cos()];
for (a, b) in values[0..4].iter().zip(expected) {
assert!((a - b).abs() < 1e-6, "{a} != {b}");
}
assert!((values[16] + 2f32.sin()).abs() < 1e-6);
assert!((values[17] - 2f32.cos()).abs() < 1e-6);
}
#[test]
fn test_rel_shift_is_the_diagonal_gather() {
let device = cpu_device();
let (b, h, t) = (2, 3, 5);
let p = 2 * t - 1;
let n = b * h * t * p;
let input: Vec<f32> = (0..n).map(|k| k as f32).collect();
let scores = Tensor::<4>::from_data(TensorData::new(input.clone(), [b, h, t, p]), &device);
let shifted = rel_shift(scores);
assert_eq!(shifted.dims(), [b, h, t, t]);
let out: Vec<f32> = shifted.into_data().try_into_vec_as::<f32>().unwrap();
for bh in 0..b * h {
for i in 0..t {
for j in 0..t {
let expected = input[(bh * t + i) * p + j + (t - 1) - i];
assert_eq!(out[(bh * t + i) * t + j], expected, "({bh}, {i}, {j})");
}
}
}
}
}