use burn::{
Tensor,
prelude::Backend,
tensor::linalg,
};
pub fn inverse_frequency_table<B: Backend>(
base: usize,
head_dim: usize,
device: &B::Device,
) -> Tensor<B, 1> {
Tensor::from_data([base as f32], device).powf(
-Tensor::arange_step(0..head_dim as i64, 2, device)
.float()
.div_scalar(head_dim as f32),
)
}
pub fn positional_frequency_table<B: Backend>(
seq_len: usize,
base: usize,
head_dim: usize,
device: &B::Device,
) -> Tensor<B, 2> {
let inv_freq = inverse_frequency_table::<B>(base, head_dim, device);
let t: Tensor<B, 1> = Tensor::arange(0..seq_len as i64, device).float();
linalg::outer::<_, 1, 2, _>(t, inv_freq)
}
#[cfg(test)]
mod tests {
use burn::tensor::Tolerance;
use serial_test::serial;
use super::*;
use crate::support::testing::{
DeviceMemoryGuard,
PerformanceBackend,
default_device,
};
#[test]
#[serial]
fn test_inverse_frequency_table() {
type B = PerformanceBackend;
let device = default_device();
let _memory = DeviceMemoryGuard::<B>::new(&device);
let base = 10000;
let head_dim = 4;
let base_f = base as f32;
let head_dim_f = head_dim as f32;
inverse_frequency_table::<B>(base, head_dim, &device)
.to_data()
.assert_approx_eq(
&Tensor::<B, 1>::from_data(
[
1.0 / base_f.powf(0.0 / head_dim_f),
1.0 / base_f.powf(2.0 / head_dim_f),
],
&device,
)
.to_data(),
Tolerance::<f32>::default(),
);
}
#[test]
#[serial]
fn test_frequency_matrix() {
type B = PerformanceBackend;
let device = default_device();
let _memory = DeviceMemoryGuard::<B>::new(&device);
let base = 10000;
let head_dim = 4;
let base_f = base as f32;
let head_dim_f = head_dim as f32;
positional_frequency_table::<B>(3, base, head_dim, &device)
.to_data()
.assert_approx_eq(
&Tensor::<B, 2>::from_data(
[
[0.0, 0.0],
[
1.0 / base_f.powf(0.0 / head_dim_f),
1.0 / base_f.powf(2.0 / head_dim_f),
],
[
2.0 / base_f.powf(0.0 / head_dim_f),
2.0 / base_f.powf(2.0 / head_dim_f),
],
],
&device,
)
.to_data(),
Tolerance::<f32>::default(),
);
}
}