use burn::{
Tensor,
prelude::{
Bool,
Int,
},
tensor::Device,
};
pub fn causal_mask(
seq_len: usize,
n_past: usize,
device: &Device,
) -> Tensor<3, Bool> {
let total = n_past + seq_len;
Tensor::<2, Bool>::tril_mask([seq_len, total], n_past as i64, device).unsqueeze::<3>()
}
pub fn causal_window_visible(
t_q: usize,
t_kv: usize,
window: Option<usize>,
device: &Device,
) -> Tensor<2, Bool> {
assert!(t_q <= t_kv, "{t_q} queries over {t_kv} keys");
assert!(window != Some(0), "a sliding window of 0 positions");
let rows =
Tensor::<1, Int>::arange((t_kv - t_q) as i64..t_kv as i64, device).unsqueeze_dim::<2>(1);
let cols = Tensor::<1, Int>::arange(0..t_kv as i64, device).unsqueeze_dim::<2>(0);
let causal = cols.clone().lower_equal(rows.clone());
match window {
None => causal,
Some(w) => causal.bool_and(cols.greater(rows.sub_scalar(w as i64))),
}
}
pub fn two_sided_window_visible(
t: usize,
window: usize,
device: &Device,
) -> Tensor<2, Bool> {
assert!(window != 0, "a sliding window of 0 positions");
let rows = Tensor::<1, Int>::arange(0..t as i64, device).unsqueeze_dim::<2>(1);
let cols = Tensor::<1, Int>::arange(0..t as i64, device).unsqueeze_dim::<2>(0);
cols.sub(rows).abs().lower_elem(window as i64)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::support::testing::cpu_device;
fn rows(mask: Tensor<2, Bool>) -> Vec<Vec<bool>> {
let [_, t_kv] = mask.dims();
let values: Vec<bool> = mask.into_data().iter::<bool>().collect();
values.chunks(t_kv).map(<[bool]>::to_vec).collect()
}
#[test]
fn test_causal_window_visible() {
let device = cpu_device();
assert_eq!(
rows(causal_window_visible(2, 5, None, &device)),
[
[true, true, true, true, false],
[true, true, true, true, true],
]
);
assert_eq!(
rows(causal_window_visible(3, 3, Some(2), &device)),
[
[true, false, false],
[true, true, false],
[false, true, true],
]
);
assert_eq!(
rows(causal_window_visible(1, 4, Some(3), &device)),
[[false, true, true, true]]
);
}
#[test]
#[should_panic(expected = "sliding window of 0")]
fn test_causal_window_visible_refuses_an_empty_window() {
let _ = causal_window_visible(1, 1, Some(0), &cpu_device());
}
#[test]
fn test_two_sided_window_visible() {
let device = cpu_device();
let visible = rows(two_sided_window_visible(5, 2, &device));
assert_eq!(
visible,
[
[true, true, false, false, false],
[true, true, true, false, false],
[false, true, true, true, false],
[false, false, true, true, true],
[false, false, false, true, true],
]
);
for i in 0..5 {
for j in 0..5 {
assert_eq!(visible[i][j], (i as i64 - j as i64).abs() < 2, "({i}, {j})");
}
}
let causal = rows(causal_window_visible(5, 5, Some(2), &device));
for i in 0..5 {
for j in 0..=i {
assert_eq!(visible[i][j], causal[i][j], "({i}, {j})");
}
}
let wide = rows(two_sided_window_visible(4, 9, &device));
assert!(
wide.iter().flatten().all(|&v| v),
"a window wider than the input"
);
}
}