use alloc::collections::BTreeMap;
use core::ops::Range;
use std::path::Path;
use burn::{
prelude::*,
store::{
ModuleStore,
SafetensorsStore,
bridge,
burn_pack::Tensor as PackTensor,
},
tensor::Tolerance,
};
use crate::ops::transformers::attention::{
KVCache,
KVCacheConfig,
KVCacheStorage,
};
pub struct ToyFixture {
io: BTreeMap<String, PackTensor>,
weights: Vec<PackTensor>,
device: Device,
}
impl ToyFixture {
pub fn load(
path: impl AsRef<Path>,
device: &Device,
) -> Self {
let (io, weights): (Vec<_>, Vec<_>) = Self::read(path.as_ref())
.into_iter()
.partition(|(name, _)| name.starts_with("io."));
Self {
io: io
.into_iter()
.map(|(name, t)| (name["io.".len()..].to_string(), t))
.collect(),
weights: weights.into_iter().map(|(_, t)| t).collect(),
device: device.clone(),
}
}
pub fn load_io(
path: impl AsRef<Path>,
device: &Device,
) -> Self {
Self {
io: Self::read(path.as_ref()).into_iter().collect(),
weights: Vec::new(),
device: device.clone(),
}
}
fn read(path: &Path) -> BTreeMap<String, PackTensor> {
SafetensorsStore::from_file(path)
.get_all_tensors()
.unwrap_or_else(|e| panic!("read the toy fixture {}: {e}", path.display()))
.clone()
}
pub fn device(&self) -> &Device {
&self.device
}
pub fn take_weights(&mut self) -> Vec<PackTensor> {
core::mem::take(&mut self.weights)
}
pub fn has(
&self,
key: &str,
) -> bool {
self.io.contains_key(key)
}
pub fn io<const D: usize>(
&self,
key: &str,
) -> Tensor<D> {
let tensor = self
.io
.get(key)
.unwrap_or_else(|| panic!("the toy fixture has no io tensor {key:?}"));
Tensor::from_data(
bridge::to_data(tensor).expect("read an io tensor"),
&self.device,
)
}
pub fn ids(&self) -> Tensor<2, Int> {
self.io::<2>("input_ids").int()
}
}
pub fn assert_io_close<const D: usize>(
what: &str,
actual: Tensor<D>,
expected: Tensor<D>,
) {
assert_io_close_within(what, actual, expected, Tolerance::rel_abs(1e-3, 1e-4));
}
pub fn assert_io_close_within<const D: usize>(
what: &str,
actual: Tensor<D>,
expected: Tensor<D>,
tolerance: Tolerance<f32>,
) -> f32 {
assert_eq!(actual.dims(), expected.dims(), "{what}: shape");
let (actual, expected) = (actual.to_data_as::<f32>(), expected.to_data_as::<f32>());
let gap = actual
.iter::<f32>()
.zip(expected.iter::<f32>())
.map(|(a, e)| (a - e).abs())
.fold(0.0, f32::max);
eprintln!("{what}: off by at most {gap:.2e}");
actual.assert_approx_eq::<f32>(&expected, tolerance);
gap
}
pub trait DecoderToy {
fn positions(&self) -> usize;
fn kv_cache_config(&self) -> KVCacheConfig;
fn logits(
&self,
range: Range<usize>,
cache: Option<&mut KVCache>,
) -> Tensor<3>;
}
pub const KV_CACHE_STORAGES: [KVCacheStorage; 4] = [
KVCacheStorage::Auto,
KVCacheStorage::Concat,
KVCacheStorage::PerLayer,
KVCacheStorage::MultiLayer,
];
pub fn check_decoder_toy(
toy: &impl DecoderToy,
uncached: Tensor<3>,
cached: Tensor<3>,
prefill: usize,
) {
let n = toy.positions();
assert!(
0 < prefill && prefill < n,
"a prefill inside the input: {prefill} of {n}"
);
assert_io_close("uncached logits", toy.logits(0..n, None), uncached);
for storage in KV_CACHE_STORAGES {
let mut cache = toy
.kv_cache_config()
.with_seq_len(2)
.with_chunk_size(2)
.with_storage(storage)
.init();
let mut parts = vec![toy.logits(0..prefill, Some(&mut cache))];
for pos in prefill..n {
parts.push(toy.logits(pos..pos + 1, Some(&mut cache)));
}
assert_eq!(cache.pos(), n, "{storage:?}: the cache's length");
assert_io_close(
&format!("{storage:?} cached logits"),
Tensor::cat(parts, 1),
cached.clone(),
);
}
}