use burn::{
Tensor,
config::Config,
module::{
Module,
Param,
},
nn::{
Embedding,
EmbeddingConfig,
LayerNorm,
LayerNormConfig,
},
prelude::s,
tensor::{
Bool,
DType,
Device,
Distribution,
Int,
},
};
use super::WHISPER_DEFAULT_D_MODEL;
use crate::{
blocks::transformers::decoders::cross::{
CrossDecoderBlock,
CrossDecoderBlockConfig,
CrossDecoderBlockMeta,
},
burner::{
module::{
HasDType,
ModuleInit,
},
tensor::device_float_dtype,
},
errors::BunsenResult,
ops::{
embedding::unembed,
transformers::attention::{
KVCache,
KVCacheConfig,
KVCacheModel,
causal_mask,
},
},
};
pub trait TextDecoderMeta {
fn vocab_size(&self) -> usize;
fn d_model(&self) -> usize;
fn max_context(&self) -> usize;
fn n_heads(&self) -> usize;
fn n_layers(&self) -> usize;
}
fn kv_cache_config(meta: &impl TextDecoderMeta) -> KVCacheConfig {
KVCacheConfig::new(
1,
meta.n_heads(),
meta.max_context(),
meta.d_model() / meta.n_heads(),
meta.n_layers(),
)
.with_max_seq_len(Some(meta.max_context()))
}
impl KVCacheModel for TextDecoderConfig {
fn kv_cache_config(&self) -> KVCacheConfig {
kv_cache_config(self)
}
}
impl KVCacheModel for TextDecoder {
fn kv_cache_config(&self) -> KVCacheConfig {
kv_cache_config(self)
}
}
#[derive(Config, Debug)]
pub struct TextDecoderConfig {
pub vocab_size: usize,
pub d_model: usize,
pub max_context: usize,
pub n_layers: usize,
#[config(defaul_value = "WHISPER_DEFAULT_D_MODEL")]
pub d_head: usize,
#[config(default = "0.0")]
pub block_dropout: f64,
}
impl TextDecoderMeta for TextDecoderConfig {
fn vocab_size(&self) -> usize {
self.vocab_size
}
fn d_model(&self) -> usize {
self.d_model
}
fn max_context(&self) -> usize {
self.max_context
}
fn n_heads(&self) -> usize {
self.d_model / self.d_head
}
fn n_layers(&self) -> usize {
self.n_layers
}
}
impl ModuleInit<TextDecoder> for TextDecoderConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<TextDecoder> {
Ok(TextDecoder {
token_embedding: EmbeddingConfig::new(self.vocab_size, self.d_model).init(device),
positional_embedding: Param::from_tensor(Tensor::<2>::random(
[self.max_context, self.d_model],
Distribution::Normal(0.0, 1.0),
device,
)),
blocks: (0..self.n_layers)
.map(|_| {
CrossDecoderBlockConfig::new(self.d_model)
.with_d_head(self.d_head)
.with_dropout(self.block_dropout)
.try_init(device)
})
.collect::<BunsenResult<Vec<CrossDecoderBlock>>>()?,
ln: LayerNormConfig::new(self.d_model).init(device),
})
}
}
#[derive(Module, Debug)]
pub struct TextDecoder {
pub token_embedding: Embedding,
pub positional_embedding: Param<Tensor<2>>,
pub blocks: Vec<CrossDecoderBlock>,
pub ln: LayerNorm,
}
impl TextDecoderMeta for TextDecoder {
fn vocab_size(&self) -> usize {
self.token_embedding.weight.val().dims()[0]
}
fn d_model(&self) -> usize {
self.blocks[0].d_model()
}
fn max_context(&self) -> usize {
self.positional_embedding.val().dims()[0]
}
fn n_heads(&self) -> usize {
self.blocks[0].n_heads()
}
fn n_layers(&self) -> usize {
self.blocks.len()
}
}
impl HasDType for TextDecoder {
fn dtype(&self) -> DType {
self.token_embedding.weight.dtype()
}
}
impl TextDecoder {
pub fn forward(
&self,
x: Tensor<2, Int>,
xa: Tensor<3>,
) -> Tensor<3> {
let xa = xa.cast(self.dtype());
let [_batch, seq_len] = x.dims();
assert!(
seq_len <= self.max_context(),
"Token sequence length {} must not exceed {}.",
seq_len,
self.max_context()
);
let x = self.embed(x);
let mask: Option<Tensor<3, Bool>> = causal_mask(seq_len, 0, &x.device()).into();
let mut x = x;
for b in self.blocks.iter() {
x = b.forward(x, xa.clone(), mask.clone()).output;
}
let x = self.ln.forward(x);
{
let dtype = device_float_dtype(&x.device());
unembed(&self.token_embedding, x).cast(dtype)
}
}
pub fn new_cache(
&self,
xa: Tensor<3>,
) -> KVCache {
self.new_cache_grouped(xa, 1)
}
pub fn new_cache_grouped(
&self,
xa: Tensor<3>,
group: usize,
) -> KVCache {
assert!(group >= 1, "at least one row per audio");
let xa = xa.cast(self.dtype());
let mut cache = self
.kv_cache_config()
.with_batch_size(xa.dims()[0] * group)
.init();
for (layer, block) in self.blocks.iter().enumerate() {
let (k, v) = block.cross_kv(xa.clone());
cache.set_cross_kv(layer, k, v);
}
cache
}
pub fn forward_cached(
&self,
x: Tensor<2, Int>,
cache: &mut KVCache,
) -> Tensor<3> {
use crate::ops::transformers::attention::KVCacheMeta;
assert_eq!(
cache.num_layers(),
self.blocks.len(),
"cache was built for a different decoder",
);
let seq_new = x.dims()[1];
let past = cache.pos();
assert!(
past + seq_new <= self.max_context(),
"Token sequence length {} must not exceed {}.",
past + seq_new,
self.max_context(),
);
let mut h = self.embed_at(x, past);
let mask: Option<Tensor<3, Bool>> = if seq_new > 1 {
Some(causal_mask(seq_new, past, &h.device()))
} else {
None
};
for (layer, block) in self.blocks.iter().enumerate() {
h = block.forward_w_kv_cache(h, mask.clone(), cache, layer);
}
{
let dtype = device_float_dtype(&h.device());
unembed(&self.token_embedding, self.ln.forward(h)).cast(dtype)
}
}
fn embed(
&self,
x: Tensor<2, Int>,
) -> Tensor<3> {
self.embed_at(x, 0)
}
fn embed_at(
&self,
x: Tensor<2, Int>,
offset: usize,
) -> Tensor<3> {
let seq_len = x.dims()[1];
self.token_embedding.forward(x)
+ self
.positional_embedding
.val()
.slice(s![offset..offset + seq_len])
.unsqueeze::<3>()
}
}
#[cfg(test)]
mod tests {
use burn::tensor::TensorData;
use serial_test::serial;
use super::*;
use crate::{
contracts::assert_shape_contract,
support::testing::{
DeviceMemoryGuard,
cpu_device,
performance_device,
},
};
#[test]
#[serial]
fn test_text_decoder_forward() {
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let d_model = 128;
let vocab_size = 64;
let max_context = 128;
let n_layers = 2;
let config = TextDecoderConfig::new(vocab_size, d_model, max_context, n_layers);
assert_eq!(config.vocab_size(), vocab_size);
assert_eq!(config.d_model(), d_model);
assert_eq!(config.max_context(), max_context);
let decoder: TextDecoder = config.init(&device);
assert_eq!(decoder.vocab_size(), vocab_size);
assert_eq!(decoder.d_model(), d_model);
assert_eq!(decoder.max_context(), max_context);
let batch = 2;
let seq_len = max_context / 2;
let x: Tensor<2, Int> = Tensor::zeros([batch, seq_len], &device);
let xa: Tensor<3> = Tensor::random([batch, seq_len, d_model], Default::default(), &device);
let output = decoder.forward(x.clone(), xa.clone());
assert_shape_contract!(
["batch", "seq", "n_vocab"],
&output,
&[("batch", batch), ("seq", seq_len), ("n_vocab", vocab_size),],
);
}
#[test]
#[serial]
fn test_forward_cached_matches_forward() {
use burn::{
prelude::TensorData,
tensor::{
Distribution,
Tolerance,
},
};
type F = f32;
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let (vocab, d_model, max_ctx, layers) = (64, 128, 16, 2);
let (batch, cross_len, seq) = (2, 5, 6);
let decoder: TextDecoder =
TextDecoderConfig::new(vocab, d_model, max_ctx, layers).init(&device);
let tokens: Tensor<2, Int> = Tensor::from_data(
TensorData::new(
(0..batch * seq)
.map(|k| (k % vocab) as i64)
.collect::<Vec<_>>(),
[batch, seq],
),
&device,
);
let xa: Tensor<3> =
Tensor::random([batch, cross_len, d_model], Distribution::Default, &device);
let whole = decoder.forward(tokens.clone(), xa.clone());
let mut cache = decoder.new_cache(xa);
let mut steps = Vec::with_capacity(seq);
for t in 0..seq {
let step = tokens.clone().slice_dim(1, t as isize..(t + 1) as isize);
steps.push(decoder.forward_cached(step, &mut cache));
}
assert_eq!(cache.pos(), seq);
let stepped: Tensor<3> = Tensor::cat(steps, 1);
assert_eq!(stepped.dims(), whole.dims());
stepped
.to_data_as::<F>()
.assert_approx_eq::<F>(&whole.to_data_as::<F>(), Tolerance::permissive());
}
#[test]
#[serial]
fn test_cached_prefill_then_step() {
use burn::{
prelude::TensorData,
tensor::{
Distribution,
Tolerance,
},
};
type F = f32;
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let (vocab, d_model, max_ctx, layers) = (64, 128, 16, 1);
let (cross_len, seq, prompt) = (4, 5, 3);
let decoder: TextDecoder =
TextDecoderConfig::new(vocab, d_model, max_ctx, layers).init(&device);
let tokens: Tensor<2, Int> = Tensor::from_data(
TensorData::new((0..seq).map(|k| k as i64).collect::<Vec<_>>(), [1, seq]),
&device,
);
let xa: Tensor<3> = Tensor::random([1, cross_len, d_model], Distribution::Default, &device);
let whole = decoder.forward(tokens.clone(), xa.clone());
let mut cache = decoder.new_cache(xa);
let mut parts = vec![
decoder.forward_cached(tokens.clone().slice_dim(1, 0..prompt as isize), &mut cache),
];
for t in prompt..seq {
let step = tokens.clone().slice_dim(1, t as isize..(t + 1) as isize);
parts.push(decoder.forward_cached(step, &mut cache));
}
let mixed: Tensor<3> = Tensor::cat(parts, 1);
mixed
.to_data_as::<F>()
.assert_approx_eq::<F>(&whole.to_data_as::<F>(), Tolerance::permissive());
}
#[test]
#[serial]
fn test_caches_take_the_decoders_dtype() {
use burn::{
module::Module,
tensor::Distribution,
};
use crate::burner::{
module::DTypeMapper,
tensor::device_float_dtype,
};
let device = cpu_device();
let float = device_float_dtype(&device);
let decoder: TextDecoder = TextDecoderConfig::new(32, 128, 8, 1).init(&device);
let decoder = decoder.map(&mut DTypeMapper::new(DType::F16));
assert_eq!(decoder.dtype(), DType::F16);
let xa: Tensor<3> = Tensor::random([1, 4, 128], Distribution::Default, &device);
assert_eq!(xa.dtype(), float);
let mut cache = decoder.new_cache(xa);
assert_eq!(cache.dtype(), Some(DType::F16), "cross-attention cache");
for layer in 0..decoder.n_layers() {
let (k, v) = cache.cross_kv(layer).expect("set");
assert_eq!((k.dtype(), v.dtype()), (DType::F16, DType::F16));
}
let tokens: Tensor<2, Int> =
Tensor::from_data(TensorData::new(vec![3i64, 5], [1, 2]), &device);
let logits = decoder.forward_cached(tokens, &mut cache);
assert_eq!(logits.dtype(), float, "logits are the interface's");
assert_eq!(cache.pos(), 2);
assert_eq!(cache.dtype(), Some(DType::F16), "self-attention cache");
}
#[test]
#[serial]
fn test_cache_reset_restarts_the_stream() {
use burn::{
prelude::TensorData,
tensor::{
Distribution,
Tolerance,
},
};
type F = f32;
let device = performance_device();
let _memory = DeviceMemoryGuard::new(&device);
let (vocab, d_model, max_ctx, layers) = (32, 128, 8, 1);
let decoder: TextDecoder =
TextDecoderConfig::new(vocab, d_model, max_ctx, layers).init(&device);
let tokens: Tensor<2, Int> =
Tensor::from_data(TensorData::new(vec![3i64, 5], [1, 2]), &device);
let xa: Tensor<3> = Tensor::random([1, 4, d_model], Distribution::Default, &device);
let mut cache = decoder.new_cache(xa);
let first = decoder.forward_cached(tokens.clone(), &mut cache);
assert_eq!(cache.pos(), 2);
cache.reset();
assert_eq!(cache.pos(), 0);
let second = decoder.forward_cached(tokens, &mut cache);
first
.to_data_as::<F>()
.assert_approx_eq::<F>(&second.to_data_as::<F>(), Tolerance::permissive());
}
}