use burn::{
Tensor,
config::Config,
module::{
Module,
Param,
},
nn::{
Embedding,
EmbeddingConfig,
LayerNorm,
LayerNormConfig,
},
prelude::{
Backend,
s,
},
tensor::{
Bool,
Distribution,
Int,
TensorData,
},
};
use super::WHISPER_DEFAULT_D_MODEL;
use crate::{
burner::{
module::ModuleInit,
store::FixPytorchLoadMappers,
},
errors::BunsenResult,
kits::speech::whisper::blocks::{
ResidualDecoderAttentionBlock,
ResidualDecoderAttentionBlockConfig,
ResidualDecoderAttentionBlockMeta,
},
ops::{
embedding::unembed,
transformers::attention::{
AttnKvPair,
causal_mask,
},
},
};
pub trait TextDecoderMeta {
fn vocab_size(&self) -> usize;
fn d_model(&self) -> usize;
fn max_context(&self) -> usize;
fn n_heads(&self) -> usize;
fn n_layers(&self) -> usize;
}
#[derive(Config, Debug)]
pub struct TextDecoderConfig {
pub vocab_size: usize,
pub d_model: usize,
pub max_context: usize,
pub n_layers: usize,
#[config(defaul_value = "WHISPER_DEFAULT_D_MODEL")]
pub d_head: usize,
#[config(default = "0.0")]
pub block_dropout: f64,
}
impl TextDecoderMeta for TextDecoderConfig {
fn vocab_size(&self) -> usize {
self.vocab_size
}
fn d_model(&self) -> usize {
self.d_model
}
fn max_context(&self) -> usize {
self.max_context
}
fn n_heads(&self) -> usize {
self.d_model / self.d_head
}
fn n_layers(&self) -> usize {
self.n_layers
}
}
impl<B: Backend> ModuleInit<B, TextDecoder<B>> for TextDecoderConfig {
fn try_init(
&self,
device: &B::Device,
) -> BunsenResult<TextDecoder<B>> {
Ok(TextDecoder {
token_embedding: EmbeddingConfig::new(self.vocab_size, self.d_model).init(device),
positional_embedding: Param::from_tensor(Tensor::<B, 2>::random(
[self.max_context, self.d_model],
Distribution::Normal(0.0, 1.0),
device,
)),
blocks: (0..self.n_layers)
.map(|_| {
ResidualDecoderAttentionBlockConfig::new(self.d_model)
.with_d_head(self.d_head)
.with_dropout(self.block_dropout)
.try_init(device)
})
.collect::<BunsenResult<Vec<ResidualDecoderAttentionBlock<B>>>>()?,
ln: LayerNormConfig::new(self.d_model).init(device),
})
}
}
#[derive(Module, Debug)]
pub struct TextDecoder<B: Backend> {
pub token_embedding: Embedding<B>,
pub positional_embedding: Param<Tensor<B, 2>>,
pub blocks: Vec<ResidualDecoderAttentionBlock<B>>,
pub ln: LayerNorm<B>,
}
impl<B: Backend> FixPytorchLoadMappers for TextDecoder<B> {
fn fix_pytorch_load_mappers(mut self) -> Self {
self.blocks = self.blocks.fix_pytorch_load_mappers();
self
}
}
impl<B: Backend> TextDecoderMeta for TextDecoder<B> {
fn vocab_size(&self) -> usize {
self.token_embedding.weight.val().dims()[0]
}
fn d_model(&self) -> usize {
self.blocks[0].d_model()
}
fn max_context(&self) -> usize {
self.positional_embedding.val().dims()[0]
}
fn n_heads(&self) -> usize {
self.blocks[0].n_heads()
}
fn n_layers(&self) -> usize {
self.blocks.len()
}
}
impl<B: Backend> TextDecoder<B> {
pub fn forward(
&self,
x: Tensor<B, 2, Int>,
xa: Tensor<B, 3>,
) -> Tensor<B, 3> {
let [_batch, seq_len] = x.dims();
assert!(
seq_len <= self.max_context(),
"Token sequence length {} must not exceed {}.",
seq_len,
self.max_context()
);
let x = self.embed(x);
let mask: Option<Tensor<B, 3, Bool>> = causal_mask(seq_len, 0, &x.device()).into();
let mut x = x;
for b in self.blocks.iter() {
x = b.forward(x, xa.clone(), mask.clone()).output;
}
let x = self.ln.forward(x);
unembed(&self.token_embedding, x)
}
pub fn new_cache(
&self,
xa: Tensor<B, 3>,
) -> TextDecoderCache<B> {
self.new_cache_grouped(xa, 1)
}
pub fn new_cache_grouped(
&self,
xa: Tensor<B, 3>,
group: usize,
) -> TextDecoderCache<B> {
assert!(group >= 1, "at least one row per audio");
TextDecoderCache {
self_kv: self.blocks.iter().map(|_| None).collect(),
cross_kv: self
.blocks
.iter()
.map(|block| block.build_cross_kv(xa.clone()))
.collect(),
group,
pos: 0,
}
}
pub fn forward_cached(
&self,
x: Tensor<B, 2, Int>,
cache: &mut TextDecoderCache<B>,
) -> Tensor<B, 3> {
assert_eq!(
cache.n_layers(),
self.blocks.len(),
"cache was built for a different decoder",
);
let seq_new = x.dims()[1];
let past = cache.pos;
assert!(
past + seq_new <= self.max_context(),
"Token sequence length {} must not exceed {}.",
past + seq_new,
self.max_context(),
);
let mut h = self.embed_at(x, past);
let mask: Option<Tensor<B, 3, Bool>> = if seq_new > 1 {
Some(causal_mask(seq_new, past, &h.device()))
} else {
None
};
for (i, block) in self.blocks.iter().enumerate() {
h = block.forward_w_kv_cache_grouped(
h,
mask.clone(),
&mut cache.self_kv[i],
&cache.cross_kv[i],
cache.group,
);
}
cache.pos += seq_new;
unembed(&self.token_embedding, self.ln.forward(h))
}
fn embed(
&self,
x: Tensor<B, 2, Int>,
) -> Tensor<B, 3> {
self.embed_at(x, 0)
}
fn embed_at(
&self,
x: Tensor<B, 2, Int>,
offset: usize,
) -> Tensor<B, 3> {
let seq_len = x.dims()[1];
self.token_embedding.forward(x)
+ self
.positional_embedding
.val()
.slice(s![offset..offset + seq_len])
.unsqueeze::<3>()
}
}
pub struct TextDecoderCache<B: Backend> {
self_kv: Vec<Option<AttnKvPair<B>>>,
cross_kv: Vec<AttnKvPair<B>>,
group: usize,
pos: usize,
}
impl<B: Backend> TextDecoderCache<B> {
pub fn pos(&self) -> usize {
self.pos
}
pub fn group(&self) -> usize {
self.group
}
pub fn n_layers(&self) -> usize {
self.cross_kv.len()
}
pub fn reset(&mut self) {
for kv in &mut self.self_kv {
*kv = None;
}
self.pos = 0;
}
pub fn reorder(
&mut self,
sources: &[usize],
) {
let Some(first) = self.self_kv.iter().flatten().next() else {
return;
};
assert_eq!(
sources.len(),
first.batch_size(),
"one source row per cached row",
);
if sources.iter().enumerate().all(|(r, &s)| r == s) {
return;
}
let device = first.key.device();
let indices: Tensor<B, 1, Int> = Tensor::from_data(
TensorData::new(
sources.iter().map(|&s| s as i64).collect::<Vec<_>>(),
[sources.len()],
),
&device,
);
for kv in self.self_kv.iter_mut().flatten() {
kv.key = kv.key.clone().select(0, indices.clone());
kv.value = kv.value.clone().select(0, indices.clone());
}
}
}
#[cfg(test)]
mod tests {
use serial_test::serial;
use super::*;
use crate::{
contracts::assert_shape_contract,
support::testing::PerformanceBackend,
};
#[test]
#[serial]
fn test_text_decoder_forward() {
type B = PerformanceBackend;
let device = Default::default();
let d_model = 128;
let vocab_size = 64;
let max_context = 128;
let n_layers = 2;
let config = TextDecoderConfig::new(vocab_size, d_model, max_context, n_layers);
assert_eq!(config.vocab_size(), vocab_size);
assert_eq!(config.d_model(), d_model);
assert_eq!(config.max_context(), max_context);
let decoder: TextDecoder<B> = config.init(&device);
assert_eq!(decoder.vocab_size(), vocab_size);
assert_eq!(decoder.d_model(), d_model);
assert_eq!(decoder.max_context(), max_context);
let batch = 2;
let seq_len = max_context / 2;
let x: Tensor<B, 2, Int> = Tensor::zeros([batch, seq_len], &device);
let xa: Tensor<B, 3> =
Tensor::random([batch, seq_len, d_model], Default::default(), &device);
let output = decoder.forward(x.clone(), xa.clone());
assert_shape_contract!(
["batch", "seq", "n_vocab"],
&output,
&[("batch", batch), ("seq", seq_len), ("n_vocab", vocab_size),],
);
}
#[test]
#[serial]
fn test_forward_cached_matches_forward() {
use burn::{
prelude::TensorData,
tensor::{
Distribution,
Tolerance,
backend::BackendTypes,
},
};
use crate::burner::tensor::TensorElemOpExt;
type B = PerformanceBackend;
type F = <B as BackendTypes>::FloatElem;
let device = Default::default();
let (vocab, d_model, max_ctx, layers) = (64, 128, 16, 2);
let (batch, cross_len, seq) = (2, 5, 6);
let decoder: TextDecoder<B> =
TextDecoderConfig::new(vocab, d_model, max_ctx, layers).init(&device);
let tokens: Tensor<B, 2, Int> = Tensor::from_data(
TensorData::new(
(0..batch * seq)
.map(|k| (k % vocab) as i64)
.collect::<Vec<_>>(),
[batch, seq],
),
&device,
);
let xa: Tensor<B, 3> =
Tensor::random([batch, cross_len, d_model], Distribution::Default, &device);
let whole = decoder.forward(tokens.clone(), xa.clone());
let mut cache = decoder.new_cache(xa);
let mut steps = Vec::with_capacity(seq);
for t in 0..seq {
let step = tokens.clone().slice_dim(1, t as isize..(t + 1) as isize);
steps.push(decoder.forward_cached(step, &mut cache));
}
assert_eq!(cache.pos(), seq);
let stepped: Tensor<B, 3> = Tensor::cat(steps, 1);
assert_eq!(stepped.dims(), whole.dims());
stepped
.to_data_as::<F>()
.assert_approx_eq::<F>(&whole.to_data_as::<F>(), Tolerance::permissive());
}
#[test]
#[serial]
fn test_cached_prefill_then_step() {
use burn::{
prelude::TensorData,
tensor::{
Distribution,
Tolerance,
backend::BackendTypes,
},
};
use crate::burner::tensor::TensorElemOpExt;
type B = PerformanceBackend;
type F = <B as BackendTypes>::FloatElem;
let device = Default::default();
let (vocab, d_model, max_ctx, layers) = (64, 128, 16, 1);
let (cross_len, seq, prompt) = (4, 5, 3);
let decoder: TextDecoder<B> =
TextDecoderConfig::new(vocab, d_model, max_ctx, layers).init(&device);
let tokens: Tensor<B, 2, Int> = Tensor::from_data(
TensorData::new((0..seq).map(|k| k as i64).collect::<Vec<_>>(), [1, seq]),
&device,
);
let xa: Tensor<B, 3> =
Tensor::random([1, cross_len, d_model], Distribution::Default, &device);
let whole = decoder.forward(tokens.clone(), xa.clone());
let mut cache = decoder.new_cache(xa);
let mut parts = vec![
decoder.forward_cached(tokens.clone().slice_dim(1, 0..prompt as isize), &mut cache),
];
for t in prompt..seq {
let step = tokens.clone().slice_dim(1, t as isize..(t + 1) as isize);
parts.push(decoder.forward_cached(step, &mut cache));
}
let mixed: Tensor<B, 3> = Tensor::cat(parts, 1);
mixed
.to_data_as::<F>()
.assert_approx_eq::<F>(&whole.to_data_as::<F>(), Tolerance::permissive());
}
#[test]
#[serial]
fn test_cache_reset_restarts_the_stream() {
use burn::{
prelude::TensorData,
tensor::{
Distribution,
Tolerance,
backend::BackendTypes,
},
};
use crate::burner::tensor::TensorElemOpExt;
type B = PerformanceBackend;
type F = <B as BackendTypes>::FloatElem;
let device = Default::default();
let (vocab, d_model, max_ctx, layers) = (32, 128, 8, 1);
let decoder: TextDecoder<B> =
TextDecoderConfig::new(vocab, d_model, max_ctx, layers).init(&device);
let tokens: Tensor<B, 2, Int> =
Tensor::from_data(TensorData::new(vec![3i64, 5], [1, 2]), &device);
let xa: Tensor<B, 3> = Tensor::random([1, 4, d_model], Distribution::Default, &device);
let mut cache = decoder.new_cache(xa);
let first = decoder.forward_cached(tokens.clone(), &mut cache);
assert_eq!(cache.pos(), 2);
cache.reset();
assert_eq!(cache.pos(), 0);
let second = decoder.forward_cached(tokens, &mut cache);
first
.to_data_as::<F>()
.assert_approx_eq::<F>(&second.to_data_as::<F>(), Tolerance::permissive());
}
}