use burn::{
Tensor,
prelude::Int,
};
use crate::{
blocks::transformers::{
decoders::stack::DecoderStackCache,
models::language_decoder::LmPlainStackOps,
},
kits::support::decoding::CachedStep,
};
pub trait StackedLm: LmPlainStackOps {
fn prefill(
&self,
embeds: Tensor<3>,
state: &mut DecoderStackCache,
) -> Tensor<2> {
let hidden = self.stack().forward_cached(embeds, state);
let [rows, seq, _] = hidden.dims();
let logits = self.logits(hidden.slice_dim(1, seq - 1..seq));
let vocab = logits.dims()[2];
logits.reshape([rows, vocab])
}
}
impl<M: LmPlainStackOps> StackedLm for M {}
impl<M: StackedLm> CachedStep for M {
type State = DecoderStackCache;
fn step(
&self,
ids: Tensor<2, Int>,
state: &mut DecoderStackCache,
) -> Tensor<2> {
self.prefill(self.embed(ids), state)
}
fn reorder(
&self,
state: &mut DecoderStackCache,
sources: &[usize],
) {
state.reorder(sources);
}
}
#[cfg(test)]
mod tests {
use burn::{
nn::norm::RmsNormConfig,
tensor::Tolerance,
};
use super::*;
use crate::{
blocks::{
norm::{
BunsenNormalizationConfig,
GatedNormConfig,
},
transformers::{
attention::csa::CausalSelfAttentionConfig,
decoders::{
causal::CausalDecoderBlockConfig,
stack::PatternedStackConfig,
},
embedding::{
RotaryEmbeddingConfig,
TokenEmbeddingConfig,
},
head::LmHeadConfig,
mixers::GatedDeltaNetConfig,
mlp::TransformerGatedMlpConfig,
models::language_decoder::{
LmDecoderModel,
LmDecoderVariantConfig,
LmHybridKind,
LmUncachedForwardHidden,
},
},
},
burner::module::ModuleInit,
kits::language::driver::CachedLm,
ops::transformers::attention::KVCacheStorage,
support::testing::{
assert_tensors_close,
cpu_device,
},
};
#[test]
fn test_lm_decoder_model_steps_as_uncached() {
let device = cpu_device();
let norm = || BunsenNormalizationConfig::Rms(RmsNormConfig::new(0));
let mlp = || TransformerGatedMlpConfig::new(12, 20).into();
let attention = CausalDecoderBlockConfig::new(
CausalSelfAttentionConfig::new(3, 1, 12)
.with_head_dim(Some(6))
.into(),
mlp(),
)
.with_norm(norm());
let linear = CausalDecoderBlockConfig::new(
GatedDeltaNetConfig::new(12, 2, 4, 4, 3)
.with_norm(GatedNormConfig::rms(0, 1e-6))
.into(),
mlp(),
)
.with_norm(norm());
let model: LmDecoderModel<LmHybridKind> = LmDecoderVariantConfig::new(
TokenEmbeddingConfig::new(40, 12),
LmHeadConfig::new(12, 40),
PatternedStackConfig::new(
vec![attention, linear],
vec![1, 0, 1],
norm(),
RotaryEmbeddingConfig::new(0, 6),
)
.into(),
)
.init(&device);
let tokens = Tensor::<2, Int>::from_data([[1, 5, 7, 2]], &device);
let mut state = model.decode_state(1, 4, KVCacheStorage::Auto, &device);
model.prefill(model.embed(tokens.clone().slice_dim(1, 0..3)), &mut state);
let stepped = model.step(tokens.clone().slice_dim(1, 3..4), &mut state);
let hidden = model.forward_hidden(model.embed(tokens), &model.rotary(4, &device));
let whole = model.logits(hidden.slice_dim(1, 3..4)).reshape([1, 40]);
assert_tensors_close(&stepped, &whole, Tolerance::rel_abs(1e-4, 1e-5));
}
}