use alloc::vec::Vec;
use core::{
marker::PhantomData,
ops::Range,
};
use burn::{
Tensor,
module::Module,
prelude::Int,
tensor::Device,
};
use super::{
LmAttentionOnlyKind,
LmDecodeKind,
LmHybridKind,
};
use crate::{
blocks::transformers::{
decoders::stack::{
DecoderStack,
DecoderStackMeta,
},
embedding::{
RotarySet,
TokenEmbedding,
TokenEmbeddingMeta,
},
head::LmHead,
},
burner::module::{
CanonicalKeys,
Rekey,
},
ops::transformers::attention::{
KVCache,
KVCacheConfig,
KVCacheModel,
},
};
pub trait LmDecoderMeta {
fn n_vocab(&self) -> usize;
fn n_logits(&self) -> usize {
self.n_vocab()
}
fn d_model(&self) -> usize;
fn n_layers(&self) -> usize;
fn n_kv_heads(&self) -> usize;
fn head_dim(&self) -> usize;
}
#[derive(Module, Debug)]
pub struct LmDecoderModel<K: LmDecodeKind = LmAttentionOnlyKind> {
pub embedding: TokenEmbedding,
pub head: LmHead,
pub stack: DecoderStack,
#[module(skip)]
pub(super) kv_cache_pre_size: usize,
#[module(skip)]
pub(super) kind: PhantomData<K>,
}
impl<K: LmDecodeKind> LmDecoderMeta for LmDecoderModel<K> {
fn n_vocab(&self) -> usize {
self.embedding.n_vocab()
}
fn n_logits(&self) -> usize {
use crate::blocks::transformers::head::LmHeadMeta;
self.head.n_vocab()
}
fn d_model(&self) -> usize {
self.embedding.d_model()
}
fn n_layers(&self) -> usize {
self.stack.n_layers()
}
fn n_kv_heads(&self) -> usize {
self.stack.n_kv_heads()
}
fn head_dim(&self) -> usize {
self.stack.head_dim()
}
}
impl<K: LmDecodeKind> LmDecoderModel<K> {
pub fn kv_cache_pre_size(&self) -> usize {
self.kv_cache_pre_size
}
pub fn float_dtype(&self) -> burn::tensor::DType {
crate::burner::tensor::param_float_dtype(&self.embedding.weight)
}
pub fn rotary(
&self,
seq_len: usize,
device: &Device,
) -> RotarySet {
self.stack.rotary(seq_len, device)
}
pub fn embed(
&self,
tokens: Tensor<2, Int>,
) -> Tensor<3> {
#[cfg(any(debug_assertions, test))]
let [batch, seq] =
crate::contracts::unpack_shape_contract!(["batch", "seq"], &tokens.dims());
let y = self.embedding.forward(tokens);
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract_periodically!(
["batch", "seq", "d_model"],
&y.dims(),
&[("batch", batch), ("seq", seq), ("d_model", self.d_model())],
);
y
}
pub fn attention_weights(
&self,
embeds: Tensor<3>,
r_emb: &RotarySet,
layers: &[usize],
rows: Range<usize>,
cols: Range<usize>,
) -> Vec<Tensor<4>> {
self.stack
.attention_weights(embeds, r_emb, layers, rows, cols)
}
pub fn logits(
&self,
hidden: Tensor<3>,
) -> Tensor<3> {
#[cfg(any(debug_assertions, test))]
let [batch, seq] = crate::contracts::unpack_shape_contract!(
["batch", "seq", "d_model"],
&hidden.dims(),
&["batch", "seq"],
&[("d_model", self.d_model())],
);
let y = self.head.forward(hidden, &self.embedding.weight);
#[cfg(any(debug_assertions, test))]
{
use crate::blocks::transformers::head::LmHeadMeta;
crate::contracts::assert_shape_contract_periodically!(
["batch", "seq", "n_vocab"],
&y.dims(),
&[
("batch", batch),
("seq", seq),
("n_vocab", self.head.n_vocab())
],
);
}
y
}
}
pub trait LmKvForwardHidden {
fn forward_hidden(
&self,
embeds: Tensor<3>,
r_emb: &RotarySet,
kv_cache: Option<&mut KVCache>,
) -> Tensor<3>;
}
impl LmKvForwardHidden for LmDecoderModel<LmAttentionOnlyKind> {
fn forward_hidden(
&self,
embeds: Tensor<3>,
r_emb: &RotarySet,
kv_cache: Option<&mut KVCache>,
) -> Tensor<3> {
self.stack.forward_hidden(embeds, r_emb, kv_cache)
}
}
pub trait LmUncachedForwardHidden {
fn forward_hidden(
&self,
embeds: Tensor<3>,
r_emb: &RotarySet,
) -> Tensor<3>;
}
impl LmUncachedForwardHidden for LmDecoderModel<LmHybridKind> {
fn forward_hidden(
&self,
embeds: Tensor<3>,
r_emb: &RotarySet,
) -> Tensor<3> {
self.stack.forward_hidden(embeds, r_emb, None)
}
}
impl KVCacheModel for LmDecoderModel<LmAttentionOnlyKind> {
fn kv_cache_config(&self) -> KVCacheConfig {
let cap = self.stack.rotary.seq_len;
self.stack
.kv_cache_config(self.kv_cache_pre_size)
.with_max_seq_len((cap > 0).then_some(cap))
}
}
impl<K: LmDecodeKind> CanonicalKeys for LmDecoderModel<K> {
fn canonical_rekey(&self) -> Rekey {
Rekey::keep_rest().mount_canonical("stack.", &self.stack)
}
}