use burn::{
Tensor,
config::Config,
module::Module,
nn::{
LayerNormConfig,
Linear,
LinearConfig,
},
prelude::{
Bool,
Int,
},
tensor::{
DType,
Device,
activation::{
gelu,
softmax,
},
},
};
use serde::{
Deserialize,
Serialize,
};
use crate::{
blocks::{
activation::BunsenActivationConfig,
norm::LayerNorm,
transformers::{
embedding::{
TokenEmbedding,
TokenEmbeddingConfig,
},
encoders::residual::{
ResidualEncoderAttentionBlock,
ResidualEncoderAttentionBlockConfig,
},
},
},
burner::{
module::{
CanonicalKeys,
ModuleInit,
Rekey,
},
tensor::param_float_dtype,
},
contracts::unpack_shape_contract,
errors::BunsenResult,
kits::text::encoders::modernbert::{
ModernBertConfig,
ModernBertMeta,
ModernBertModel,
},
ops::transformers::attention::length_mask,
};
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum LayaQuestionKind {
Choice,
Score,
Noul,
}
impl LayaQuestionKind {
pub const ALL: [LayaQuestionKind; 3] = [Self::Choice, Self::Score, Self::Noul];
pub fn index(self) -> usize {
self as usize
}
pub fn name(self) -> &'static str {
match self {
Self::Choice => "choice",
Self::Score => "score",
Self::Noul => "noul",
}
}
}
#[derive(Config, Debug)]
pub struct LayaConfig {
pub encoder: ModernBertConfig,
#[config(default = "2")]
pub head_layers: usize,
#[config(default = "2")]
pub n_act: usize,
#[config(default = "256")]
pub act_hidden: usize,
}
impl LayaConfig {
pub fn d_model(&self) -> usize {
self.encoder.d_model
}
pub fn head_layer(&self) -> ResidualEncoderAttentionBlockConfig {
let d = self.d_model();
let heads = (d / 64).max(1);
ResidualEncoderAttentionBlockConfig::new(d)
.with_d_head(d / heads)
.with_key_bias(true)
.with_activation(BunsenActivationConfig::Relu)
}
}
impl ModuleInit<LayaModel> for LayaConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<LayaModel> {
let d = self.d_model();
let head = (0..self.head_layers)
.map(|_| self.head_layer().try_init(device))
.collect::<BunsenResult<_>>()?;
Ok(LayaModel {
encoder: self.encoder.try_init(device)?,
type_emb: TokenEmbeddingConfig::new(LayaQuestionKind::ALL.len(), d).try_init(device)?,
head,
scorer: LayaScorer {
norm: LayerNorm::init(&LayerNormConfig::new(d), device),
dense: LinearConfig::new(d, d).init(device),
out: LinearConfig::new(d, 1).init(device),
},
act_head: LayaActHead {
dense: LinearConfig::new(d + 4, self.act_hidden).init(device),
out: LinearConfig::new(self.act_hidden, self.n_act).init(device),
},
})
}
}
#[derive(Module, Debug)]
pub struct LayaScorer {
pub norm: LayerNorm,
pub dense: Linear,
pub out: Linear,
}
impl LayaScorer {
pub fn forward<const D: usize>(
&self,
x: Tensor<D>,
) -> Tensor<D> {
self.out
.forward(gelu(self.dense.forward(self.norm.forward(x))))
}
}
#[derive(Module, Debug)]
pub struct LayaActHead {
pub dense: Linear,
pub out: Linear,
}
#[derive(Debug, Clone)]
pub struct LayaOutput {
pub logits: Tensor<2>,
pub act_logits: Tensor<2>,
}
#[derive(Module, Debug)]
pub struct LayaModel {
pub encoder: ModernBertModel,
pub type_emb: TokenEmbedding,
pub head: Vec<ResidualEncoderAttentionBlock>,
pub scorer: LayaScorer,
pub act_head: LayaActHead,
}
impl LayaModel {
pub fn d_model(&self) -> usize {
self.encoder.d_model()
}
pub fn head_states(
&self,
ids: Tensor<2, Int>,
lengths: &[usize],
kinds: Tensor<1, Int>,
) -> Tensor<3> {
let [b, t] = unpack_shape_contract!(["B", "T"], &ids.dims());
let device = ids.device();
let h = self.encoder.forward(ids, Some(lengths));
let h = h + self.type_emb.forward(kinds.unsqueeze_dim::<2>(1));
let padding = length_mask(lengths, t, &device)
.bool_not()
.unsqueeze_dim::<3>(1)
.expand([b, t, t]);
self.head
.iter()
.fold(h, |h, layer| layer.forward_masked(h, Some(padding.clone())))
}
pub fn forward(
&self,
ids: Tensor<2, Int>,
lengths: &[usize],
kinds: Tensor<1, Int>,
markers: Tensor<2, Int>,
options: Tensor<2, Bool>,
) -> LayaOutput {
let [b, k] = unpack_shape_contract!(["B", "K"], &markers.dims());
let d = self.d_model();
let h = self.head_states(ids, lengths, kinds);
let at = markers.clamp_min(0).unsqueeze_dim::<3>(2).expand([b, k, d]);
let marked = h.clone().gather(1, at);
let logits = self
.scorer
.forward(marked)
.squeeze_dim::<2>(2)
.cast(DType::F32)
.mask_fill(options.clone().bool_not(), -1e4);
let p = softmax(logits.clone(), 1);
let count = options
.int()
.sum_dim(1)
.float()
.cast(DType::F32)
.clamp_min(2.0);
let entropy = (p.clone() * p.clone().clamp_min(1e-9).log())
.sum_dim(1)
.neg()
/ count.clone().log();
let top = p.topk(2, 1);
let first = top.clone().slice_dim(1, 0..1);
let margin = first.clone() - top.slice_dim(1, 1..2);
let feats = Tensor::cat(vec![first, margin, entropy, count / 255.0], 1);
let pooled = h.slice_dim(1, 0..1).squeeze_dim::<2>(1).cast(DType::F32);
let act_in = Tensor::cat(vec![pooled, feats], 1)
.cast(param_float_dtype(&self.act_head.dense.weight));
let act_logits = self
.act_head
.out
.forward(gelu(self.act_head.dense.forward(act_in)))
.cast(DType::F32);
LayaOutput { logits, act_logits }
}
}
impl CanonicalKeys for LayaModel {
fn canonical_rekey(&self) -> Rekey {
Rekey::keep_rest()
.mount_canonical("encoder.", &self.encoder)
.mount_canonical("head.", &self.head)
}
}