use burn::config::Config;
use burn::module::{Module, Param};
use burn::nn::{
Dropout, DropoutConfig, EmbeddingConfig, Gelu, LayerNorm, LayerNormConfig, Linear, LinearConfig,
};
use burn::tensor::{Bool, Int, Tensor, backend::Backend};
use super::ConsFormerInputs;
use crate::neural::*;
#[derive(Module, Debug)]
pub struct AssignmentEmbedding<B: Backend> {
embedding: burn::nn::Embedding<B>,
}
#[derive(Config, Debug)]
pub(crate) struct AssignmentEmbeddingConfig {
pub domain_size: usize,
pub embedding_size: usize,
}
impl AssignmentEmbeddingConfig {
pub(crate) fn init<B: Backend>(&self, device: &B::Device) -> AssignmentEmbedding<B> {
AssignmentEmbedding {
embedding: EmbeddingConfig::new(self.domain_size, self.embedding_size).init(device),
}
}
}
impl<B: Backend> AssignmentEmbedding<B> {
pub fn forward(&self, x: Tensor<B, 2, Int>) -> Tensor<B, 3> {
self.embedding.forward(x)
}
}
#[derive(Module, Debug)]
pub struct FixedPositionalEmbedding<B: Backend> {
table: Tensor<B, 2>,
}
impl<B: Backend> FixedPositionalEmbedding<B> {
pub fn new(embedding_size: usize, device: &B::Device) -> Self {
let half = embedding_size / 2;
let inv_freq: Vec<f32> = (0..half)
.map(|i| 1.0f32 / 10000f32.powf((2 * i) as f32 / embedding_size as f32))
.collect();
let inv_freq: Tensor<B, 2> =
Tensor::<B, 1>::from_floats(inv_freq.as_slice(), device).reshape([1, half]);
let t: Tensor<B, 2> = Tensor::<B, 1, Int>::arange(0..16384, device)
.float()
.reshape([16384, 1]);
let sinusoid_inp = t.matmul(inv_freq);
let table = Tensor::cat(vec![sinusoid_inp.clone().sin(), sinusoid_inp.cos()], 1);
FixedPositionalEmbedding { table }
}
pub fn forward(&self, x: Tensor<B, 2, Int>) -> Tensor<B, 3> {
let [batch_size, number_variables] = x.dims();
let embedding_size = self.table.dims()[1];
let flat_ids: Tensor<B, 1, Int> = x.reshape([batch_size * number_variables]);
let gathered = self.table.clone().select(0, flat_ids);
gathered.reshape([batch_size, number_variables, embedding_size])
}
}
#[derive(Module, Debug)]
pub struct StructuredPositionalEmbedding<B: Backend> {
axes: Vec<FixedPositionalEmbedding<B>>,
#[module(skip)]
axis_ids: Vec<Vec<usize>>,
}
impl<B: Backend> StructuredPositionalEmbedding<B> {
pub fn new(
embedding_size: usize,
num_axes: usize,
positions: Vec<Vec<usize>>,
device: &B::Device,
) -> Self {
let axes = vec![FixedPositionalEmbedding::new(embedding_size, device); num_axes];
let axis_ids: Vec<Vec<usize>> = (0..num_axes)
.map(|a| positions.iter().map(|coords| coords[a]).collect())
.collect();
StructuredPositionalEmbedding { axes, axis_ids }
}
pub fn forward(&self, batch_size: usize, device: &B::Device) -> Tensor<B, 3> {
let mut acc: Option<Tensor<B, 3>> = None;
for (axis_embed, ids) in self.axes.iter().zip(self.axis_ids.iter()) {
let number_vars = ids.len();
let ids_flat: Vec<i64> = ids.iter().map(|&v| v as i64).collect();
let ids_2d: Tensor<B, 2, Int> =
Tensor::<B, 1, Int>::from_data(ids_flat.as_slice(), device)
.reshape([1, number_vars])
.repeat_dim(0, batch_size);
let embed = axis_embed.forward(ids_2d);
acc = Some(match acc {
None => embed,
Some(prev) => prev + embed,
});
}
acc.expect("PositionalStructure must have at least one axis")
}
}
#[derive(Module, Debug)]
pub struct EmbeddingMixer<B: Backend> {
assignment_weight: Param<Tensor<B, 3>>,
mask_weight: Param<Tensor<B, 3>>,
position_weight: Param<Tensor<B, 3>>,
}
#[derive(Config, Debug)]
pub(crate) struct EmbeddingMixerConfig {
pub embedding_size: usize,
}
impl EmbeddingMixerConfig {
pub(crate) fn init<B: Backend>(&self, device: &B::Device) -> EmbeddingMixer<B> {
EmbeddingMixer {
assignment_weight: Param::from_tensor(Tensor::ones([1, 1, 1], device)),
mask_weight: Param::from_tensor(Tensor::ones([1, 1, 1], device)),
position_weight: Param::from_tensor(Tensor::ones([1, 1, 1], device)),
}
}
}
impl<B: Backend> EmbeddingMixer<B> {
pub fn forward(
&self,
assignment_embeddings: Tensor<B, 3>,
mask_embedding: Tensor<B, 1>,
position_embeddings: Option<Tensor<B, 3>>,
mask_vars: Tensor<B, 2, Bool>,
) -> Tensor<B, 3> {
let [batch, seq_len, embed_dim] = assignment_embeddings.dims();
let mask_inds_3d = mask_vars.reshape([batch, seq_len, 1]);
let mask_embed_bc = mask_embedding.reshape([1, 1, embed_dim]);
let combined = match position_embeddings {
None => assignment_embeddings * self.assignment_weight.val(),
Some(pos) => {
assignment_embeddings * self.assignment_weight.val()
+ pos * self.position_weight.val()
}
};
let masked_component = mask_embed_bc * self.mask_weight.val();
let mask_inds_f = mask_inds_3d.float();
combined + masked_component * mask_inds_f
}
}
#[derive(Module, Debug)]
pub struct FeedForward<B: Backend> {
fc1: Linear<B>,
act: Gelu,
fc2: Linear<B>,
drop: Dropout,
}
#[derive(Config, Debug)]
pub(crate) struct FeedForwardConfig {
pub hidden_size: usize,
pub expand_size: usize,
#[config(default = 0.1)]
pub drop: f64,
#[config(default = true)]
pub bias: bool,
}
impl FeedForwardConfig {
pub(crate) fn init<B: Backend>(&self, device: &B::Device) -> FeedForward<B> {
FeedForward {
fc1: LinearConfig::new(self.hidden_size, self.expand_size)
.with_bias(self.bias)
.init(device),
act: Gelu::new(),
fc2: LinearConfig::new(self.expand_size, self.hidden_size)
.with_bias(self.bias)
.init(device),
drop: DropoutConfig::new(self.drop).init(),
}
}
}
impl<B: Backend> FeedForward<B> {
pub fn forward(&self, x: Tensor<B, 3>) -> Tensor<B, 3> {
let x = self.fc1.forward(x);
let x = self.act.forward(x);
let x = self.fc2.forward(x);
self.drop.forward(x)
}
}
#[derive(Module, Debug)]
pub struct LearnedMaskBias<B: Backend> {
connected: Param<Tensor<B, 1>>,
unconnected: Param<Tensor<B, 1>>,
}
const UNCONNECTED_BIAS_INIT: f32 = -1.0;
impl<B: Backend> LearnedMaskBias<B> {
fn new(head_count: usize, device: &B::Device) -> Self {
LearnedMaskBias {
connected: Param::from_tensor(Tensor::zeros([head_count], device)),
unconnected: Param::from_tensor(
Tensor::ones([head_count], device).mul_scalar(UNCONNECTED_BIAS_INIT),
),
}
}
fn bias_for(&self, mask: Tensor<B, 4, Bool>) -> Tensor<B, 4> {
let head_count = self.connected.val().dims()[0];
let mask_f = mask.float();
let connected = self.connected.val().reshape([1, head_count, 1, 1]);
let unconnected = self.unconnected.val().reshape([1, head_count, 1, 1]);
mask_f.clone() * connected + (mask_f.neg().add_scalar(1.0)) * unconnected
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
pub enum MaskingKind {
Hard,
Learned,
}
#[derive(Module, Debug)]
pub struct MultiHeadAttention<B: Backend> {
q_proj: Linear<B>,
k_proj: Linear<B>,
v_proj: Linear<B>,
drop_out: Dropout,
projection: Linear<B>,
hidden_size: usize,
head_count: usize,
learned_bias: Option<LearnedMaskBias<B>>,
}
#[derive(Config, Debug)]
pub(crate) struct MultiHeadAttentionConfig {
pub embedding_size: usize,
pub hidden_size: usize,
#[config(default = 8)]
pub head_count: usize,
#[config(default = 0.1)]
pub dropout: f64,
#[config(default = "MaskingKind::Hard")]
pub masking_kind: MaskingKind,
}
impl MultiHeadAttentionConfig {
pub(crate) fn init<B: Backend>(&self, device: &B::Device) -> MultiHeadAttention<B> {
MultiHeadAttention {
q_proj: LinearConfig::new(self.embedding_size, self.hidden_size * self.head_count)
.init(device),
k_proj: LinearConfig::new(self.embedding_size, self.hidden_size * self.head_count)
.init(device),
v_proj: LinearConfig::new(self.embedding_size, self.hidden_size * self.head_count)
.init(device),
drop_out: DropoutConfig::new(self.dropout).init(),
projection: LinearConfig::new(self.hidden_size * self.head_count, self.hidden_size)
.init(device),
hidden_size: self.hidden_size,
head_count: self.head_count,
learned_bias: match self.masking_kind {
MaskingKind::Hard => None,
MaskingKind::Learned => Some(LearnedMaskBias::new(self.head_count, device)),
},
}
}
}
impl<B: Backend> MultiHeadAttention<B> {
pub fn forward(&self, x: Tensor<B, 3>, attention_mask: Tensor<B, 3, Bool>) -> Tensor<B, 3> {
let [batch_size, number_vars, _] = x.dims();
let queries = self.reshape_qkv(self.q_proj.forward(x.clone()), batch_size, number_vars);
let keys = self.reshape_qkv(self.k_proj.forward(x.clone()), batch_size, number_vars);
let values = self.reshape_qkv(self.v_proj.forward(x), batch_size, number_vars);
let mut energy = queries.matmul(keys.transpose()) / (self.hidden_size as f64).sqrt();
let mask_4d: Tensor<B, 4, Bool> =
attention_mask.reshape([batch_size, 1, number_vars, number_vars]);
energy = match &self.learned_bias {
None => energy.mask_fill(mask_4d.bool_not(), f32::NEG_INFINITY),
Some(bias) => energy + bias.bias_for(mask_4d),
};
let attention = burn::tensor::activation::softmax(energy, 3);
let attention = self.drop_out.forward(attention);
let out = attention.matmul(values);
let out = out.swap_dims(1, 2);
let out = out.reshape([batch_size, number_vars, self.head_count * self.hidden_size]);
self.projection.forward(out)
}
fn reshape_qkv(&self, x: Tensor<B, 3>, batch_size: usize, number_vars: usize) -> Tensor<B, 4> {
x.reshape([batch_size, number_vars, self.head_count, self.hidden_size])
.swap_dims(1, 2)
}
}
#[derive(Module, Debug)]
pub struct TransformerBlock<B: Backend> {
norm1: LayerNorm<B>,
attn: MultiHeadAttention<B>,
norm2: LayerNorm<B>,
ffn: FeedForward<B>,
}
#[derive(Config, Debug)]
pub(crate) struct TransformerBlockConfig {
pub embedding_size: usize,
pub hidden_size: usize,
pub num_heads: usize,
pub expand_size: usize,
#[config(default = 0.1)]
pub attn_drop: f64,
#[config(default = 0.1)]
pub ffn_drop: f64,
#[config(default = true)]
pub bias: bool,
#[config(default = "MaskingKind::Hard")]
pub masking_kind: MaskingKind,
}
impl TransformerBlockConfig {
pub(crate) fn init<B: Backend>(&self, device: &B::Device) -> TransformerBlock<B> {
TransformerBlock {
norm1: LayerNormConfig::new(self.hidden_size).init(device),
attn: MultiHeadAttentionConfig::new(self.embedding_size, self.hidden_size)
.with_head_count(self.num_heads)
.with_dropout(self.attn_drop)
.with_masking_kind(self.masking_kind)
.init(device),
norm2: LayerNormConfig::new(self.hidden_size).init(device),
ffn: FeedForwardConfig::new(self.hidden_size, self.expand_size)
.with_drop(self.ffn_drop)
.with_bias(self.bias)
.init(device),
}
}
}
impl<B: Backend> TransformerBlock<B> {
pub fn forward(&self, x: Tensor<B, 3>, attention_mask: Tensor<B, 3, Bool>) -> Tensor<B, 3> {
let x = x.clone() + self.attn.forward(self.norm1.forward(x), attention_mask);
x.clone() + self.ffn.forward(self.norm2.forward(x))
}
}
#[derive(Module, Debug)]
pub struct ConsFormer<B: Backend> {
pub mask_embedding: Param<Tensor<B, 1>>,
pub assignment_embedding: AssignmentEmbedding<B>,
pub embedding_mixer: EmbeddingMixer<B>,
pub transformer_blocks: Vec<TransformerBlock<B>>,
pub head: Linear<B>,
pub position_embedding: Option<StructuredPositionalEmbedding<B>>,
#[module(skip)]
pub tau: f64,
}
impl<B: Backend, Ba: ConsFormerInputs<B>> Network<B, Ba> for ConsFormer<B> {
fn forward(&self, batch: &Ba) -> Tensor<B, 3> {
let assignments = batch.assignments();
let [batch_size, _seq_len] = assignments.dims();
let device = assignments.device();
let position_embeds = self
.position_embedding
.as_ref()
.map(|pe| pe.forward(batch_size, &device));
let x = self.assignment_embedding.forward(assignments);
let mut x = self.embedding_mixer.forward(
x,
self.mask_embedding.val(),
position_embeds,
batch.var_masks(),
);
for block in &self.transformer_blocks {
x = block.forward(x, batch.attention_masks());
}
self.head.forward(x).div_scalar(self.tau)
}
}
#[cfg(test)]
mod tests {
use super::*;
use burn::backend::ndarray::{NdArray, NdArrayDevice};
fn toy_mask<B: Backend>(device: &B::Device) -> Tensor<B, 3, Bool> {
let data = vec![
true, true, false, true, true, false, false, false, true,
];
Tensor::<B, 1, Bool>::from_data(data.as_slice(), device).reshape([1, 3, 3])
}
#[test]
fn learned_mask_bias_starts_near_hard_masking() {
let device = NdArrayDevice::default();
let bias = LearnedMaskBias::<NdArray>::new(4, &device);
let connected: Vec<f32> = bias.connected.val().into_data().to_vec::<f32>().unwrap();
let unconnected: Vec<f32> = bias.unconnected.val().into_data().to_vec::<f32>().unwrap();
assert_eq!(connected, vec![0.0; 4]);
assert_eq!(unconnected, vec![UNCONNECTED_BIAS_INIT; 4]);
}
#[test]
fn both_masking_kinds_produce_finite_output() {
let device = NdArrayDevice::default();
let mask = toy_mask::<NdArray>(&device);
for masking_kind in [MaskingKind::Hard, MaskingKind::Learned] {
let attn = MultiHeadAttentionConfig::new(8, 8)
.with_head_count(2)
.with_dropout(0.0)
.with_masking_kind(masking_kind)
.init::<NdArray>(&device);
let x: Tensor<NdArray, 3> = Tensor::random(
[1, 3, 8],
burn::tensor::Distribution::Uniform(-1.0, 1.0),
&device,
);
let out = attn.forward(x, mask.clone());
let data: Vec<f32> = out.into_data().to_vec::<f32>().unwrap();
assert!(
data.iter().all(|v| v.is_finite()),
"masking_kind={masking_kind:?} produced a non-finite output: {data:?}"
);
}
}
#[test]
fn learned_mask_bias_is_trainable() {
use burn::backend::Autodiff;
type ADBackend = Autodiff<NdArray>;
let device = NdArrayDevice::default();
let mask = toy_mask::<ADBackend>(&device).reshape([1, 1, 3, 3]);
let bias = LearnedMaskBias::<ADBackend>::new(2, &device);
let loss = bias.bias_for(mask).sum();
let grads = loss.backward();
let connected_grad = bias
.connected
.val()
.grad(&grads)
.expect("connected bias should have a gradient after backward()");
let unconnected_grad = bias
.unconnected
.val()
.grad(&grads)
.expect("unconnected bias should have a gradient after backward()");
let connected_grad: Vec<f32> = connected_grad.into_data().to_vec::<f32>().unwrap();
let unconnected_grad: Vec<f32> = unconnected_grad.into_data().to_vec::<f32>().unwrap();
assert!(connected_grad.iter().any(|&v| v != 0.0));
assert!(unconnected_grad.iter().any(|&v| v != 0.0));
}
}