mod architecture;
mod dataset;
use super::*;
use crate::modelling::Problem;
use architecture::*;
pub use architecture::{ConsFormer, MaskingKind};
pub use dataset::{ConsFormerBatch, ConsFormerBatcher, ConsFormerDataset, ConsFormerSample};
use burn::config::Config;
use burn::module::Param;
use burn::nn::LinearConfig;
use burn::tensor::Bool;
use std::sync::Arc;
pub trait ConsFormerInputs<B: Backend>: Batch<B> {
fn attention_masks(&self) -> Tensor<B, 3, Bool>;
}
#[derive(Config, Debug)]
pub struct ConsFormerConfig {
pub domain_size: usize,
pub embedding_size: usize,
pub hidden_size: usize,
pub num_heads: usize,
pub expand_size: usize,
#[config(default = 1)]
pub num_layers: usize,
#[config(default = 0.0)]
pub drop_out: f64,
#[config(default = true)]
pub bias: bool,
pub positional_encoding_dimensions: usize,
pub mask_fraction: f64,
pub tau: f64,
#[config(default = "MaskingKind::Hard")]
pub masking_kind: MaskingKind,
}
impl<B: Backend> NetworkConfig<B> for ConsFormerConfig {
type N = ConsFormer<B>;
fn init(&self, problems: &[Arc<Problem>], device: &B::Device) -> Self::N {
if self.hidden_size != self.embedding_size {
panic!(
"ConsFormer architecture initialised with hidden size different than embedding size ({} != {}). Currently, it is not supported.",
self.hidden_size, self.embedding_size
);
}
let transformer_blocks = (0..self.num_layers)
.map(|_| {
TransformerBlockConfig::new(
self.embedding_size,
self.hidden_size,
self.num_heads,
self.expand_size,
)
.with_attn_drop(self.drop_out)
.with_ffn_drop(self.drop_out)
.with_bias(self.bias)
.with_masking_kind(self.masking_kind)
.init(device)
})
.collect();
let position_embedding = if self.positional_encoding_dimensions == 0 {
None
} else {
dataset::shared_number_variables(problems);
let positions = problems[0]
.iter_variables()
.map(|variable| problems[0][variable].position().clone().unwrap())
.collect();
Some(StructuredPositionalEmbedding::new(
self.embedding_size,
self.positional_encoding_dimensions,
positions,
device,
))
};
ConsFormer {
mask_embedding: Param::from_tensor(Tensor::random(
[self.embedding_size],
burn::tensor::Distribution::Uniform(-1.0, 1.0),
device,
)),
assignment_embedding: AssignmentEmbeddingConfig::new(
self.domain_size,
self.embedding_size,
)
.init(device),
embedding_mixer: EmbeddingMixerConfig::new(self.embedding_size).init(device),
transformer_blocks,
head: LinearConfig::new(self.hidden_size, self.domain_size).init(device),
position_embedding,
tau: self.tau,
}
}
}