use burn::{
Tensor,
config::Config,
module::Module,
nn::{
Linear,
LinearConfig,
},
tensor::Device,
};
use crate::{
blocks::activation::{
BunsenActivation,
BunsenActivationConfig,
},
burner::module::ModuleInit,
errors::BunsenResult,
};
pub trait AdaptiveScaleMeta {
fn d_model(&self) -> usize;
fn d_hidden(&self) -> usize;
}
#[derive(Config, Debug)]
pub struct AdaptiveScaleConfig {
pub d_model: usize,
pub d_hidden: usize,
#[config(default = "BunsenActivationConfig::Gelu")]
pub activation: BunsenActivationConfig,
#[config(default = "false")]
pub bias: bool,
}
impl AdaptiveScaleMeta for AdaptiveScaleConfig {
fn d_model(&self) -> usize {
self.d_model
}
fn d_hidden(&self) -> usize {
self.d_hidden
}
}
impl ModuleInit<AdaptiveScale> for AdaptiveScaleConfig {
fn try_init(
&self,
device: &Device,
) -> BunsenResult<AdaptiveScale> {
Ok(AdaptiveScale {
linear1: LinearConfig::new(self.d_model, self.d_hidden)
.with_bias(self.bias)
.init(device),
act: self.activation.init(device),
linear2: LinearConfig::new(self.d_hidden, self.d_model)
.with_bias(self.bias)
.init(device),
})
}
}
#[derive(Module, Debug)]
pub struct AdaptiveScale {
pub linear1: Linear,
pub act: BunsenActivation,
pub linear2: Linear,
}
impl crate::burner::module::InputDTypes for AdaptiveScale {
fn preferred_dtype(&self) -> burn::tensor::FloatDType {
burn::tensor::FloatDType::from(self.linear1.weight.val().dtype())
}
}
impl crate::burner::module::CoercesInput for AdaptiveScale {}
impl crate::burner::module::ResultDType for AdaptiveScale {
fn result_dtype(
&self,
_input: burn::tensor::FloatDType,
) -> burn::tensor::FloatDType {
crate::burner::module::InputDTypes::preferred_dtype(self)
}
}
impl AdaptiveScaleMeta for AdaptiveScale {
fn d_model(&self) -> usize {
self.linear1.weight.dims()[0]
}
fn d_hidden(&self) -> usize {
self.linear1.weight.dims()[1]
}
}
impl AdaptiveScale {
pub fn scale(
&self,
cond: Tensor<3>,
) -> Tensor<3> {
let [batch, seq] = crate::contracts::unpack_shape_contract!(
["batch", "seq", "d_model"],
&cond.dims(),
&["batch", "seq"],
&[("d_model", self.d_model())]
);
let cond = crate::burner::module::InputDTypes::coerce_input(self, cond);
let y = self
.linear2
.forward(self.act.forward(self.linear1.forward(cond)))
.add_scalar(1.0);
#[cfg(any(debug_assertions, test))]
crate::contracts::assert_shape_contract_periodically!(
["batch", "seq", "d_model"],
&y.dims(),
&[("batch", batch), ("seq", seq), ("d_model", self.d_model())]
);
#[cfg(not(any(debug_assertions, test)))]
let _ = (batch, seq);
y
}
pub fn forward(
&self,
x: Tensor<3>,
cond: Tensor<3>,
) -> Tensor<3> {
crate::contracts::assert_shape_contract_periodically!(
["batch", "seq", "d_model"],
&x.dims(),
&[("d_model", self.d_model())]
);
x * self.scale(cond)
}
}
#[cfg(test)]
mod tests {
use burn::{
module::Param,
tensor::{
Distribution,
Tolerance,
},
};
use super::*;
use crate::support::testing::{
assert_tensors_close,
cpu_device,
seeded_tensor,
};
#[test]
fn test_meta() {
let config = AdaptiveScaleConfig::new(12, 4)
.with_activation(BunsenActivationConfig::Relu)
.with_bias(true);
let block: AdaptiveScale = config.init(&cpu_device());
assert_eq!(block.d_model(), config.d_model());
assert_eq!(block.d_hidden(), config.d_hidden());
assert!(block.linear1.bias.is_some() && block.linear2.bias.is_some());
assert!(matches!(block.act, BunsenActivation::Relu(_)));
}
#[test]
fn test_scale() {
let device = cpu_device();
let mut block: AdaptiveScale = AdaptiveScaleConfig::new(6, 3).init(&device);
let x = seeded_tensor::<3>(1, [2, 5, 6], Distribution::Default, &device);
let cond = seeded_tensor::<3>(2, [1, 1, 6], Distribution::Default, &device);
let expected = x.clone()
* (block.linear2.forward(burn::tensor::activation::gelu(
block.linear1.forward(cond.clone()),
)) + 1.0);
assert_tensors_close(
&block.forward(x.clone(), cond.clone()),
&expected,
Tolerance::default(),
);
block.linear2.weight = Param::from_tensor(block.linear2.weight.val().zeros_like());
assert_tensors_close(&block.forward(x.clone(), cond), &x, Tolerance::default());
}
}