use burn::{
Tensor,
config::Config,
module::Module,
nn::{
Linear,
LinearConfig,
},
prelude::Backend,
tensor::activation::{
sigmoid,
tanh,
},
};
use crate::burner::module::ModuleInit;
pub trait FusedLstmMeta {
fn d_model(&self) -> usize;
}
#[derive(Config, Debug)]
pub struct FusedLstmConfig {
pub d_model: usize,
#[config(default = "true")]
pub bias: bool,
#[config(default = "burn::nn::LinearLayout::Row")]
pub layout: burn::nn::LinearLayout,
}
impl From<FusedLstmConfig> for LinearConfig {
fn from(config: FusedLstmConfig) -> Self {
LinearConfig::new(config.d_model, 4 * config.d_model)
.with_bias(config.bias)
.with_layout(config.layout)
}
}
impl FusedLstmMeta for FusedLstmConfig {
fn d_model(&self) -> usize {
self.d_model
}
}
impl<B: Backend> ModuleInit<B, FusedLstm<B>> for FusedLstmConfig {
fn try_init(
&self,
device: &B::Device,
) -> crate::errors::BunsenResult<FusedLstm<B>> {
let cfg: LinearConfig = self.clone().into();
Ok(FusedLstm {
features: cfg.init(device),
hidden: cfg.init(device),
})
}
}
#[derive(Module, Debug)]
pub struct FusedLstm<B: Backend> {
pub features: Linear<B>,
pub hidden: Linear<B>,
}
impl<B: Backend> FusedLstmMeta for FusedLstm<B> {
fn d_model(&self) -> usize {
self.features.weight.dims()[0]
}
}
impl<B: Backend> FusedLstm<B> {
pub fn step(
&self,
features: Tensor<B, 2>,
hidden: Tensor<B, 2>,
cell: Tensor<B, 2>,
) -> (Tensor<B, 2>, Tensor<B, 2>) {
#[cfg(any(test, debug_assertions))]
use crate::contracts::{
assert_shape_contract_periodically,
unpack_shape_contract,
};
#[cfg(any(test, debug_assertions))]
let batch = {
let [batch] = unpack_shape_contract!(
["batch", "d_model"],
&features,
&["batch"],
&[("d_model", self.d_model())],
);
assert_shape_contract_periodically!(
["batch", "d_model"],
&cell,
&[("batch", batch), ("d_model", self.d_model())]
);
assert_shape_contract_periodically!(
["batch", "d_model"],
&hidden,
&[("batch", batch), ("d_model", self.d_model())]
);
batch
};
let gates = self.features.forward(features) + self.hidden.forward(hidden);
#[cfg(any(test, debug_assertions))]
assert_shape_contract_periodically!(
["batch", 4 * "d_model"],
&gates,
&[("batch", batch), ("d_model", self.d_model())]
);
let [g_i_f, g_c_o] = gates.chunk(2, 1).try_into().unwrap();
let [g_c, g_o] = g_c_o.chunk(2, 1).try_into().unwrap();
let [input_values, forget_values] = sigmoid(g_i_f).chunk(2, 1).try_into().unwrap();
let candidate_cell_values = tanh(g_c);
let output_values = sigmoid(g_o);
let cell = forget_values * cell + input_values * candidate_cell_values;
let hidden = output_values * tanh(cell.clone());
(hidden, cell)
}
}