use burn::{
Tensor,
config::Config,
module::Module,
nn::{
Linear,
LinearConfig,
},
tensor::{
Device,
activation::{
sigmoid,
tanh,
},
},
};
use crate::burner::module::{
ModuleInit,
Rekey,
};
pub trait FusedLstmMeta {
fn d_model(&self) -> usize;
}
#[derive(Config, Debug)]
pub struct FusedLstmConfig {
pub d_model: usize,
#[config(default = "true")]
pub bias: bool,
#[config(default = "burn::nn::LinearLayout::Row")]
pub layout: burn::nn::LinearLayout,
}
impl From<FusedLstmConfig> for LinearConfig {
fn from(config: FusedLstmConfig) -> Self {
LinearConfig::new(config.d_model, 4 * config.d_model)
.with_bias(config.bias)
.with_layout(config.layout)
}
}
impl FusedLstmMeta for FusedLstmConfig {
fn d_model(&self) -> usize {
self.d_model
}
}
impl ModuleInit<FusedLstm> for FusedLstmConfig {
fn try_init(
&self,
device: &Device,
) -> crate::errors::BunsenResult<FusedLstm> {
let cfg: LinearConfig = self.clone().into();
Ok(FusedLstm {
features: cfg.init(device),
hidden: cfg.init(device),
})
}
}
#[derive(Module, Debug)]
pub struct FusedLstm {
pub features: Linear,
pub hidden: Linear,
}
impl FusedLstmMeta for FusedLstm {
fn d_model(&self) -> usize {
self.features.weight.dims()[0]
}
}
impl FusedLstm {
pub fn step(
&self,
features: Tensor<2>,
hidden: Tensor<2>,
cell: Tensor<2>,
) -> (Tensor<2>, Tensor<2>) {
#[cfg(any(test, debug_assertions))]
use crate::contracts::{
assert_shape_contract_periodically,
unpack_shape_contract,
};
#[cfg(any(test, debug_assertions))]
let batch = {
let [batch] = unpack_shape_contract!(
["batch", "d_model"],
&features,
&["batch"],
&[("d_model", self.d_model())],
);
assert_shape_contract_periodically!(
["batch", "d_model"],
&cell,
&[("batch", batch), ("d_model", self.d_model())]
);
assert_shape_contract_periodically!(
["batch", "d_model"],
&hidden,
&[("batch", batch), ("d_model", self.d_model())]
);
batch
};
let gates = self.features.forward(features) + self.hidden.forward(hidden);
#[cfg(any(test, debug_assertions))]
assert_shape_contract_periodically!(
["batch", 4 * "d_model"],
&gates,
&[("batch", batch), ("d_model", self.d_model())]
);
let [g_i_f, g_c_o] = gates.chunk(2, 1).try_into().unwrap();
let [g_c, g_o] = g_c_o.chunk(2, 1).try_into().unwrap();
let [input_values, forget_values] = sigmoid(g_i_f).chunk(2, 1).try_into().unwrap();
let candidate_cell_values = tanh(g_c);
let output_values = sigmoid(g_o);
let cell = forget_values * cell + input_values * candidate_cell_values;
let hidden = output_values * tanh(cell.clone());
(hidden, cell)
}
}
impl FusedLstm {
pub fn torch_cell_rekey() -> Rekey {
Rekey::drop_rest()
.exact("weight_ih", "features.weight")
.exact("bias_ih", "features.bias")
.exact("weight_hh", "hidden.weight")
.exact("bias_hh", "hidden.bias")
}
pub fn torch_stack_rekey() -> Rekey {
let cell = Self::torch_cell_rekey();
Rekey::drop_rest().custom(move |name| {
let (param, layer) = name.rsplit_once("_l")?;
Some(cell.apply(param).map(|path| format!("{layer}.{path}")))
})
}
}