use crate::linear::Linear;
use crate::module::{prefixed_parameters, Module, ModuleError, ModuleResult};
use crate::rnn::validation;
use crate::rnn::VarPair;
use coeus_autograd::Var;
use coeus_core::{Float, MoiraiBackend};
use coeus_tensor::Tensor;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum RnnNonlinearity {
Tanh,
Relu,
}
#[derive(Clone)]
pub struct RNNCell<T: Float, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
pub w_ih: Linear<T, B>,
pub w_hh: Linear<T, B>,
pub nonlinearity: RnnNonlinearity,
pub input_size: usize,
pub hidden_size: usize,
}
impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> RNNCell<T, B> {
pub fn new(
input_size: usize,
hidden_size: usize,
nonlinearity: RnnNonlinearity,
) -> Result<Self, crate::init::InitializationError<B::Error>>
where
B: coeus_ops::RandomInitOps<T>,
{
Ok(Self {
w_ih: Linear::new(input_size, hidden_size, true)?,
w_hh: Linear::new(hidden_size, hidden_size, true)?,
nonlinearity,
input_size,
hidden_size,
})
}
pub fn step(&self, x: &Var<T, B>, h: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
let batch = validation::cell_input(x.tensor.shape(), self.input_size, "RNNCell")?;
validation::state(
h.tensor.shape(),
batch,
self.hidden_size,
"RNNCell",
"hidden state",
)?;
self.step_validated(x, h)
}
fn step_validated(
&self,
x: &Var<T, B>,
h: &Var<T, B>,
) -> Result<Var<T, B>, ModuleError<B::Error>> {
let input = self.w_ih.forward(x)?;
let hidden = self.w_hh.forward(h)?;
let pre = coeus_autograd::add(&input, &hidden)?;
Ok(match self.nonlinearity {
RnnNonlinearity::Tanh => coeus_autograd::tanh(&pre)?,
RnnNonlinearity::Relu => coeus_autograd::relu(&pre)?,
})
}
}
impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B>
for RNNCell<T, B>
{
fn parameters(&self) -> Vec<Var<T, B>> {
let mut p = self.w_ih.parameters();
p.extend(self.w_hh.parameters());
p
}
fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
let mut parameters = prefixed_parameters("input", &self.w_ih);
parameters.extend(prefixed_parameters("hidden", &self.w_hh));
parameters
}
fn forward(&self, x: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
let batch = validation::cell_input(x.tensor.shape(), self.input_size, "RNNCell")?;
let backend = B::default();
let h = Var::new(
Tensor::zeros_on([batch, self.hidden_size], &backend)?,
false,
)?;
self.step_validated(x, &h)
}
}
#[derive(Clone)]
pub struct Rnn<T: Float, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
cell: RNNCell<T, B>,
pub input_size: usize,
pub hidden_size: usize,
}
impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Rnn<T, B>
where
B::DeviceBuffer<T>:
coeus_core::CpuAddressableStorage<T> + coeus_core::CpuAddressableStorageMut<T>,
{
pub fn new(
input_size: usize,
hidden_size: usize,
nonlinearity: RnnNonlinearity,
) -> Result<Self, crate::init::InitializationError<B::Error>>
where
B: coeus_ops::RandomInitOps<T>,
{
Ok(Self {
cell: RNNCell::new(input_size, hidden_size, nonlinearity)?,
input_size,
hidden_size,
})
}
pub fn forward_seq(&self, x: &Var<T, B>) -> ModuleResult<VarPair<T, B>, B> {
let (batch, seq_len) =
validation::sequence_input(x.tensor.shape(), self.input_size, "Rnn")?;
let backend = B::default();
let mut h = Var::new(
Tensor::zeros_on([batch, self.hidden_size], &backend)?,
false,
)?;
let mut outputs: Vec<Var<T, B>> = Vec::with_capacity(seq_len);
for t in 0..seq_len {
let x_t_3d = coeus_autograd::slice(x, &[(0, batch), (t, t + 1), (0, self.input_size)])?;
let x_t = coeus_autograd::reshape(&x_t_3d, vec![batch, self.input_size])?;
let h_new = self.cell.step_validated(&x_t, &h)?;
outputs.push(coeus_autograd::reshape(
&h_new,
vec![batch, 1, self.hidden_size],
)?);
h = h_new;
}
let refs: Vec<&Var<T, B>> = outputs.iter().collect();
let output = coeus_autograd::cat(&refs, 1)?;
Ok((output, h))
}
}
impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B>
for Rnn<T, B>
where
B::DeviceBuffer<T>:
coeus_core::CpuAddressableStorage<T> + coeus_core::CpuAddressableStorageMut<T>,
{
fn parameters(&self) -> Vec<Var<T, B>> {
self.cell.parameters()
}
fn named_parameters(&self) -> Vec<coeus_autograd::Parameter<T, B>> {
prefixed_parameters("cell", &self.cell)
}
fn forward(&self, x: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
Ok(self.forward_seq(x)?.0)
}
}