use crate::init::InitializationError;
use crate::module::{Module, ModuleError};
use coeus_autograd::Var;
use coeus_core::{Float, MoiraiBackend, Scalar};
use coeus_ops::RandomInitOps;
use coeus_tensor::Tensor;
#[derive(Clone)]
pub struct Linear<T: Scalar, B: coeus_ops::BackendOps<T> + Default = MoiraiBackend> {
pub weight: Var<T, B>,
pub bias: Option<Var<T, B>>,
}
impl<T, B> Linear<T, B>
where
T: Scalar + Float + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + RandomInitOps<T> + Default,
{
pub fn new(
in_features: usize,
out_features: usize,
bias: bool,
) -> Result<Self, InitializationError<B::Error>> {
Self::build(in_features, out_features, bias, None)
}
pub fn with_seed(
in_features: usize,
out_features: usize,
bias: bool,
seed: u64,
) -> Result<Self, InitializationError<B::Error>> {
Self::build(in_features, out_features, bias, Some(seed))
}
fn build(
in_features: usize,
out_features: usize,
bias: bool,
seed: Option<u64>,
) -> Result<Self, InitializationError<B::Error>> {
let backend = B::default();
let w_tensor = Tensor::zeros_on([out_features, in_features], &backend)?;
let mut weight = Var::new(w_tensor, true)?;
match seed {
Some(seed) => crate::init::kaiming_uniform_with_seed(&mut weight, in_features, seed)?,
None => crate::init::kaiming_uniform(&mut weight, in_features)?,
}
let bias_var = if bias {
Some(Var::new(Tensor::zeros_on([out_features], &backend)?, true)?)
} else {
None
};
Ok(Self {
weight,
bias: bias_var,
})
}
}
impl<T: Scalar + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B>
for Linear<T, B>
{
fn parameters(&self) -> Vec<Var<T, B>> {
let mut params = vec![self.weight.clone()];
if let Some(ref b) = self.bias {
params.push(b.clone());
}
params
}
fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
let input_shape = input.tensor.shape();
if input_shape.len() < 2 {
return Err(ModuleError::InvalidRank {
module: "Linear",
expected: "at least 2",
actual: input_shape.len(),
});
}
let in_features = self.weight.tensor.shape()[1];
let actual_features = input_shape[input_shape.len() - 1];
if actual_features != in_features {
return Err(ModuleError::ShapeMismatch {
module: "Linear",
parameter: "input trailing dimension",
expected: vec![in_features],
actual: vec![actual_features],
});
}
let rows = input_shape[..input_shape.len() - 1]
.iter()
.copied()
.product::<usize>();
let w_t = coeus_autograd::transpose_2d(&self.weight)?;
let projected = if input_shape.len() == 2 {
coeus_autograd::matmul(input, &w_t)?
} else {
let flattened = coeus_autograd::reshape(input, [rows, in_features])?;
coeus_autograd::matmul(&flattened, &w_t)?
};
let projected = if let Some(ref bias) = self.bias {
coeus_autograd::add(&projected, bias)?
} else {
projected
};
let output = if input_shape.len() == 2 {
projected
} else {
let mut output_shape = input_shape.to_vec();
*output_shape
.last_mut()
.expect("invariant: rank was validated as at least two") =
self.weight.tensor.shape()[0];
coeus_autograd::reshape(&projected, output_shape)?
};
Ok(output)
}
fn load_parameters(&mut self, params: &[Var<T, B>]) {
self.weight = params[0].clone();
if self.bias.is_some() {
self.bias = Some(params[1].clone());
}
}
}