use crate::err::{Error, Result};
use crate::nn::module::{Module, ShapeInfo, TrainMode};
use crate::shapes::{DynShape, Shape};
use crate::tensor::base::Tensor;
use crate::tensor::device::Device;
use crate::tensor::grad::RequiresGrad;
use alloc::{
string::{String, ToString},
vec::Vec,
};
#[derive(Debug, Clone, Default)]
pub struct ReLU;
macro_rules! impl_stateless_shape_info {
($($ty:ty),+ $(,)?) => {
$(impl ShapeInfo for $ty {
fn shape_info(&self) -> Option<String> {
None
}
})+
};
}
impl_stateless_shape_info!(ReLU);
impl TrainMode for ReLU {}
use crate::exec::catalog::op;
use crate::tensor::backend::Execute;
impl<
S: Shape + DynShape,
B: crate::tensor::backend::VariableBackend + Execute<op::Relu>,
G: RequiresGrad,
> Module<Tensor<S, B, f32, G>> for ReLU
where
<B as Execute<op::Relu>>::Output: Into<B::Storage<f32>>,
{
type Output = Tensor<S, B, f32, G>;
type Error = Error;
#[inline]
fn forward(
&self,
x: Tensor<S, B, f32, G>,
) -> core::result::Result<Tensor<S, B, f32, G>, Error> {
x.relu()
}
}
#[derive(Debug, Clone, Default)]
pub struct GELU;
impl_stateless_shape_info!(GELU);
impl TrainMode for GELU {}
impl<S: Shape + DynShape, B: crate::tensor::backend::VariableBackend + Execute<op::Gelu>>
Module<Tensor<S, B>> for GELU
where
<B as Execute<op::Gelu>>::Output: Into<B::Storage<f32>>,
{
type Output = Tensor<S, B>;
type Error = Error;
#[inline]
fn forward(&self, x: Tensor<S, B>) -> core::result::Result<Tensor<S, B>, Error> {
x.gelu()
}
}
#[derive(Debug, Clone, Default)]
pub struct Swish;
impl_stateless_shape_info!(Swish);
impl TrainMode for Swish {}
impl<S: Shape + DynShape, B: crate::tensor::backend::VariableBackend + Execute<op::Swish>>
Module<Tensor<S, B>> for Swish
where
<B as Execute<op::Swish>>::Output: Into<B::Storage<f32>>,
{
type Output = Tensor<S, B>;
type Error = Error;
#[inline]
fn forward(&self, x: Tensor<S, B>) -> core::result::Result<Tensor<S, B>, Error> {
x.swish()
}
}
#[derive(Debug, Clone, Default)]
pub struct Mish;
impl_stateless_shape_info!(Mish);
impl TrainMode for Mish {}
impl<S: Shape + DynShape, B: crate::tensor::backend::VariableBackend + Execute<op::Mish>>
Module<Tensor<S, B>> for Mish
where
<B as Execute<op::Mish>>::Output: Into<B::Storage<f32>>,
{
type Output = Tensor<S, B>;
type Error = Error;
#[inline]
fn forward(&self, x: Tensor<S, B>) -> core::result::Result<Tensor<S, B>, Error> {
x.mish()
}
}
#[derive(Debug, Clone, Default)]
#[allow(clippy::upper_case_acronyms)]
pub struct ELU;
impl_stateless_shape_info!(ELU);
impl TrainMode for ELU {}
impl<S: Shape + DynShape, B: crate::tensor::backend::VariableBackend + Execute<op::Elu>>
Module<Tensor<S, B>> for ELU
where
<B as Execute<op::Elu>>::Output: Into<B::Storage<f32>>,
{
type Output = Tensor<S, B>;
type Error = Error;
#[inline]
fn forward(&self, x: Tensor<S, B>) -> core::result::Result<Tensor<S, B>, Error> {
x.elu()
}
}
#[derive(Debug, Clone)]
pub struct Softmax {
pub dim: usize,
}
impl ShapeInfo for Softmax {
fn shape_info(&self) -> Option<String> {
None
}
}
impl Softmax {
pub fn new(dim: usize) -> Self {
Self { dim }
}
}
impl TrainMode for Softmax {}
impl<
S: Shape + DynShape + crate::shapes::RuntimeRankProjection,
B: crate::tensor::backend::VariableBackend
+ crate::tensor::backend::Execute<crate::exec::catalog::op::Softmax>,
> Module<Tensor<S, B>> for Softmax
where
<B as crate::tensor::backend::Execute<crate::exec::catalog::op::Softmax>>::Output:
Into<B::Storage<f32>>,
{
type Output = Tensor<S, B>;
type Error = Error;
#[inline]
fn forward(&self, x: Tensor<S, B>) -> core::result::Result<Tensor<S, B>, Error> {
x.softmax(self.dim as isize)
}
}
#[derive(Debug, Clone, Default)]
pub struct Sigmoid;
impl_stateless_shape_info!(Sigmoid);
impl TrainMode for Sigmoid {}
impl<S: Shape + DynShape, B: crate::tensor::backend::VariableBackend + Execute<op::Sigmoid>>
Module<Tensor<S, B>> for Sigmoid
where
<B as Execute<op::Sigmoid>>::Output: Into<B::Storage<f32>>,
{
type Output = Tensor<S, B>;
type Error = Error;
#[inline]
fn forward(&self, x: Tensor<S, B>) -> core::result::Result<Tensor<S, B>, Error> {
x.sigmoid()
}
}
#[derive(Debug, Clone, Default)]
pub struct Tanh;
impl_stateless_shape_info!(Tanh);
impl TrainMode for Tanh {}
impl<S: Shape + DynShape, B: crate::tensor::backend::VariableBackend + Execute<op::Tanh>>
Module<Tensor<S, B>> for Tanh
where
<B as Execute<op::Tanh>>::Output: Into<B::Storage<f32>>,
{
type Output = Tensor<S, B>;
type Error = Error;
#[inline]
fn forward(&self, x: Tensor<S, B>) -> core::result::Result<Tensor<S, B>, Error> {
x.tanh()
}
}
macro_rules! impl_stateless_state_visitors {
($($t:ty),+ $(,)?) => {$ (
impl<B: crate::tensor::backend::VariableBackend> crate::nn::VisitState<B> for $t {
fn visit_state<V: crate::nn::StateVisitor<B>>(
&self,
_: &crate::nn::StatePath,
_: &mut V,
) -> crate::err::Result<()> { Ok(()) }
}
impl<B: crate::tensor::backend::VariableBackend> crate::nn::VisitStateMut<B> for $t {
fn visit_state_mut<V: crate::nn::StateMutVisitor<B>>(
&mut self,
_: &crate::nn::StatePath,
_: &mut V,
) -> crate::err::Result<()> { Ok(()) }
}
impl<B: crate::tensor::backend::VariableBackend> crate::nn::VisitParameters<B> for $t {
fn visit_parameters<V: crate::nn::ParameterVisitor<B>>(
&self,
_: &crate::nn::StatePath,
_: &mut V,
) -> crate::err::Result<()> { Ok(()) }
}
)+ };
}
impl_stateless_state_visitors!(ReLU, GELU, Swish, Mish, ELU, Softmax, Sigmoid, Tanh);
impl crate::nn::module::NamedLayers for ReLU {
fn layer_structure(&self, prefix: &str) -> Vec<crate::nn::module::LayerNode> {
vec![crate::nn::module::LayerNode {
name: prefix.to_string(),
type_name: alloc::string::String::from("ReLU"),
shape_info: "".to_string(),
children: vec![],
}]
}
}
impl crate::nn::module::NamedLayers for GELU {
fn layer_structure(&self, prefix: &str) -> Vec<crate::nn::module::LayerNode> {
vec![crate::nn::module::LayerNode {
name: prefix.to_string(),
type_name: alloc::string::String::from("GELU"),
shape_info: "".to_string(),
children: vec![],
}]
}
}
impl crate::nn::module::NamedLayers for Swish {
fn layer_structure(&self, prefix: &str) -> Vec<crate::nn::module::LayerNode> {
vec![crate::nn::module::LayerNode {
name: prefix.to_string(),
type_name: alloc::string::String::from("Swish"),
shape_info: "".to_string(),
children: vec![],
}]
}
}
impl crate::nn::module::NamedLayers for Sigmoid {
fn layer_structure(&self, prefix: &str) -> Vec<crate::nn::module::LayerNode> {
vec![crate::nn::module::LayerNode {
name: prefix.to_string(),
type_name: alloc::string::String::from("Sigmoid"),
shape_info: "".to_string(),
children: vec![],
}]
}
}
impl crate::nn::module::NamedLayers for Tanh {
fn layer_structure(&self, prefix: &str) -> Vec<crate::nn::module::LayerNode> {
vec![crate::nn::module::LayerNode {
name: prefix.to_string(),
type_name: alloc::string::String::from("Tanh"),
shape_info: "".to_string(),
children: vec![],
}]
}
}
impl crate::nn::module::NamedLayers for Softmax {
fn layer_structure(&self, prefix: &str) -> Vec<crate::nn::module::LayerNode> {
vec![crate::nn::module::LayerNode {
name: prefix.to_string(),
type_name: alloc::string::String::from("Softmax"),
shape_info: format!("dim={}", self.dim),
children: vec![],
}]
}
}
macro_rules! impl_unit_to_device {
($($t:ty),+) => {
$(
impl<B: crate::tensor::backend::VariableBackend, NewD: Device> crate::tensor::transfer::ToDevice<B, NewD> for $t {
type Output = $t;
fn to_device(self, _arg: &NewD::Arg) -> Result<Self::Output> {
Ok(self)
}
}
)+
};
}
impl_unit_to_device!(ReLU, GELU, Swish, Mish, ELU, Softmax, Sigmoid, Tanh);