use super::Module;
use crate::{Result, RustTorchError};
use tch::Tensor;
fn defined(input: &Tensor) -> Result<()> {
if input.defined() {
Ok(())
} else {
Err(RustTorchError::InvalidConfiguration {
field: "activation input",
reason: "must be a defined tensor".to_owned(),
})
}
}
fn floating(input: &Tensor) -> Result<()> {
defined(input)?;
if input.f_is_floating_point()? {
Ok(())
} else {
Err(RustTorchError::InvalidConfiguration {
field: "activation dtype",
reason: "requires a real floating-point tensor".to_owned(),
})
}
}
macro_rules! activation {
($name:ident, $operation:ident, $description:literal) => {
#[doc = $description]
#[doc = concat!("let output = rusttorch::nn::", stringify!($name), ".forward(&Tensor::from_slice(&[-1_f32, 0., 1.]))?;")]
#[derive(Debug, Default, Clone, Copy)]
pub struct $name;
impl Module for $name {
fn forward(&self, input: &Tensor) -> Result<Tensor> { defined(input)?; Ok(input.$operation()?) }
}
};
}
activation!(
Sigmoid,
f_sigmoid,
"Maps logits into `[0, 1]` with `1 / (1 + exp(-x))`; useful for binary predictions."
);
activation!(
Tanh,
f_tanh,
"Maps real features into `[-1, 1]`; useful in bounded predictions and recurrent models."
);
activation!(
SiLU,
f_silu,
"Applies `x * sigmoid(x)`, a smooth nonlinearity for hidden image or token features."
);
macro_rules! softmax {
($name:ident, $operation:ident, $description:literal) => {
#[doc = $description]
#[doc = concat!("let layer = rusttorch::nn::", stringify!($name), "::new(-1);")]
#[derive(Debug, Clone, Copy)]
pub struct $name {
dim: i64,
}
impl $name {
pub const fn new(dim: i64) -> Self {
Self { dim }
}
}
impl Module for $name {
fn forward(&self, input: &Tensor) -> Result<Tensor> {
defined(input)?;
Ok(input.$operation(self.dim, None)?)
}
}
};
}
softmax!(
Softmax,
f_softmax,
"Converts scores to probabilities that sum to one along the selected axis."
);
softmax!(
LogSoftmax,
f_log_softmax,
"Converts scores to stable log probabilities for likelihood-based models."
);
#[derive(Debug, Clone, Copy)]
pub struct LeakyReLU {
negative_slope: f64,
}
impl Default for LeakyReLU {
fn default() -> Self {
Self {
negative_slope: 0.01,
}
}
}
impl LeakyReLU {
pub fn new(negative_slope: f64) -> Result<Self> {
if !negative_slope.is_finite() {
return Err(RustTorchError::InvalidConfiguration {
field: "negative_slope",
reason: "must be finite".to_owned(),
});
}
Ok(Self { negative_slope })
}
}
impl Module for LeakyReLU {
fn forward(&self, input: &Tensor) -> Result<Tensor> {
floating(input)?;
Ok(input.f_where_self(&input.f_gt(0.0)?, &input.f_mul_scalar(self.negative_slope)?)?)
}
}
#[derive(Debug, Clone, Copy)]
pub struct ELU {
alpha: f64,
}
impl Default for ELU {
fn default() -> Self {
Self { alpha: 1.0 }
}
}
impl ELU {
pub fn new(alpha: f64) -> Result<Self> {
if !alpha.is_finite() {
return Err(RustTorchError::InvalidConfiguration {
field: "alpha",
reason: "must be finite".to_owned(),
});
}
Ok(Self { alpha })
}
}
impl Module for ELU {
fn forward(&self, input: &Tensor) -> Result<Tensor> {
floating(input)?;
let negative = input
.f_clamp_max(0.0)?
.f_expm1()?
.f_mul_scalar(self.alpha)?;
Ok(input.f_where_self(&input.f_gt(0.0)?, &negative)?)
}
}