use crate::module::{Module, ModuleError};
use coeus_autograd::Var;
use coeus_core::Float;
mod prelu;
pub use prelu::{prelu, PReLU};
#[inline]
pub fn elu<T: Float, B: coeus_ops::BackendOps<T> + Default>(
input: &Var<T, B>,
) -> Result<Var<T, B>, B::Error> {
coeus_autograd::elu(input)
}
#[derive(Clone, Debug, Default)]
pub struct ELU;
impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for ELU {
#[inline]
fn parameters(&self) -> Vec<Var<T, B>> {
vec![]
}
#[inline]
fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
Ok(elu(input)?)
}
}
#[inline]
pub fn gelu_tanh<T: Float, B: coeus_ops::BackendOps<T> + Default>(
input: &Var<T, B>,
) -> Result<Var<T, B>, B::Error> {
coeus_autograd::gelu_tanh(input)
}
#[derive(Clone, Debug, Default)]
pub struct GeLUTanh;
impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for GeLUTanh {
#[inline]
fn parameters(&self) -> Vec<Var<T, B>> {
vec![]
}
#[inline]
fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
Ok(gelu_tanh(input)?)
}
}
pub fn glu<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default>(
input: &Var<T, B>,
dim: usize,
) -> Result<Var<T, B>, B::Error> {
let shape = input.tensor.shape();
let ndim = shape.len();
assert!(dim < ndim, "glu: dim {dim} out of range for rank {ndim}");
let axis = shape[dim];
assert!(
axis.is_multiple_of(2),
"glu: dim {dim} must have even extent, got {axis}"
);
let half = axis / 2;
let mut first: Vec<(usize, usize)> = shape.iter().map(|&extent| (0, extent)).collect();
let mut second = first.clone();
first[dim] = (0, half);
second[dim] = (half, axis);
let a = coeus_autograd::slice(input, &first)?;
let b = coeus_autograd::slice(input, &second)?;
coeus_autograd::mul(&a, &coeus_autograd::sigmoid(&b)?)
}
#[derive(Clone, Copy, Debug)]
pub struct GLU {
dim: usize,
}
impl GLU {
#[inline]
#[must_use]
pub fn new(dim: usize) -> Self {
Self { dim }
}
}
impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for GLU {
#[inline]
fn parameters(&self) -> Vec<Var<T, B>> {
vec![]
}
#[inline]
fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
let shape = input.tensor.shape();
let Some(&extent) = shape.get(self.dim) else {
return Err(ModuleError::InvalidAxis {
module: "GLU",
axis: self.dim,
rank: shape.len(),
});
};
if !extent.is_multiple_of(2) {
return Err(ModuleError::UnevenSplit {
module: "GLU",
axis: self.dim,
extent,
});
}
Ok(glu(input, self.dim)?)
}
}
#[inline]
pub fn leaky_relu<T: Float, B: coeus_ops::BackendOps<T> + Default>(
input: &Var<T, B>,
negative_slope: f64,
) -> Result<Var<T, B>, B::Error> {
coeus_autograd::leaky_relu(input, negative_slope)
}
#[derive(Clone, Debug)]
pub struct LeakyReLU {
pub negative_slope: f64,
}
impl LeakyReLU {
pub fn new(negative_slope: f64) -> Self {
Self { negative_slope }
}
}
impl Default for LeakyReLU {
fn default() -> Self {
Self {
negative_slope: 0.01,
}
}
}
impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for LeakyReLU {
#[inline]
fn parameters(&self) -> Vec<Var<T, B>> {
vec![]
}
#[inline]
fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
Ok(leaky_relu(input, self.negative_slope)?)
}
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct HardtanhOp {
pub min_val: f64,
pub max_val: f64,
}
impl HardtanhOp {
#[must_use]
pub const fn from_bounds(min_val: f64, max_val: f64) -> Self {
Self { min_val, max_val }
}
}
#[inline]
pub fn hardtanh<T: Float, B: coeus_ops::BackendOps<T> + Default>(
input: &Var<T, B>,
min_val: f64,
max_val: f64,
) -> Result<Var<T, B>, B::Error> {
coeus_autograd::hardtanh(input, min_val, max_val)
}
#[derive(Clone, Debug)]
pub struct Hardtanh {
pub min_val: f64,
pub max_val: f64,
}
impl Hardtanh {
pub fn new(min_val: f64, max_val: f64) -> Self {
Self { min_val, max_val }
}
}
impl Default for Hardtanh {
fn default() -> Self {
Self {
min_val: -1.0,
max_val: 1.0,
}
}
}
impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for Hardtanh {
#[inline]
fn parameters(&self) -> Vec<Var<T, B>> {
vec![]
}
#[inline]
fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
Ok(hardtanh(input, self.min_val, self.max_val)?)
}
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct HardshrinkOp(pub f64);
impl HardshrinkOp {
#[must_use]
pub const fn from_lambda(lambda: f64) -> Self {
Self(lambda)
}
}
#[inline]
pub fn hardshrink<T: Float, B: coeus_ops::BackendOps<T> + Default>(
input: &Var<T, B>,
lambda: f64,
) -> Result<Var<T, B>, B::Error> {
coeus_autograd::hardshrink(input, lambda)
}
#[derive(Clone, Debug)]
pub struct Hardshrink {
pub lambda: f64,
}
impl Hardshrink {
pub fn new(lambda: f64) -> Self {
Self { lambda }
}
}
impl Default for Hardshrink {
fn default() -> Self {
Self { lambda: 0.5 }
}
}
impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for Hardshrink {
#[inline]
fn parameters(&self) -> Vec<Var<T, B>> {
vec![]
}
#[inline]
fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
Ok(hardshrink(input, self.lambda)?)
}
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct SoftshrinkOp(pub f64);
impl SoftshrinkOp {
#[must_use]
pub const fn from_lambda(lambda: f64) -> Self {
Self(lambda)
}
}
#[inline]
pub fn softshrink<T: Float, B: coeus_ops::BackendOps<T> + Default>(
input: &Var<T, B>,
lambda: f64,
) -> Result<Var<T, B>, B::Error> {
coeus_autograd::softshrink(input, lambda)
}
#[derive(Clone, Debug)]
pub struct Softshrink {
pub lambda: f64,
}
impl Softshrink {
pub fn new(lambda: f64) -> Self {
Self { lambda }
}
}
impl Default for Softshrink {
fn default() -> Self {
Self { lambda: 0.5 }
}
}
impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for Softshrink {
#[inline]
fn parameters(&self) -> Vec<Var<T, B>> {
vec![]
}
#[inline]
fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
Ok(softshrink(input, self.lambda)?)
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ThresholdNode;
impl ThresholdNode {
#[must_use]
pub const fn op_name() -> &'static str {
"threshold"
}
}
#[inline]
pub fn threshold<T: Float, B: coeus_ops::BackendOps<T> + Default>(
input: &Var<T, B>,
thresh: f64,
value: f64,
) -> Result<Var<T, B>, B::Error> {
coeus_autograd::threshold(input, thresh, value)
}
#[derive(Clone, Debug)]
pub struct Threshold {
pub threshold: f64,
pub value: f64,
}
impl Threshold {
pub fn new(threshold: f64, value: f64) -> Self {
Self { threshold, value }
}
}
impl Default for Threshold {
fn default() -> Self {
Self {
threshold: 0.0,
value: 0.0,
}
}
}
impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for Threshold {
#[inline]
fn parameters(&self) -> Vec<Var<T, B>> {
vec![]
}
#[inline]
fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
Ok(threshold(input, self.threshold, self.value)?)
}
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct CeluOp(pub f64);
impl CeluOp {
#[must_use]
pub const fn from_alpha(alpha: f64) -> Self {
Self(alpha)
}
}
#[inline]
pub fn celu<T: Float, B: coeus_ops::BackendOps<T> + Default>(
input: &Var<T, B>,
alpha: f64,
) -> Result<Var<T, B>, B::Error> {
coeus_autograd::celu(input, alpha)
}
#[derive(Clone, Debug)]
pub struct Celu {
pub alpha: f64,
}
impl Celu {
pub fn new(alpha: f64) -> Self {
Self { alpha }
}
}
impl Default for Celu {
fn default() -> Self {
Self { alpha: 1.0 }
}
}
impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for Celu {
#[inline]
fn parameters(&self) -> Vec<Var<T, B>> {
vec![]
}
#[inline]
fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
Ok(celu(input, self.alpha)?)
}
}