Skip to main content

coeus_nn/activation/
parametric.rs

1use crate::module::{Module, ModuleError};
2use coeus_autograd::Var;
3use coeus_core::Float;
4
5mod prelu;
6
7pub use prelu::{prelu, PReLU};
8
9/// Functional ELU activation.
10#[inline]
11pub fn elu<T: Float, B: coeus_ops::BackendOps<T> + Default>(
12    input: &Var<T, B>,
13) -> Result<Var<T, B>, B::Error> {
14    coeus_autograd::elu(input)
15}
16
17/// ELU activation module (alpha = 1.0).
18#[derive(Clone, Debug, Default)]
19pub struct ELU;
20
21impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for ELU {
22    #[inline]
23    fn parameters(&self) -> Vec<Var<T, B>> {
24        vec![]
25    }
26
27    #[inline]
28    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
29        Ok(elu(input)?)
30    }
31}
32
33/// Functional GELU tanh approximation.
34#[inline]
35pub fn gelu_tanh<T: Float, B: coeus_ops::BackendOps<T> + Default>(
36    input: &Var<T, B>,
37) -> Result<Var<T, B>, B::Error> {
38    coeus_autograd::gelu_tanh(input)
39}
40
41/// GELU tanh approximation module.
42#[derive(Clone, Debug, Default)]
43pub struct GeLUTanh;
44
45impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for GeLUTanh {
46    #[inline]
47    fn parameters(&self) -> Vec<Var<T, B>> {
48        vec![]
49    }
50
51    #[inline]
52    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
53        Ok(gelu_tanh(input)?)
54    }
55}
56
57/// Functional Gated Linear Unit along `dim` (`torch.nn.functional.glu`).
58///
59/// Splits `input` into two equal halves `[a, b]` along `dim` and gates the first
60/// by the sigmoid of the second: `a * sigmoid(b)`. Differentiable — composed from
61/// tracked `slice`, `sigmoid`, and `mul`, so gradients flow to `input`.
62///
63/// # Panics
64/// If `dim` is out of range or the extent along `dim` is odd.
65pub fn glu<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default>(
66    input: &Var<T, B>,
67    dim: usize,
68) -> Result<Var<T, B>, B::Error> {
69    let shape = input.tensor.shape();
70    let ndim = shape.len();
71    assert!(dim < ndim, "glu: dim {dim} out of range for rank {ndim}");
72    let axis = shape[dim];
73    assert!(
74        axis.is_multiple_of(2),
75        "glu: dim {dim} must have even extent, got {axis}"
76    );
77    let half = axis / 2;
78    let mut first: Vec<(usize, usize)> = shape.iter().map(|&extent| (0, extent)).collect();
79    let mut second = first.clone();
80    first[dim] = (0, half);
81    second[dim] = (half, axis);
82    let a = coeus_autograd::slice(input, &first)?;
83    let b = coeus_autograd::slice(input, &second)?;
84    coeus_autograd::mul(&a, &coeus_autograd::sigmoid(&b)?)
85}
86
87/// Gated Linear Unit module gating along a fixed `dim` (see [`glu`]).
88///
89/// Parameter-free; the split dimension is captured at construction so `GLU` can
90/// participate in a [`Sequential`](crate::Sequential) stack like other activations.
91#[derive(Clone, Copy, Debug)]
92pub struct GLU {
93    dim: usize,
94}
95
96impl GLU {
97    /// Create a GLU module that gates along `dim`.
98    #[inline]
99    #[must_use]
100    pub fn new(dim: usize) -> Self {
101        Self { dim }
102    }
103}
104
105impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for GLU {
106    #[inline]
107    fn parameters(&self) -> Vec<Var<T, B>> {
108        vec![]
109    }
110
111    #[inline]
112    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
113        let shape = input.tensor.shape();
114        let Some(&extent) = shape.get(self.dim) else {
115            return Err(ModuleError::InvalidAxis {
116                module: "GLU",
117                axis: self.dim,
118                rank: shape.len(),
119            });
120        };
121        if !extent.is_multiple_of(2) {
122            return Err(ModuleError::UnevenSplit {
123                module: "GLU",
124                axis: self.dim,
125                extent,
126            });
127        }
128        Ok(glu(input, self.dim)?)
129    }
130}
131
132/// Functional LeakyReLU activation.
133#[inline]
134pub fn leaky_relu<T: Float, B: coeus_ops::BackendOps<T> + Default>(
135    input: &Var<T, B>,
136    negative_slope: f64,
137) -> Result<Var<T, B>, B::Error> {
138    coeus_autograd::leaky_relu(input, negative_slope)
139}
140
141/// LeakyReLU activation module.
142#[derive(Clone, Debug)]
143pub struct LeakyReLU {
144    /// Slope for negative inputs.
145    pub negative_slope: f64,
146}
147
148impl LeakyReLU {
149    /// Create a LeakyReLU module.
150    pub fn new(negative_slope: f64) -> Self {
151        Self { negative_slope }
152    }
153}
154
155impl Default for LeakyReLU {
156    fn default() -> Self {
157        Self {
158            negative_slope: 0.01,
159        }
160    }
161}
162
163impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for LeakyReLU {
164    #[inline]
165    fn parameters(&self) -> Vec<Var<T, B>> {
166        vec![]
167    }
168
169    #[inline]
170    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
171        Ok(leaky_relu(input, self.negative_slope)?)
172    }
173}
174
175/// Public descriptor for the functional Hardtanh parameter set.
176#[derive(Clone, Copy, Debug, PartialEq)]
177pub struct HardtanhOp {
178    /// Lower clamp bound.
179    pub min_val: f64,
180    /// Upper clamp bound.
181    pub max_val: f64,
182}
183
184impl HardtanhOp {
185    /// Construct a Hardtanh descriptor from explicit bounds.
186    #[must_use]
187    pub const fn from_bounds(min_val: f64, max_val: f64) -> Self {
188        Self { min_val, max_val }
189    }
190}
191
192/// Functional Hardtanh activation.
193#[inline]
194pub fn hardtanh<T: Float, B: coeus_ops::BackendOps<T> + Default>(
195    input: &Var<T, B>,
196    min_val: f64,
197    max_val: f64,
198) -> Result<Var<T, B>, B::Error> {
199    coeus_autograd::hardtanh(input, min_val, max_val)
200}
201
202/// Hardtanh activation module (default range [-1, 1], matching PyTorch's
203/// `torch.nn.Hardtanh`).
204#[derive(Clone, Debug)]
205pub struct Hardtanh {
206    /// Lower clamp bound.
207    pub min_val: f64,
208    /// Upper clamp bound.
209    pub max_val: f64,
210}
211
212impl Hardtanh {
213    /// Create a Hardtanh module with custom bounds.
214    pub fn new(min_val: f64, max_val: f64) -> Self {
215        Self { min_val, max_val }
216    }
217}
218
219impl Default for Hardtanh {
220    fn default() -> Self {
221        Self {
222            min_val: -1.0,
223            max_val: 1.0,
224        }
225    }
226}
227
228impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for Hardtanh {
229    #[inline]
230    fn parameters(&self) -> Vec<Var<T, B>> {
231        vec![]
232    }
233
234    #[inline]
235    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
236        Ok(hardtanh(input, self.min_val, self.max_val)?)
237    }
238}
239
240/// Public descriptor for the functional Hardshrink parameter set.
241#[derive(Clone, Copy, Debug, PartialEq)]
242pub struct HardshrinkOp(pub f64);
243
244impl HardshrinkOp {
245    /// Construct a Hardshrink descriptor from lambda.
246    #[must_use]
247    pub const fn from_lambda(lambda: f64) -> Self {
248        Self(lambda)
249    }
250}
251
252/// Functional Hardshrink activation.
253#[inline]
254pub fn hardshrink<T: Float, B: coeus_ops::BackendOps<T> + Default>(
255    input: &Var<T, B>,
256    lambda: f64,
257) -> Result<Var<T, B>, B::Error> {
258    coeus_autograd::hardshrink(input, lambda)
259}
260
261/// Hardshrink activation module (default λ = 0.5, matching PyTorch's
262/// `torch.nn.Hardshrink`).
263#[derive(Clone, Debug)]
264pub struct Hardshrink {
265    /// Threshold λ below which the activation is replaced with 0.
266    pub lambda: f64,
267}
268
269impl Hardshrink {
270    /// Create a Hardshrink module with custom λ.
271    pub fn new(lambda: f64) -> Self {
272        Self { lambda }
273    }
274}
275
276impl Default for Hardshrink {
277    fn default() -> Self {
278        Self { lambda: 0.5 }
279    }
280}
281
282impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for Hardshrink {
283    #[inline]
284    fn parameters(&self) -> Vec<Var<T, B>> {
285        vec![]
286    }
287
288    #[inline]
289    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
290        Ok(hardshrink(input, self.lambda)?)
291    }
292}
293
294/// Public descriptor for the functional Softshrink parameter set.
295#[derive(Clone, Copy, Debug, PartialEq)]
296pub struct SoftshrinkOp(pub f64);
297
298impl SoftshrinkOp {
299    /// Construct a Softshrink descriptor from lambda.
300    #[must_use]
301    pub const fn from_lambda(lambda: f64) -> Self {
302        Self(lambda)
303    }
304}
305
306/// Functional Softshrink activation.
307#[inline]
308pub fn softshrink<T: Float, B: coeus_ops::BackendOps<T> + Default>(
309    input: &Var<T, B>,
310    lambda: f64,
311) -> Result<Var<T, B>, B::Error> {
312    coeus_autograd::softshrink(input, lambda)
313}
314
315/// Softshrink activation module (default λ = 0.5).
316#[derive(Clone, Debug)]
317pub struct Softshrink {
318    /// Threshold λ below which the activation maps to 0.
319    pub lambda: f64,
320}
321
322impl Softshrink {
323    /// Create a Softshrink module with custom λ.
324    pub fn new(lambda: f64) -> Self {
325        Self { lambda }
326    }
327}
328
329impl Default for Softshrink {
330    fn default() -> Self {
331        Self { lambda: 0.5 }
332    }
333}
334
335impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for Softshrink {
336    #[inline]
337    fn parameters(&self) -> Vec<Var<T, B>> {
338        vec![]
339    }
340
341    #[inline]
342    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
343        Ok(softshrink(input, self.lambda)?)
344    }
345}
346
347/// Public descriptor for the functional Threshold operation.
348#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
349pub struct ThresholdNode;
350
351impl ThresholdNode {
352    /// Operation name used by the autograd node.
353    #[must_use]
354    pub const fn op_name() -> &'static str {
355        "threshold"
356    }
357}
358
359/// Functional Threshold activation.
360#[inline]
361pub fn threshold<T: Float, B: coeus_ops::BackendOps<T> + Default>(
362    input: &Var<T, B>,
363    thresh: f64,
364    value: f64,
365) -> Result<Var<T, B>, B::Error> {
366    coeus_autograd::threshold(input, thresh, value)
367}
368
369/// Threshold activation module (default threshold=0, value=0).
370#[derive(Clone, Debug)]
371pub struct Threshold {
372    /// Lower-bound threshold.
373    pub threshold: f64,
374    /// Replacement value below threshold.
375    pub value: f64,
376}
377
378impl Threshold {
379    /// Create a Threshold module with custom `threshold` and `value`.
380    pub fn new(threshold: f64, value: f64) -> Self {
381        Self { threshold, value }
382    }
383}
384
385impl Default for Threshold {
386    fn default() -> Self {
387        Self {
388            threshold: 0.0,
389            value: 0.0,
390        }
391    }
392}
393
394impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for Threshold {
395    #[inline]
396    fn parameters(&self) -> Vec<Var<T, B>> {
397        vec![]
398    }
399
400    #[inline]
401    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
402        Ok(threshold(input, self.threshold, self.value)?)
403    }
404}
405
406/// Public descriptor for the functional Celu parameter set.
407#[derive(Clone, Copy, Debug, PartialEq)]
408pub struct CeluOp(pub f64);
409
410impl CeluOp {
411    /// Construct a Celu descriptor from alpha.
412    #[must_use]
413    pub const fn from_alpha(alpha: f64) -> Self {
414        Self(alpha)
415    }
416}
417
418/// Functional Celu activation.
419#[inline]
420pub fn celu<T: Float, B: coeus_ops::BackendOps<T> + Default>(
421    input: &Var<T, B>,
422    alpha: f64,
423) -> Result<Var<T, B>, B::Error> {
424    coeus_autograd::celu(input, alpha)
425}
426
427/// Celu activation module (default α = 1.0, matching PyTorch).
428#[derive(Clone, Debug)]
429pub struct Celu {
430    /// Continuity-preserving α constant.
431    pub alpha: f64,
432}
433
434impl Celu {
435    /// Create a Celu module with custom α.
436    pub fn new(alpha: f64) -> Self {
437        Self { alpha }
438    }
439}
440
441impl Default for Celu {
442    fn default() -> Self {
443        Self { alpha: 1.0 }
444    }
445}
446
447impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for Celu {
448    #[inline]
449    fn parameters(&self) -> Vec<Var<T, B>> {
450        vec![]
451    }
452
453    #[inline]
454    fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
455        Ok(celu(input, self.alpha)?)
456    }
457}