1use crate::module::{Module, ModuleError};
2use coeus_autograd::Var;
3use coeus_core::Float;
4
5mod prelu;
6
7pub use prelu::{prelu, PReLU};
8
9#[inline]
11pub fn elu<T: Float, B: coeus_ops::BackendOps<T> + Default>(
12 input: &Var<T, B>,
13) -> Result<Var<T, B>, B::Error> {
14 coeus_autograd::elu(input)
15}
16
17#[derive(Clone, Debug, Default)]
19pub struct ELU;
20
21impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for ELU {
22 #[inline]
23 fn parameters(&self) -> Vec<Var<T, B>> {
24 vec![]
25 }
26
27 #[inline]
28 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
29 Ok(elu(input)?)
30 }
31}
32
33#[inline]
35pub fn gelu_tanh<T: Float, B: coeus_ops::BackendOps<T> + Default>(
36 input: &Var<T, B>,
37) -> Result<Var<T, B>, B::Error> {
38 coeus_autograd::gelu_tanh(input)
39}
40
41#[derive(Clone, Debug, Default)]
43pub struct GeLUTanh;
44
45impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for GeLUTanh {
46 #[inline]
47 fn parameters(&self) -> Vec<Var<T, B>> {
48 vec![]
49 }
50
51 #[inline]
52 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
53 Ok(gelu_tanh(input)?)
54 }
55}
56
57pub fn glu<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default>(
66 input: &Var<T, B>,
67 dim: usize,
68) -> Result<Var<T, B>, B::Error> {
69 let shape = input.tensor.shape();
70 let ndim = shape.len();
71 assert!(dim < ndim, "glu: dim {dim} out of range for rank {ndim}");
72 let axis = shape[dim];
73 assert!(
74 axis.is_multiple_of(2),
75 "glu: dim {dim} must have even extent, got {axis}"
76 );
77 let half = axis / 2;
78 let mut first: Vec<(usize, usize)> = shape.iter().map(|&extent| (0, extent)).collect();
79 let mut second = first.clone();
80 first[dim] = (0, half);
81 second[dim] = (half, axis);
82 let a = coeus_autograd::slice(input, &first)?;
83 let b = coeus_autograd::slice(input, &second)?;
84 coeus_autograd::mul(&a, &coeus_autograd::sigmoid(&b)?)
85}
86
87#[derive(Clone, Copy, Debug)]
92pub struct GLU {
93 dim: usize,
94}
95
96impl GLU {
97 #[inline]
99 #[must_use]
100 pub fn new(dim: usize) -> Self {
101 Self { dim }
102 }
103}
104
105impl<T: Float + coeus_cpu::RealScalar, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for GLU {
106 #[inline]
107 fn parameters(&self) -> Vec<Var<T, B>> {
108 vec![]
109 }
110
111 #[inline]
112 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
113 let shape = input.tensor.shape();
114 let Some(&extent) = shape.get(self.dim) else {
115 return Err(ModuleError::InvalidAxis {
116 module: "GLU",
117 axis: self.dim,
118 rank: shape.len(),
119 });
120 };
121 if !extent.is_multiple_of(2) {
122 return Err(ModuleError::UnevenSplit {
123 module: "GLU",
124 axis: self.dim,
125 extent,
126 });
127 }
128 Ok(glu(input, self.dim)?)
129 }
130}
131
132#[inline]
134pub fn leaky_relu<T: Float, B: coeus_ops::BackendOps<T> + Default>(
135 input: &Var<T, B>,
136 negative_slope: f64,
137) -> Result<Var<T, B>, B::Error> {
138 coeus_autograd::leaky_relu(input, negative_slope)
139}
140
141#[derive(Clone, Debug)]
143pub struct LeakyReLU {
144 pub negative_slope: f64,
146}
147
148impl LeakyReLU {
149 pub fn new(negative_slope: f64) -> Self {
151 Self { negative_slope }
152 }
153}
154
155impl Default for LeakyReLU {
156 fn default() -> Self {
157 Self {
158 negative_slope: 0.01,
159 }
160 }
161}
162
163impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for LeakyReLU {
164 #[inline]
165 fn parameters(&self) -> Vec<Var<T, B>> {
166 vec![]
167 }
168
169 #[inline]
170 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
171 Ok(leaky_relu(input, self.negative_slope)?)
172 }
173}
174
175#[derive(Clone, Copy, Debug, PartialEq)]
177pub struct HardtanhOp {
178 pub min_val: f64,
180 pub max_val: f64,
182}
183
184impl HardtanhOp {
185 #[must_use]
187 pub const fn from_bounds(min_val: f64, max_val: f64) -> Self {
188 Self { min_val, max_val }
189 }
190}
191
192#[inline]
194pub fn hardtanh<T: Float, B: coeus_ops::BackendOps<T> + Default>(
195 input: &Var<T, B>,
196 min_val: f64,
197 max_val: f64,
198) -> Result<Var<T, B>, B::Error> {
199 coeus_autograd::hardtanh(input, min_val, max_val)
200}
201
202#[derive(Clone, Debug)]
205pub struct Hardtanh {
206 pub min_val: f64,
208 pub max_val: f64,
210}
211
212impl Hardtanh {
213 pub fn new(min_val: f64, max_val: f64) -> Self {
215 Self { min_val, max_val }
216 }
217}
218
219impl Default for Hardtanh {
220 fn default() -> Self {
221 Self {
222 min_val: -1.0,
223 max_val: 1.0,
224 }
225 }
226}
227
228impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for Hardtanh {
229 #[inline]
230 fn parameters(&self) -> Vec<Var<T, B>> {
231 vec![]
232 }
233
234 #[inline]
235 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
236 Ok(hardtanh(input, self.min_val, self.max_val)?)
237 }
238}
239
240#[derive(Clone, Copy, Debug, PartialEq)]
242pub struct HardshrinkOp(pub f64);
243
244impl HardshrinkOp {
245 #[must_use]
247 pub const fn from_lambda(lambda: f64) -> Self {
248 Self(lambda)
249 }
250}
251
252#[inline]
254pub fn hardshrink<T: Float, B: coeus_ops::BackendOps<T> + Default>(
255 input: &Var<T, B>,
256 lambda: f64,
257) -> Result<Var<T, B>, B::Error> {
258 coeus_autograd::hardshrink(input, lambda)
259}
260
261#[derive(Clone, Debug)]
264pub struct Hardshrink {
265 pub lambda: f64,
267}
268
269impl Hardshrink {
270 pub fn new(lambda: f64) -> Self {
272 Self { lambda }
273 }
274}
275
276impl Default for Hardshrink {
277 fn default() -> Self {
278 Self { lambda: 0.5 }
279 }
280}
281
282impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for Hardshrink {
283 #[inline]
284 fn parameters(&self) -> Vec<Var<T, B>> {
285 vec![]
286 }
287
288 #[inline]
289 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
290 Ok(hardshrink(input, self.lambda)?)
291 }
292}
293
294#[derive(Clone, Copy, Debug, PartialEq)]
296pub struct SoftshrinkOp(pub f64);
297
298impl SoftshrinkOp {
299 #[must_use]
301 pub const fn from_lambda(lambda: f64) -> Self {
302 Self(lambda)
303 }
304}
305
306#[inline]
308pub fn softshrink<T: Float, B: coeus_ops::BackendOps<T> + Default>(
309 input: &Var<T, B>,
310 lambda: f64,
311) -> Result<Var<T, B>, B::Error> {
312 coeus_autograd::softshrink(input, lambda)
313}
314
315#[derive(Clone, Debug)]
317pub struct Softshrink {
318 pub lambda: f64,
320}
321
322impl Softshrink {
323 pub fn new(lambda: f64) -> Self {
325 Self { lambda }
326 }
327}
328
329impl Default for Softshrink {
330 fn default() -> Self {
331 Self { lambda: 0.5 }
332 }
333}
334
335impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for Softshrink {
336 #[inline]
337 fn parameters(&self) -> Vec<Var<T, B>> {
338 vec![]
339 }
340
341 #[inline]
342 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
343 Ok(softshrink(input, self.lambda)?)
344 }
345}
346
347#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
349pub struct ThresholdNode;
350
351impl ThresholdNode {
352 #[must_use]
354 pub const fn op_name() -> &'static str {
355 "threshold"
356 }
357}
358
359#[inline]
361pub fn threshold<T: Float, B: coeus_ops::BackendOps<T> + Default>(
362 input: &Var<T, B>,
363 thresh: f64,
364 value: f64,
365) -> Result<Var<T, B>, B::Error> {
366 coeus_autograd::threshold(input, thresh, value)
367}
368
369#[derive(Clone, Debug)]
371pub struct Threshold {
372 pub threshold: f64,
374 pub value: f64,
376}
377
378impl Threshold {
379 pub fn new(threshold: f64, value: f64) -> Self {
381 Self { threshold, value }
382 }
383}
384
385impl Default for Threshold {
386 fn default() -> Self {
387 Self {
388 threshold: 0.0,
389 value: 0.0,
390 }
391 }
392}
393
394impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for Threshold {
395 #[inline]
396 fn parameters(&self) -> Vec<Var<T, B>> {
397 vec![]
398 }
399
400 #[inline]
401 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
402 Ok(threshold(input, self.threshold, self.value)?)
403 }
404}
405
406#[derive(Clone, Copy, Debug, PartialEq)]
408pub struct CeluOp(pub f64);
409
410impl CeluOp {
411 #[must_use]
413 pub const fn from_alpha(alpha: f64) -> Self {
414 Self(alpha)
415 }
416}
417
418#[inline]
420pub fn celu<T: Float, B: coeus_ops::BackendOps<T> + Default>(
421 input: &Var<T, B>,
422 alpha: f64,
423) -> Result<Var<T, B>, B::Error> {
424 coeus_autograd::celu(input, alpha)
425}
426
427#[derive(Clone, Debug)]
429pub struct Celu {
430 pub alpha: f64,
432}
433
434impl Celu {
435 pub fn new(alpha: f64) -> Self {
437 Self { alpha }
438 }
439}
440
441impl Default for Celu {
442 fn default() -> Self {
443 Self { alpha: 1.0 }
444 }
445}
446
447impl<T: Float, B: coeus_ops::BackendOps<T> + Default> Module<T, B> for Celu {
448 #[inline]
449 fn parameters(&self) -> Vec<Var<T, B>> {
450 vec![]
451 }
452
453 #[inline]
454 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
455 Ok(celu(input, self.alpha)?)
456 }
457}