1use crate::module::{Module, ModuleError};
7use coeus_autograd::Var;
8use coeus_core::Float;
9use coeus_tensor::Tensor;
10
11#[derive(Clone, Debug)]
22pub struct AlphaDropout {
23 pub p: f64,
25 pub is_training: bool,
27 pub seed: u64,
29}
30
31impl AlphaDropout {
32 const SELU_ALPHA: f64 = 1.673_263_242_354_377_2;
34 const SELU_LAMBDA: f64 = 1.050_700_987_355_480_5;
36
37 pub fn new(p: f64) -> Self {
39 assert!((0.0..1.0).contains(&p), "p must be in [0, 1)");
40 Self {
41 p,
42 is_training: true,
43 seed: 42,
44 }
45 }
46
47 pub fn set_training(&mut self, mode: bool) {
49 self.is_training = mode;
50 }
51}
52
53fn alpha_dropout_with_mask<
54 T: Float + coeus_core::FloatElement + coeus_cpu::RealScalar,
55 B: coeus_ops::BackendOps<T> + Default,
56>(
57 input: &Var<T, B>,
58 p: f64,
59 seed: u64,
60 feature_wise: bool,
61) -> Result<Var<T, B>, ModuleError<B::Error>> {
62 let shape = input.tensor.shape_cloned();
63 if feature_wise && shape.len() < 2 {
64 return Err(ModuleError::InvalidRank {
65 module: "FeatureAlphaDropout",
66 expected: "at least 2",
67 actual: shape.len(),
68 });
69 }
70
71 let numel = shape.iter().product::<usize>();
72 let spatial = shape.get(2..).map_or(1, |dims| dims.iter().product());
73 let channels = shape.get(1).copied().unwrap_or(1);
74 let mut rng = coeus_autograd::ops::nn::dropout::Xorshift64::new(seed);
75 let mut keep = Vec::with_capacity(numel);
76
77 if feature_wise {
78 for _batch in 0..shape[0] {
79 for _channel in 0..channels {
80 let kept = rng.next_unit() >= p;
81 keep.extend(std::iter::repeat_n(
82 if kept { T::ONE } else { T::ZERO },
83 spatial,
84 ));
85 }
86 }
87 } else {
88 keep.extend((0..numel).map(|_| {
89 if rng.next_unit() >= p {
90 T::ONE
91 } else {
92 T::ZERO
93 }
94 }));
95 }
96
97 let alpha_prime = -(AlphaDropout::SELU_ALPHA * AlphaDropout::SELU_LAMBDA);
98 let keep_probability = 1.0 - p;
99 let scale = 1.0 / (keep_probability * (1.0 + p * alpha_prime * alpha_prime)).sqrt();
100 let shift = -scale * p * alpha_prime;
101 let saturation = <T as coeus_core::FloatElement>::from_f64(alpha_prime);
102 let backend = B::default();
103 let keep_tensor = Tensor::from_slice_on(shape.clone(), &keep, &backend)?;
104 let saturation_data = keep
105 .iter()
106 .map(|&value| {
107 if value == T::ZERO {
108 saturation
109 } else {
110 T::ZERO
111 }
112 })
113 .collect::<Vec<_>>();
114 let keep_var = Var::new(keep_tensor, false)?;
115 let saturation_var = Var::new(
116 Tensor::from_slice_on(shape, &saturation_data, &backend)?,
117 false,
118 )?;
119 let selected = coeus_autograd::add(&coeus_autograd::mul(input, &keep_var)?, &saturation_var)?;
120 Ok(coeus_autograd::scalar_add(
121 &coeus_autograd::scalar_mul(&selected, <T as coeus_core::FloatElement>::from_f64(scale))?,
122 <T as coeus_core::FloatElement>::from_f64(shift),
123 )?)
124}
125
126impl<
127 T: Float + coeus_core::FloatElement + coeus_cpu::RealScalar,
128 B: coeus_ops::BackendOps<T> + Default,
129 > Module<T, B> for AlphaDropout
130{
131 fn parameters(&self) -> Vec<Var<T, B>> {
132 vec![]
133 }
134
135 fn train(&mut self, mode: bool) {
136 self.set_training(mode);
137 }
138
139 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
147 if !self.is_training || self.p == 0.0 {
148 return Ok(input.clone());
149 }
150 alpha_dropout_with_mask(input, self.p, self.seed, false)
151 }
152}
153
154#[derive(Clone, Debug)]
161pub struct FeatureAlphaDropout {
162 pub p: f64,
164 pub is_training: bool,
166 pub seed: u64,
168}
169
170impl FeatureAlphaDropout {
171 pub fn new(p: f64) -> Self {
173 assert!((0.0..1.0).contains(&p), "p must be in [0, 1)");
174 Self {
175 p,
176 is_training: true,
177 seed: 42,
178 }
179 }
180
181 pub fn set_training(&mut self, mode: bool) {
183 self.is_training = mode;
184 }
185}
186
187impl<
188 T: Float + coeus_core::FloatElement + coeus_cpu::RealScalar,
189 B: coeus_ops::BackendOps<T> + Default,
190 > Module<T, B> for FeatureAlphaDropout
191{
192 fn parameters(&self) -> Vec<Var<T, B>> {
193 vec![]
194 }
195
196 fn train(&mut self, mode: bool) {
197 self.set_training(mode);
198 }
199
200 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
202 if !self.is_training || self.p == 0.0 {
203 return Ok(input.clone());
204 }
205 alpha_dropout_with_mask(input, self.p, self.seed, true)
206 }
207}
208
209#[derive(Clone, Debug)]
216pub struct GaussianNoise {
217 pub std: f64,
219 pub is_training: bool,
221}
222
223impl GaussianNoise {
224 pub fn new(std: f64) -> Self {
226 assert!(std >= 0.0, "std must be non-negative");
227 Self {
228 std,
229 is_training: true,
230 }
231 }
232
233 pub fn set_training(&mut self, mode: bool) {
235 self.is_training = mode;
236 }
237}
238
239impl<
240 T: Float + coeus_core::FloatElement + coeus_cpu::RealScalar,
241 B: coeus_ops::BackendOps<T> + Default,
242 > Module<T, B> for GaussianNoise
243{
244 fn parameters(&self) -> Vec<Var<T, B>> {
245 vec![]
246 }
247
248 fn train(&mut self, mode: bool) {
249 self.set_training(mode);
250 }
251
252 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
254 if !self.is_training || self.std == 0.0 {
255 return Ok(input.clone());
256 }
257 let backend = B::default();
258 let shape = input.tensor.shape_cloned();
259 let numel = shape.iter().product::<usize>();
260
261 let mut rng = coeus_autograd::ops::nn::dropout::Xorshift64::new(42);
263 let mut noise = vec![T::ZERO; numel];
264 let std = self.std;
265 let mut i = 0;
266 while i < numel {
267 let u1 = rng.next_unit().max(1e-10);
268 let u2 = rng.next_unit();
269 let r = (-2.0 * u1.ln()).sqrt() * std;
270 noise[i] = <T as coeus_core::FloatElement>::from_f64(
271 r * (2.0 * std::f64::consts::PI * u2).cos(),
272 );
273 if i + 1 < numel {
274 noise[i + 1] = <T as coeus_core::FloatElement>::from_f64(
275 r * (2.0 * std::f64::consts::PI * u2).sin(),
276 );
277 }
278 i += 2;
279 }
280
281 let noise_tensor = Tensor::from_slice_on(shape, &noise, &backend)?;
282 let noise_var = Var::new(noise_tensor, false)?;
283 Ok(coeus_autograd::add(input, &noise_var)?)
284 }
285}
286
287#[derive(Clone, Debug)]
297pub struct LocalResponseNorm {
298 pub size: usize,
300 pub alpha: f64,
302 pub beta: f64,
304 pub k: f64,
306}
307
308impl LocalResponseNorm {
309 pub fn new(size: usize) -> Self {
313 Self {
314 size,
315 alpha: 0.0001,
316 beta: 0.75,
317 k: 1.0,
318 }
319 }
320
321 pub fn with_params(size: usize, alpha: f64, beta: f64, k: f64) -> Self {
323 assert!(size >= 1, "size must be >= 1");
324 Self {
325 size,
326 alpha,
327 beta,
328 k,
329 }
330 }
331}
332
333impl<
334 T: Float + coeus_core::FloatElement + std::ops::Neg<Output = T> + coeus_cpu::RealScalar,
335 B: coeus_ops::BackendOps<T> + Default,
336 > Module<T, B> for LocalResponseNorm
337where
338 B::DeviceBuffer<T>:
339 coeus_core::CpuAddressableStorage<T> + coeus_core::CpuAddressableStorageMut<T>,
340{
341 fn parameters(&self) -> Vec<Var<T, B>> {
342 vec![]
343 }
344
345 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
354 let shape = input.tensor.shape_cloned();
355 if !(2..=4).contains(&shape.len()) {
356 return Err(ModuleError::InvalidRank {
357 module: "LocalResponseNorm",
358 expected: "2 to 4",
359 actual: shape.len(),
360 });
361 }
362 let n = shape[0];
363 let c = shape[1];
364 let spatial: usize = shape[2..].iter().product::<usize>().max(1);
365 let half = self.size / 2;
366 let backend = B::default();
367
368 let x3 = coeus_autograd::reshape(input, [n, c, spatial])?;
370 let sq = coeus_autograd::mul(&x3, &x3)?;
371
372 let mut m_data = vec![T::ZERO; c * c];
377 for i in 0..c {
378 let lo = i.saturating_sub(half);
379 let hi = (i + half + 1).min(c);
380 for cell in m_data[i * c + lo..i * c + hi].iter_mut() {
381 *cell = T::ONE;
382 }
383 }
384 let m = Var::new(Tensor::from_slice_on([c, c], &m_data, &backend)?, false)?;
385
386 let sq_cns = coeus_autograd::permute(&sq, &[1, 0, 2])?;
388 let sq_2d = coeus_autograd::reshape(&sq_cns, [c, n * spatial])?;
389 let win_2d = coeus_autograd::matmul(&m, &sq_2d)?;
390 let win_cns = coeus_autograd::reshape(&win_2d, [c, n, spatial])?;
391 let windowed = coeus_autograd::permute(&win_cns, &[1, 0, 2])?;
392
393 let scaled = coeus_autograd::scalar_mul(
397 &windowed,
398 <T as coeus_core::FloatElement>::from_f64(
399 self.alpha / <f64 as coeus_core::FloatElement>::from_count(self.size),
400 ),
401 )?;
402 let denom = coeus_autograd::pow(
403 &coeus_autograd::scalar_add(
404 &scaled,
405 <T as coeus_core::FloatElement>::from_f64(self.k),
406 )?,
407 self.beta,
408 )?;
409 let y3 = coeus_autograd::div(&x3, &denom)?;
410 Ok(coeus_autograd::reshape(&y3, shape)?)
411 }
412}