use crate::module::{Module, ModuleError};
use coeus_autograd::Var;
use coeus_core::Float;
use coeus_tensor::Tensor;
#[derive(Clone, Debug)]
pub struct AlphaDropout {
pub p: f64,
pub is_training: bool,
pub seed: u64,
}
impl AlphaDropout {
const SELU_ALPHA: f64 = 1.673_263_242_354_377_2;
const SELU_LAMBDA: f64 = 1.050_700_987_355_480_5;
pub fn new(p: f64) -> Self {
assert!((0.0..1.0).contains(&p), "p must be in [0, 1)");
Self {
p,
is_training: true,
seed: 42,
}
}
pub fn set_training(&mut self, mode: bool) {
self.is_training = mode;
}
}
fn alpha_dropout_with_mask<
T: Float + coeus_core::FloatElement + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + Default,
>(
input: &Var<T, B>,
p: f64,
seed: u64,
feature_wise: bool,
) -> Result<Var<T, B>, ModuleError<B::Error>> {
let shape = input.tensor.shape_cloned();
if feature_wise && shape.len() < 2 {
return Err(ModuleError::InvalidRank {
module: "FeatureAlphaDropout",
expected: "at least 2",
actual: shape.len(),
});
}
let numel = shape.iter().product::<usize>();
let spatial = shape.get(2..).map_or(1, |dims| dims.iter().product());
let channels = shape.get(1).copied().unwrap_or(1);
let mut rng = coeus_autograd::ops::nn::dropout::Xorshift64::new(seed);
let mut keep = Vec::with_capacity(numel);
if feature_wise {
for _batch in 0..shape[0] {
for _channel in 0..channels {
let kept = rng.next_unit() >= p;
keep.extend(std::iter::repeat_n(
if kept { T::ONE } else { T::ZERO },
spatial,
));
}
}
} else {
keep.extend((0..numel).map(|_| {
if rng.next_unit() >= p {
T::ONE
} else {
T::ZERO
}
}));
}
let alpha_prime = -(AlphaDropout::SELU_ALPHA * AlphaDropout::SELU_LAMBDA);
let keep_probability = 1.0 - p;
let scale = 1.0 / (keep_probability * (1.0 + p * alpha_prime * alpha_prime)).sqrt();
let shift = -scale * p * alpha_prime;
let saturation = <T as coeus_core::FloatElement>::from_f64(alpha_prime);
let backend = B::default();
let keep_tensor = Tensor::from_slice_on(shape.clone(), &keep, &backend)?;
let saturation_data = keep
.iter()
.map(|&value| {
if value == T::ZERO {
saturation
} else {
T::ZERO
}
})
.collect::<Vec<_>>();
let keep_var = Var::new(keep_tensor, false)?;
let saturation_var = Var::new(
Tensor::from_slice_on(shape, &saturation_data, &backend)?,
false,
)?;
let selected = coeus_autograd::add(&coeus_autograd::mul(input, &keep_var)?, &saturation_var)?;
Ok(coeus_autograd::scalar_add(
&coeus_autograd::scalar_mul(&selected, <T as coeus_core::FloatElement>::from_f64(scale))?,
<T as coeus_core::FloatElement>::from_f64(shift),
)?)
}
impl<
T: Float + coeus_core::FloatElement + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + Default,
> Module<T, B> for AlphaDropout
{
fn parameters(&self) -> Vec<Var<T, B>> {
vec![]
}
fn train(&mut self, mode: bool) {
self.set_training(mode);
}
fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
if !self.is_training || self.p == 0.0 {
return Ok(input.clone());
}
alpha_dropout_with_mask(input, self.p, self.seed, false)
}
}
#[derive(Clone, Debug)]
pub struct FeatureAlphaDropout {
pub p: f64,
pub is_training: bool,
pub seed: u64,
}
impl FeatureAlphaDropout {
pub fn new(p: f64) -> Self {
assert!((0.0..1.0).contains(&p), "p must be in [0, 1)");
Self {
p,
is_training: true,
seed: 42,
}
}
pub fn set_training(&mut self, mode: bool) {
self.is_training = mode;
}
}
impl<
T: Float + coeus_core::FloatElement + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + Default,
> Module<T, B> for FeatureAlphaDropout
{
fn parameters(&self) -> Vec<Var<T, B>> {
vec![]
}
fn train(&mut self, mode: bool) {
self.set_training(mode);
}
fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
if !self.is_training || self.p == 0.0 {
return Ok(input.clone());
}
alpha_dropout_with_mask(input, self.p, self.seed, true)
}
}
#[derive(Clone, Debug)]
pub struct GaussianNoise {
pub std: f64,
pub is_training: bool,
}
impl GaussianNoise {
pub fn new(std: f64) -> Self {
assert!(std >= 0.0, "std must be non-negative");
Self {
std,
is_training: true,
}
}
pub fn set_training(&mut self, mode: bool) {
self.is_training = mode;
}
}
impl<
T: Float + coeus_core::FloatElement + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + Default,
> Module<T, B> for GaussianNoise
{
fn parameters(&self) -> Vec<Var<T, B>> {
vec![]
}
fn train(&mut self, mode: bool) {
self.set_training(mode);
}
fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
if !self.is_training || self.std == 0.0 {
return Ok(input.clone());
}
let backend = B::default();
let shape = input.tensor.shape_cloned();
let numel = shape.iter().product::<usize>();
let mut rng = coeus_autograd::ops::nn::dropout::Xorshift64::new(42);
let mut noise = vec![T::ZERO; numel];
let std = self.std;
let mut i = 0;
while i < numel {
let u1 = rng.next_unit().max(1e-10);
let u2 = rng.next_unit();
let r = (-2.0 * u1.ln()).sqrt() * std;
noise[i] = <T as coeus_core::FloatElement>::from_f64(
r * (2.0 * std::f64::consts::PI * u2).cos(),
);
if i + 1 < numel {
noise[i + 1] = <T as coeus_core::FloatElement>::from_f64(
r * (2.0 * std::f64::consts::PI * u2).sin(),
);
}
i += 2;
}
let noise_tensor = Tensor::from_slice_on(shape, &noise, &backend)?;
let noise_var = Var::new(noise_tensor, false)?;
Ok(coeus_autograd::add(input, &noise_var)?)
}
}
#[derive(Clone, Debug)]
pub struct LocalResponseNorm {
pub size: usize,
pub alpha: f64,
pub beta: f64,
pub k: f64,
}
impl LocalResponseNorm {
pub fn new(size: usize) -> Self {
Self {
size,
alpha: 0.0001,
beta: 0.75,
k: 1.0,
}
}
pub fn with_params(size: usize, alpha: f64, beta: f64, k: f64) -> Self {
assert!(size >= 1, "size must be >= 1");
Self {
size,
alpha,
beta,
k,
}
}
}
impl<
T: Float + coeus_core::FloatElement + std::ops::Neg<Output = T> + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + Default,
> Module<T, B> for LocalResponseNorm
where
B::DeviceBuffer<T>:
coeus_core::CpuAddressableStorage<T> + coeus_core::CpuAddressableStorageMut<T>,
{
fn parameters(&self) -> Vec<Var<T, B>> {
vec![]
}
fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
let shape = input.tensor.shape_cloned();
if !(2..=4).contains(&shape.len()) {
return Err(ModuleError::InvalidRank {
module: "LocalResponseNorm",
expected: "2 to 4",
actual: shape.len(),
});
}
let n = shape[0];
let c = shape[1];
let spatial: usize = shape[2..].iter().product::<usize>().max(1);
let half = self.size / 2;
let backend = B::default();
let x3 = coeus_autograd::reshape(input, [n, c, spatial])?;
let sq = coeus_autograd::mul(&x3, &x3)?;
let mut m_data = vec![T::ZERO; c * c];
for i in 0..c {
let lo = i.saturating_sub(half);
let hi = (i + half + 1).min(c);
for cell in m_data[i * c + lo..i * c + hi].iter_mut() {
*cell = T::ONE;
}
}
let m = Var::new(Tensor::from_slice_on([c, c], &m_data, &backend)?, false)?;
let sq_cns = coeus_autograd::permute(&sq, &[1, 0, 2])?;
let sq_2d = coeus_autograd::reshape(&sq_cns, [c, n * spatial])?;
let win_2d = coeus_autograd::matmul(&m, &sq_2d)?;
let win_cns = coeus_autograd::reshape(&win_2d, [c, n, spatial])?;
let windowed = coeus_autograd::permute(&win_cns, &[1, 0, 2])?;
let scaled = coeus_autograd::scalar_mul(
&windowed,
<T as coeus_core::FloatElement>::from_f64(
self.alpha / <f64 as coeus_core::FloatElement>::from_count(self.size),
),
)?;
let denom = coeus_autograd::pow(
&coeus_autograd::scalar_add(
&scaled,
<T as coeus_core::FloatElement>::from_f64(self.k),
)?,
self.beta,
)?;
let y3 = coeus_autograd::div(&x3, &denom)?;
Ok(coeus_autograd::reshape(&y3, shape)?)
}
}