use super::validation;
use crate::module::{Module, ModuleError, ModuleResult};
use coeus_autograd::Var;
use coeus_core::{CpuAddressableStorage, CpuAddressableStorageMut, Float, MoiraiBackend};
use coeus_tensor::Tensor;
use std::cell::{RefCell, RefMut};
type GroupNormCacheRef<'a, T, B> = RefMut<'a, Option<GroupNormCache<T, B>>>;
#[derive(Clone)]
struct GroupNormCache<T: Float, B: coeus_ops::BackendOps<T> + Default> {
group_size: usize,
ln_weight: Var<T, B>,
ln_bias: Var<T, B>,
eps_t: Tensor<T, B>,
d_const: Tensor<T, B>,
ones_cache: RefCell<Option<(usize, Tensor<T, B>)>>,
}
#[derive(Clone)]
pub struct GroupNorm<
T: Float,
B: coeus_ops::BackendOps<T> + Default = MoiraiBackend,
const G: usize = 1,
> {
pub weight: Var<T, B>,
pub bias: Var<T, B>,
pub num_features: usize,
pub eps: f64,
cache: RefCell<Option<GroupNormCache<T, B>>>,
}
impl<
T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + Default,
const G: usize,
> GroupNorm<T, B, G>
{
pub fn new(
num_features: usize,
eps: f64,
) -> Result<Self, crate::init::InitializationError<B::Error>> {
assert!(
G > 0 && num_features.is_multiple_of(G),
"GroupNorm: num_features ({num_features}) must be divisible by G ({G})"
);
let backend = B::default();
let weight = Var::new(Tensor::ones_on([num_features], &backend)?, true)?;
let bias = Var::new(Tensor::zeros_on([num_features], &backend)?, true)?;
Ok(Self {
weight,
bias,
num_features,
eps,
cache: RefCell::new(None),
})
}
fn get_cache(&self, group_size: usize) -> ModuleResult<GroupNormCacheRef<'_, T, B>, B> {
let mut cache = self
.cache
.try_borrow_mut()
.map_err(|_| validation::state_borrow("GroupNorm", "cache"))?;
let need_recreate = match &*cache {
Some(c) => c.group_size != group_size,
None => true,
};
if need_recreate {
let backend = B::default();
let ln_weight = Var::new(Tensor::ones_on([group_size], &backend)?, false)?;
let ln_bias = Var::new(Tensor::zeros_on([group_size], &backend)?, false)?;
let eps_t =
Tensor::full_on([1], coeus_core::FloatElement::from_f64(self.eps), &backend)?;
let d_const = Tensor::full_on(
[1],
coeus_core::FloatElement::from_count(group_size),
&backend,
)?;
*cache = Some(GroupNormCache {
group_size,
ln_weight,
ln_bias,
eps_t,
d_const,
ones_cache: RefCell::new(None),
});
}
Ok(cache)
}
}
impl<
T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + Default,
const G: usize,
> Module<T, B> for GroupNorm<T, B, G>
{
fn parameters(&self) -> Vec<Var<T, B>> {
vec![self.weight.clone(), self.bias.clone()]
}
fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
const MODULE: &str = "GroupNorm";
let shape = input.tensor.shape_cloned();
if shape.len() < 2 {
return Err(validation::invalid_rank(MODULE, "at least 2", shape.len()));
}
let n = shape[0];
let c = shape[1];
if c != self.num_features {
return Err(validation::channel_mismatch(MODULE, self.num_features, c));
}
if G == 0 || !c.is_multiple_of(G) {
return Err(ModuleError::InvalidGroupCount {
module: MODULE,
groups: G,
channels: c,
});
}
for (parameter, actual) in [
("weight", self.weight.tensor.shape()),
("bias", self.bias.tensor.shape()),
] {
if actual != [c] {
return Err(validation::shape_mismatch(MODULE, parameter, &[c], actual));
}
}
let c_per_g = c / G;
let spatial: usize = if shape.len() > 2 {
shape[2..].iter().product()
} else {
1
};
let group_size = c_per_g * spatial;
let flat = coeus_autograd::reshape(input, [n * G, group_size])?;
let cache_borrow = self.get_cache(group_size)?;
let Some(cache) = cache_borrow.as_ref() else {
unreachable!("invariant: get_cache initializes the GroupNorm cache")
};
let backend = B::default();
let mean_t = coeus_ops::mean_axis(&flat.tensor, 1, &backend)
.map_err(|source| validation::backend(MODULE, source))?;
let xmu = coeus_ops::sub(&flat.tensor, &mean_t, &backend)?;
let xmu_sq = coeus_ops::mul(&xmu, &xmu, &backend)?;
let mut stdev = coeus_ops::mean_axis(&xmu_sq, 1, &backend)
.map_err(|source| validation::backend(MODULE, source))?;
coeus_ops::add_assign(&mut stdev, &cache.eps_t, &backend)
.map_err(|source| validation::backend(MODULE, source))?;
coeus_ops::sqrt_assign(&mut stdev, &backend)
.map_err(|source| validation::backend(MODULE, source))?;
let ones = {
let mut o_cache = cache
.ones_cache
.try_borrow_mut()
.map_err(|_| validation::state_borrow(MODULE, "ones_cache"))?;
let n_groups = n * G;
if let Some((cached_n, ref cached_ones)) = *o_cache {
if cached_n == n_groups {
cached_ones.clone()
} else {
let ones = Tensor::ones_on([n_groups, 1], &backend)?;
*o_cache = Some((n_groups, ones.clone()));
ones
}
} else {
let ones = Tensor::ones_on([n_groups, 1], &backend)?;
*o_cache = Some((n_groups, ones.clone()));
ones
}
};
let mut istdev = ones;
coeus_ops::div_assign(&mut istdev, &stdev, &backend)
.map_err(|source| validation::backend(MODULE, source))?;
let x_hat = coeus_ops::mul(&xmu, &istdev, &backend)?;
let w_reshaped = cache.ln_weight.tensor.reshape([1, group_size]);
let b_reshaped = cache.ln_bias.tensor.reshape([1, group_size]);
let mut out_tensor = coeus_ops::mul(&x_hat, &w_reshaped, &backend)?;
coeus_ops::add_assign(&mut out_tensor, &b_reshaped, &backend)
.map_err(|source| validation::backend(MODULE, source))?;
let normed_flat = coeus_autograd::layernorm(
&flat,
&cache.ln_weight,
&cache.ln_bias,
out_tensor,
x_hat,
istdev,
cache.d_const.clone(),
)?;
let normed = coeus_autograd::reshape(&normed_flat, shape.clone())?;
let mut broadcast_shape = vec![1usize; shape.len()];
broadcast_shape[1] = c;
let w_reshaped = coeus_autograd::reshape(&self.weight, broadcast_shape.as_slice())?;
let b_reshaped = coeus_autograd::reshape(&self.bias, broadcast_shape.as_slice())?;
let scaled = coeus_autograd::mul(&normed, &w_reshaped)?;
Ok(coeus_autograd::add(&scaled, &b_reshaped)?)
}
}
pub fn group_norm<
T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
B: coeus_ops::BackendOps<T> + Default,
>(
input: &Tensor<T, B>,
num_groups: usize,
weight: Option<&Tensor<T, B>>,
bias: Option<&Tensor<T, B>>,
eps: f64,
) -> Result<Tensor<T, B>, ModuleError<B::Error>>
where
B::DeviceBuffer<T>: CpuAddressableStorage<T> + CpuAddressableStorageMut<T>,
{
const MODULE: &str = "GroupNorm";
use coeus_ops::{add_assign, div_assign, mean_axis, mul, sqrt_assign, sub};
let backend = B::default();
let shape = input.shape_cloned();
if shape.len() < 2 {
return Err(validation::invalid_rank(MODULE, "at least 2", shape.len()));
}
let n = shape[0];
let c = shape[1];
if num_groups == 0 || !c.is_multiple_of(num_groups) {
return Err(ModuleError::InvalidGroupCount {
module: MODULE,
groups: num_groups,
channels: c,
});
}
if !eps.is_finite() || eps < 0.0 {
return Err(ModuleError::InvalidEpsilon { module: MODULE });
}
if let Some(w) = weight {
if w.shape() != [c] {
return Err(validation::shape_mismatch(
MODULE,
"weight",
&[c],
w.shape(),
));
}
}
if let Some(b) = bias {
if b.shape() != [c] {
return Err(validation::shape_mismatch(MODULE, "bias", &[c], b.shape()));
}
}
let c_per_g = c / num_groups;
let spatial: usize = if shape.len() > 2 {
shape[2..].iter().product()
} else {
1
};
let group_size = c_per_g * spatial;
let flat = input.reshape([n * num_groups, group_size]);
let mean =
mean_axis(&flat, 1, &backend).map_err(|source| validation::backend(MODULE, source))?;
let xmu = sub(&flat, &mean, &backend)?;
let xmu_sq = mul(&xmu, &xmu, &backend)?;
let mut var =
mean_axis(&xmu_sq, 1, &backend).map_err(|source| validation::backend(MODULE, source))?;
let eps_t = Tensor::full_on([1], coeus_core::FloatElement::from_f64(eps), &backend)?;
add_assign(&mut var, &eps_t, &backend).map_err(|source| validation::backend(MODULE, source))?;
sqrt_assign(&mut var, &backend).map_err(|source| validation::backend(MODULE, source))?;
let ones = Tensor::ones_on([n * num_groups, 1], &backend)?;
let mut istdev = ones;
div_assign(&mut istdev, &var, &backend)
.map_err(|source| validation::backend(MODULE, source))?;
let x_hat = mul(&xmu, &istdev, &backend)?;
let mut out = x_hat.reshape(shape.clone());
let mut broadcast_shape = vec![1usize; shape.len()];
broadcast_shape[1] = c;
if let Some(w) = weight {
let w_bc = w.reshape(broadcast_shape.clone());
out = mul(&out, &w_bc, &backend)?;
}
if let Some(b) = bias {
let b_bc = b.reshape(broadcast_shape.clone());
add_assign(&mut out, &b_bc, &backend)
.map_err(|source| validation::backend(MODULE, source))?;
}
Ok(out)
}