1use super::validation;
9use crate::module::{Module, ModuleError, ModuleResult};
10use coeus_autograd::Var;
11use coeus_core::{CpuAddressableStorage, CpuAddressableStorageMut, Float, MoiraiBackend};
12use coeus_tensor::Tensor;
13use std::cell::{RefCell, RefMut};
14
15type GroupNormCacheRef<'a, T, B> = RefMut<'a, Option<GroupNormCache<T, B>>>;
17
18#[derive(Clone)]
19struct GroupNormCache<T: Float, B: coeus_ops::BackendOps<T> + Default> {
20 group_size: usize,
21 ln_weight: Var<T, B>,
22 ln_bias: Var<T, B>,
23 eps_t: Tensor<T, B>,
24 d_const: Tensor<T, B>,
25 ones_cache: RefCell<Option<(usize, Tensor<T, B>)>>,
26}
27
28#[derive(Clone)]
39pub struct GroupNorm<
40 T: Float,
41 B: coeus_ops::BackendOps<T> + Default = MoiraiBackend,
42 const G: usize = 1,
43> {
44 pub weight: Var<T, B>,
46 pub bias: Var<T, B>,
48 pub num_features: usize,
50 pub eps: f64,
52 cache: RefCell<Option<GroupNormCache<T, B>>>,
53}
54
55impl<
56 T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
57 B: coeus_ops::BackendOps<T> + Default,
58 const G: usize,
59 > GroupNorm<T, B, G>
60{
61 pub fn new(
66 num_features: usize,
67 eps: f64,
68 ) -> Result<Self, crate::init::InitializationError<B::Error>> {
69 assert!(
70 G > 0 && num_features.is_multiple_of(G),
71 "GroupNorm: num_features ({num_features}) must be divisible by G ({G})"
72 );
73 let backend = B::default();
74 let weight = Var::new(Tensor::ones_on([num_features], &backend)?, true)?;
75 let bias = Var::new(Tensor::zeros_on([num_features], &backend)?, true)?;
76 Ok(Self {
77 weight,
78 bias,
79 num_features,
80 eps,
81 cache: RefCell::new(None),
82 })
83 }
84
85 fn get_cache(&self, group_size: usize) -> ModuleResult<GroupNormCacheRef<'_, T, B>, B> {
86 let mut cache = self
87 .cache
88 .try_borrow_mut()
89 .map_err(|_| validation::state_borrow("GroupNorm", "cache"))?;
90 let need_recreate = match &*cache {
91 Some(c) => c.group_size != group_size,
92 None => true,
93 };
94 if need_recreate {
95 let backend = B::default();
96 let ln_weight = Var::new(Tensor::ones_on([group_size], &backend)?, false)?;
97 let ln_bias = Var::new(Tensor::zeros_on([group_size], &backend)?, false)?;
98 let eps_t =
99 Tensor::full_on([1], coeus_core::FloatElement::from_f64(self.eps), &backend)?;
100 let d_const = Tensor::full_on(
101 [1],
102 coeus_core::FloatElement::from_count(group_size),
103 &backend,
104 )?;
105 *cache = Some(GroupNormCache {
106 group_size,
107 ln_weight,
108 ln_bias,
109 eps_t,
110 d_const,
111 ones_cache: RefCell::new(None),
112 });
113 }
114 Ok(cache)
115 }
116}
117
118impl<
120 T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
121 B: coeus_ops::BackendOps<T> + Default,
122 const G: usize,
123 > Module<T, B> for GroupNorm<T, B, G>
124{
125 fn parameters(&self) -> Vec<Var<T, B>> {
126 vec![self.weight.clone(), self.bias.clone()]
127 }
128
129 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
135 const MODULE: &str = "GroupNorm";
136 let shape = input.tensor.shape_cloned();
137 if shape.len() < 2 {
138 return Err(validation::invalid_rank(MODULE, "at least 2", shape.len()));
139 }
140 let n = shape[0];
141 let c = shape[1];
142 if c != self.num_features {
143 return Err(validation::channel_mismatch(MODULE, self.num_features, c));
144 }
145 if G == 0 || !c.is_multiple_of(G) {
146 return Err(ModuleError::InvalidGroupCount {
147 module: MODULE,
148 groups: G,
149 channels: c,
150 });
151 }
152 for (parameter, actual) in [
153 ("weight", self.weight.tensor.shape()),
154 ("bias", self.bias.tensor.shape()),
155 ] {
156 if actual != [c] {
157 return Err(validation::shape_mismatch(MODULE, parameter, &[c], actual));
158 }
159 }
160 let c_per_g = c / G;
161
162 let spatial: usize = if shape.len() > 2 {
164 shape[2..].iter().product()
165 } else {
166 1
167 };
168 let group_size = c_per_g * spatial;
169
170 let flat = coeus_autograd::reshape(input, [n * G, group_size])?;
172
173 let cache_borrow = self.get_cache(group_size)?;
175 let Some(cache) = cache_borrow.as_ref() else {
176 unreachable!("invariant: get_cache initializes the GroupNorm cache")
177 };
178
179 let backend = B::default();
180
181 let mean_t = coeus_ops::mean_axis(&flat.tensor, 1, &backend)
183 .map_err(|source| validation::backend(MODULE, source))?; let xmu = coeus_ops::sub(&flat.tensor, &mean_t, &backend)?; let xmu_sq = coeus_ops::mul(&xmu, &xmu, &backend)?;
190 let mut stdev = coeus_ops::mean_axis(&xmu_sq, 1, &backend)
191 .map_err(|source| validation::backend(MODULE, source))?; coeus_ops::add_assign(&mut stdev, &cache.eps_t, &backend)
195 .map_err(|source| validation::backend(MODULE, source))?;
196 coeus_ops::sqrt_assign(&mut stdev, &backend)
197 .map_err(|source| validation::backend(MODULE, source))?;
198
199 let ones = {
200 let mut o_cache = cache
201 .ones_cache
202 .try_borrow_mut()
203 .map_err(|_| validation::state_borrow(MODULE, "ones_cache"))?;
204 let n_groups = n * G;
205 if let Some((cached_n, ref cached_ones)) = *o_cache {
206 if cached_n == n_groups {
207 cached_ones.clone()
208 } else {
209 let ones = Tensor::ones_on([n_groups, 1], &backend)?;
210 *o_cache = Some((n_groups, ones.clone()));
211 ones
212 }
213 } else {
214 let ones = Tensor::ones_on([n_groups, 1], &backend)?;
215 *o_cache = Some((n_groups, ones.clone()));
216 ones
217 }
218 };
219 let mut istdev = ones;
220 coeus_ops::div_assign(&mut istdev, &stdev, &backend)
221 .map_err(|source| validation::backend(MODULE, source))?; let x_hat = coeus_ops::mul(&xmu, &istdev, &backend)?; let w_reshaped = cache.ln_weight.tensor.reshape([1, group_size]);
228 let b_reshaped = cache.ln_bias.tensor.reshape([1, group_size]);
229 let mut out_tensor = coeus_ops::mul(&x_hat, &w_reshaped, &backend)?;
230 coeus_ops::add_assign(&mut out_tensor, &b_reshaped, &backend)
231 .map_err(|source| validation::backend(MODULE, source))?;
232
233 let normed_flat = coeus_autograd::layernorm(
234 &flat,
235 &cache.ln_weight,
236 &cache.ln_bias,
237 out_tensor,
238 x_hat,
239 istdev,
240 cache.d_const.clone(),
241 )?;
242
243 let normed = coeus_autograd::reshape(&normed_flat, shape.clone())?;
245
246 let mut broadcast_shape = vec![1usize; shape.len()];
249 broadcast_shape[1] = c;
250 let w_reshaped = coeus_autograd::reshape(&self.weight, broadcast_shape.as_slice())?;
251 let b_reshaped = coeus_autograd::reshape(&self.bias, broadcast_shape.as_slice())?;
252
253 let scaled = coeus_autograd::mul(&normed, &w_reshaped)?;
254 Ok(coeus_autograd::add(&scaled, &b_reshaped)?)
255 }
256}
257
258pub fn group_norm<
274 T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
275 B: coeus_ops::BackendOps<T> + Default,
276>(
277 input: &Tensor<T, B>,
278 num_groups: usize,
279 weight: Option<&Tensor<T, B>>,
280 bias: Option<&Tensor<T, B>>,
281 eps: f64,
282) -> Result<Tensor<T, B>, ModuleError<B::Error>>
283where
284 B::DeviceBuffer<T>: CpuAddressableStorage<T> + CpuAddressableStorageMut<T>,
285{
286 const MODULE: &str = "GroupNorm";
287 use coeus_ops::{add_assign, div_assign, mean_axis, mul, sqrt_assign, sub};
288 let backend = B::default();
289 let shape = input.shape_cloned();
290 if shape.len() < 2 {
291 return Err(validation::invalid_rank(MODULE, "at least 2", shape.len()));
292 }
293 let n = shape[0];
294 let c = shape[1];
295 if num_groups == 0 || !c.is_multiple_of(num_groups) {
296 return Err(ModuleError::InvalidGroupCount {
297 module: MODULE,
298 groups: num_groups,
299 channels: c,
300 });
301 }
302 if !eps.is_finite() || eps < 0.0 {
303 return Err(ModuleError::InvalidEpsilon { module: MODULE });
304 }
305 if let Some(w) = weight {
306 if w.shape() != [c] {
307 return Err(validation::shape_mismatch(
308 MODULE,
309 "weight",
310 &[c],
311 w.shape(),
312 ));
313 }
314 }
315 if let Some(b) = bias {
316 if b.shape() != [c] {
317 return Err(validation::shape_mismatch(MODULE, "bias", &[c], b.shape()));
318 }
319 }
320 let c_per_g = c / num_groups;
321 let spatial: usize = if shape.len() > 2 {
322 shape[2..].iter().product()
323 } else {
324 1
325 };
326 let group_size = c_per_g * spatial;
327
328 let flat = input.reshape([n * num_groups, group_size]);
330
331 let mean =
333 mean_axis(&flat, 1, &backend).map_err(|source| validation::backend(MODULE, source))?;
334
335 let xmu = sub(&flat, &mean, &backend)?;
337
338 let xmu_sq = mul(&xmu, &xmu, &backend)?;
340 let mut var =
341 mean_axis(&xmu_sq, 1, &backend).map_err(|source| validation::backend(MODULE, source))?;
342
343 let eps_t = Tensor::full_on([1], coeus_core::FloatElement::from_f64(eps), &backend)?;
345 add_assign(&mut var, &eps_t, &backend).map_err(|source| validation::backend(MODULE, source))?;
346 sqrt_assign(&mut var, &backend).map_err(|source| validation::backend(MODULE, source))?; let ones = Tensor::ones_on([n * num_groups, 1], &backend)?;
350 let mut istdev = ones;
351 div_assign(&mut istdev, &var, &backend)
352 .map_err(|source| validation::backend(MODULE, source))?;
353
354 let x_hat = mul(&xmu, &istdev, &backend)?;
356
357 let mut out = x_hat.reshape(shape.clone());
359
360 let mut broadcast_shape = vec![1usize; shape.len()];
362 broadcast_shape[1] = c;
363
364 if let Some(w) = weight {
365 let w_bc = w.reshape(broadcast_shape.clone());
366 out = mul(&out, &w_bc, &backend)?;
367 }
368 if let Some(b) = bias {
369 let b_bc = b.reshape(broadcast_shape.clone());
370 add_assign(&mut out, &b_bc, &backend)
371 .map_err(|source| validation::backend(MODULE, source))?;
372 }
373
374 Ok(out)
375}