1use super::validation;
16use crate::module::{Module, ModuleError, ModuleResult};
17use coeus_autograd::Var;
18use coeus_core::{Float, MoiraiBackend, Scalar};
19use coeus_tensor::Tensor;
20use std::cell::RefCell;
21
22type TensorPair<T, B> = (Tensor<T, B>, Tensor<T, B>);
24
25type MCacheEntry<T, B> = (usize, Tensor<T, B>, Tensor<T, B>);
27
28type MCache<T, B> = RefCell<Option<MCacheEntry<T, B>>>;
30
31const fn module_name<const DIM: usize>() -> &'static str {
33 match DIM {
34 1 => "BatchNorm1d",
35 2 => "BatchNorm2d",
36 3 => "BatchNorm3d",
37 _ => "BatchNormNd",
38 }
39}
40
41const fn rank_str<const DIM: usize>() -> &'static str {
43 match DIM {
44 1 => "2 or 3",
45 2 => "4",
46 3 => "5",
47 _ => "configured batch-normalization rank",
48 }
49}
50
51fn permute_to_nhwc<T: Scalar, B: coeus_ops::BackendOps<T> + Default, const DIM: usize>(
55 tensor: &Tensor<T, B>,
56 backend: &B,
57) -> Result<Tensor<T, B>, B::Error> {
58 match DIM {
59 1 => tensor.permute(&[0, 2, 1]).to_contiguous_on(backend),
60 2 => tensor.permute(&[0, 2, 3, 1]).to_contiguous_on(backend),
61 3 => tensor.permute(&[0, 2, 3, 4, 1]).to_contiguous_on(backend),
62 _ => panic!("BatchNorm permute_to_nhwc: unsupported DIM {DIM}"),
63 }
64}
65
66fn permute_from_nhwc<T: Scalar, B: coeus_ops::BackendOps<T> + Default, const DIM: usize>(
68 tensor: &Tensor<T, B>,
69 backend: &B,
70) -> Result<Tensor<T, B>, B::Error> {
71 match DIM {
72 1 => tensor.permute(&[0, 2, 1]).to_contiguous_on(backend),
73 2 => tensor.permute(&[0, 3, 1, 2]).to_contiguous_on(backend),
74 3 => tensor.permute(&[0, 4, 1, 2, 3]).to_contiguous_on(backend),
75 _ => panic!("BatchNorm permute_from_nhwc: unsupported DIM {DIM}"),
76 }
77}
78
79fn reshape_from_flat<T: Scalar, B: coeus_ops::BackendOps<T>, const DIM: usize>(
81 tensor: Tensor<T, B>,
82 n: usize,
83 spatial: &[usize],
84 c: usize,
85) -> Result<Tensor<T, B>, B::Error> {
86 match DIM {
87 1 => Ok(tensor.reshape([n, spatial[0], c])),
88 2 => Ok(tensor.reshape([n, spatial[0], spatial[1], c])),
89 3 => Ok(tensor.reshape([n, spatial[0], spatial[1], spatial[2], c])),
90 _ => panic!("BatchNorm reshape_from_flat: unsupported DIM {DIM}"),
91 }
92}
93
94#[derive(Clone)]
108pub struct BatchNorm<
109 T: Float,
110 B: coeus_ops::BackendOps<T> + Default = MoiraiBackend,
111 const DIM: usize = 1,
112> {
113 pub num_features: usize,
115 pub weight: Var<T, B>,
117 pub bias: Var<T, B>,
119 pub eps: f64,
121 pub momentum: f64,
123 pub is_training: bool,
125 pub running_mean: RefCell<Tensor<T, B>>,
127 pub running_var: RefCell<Tensor<T, B>>,
129 eps_t: Tensor<T, B>,
131 mom_t: Tensor<T, B>,
133 one_minus_mom_t: Tensor<T, B>,
135 minus_half: Tensor<T, B>,
137 two_const: Tensor<T, B>,
139 ones_c: Tensor<T, B>,
141 m_cache: MCache<T, B>,
143}
144
145impl<
146 T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
147 B: coeus_ops::BackendOps<T> + Default,
148 const DIM: usize,
149 > BatchNorm<T, B, DIM>
150{
151 pub fn new(
153 num_features: usize,
154 eps: f64,
155 momentum: f64,
156 ) -> Result<Self, crate::init::InitializationError<B::Error>> {
157 let backend = B::default();
158 Self::from_parts(
159 num_features,
160 Var::new(Tensor::ones_on([num_features], &backend)?, true)?,
161 Var::new(Tensor::zeros_on([num_features], &backend)?, true)?,
162 eps,
163 momentum,
164 Tensor::zeros_on([num_features], &backend)?,
165 Tensor::ones_on([num_features], &backend)?,
166 )
167 }
168
169 pub fn from_parts(
171 num_features: usize,
172 weight: Var<T, B>,
173 bias: Var<T, B>,
174 eps: f64,
175 momentum: f64,
176 running_mean: Tensor<T, B>,
177 running_var: Tensor<T, B>,
178 ) -> Result<Self, crate::init::InitializationError<B::Error>> {
179 let backend = B::default();
180 let eps_t = Tensor::full_on([1], coeus_core::FloatElement::from_f64(eps), &backend)?;
181 let mom_t = Tensor::full_on([1], coeus_core::FloatElement::from_f64(momentum), &backend)?;
182 let one_minus_mom_t = Tensor::full_on(
183 [1],
184 coeus_core::FloatElement::from_f64(1.0 - momentum),
185 &backend,
186 )?;
187 let minus_half = Tensor::full_on([1], coeus_core::FloatElement::from_f64(-0.5), &backend)?;
188 let two_const = Tensor::full_on([1], coeus_core::FloatElement::from_f64(2.0), &backend)?;
189 let ones_c = Tensor::ones_on([1, num_features], &backend)?;
190 Ok(Self {
191 num_features,
192 weight,
193 bias,
194 eps,
195 momentum,
196 is_training: true,
197 running_mean: RefCell::new(running_mean),
198 running_var: RefCell::new(running_var),
199 eps_t,
200 mom_t,
201 one_minus_mom_t,
202 minus_half,
203 two_const,
204 ones_c,
205 m_cache: RefCell::new(None),
206 })
207 }
208
209 pub fn set_training(&mut self, mode: bool) {
211 self.is_training = mode;
212 }
213
214 fn m_constants(&self, m: usize, backend: &B) -> ModuleResult<TensorPair<T, B>, B> {
216 let module: &str = module_name::<DIM>();
217 let mut cache = self
218 .m_cache
219 .try_borrow_mut()
220 .map_err(|_| validation::state_borrow(module, "m_cache"))?;
221 if let Some((cached_m, ref cached_m_const, ref cached_corr_t)) = *cache {
222 if cached_m == m {
223 return Ok((cached_m_const.clone(), cached_corr_t.clone()));
224 }
225 }
226 let m_const = Tensor::full_on([1], coeus_core::FloatElement::from_count(m), backend)?;
227 let correction = if m > 1 {
230 <f64 as coeus_core::FloatElement>::from_count(m)
231 / <f64 as coeus_core::FloatElement>::from_count(m - 1)
232 } else {
233 1.0
234 };
235 let corr_t = Tensor::full_on(
236 [1],
237 <T as coeus_core::FloatElement>::from_f64(correction),
238 backend,
239 )?;
240 *cache = Some((m, m_const.clone(), corr_t.clone()));
241 Ok((m_const, corr_t))
242 }
243}
244
245impl<
246 T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
247 B: coeus_ops::BackendOps<T> + Default,
248 const DIM: usize,
249 > Module<T, B> for BatchNorm<T, B, DIM>
250{
251 fn parameters(&self) -> Vec<Var<T, B>> {
252 vec![self.weight.clone(), self.bias.clone()]
253 }
254
255 fn train(&mut self, mode: bool) {
256 self.is_training = mode;
257 }
258
259 fn forward(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
260 let module: &str = module_name::<DIM>();
261 let shape = input.tensor.shape();
262 let rank_ok = shape.len() == DIM + 2 || (DIM == 1 && shape.len() == 2);
264 if !rank_ok {
265 return Err(validation::invalid_rank(
266 module,
267 rank_str::<DIM>(),
268 shape.len(),
269 ));
270 }
271 let c = shape[1];
272 if c != self.num_features {
273 return Err(validation::channel_mismatch(module, self.num_features, c));
274 }
275 if !self.eps.is_finite() || self.eps < 0.0 {
276 return Err(ModuleError::InvalidEpsilon { module });
277 }
278 for (parameter, actual) in [
279 ("weight", self.weight.tensor.shape()),
280 ("bias", self.bias.tensor.shape()),
281 ] {
282 if actual != [self.num_features] {
283 return Err(validation::shape_mismatch(
284 module,
285 parameter,
286 &[self.num_features],
287 actual,
288 ));
289 }
290 }
291 let input_is_2d = DIM == 1 && shape.len() == 2;
296 let upstream: Var<T, B> = if input_is_2d {
297 let n = shape[0];
298 let c = shape[1];
299 coeus_autograd::reshape(input, vec![n, c, 1])?
301 } else {
302 input.clone()
303 };
304 let out = self.forward_nd(&upstream)?;
305 if input_is_2d {
306 let n = shape[0];
307 let c = shape[1];
308 Ok(coeus_autograd::reshape(&out, vec![n, c])?)
310 } else {
311 Ok(out)
312 }
313 }
314}
315
316impl<
317 T: coeus_core::FloatElement + Float + coeus_cpu::RealScalar,
318 B: coeus_ops::BackendOps<T> + Default,
319 const DIM: usize,
320 > BatchNorm<T, B, DIM>
321{
322 fn forward_nd(&self, input: &Var<T, B>) -> Result<Var<T, B>, ModuleError<B::Error>> {
326 let module: &str = module_name::<DIM>();
327 let shape = input.tensor.shape();
328 let n = shape[0];
329 let c = shape[1];
330 let mut spatial = [1usize; 3];
331 for (slot, &extent) in spatial.iter_mut().zip(shape[2..].iter()) {
332 *slot = extent;
333 }
334 let backend = B::default();
335 let m_spatial: usize = spatial.iter().product();
336 let m = n * m_spatial; if !self.is_training {
340 let rm = self
341 .running_mean
342 .try_borrow()
343 .map_err(|_| validation::state_borrow(module, "running_mean"))?;
344 let rv = self
345 .running_var
346 .try_borrow()
347 .map_err(|_| validation::state_borrow(module, "running_var"))?;
348 for (parameter, actual) in [("running_mean", rm.shape()), ("running_var", rv.shape())] {
349 if actual != [self.num_features] {
350 return Err(validation::shape_mismatch(
351 module,
352 parameter,
353 &[self.num_features],
354 actual,
355 ));
356 }
357 }
358 let nhwc = permute_to_nhwc::<T, B, DIM>(&input.tensor, &backend);
360 let flat = nhwc?.reshape([m, c]);
361 let rm_row = rm.reshape([1, c]);
362 let rv_row = rv.reshape([1, c]);
363 let mut istdev = rv_row.clone();
364 coeus_ops::add_assign(&mut istdev, &self.eps_t, &backend)
365 .map_err(|source| validation::backend(module, source))?;
366 coeus_ops::sqrt_assign(&mut istdev, &backend)
367 .map_err(|source| validation::backend(module, source))?;
368 let ones = Tensor::ones_on([1, c], &backend)?;
369 let mut istdev_inv = ones;
370 coeus_ops::div_assign(&mut istdev_inv, &istdev, &backend)
371 .map_err(|source| validation::backend(module, source))?;
372 let xmu = coeus_ops::sub(&flat, &rm_row, &backend)?;
373 let x_hat = coeus_ops::mul(&xmu, &istdev_inv, &backend)?;
374 let w_r = self.weight.tensor.reshape([1, c]);
375 let b_r = self.bias.tensor.reshape([1, c]);
376 let mut y = coeus_ops::mul(&x_hat, &w_r, &backend)?;
377 coeus_ops::add_assign(&mut y, &b_r, &backend)
378 .map_err(|source| validation::backend(module, source))?;
379 let y_nhwc = reshape_from_flat::<T, B, DIM>(y, n, &spatial, c)?;
380 let out_tensor = permute_from_nhwc::<T, B, DIM>(&y_nhwc, &backend)?;
381 return Ok(Var::new(out_tensor, false)?);
382 }
383
384 if m < 2 {
385 return Err(validation::insufficient_elements(module, 2, m));
386 }
387
388 let (m_const, corr_t) = self.m_constants(m, &backend)?;
389
390 let nhwc = permute_to_nhwc::<T, B, DIM>(&input.tensor, &backend);
392 let flat = nhwc?.reshape([m, c]); let mean_t = coeus_ops::mean_axis(&flat, 0, &backend)
396 .map_err(|source| validation::backend(module, source))?; let xmu = coeus_ops::sub(&flat, &mean_t, &backend)?;
400
401 let xmu_sq = coeus_ops::mul(&xmu, &xmu, &backend)?;
403 let var_t = coeus_ops::mean_axis(&xmu_sq, 0, &backend)
404 .map_err(|source| validation::backend(module, source))?; let mut stdev = var_t.clone();
408 coeus_ops::add_assign(&mut stdev, &self.eps_t, &backend)
409 .map_err(|source| validation::backend(module, source))?;
410 coeus_ops::sqrt_assign(&mut stdev, &backend)
411 .map_err(|source| validation::backend(module, source))?;
412
413 let mut istdev = self.ones_c.clone();
414 coeus_ops::div_assign(&mut istdev, &stdev, &backend)
415 .map_err(|source| validation::backend(module, source))?; let x_hat = coeus_ops::mul(&xmu, &istdev, &backend)?;
419
420 let w_reshaped = self.weight.tensor.reshape([1, c]);
422 let b_reshaped = self.bias.tensor.reshape([1, c]);
423 let mut y_flat = coeus_ops::mul(&x_hat, &w_reshaped, &backend)?;
424 coeus_ops::add_assign(&mut y_flat, &b_reshaped, &backend)
425 .map_err(|source| validation::backend(module, source))?;
426
427 let y_nhwc = reshape_from_flat::<T, B, DIM>(y_flat, n, &spatial, c)?;
429 let out_tensor = permute_from_nhwc::<T, B, DIM>(&y_nhwc, &backend)?;
430
431 let mut next_mean = self
433 .running_mean
434 .try_borrow()
435 .map_err(|_| validation::state_borrow(module, "running_mean"))?
436 .clone();
437 let mut next_var = self
438 .running_var
439 .try_borrow()
440 .map_err(|_| validation::state_borrow(module, "running_var"))?
441 .clone();
442 for (parameter, actual) in [
443 ("running_mean", next_mean.shape()),
444 ("running_var", next_var.shape()),
445 ] {
446 if actual != [self.num_features] {
447 return Err(validation::shape_mismatch(
448 module,
449 parameter,
450 &[self.num_features],
451 actual,
452 ));
453 }
454 }
455 let mean_c = mean_t.reshape([c]);
456 let var_c = var_t.reshape([c]);
457 coeus_ops::mul_assign(&mut next_mean, &self.one_minus_mom_t, &backend)
458 .map_err(|source| validation::backend(module, source))?;
459 let term_mean = coeus_ops::mul(&mean_c, &self.mom_t, &backend)?;
460 coeus_ops::add_assign(&mut next_mean, &term_mean, &backend)
461 .map_err(|source| validation::backend(module, source))?;
462 coeus_ops::mul_assign(&mut next_var, &self.one_minus_mom_t, &backend)
463 .map_err(|source| validation::backend(module, source))?;
464 let var_corrected = coeus_ops::mul(&var_c, &corr_t, &backend)?;
465 let term_var = coeus_ops::mul(&var_corrected, &self.mom_t, &backend)?;
466 coeus_ops::add_assign(&mut next_var, &term_var, &backend)
467 .map_err(|source| validation::backend(module, source))?;
468 let mut running_mean = self
469 .running_mean
470 .try_borrow_mut()
471 .map_err(|_| validation::state_borrow(module, "running_mean"))?;
472 let mut running_var = self
473 .running_var
474 .try_borrow_mut()
475 .map_err(|_| validation::state_borrow(module, "running_var"))?;
476 *running_mean = next_mean;
477 *running_var = next_var;
478
479 Ok(coeus_autograd::batchnorm_nd::<T, B, DIM>(
480 input,
481 &self.weight,
482 &self.bias,
483 coeus_autograd::BatchNormArgs {
484 out_tensor,
485 x_hat,
486 xmu,
487 istdev,
488 m_const,
489 minus_half: self.minus_half.clone(),
490 two_const: self.two_const.clone(),
491 n,
492 c,
493 spatial,
494 m,
495 },
496 )?)
497 }
498}