Skip to main content

rusting_brain/
conv.rs

1//! Convolutions and the layers that surround them in an image decoder.
2//!
3//! A diffusion model's latent becomes pixels through a stack that is almost
4//! entirely convolutional: `Conv2d`, [`GroupNorm`], SiLU, nearest-neighbour
5//! upsampling and, in the FLUX.2 line, a pixel shuffle. None of that existed
6//! here, because a language model needs none of it.
7//!
8//! The convolution is lowered to the matrix multiply the crate already has.
9//! `im2col` copies each patch the kernel will see into a row, so one
10//! `[pixels, in_channels * kh * kw]` by `[out_channels, in_channels * kh * kw]`
11//! multiply produces the whole output. That costs `kh * kw` times the memory of
12//! the input for the copy, and buys the tuned `matrixmultiply` kernel and the
13//! rayon threading that comes with it — a much better trade than a hand-written
14//! seven-deep loop nest.
15//!
16//! These layers are inference only: they hold plain buffers rather than
17//! [`crate::param::Param`]s and have no backward pass. Hosting a published
18//! decoder is a forward pass; training one is a different project.
19//!
20//! ponytail: [`Conv2d`] has no backward pass and no CUDA. Both are additive —
21//! `Param` and a device buffer can replace the plain `Matrix` without changing
22//! the shapes — and neither is needed to run a checkpoint someone else trained.
23//!
24//! # Training a convolution
25//!
26//! [`TrainableConv2d`] is the other half: a batched convolution whose weight
27//! and bias are [`Param`]s, with a backward pass and a
28//! device path. It is a separate type rather than a mode of [`Conv2d`] because
29//! the two want different layouts. A published checkpoint stores
30//! `[out, in, kh, kw]`, which flattens to a column ordered channel-slowest, and
31//! that is what [`Conv2d`] reads. A trained weight is only ever read by the
32//! code that wrote it, so [`TrainableConv2d`] orders its columns
33//! channel-fastest over an [`ImageBatch`] of `[batch * height * width,
34//! channels]` rows, which is the layout the rest of an image training stack
35//! already holds its activations in and which makes im2col one contiguous copy
36//! per pixel.
37//!
38//! The decision behind the device path, since the alternative was to publish
39//! `im2col`/`col2im` for an outside crate to drive: a convolution's column
40//! matrix is `kernel * kernel` times its input — 452 MiB for one 3x3 layer over
41//! 96 channels of 256x256 at batch 2 — and a caller that drove the pair itself
42//! would still have to hand the columns to the GEMM across the bus. Owning both
43//! sides here means the columns are built on the device and consumed there, and
44//! only the activation and its gradient cross, which is the nine-fold
45//! difference the layer's cost is made of.
46
47use crate::matrix::Matrix;
48use crate::network::NetworkError;
49use crate::param::{Linear, Param};
50#[cfg(feature = "cuda")]
51use cudarc::driver::PushKernelArg;
52use rand::rngs::StdRng;
53use rayon::prelude::*;
54
55/// An image-shaped buffer: channels, then rows, then columns, which is the
56/// layout every published checkpoint stores and every convolution reads.
57#[derive(Clone, Debug, PartialEq)]
58pub struct FeatureMap {
59    pub channels: usize,
60    pub height: usize,
61    pub width: usize,
62    pub data: Vec<f32>,
63}
64
65impl FeatureMap {
66    /// A zeroed map of the given shape.
67    pub fn new(channels: usize, height: usize, width: usize) -> Self {
68        Self {
69            channels,
70            height,
71            width,
72            data: vec![0.0; channels * height * width],
73        }
74    }
75
76    /// Wraps data that is already in channel-height-width order.
77    pub fn from_vec(
78        channels: usize,
79        height: usize,
80        width: usize,
81        data: Vec<f32>,
82    ) -> Result<Self, NetworkError> {
83        if data.len() != channels * height * width {
84            return Err(NetworkError::InvalidTarget {
85                expected: channels * height * width,
86                actual: data.len(),
87            });
88        }
89        Ok(Self {
90            channels,
91            height,
92            width,
93            data,
94        })
95    }
96
97    /// Pixels per channel.
98    pub fn pixels(&self) -> usize {
99        self.height * self.width
100    }
101
102    /// One channel's plane.
103    pub fn plane(&self, channel: usize) -> &[f32] {
104        let pixels = self.pixels();
105        &self.data[channel * pixels..(channel + 1) * pixels]
106    }
107
108    /// The map as a `[pixels, channels]` matrix, which is the shape attention
109    /// and a linear layer want.
110    pub fn to_tokens(&self) -> Matrix {
111        let pixels = self.pixels();
112        let mut tokens = Matrix::new(pixels, self.channels);
113        for channel in 0..self.channels {
114            let plane = self.plane(channel);
115            for (pixel, value) in plane.iter().enumerate() {
116                tokens.data[pixel * self.channels + channel] = *value;
117            }
118        }
119        tokens
120    }
121
122    /// The reverse of [`FeatureMap::to_tokens`].
123    pub fn from_tokens(tokens: &Matrix, height: usize, width: usize) -> Result<Self, NetworkError> {
124        if tokens.rows != height * width {
125            return Err(NetworkError::InvalidTarget {
126                expected: height * width,
127                actual: tokens.rows,
128            });
129        }
130        let mut map = Self::new(tokens.cols, height, width);
131        for channel in 0..tokens.cols {
132            for pixel in 0..tokens.rows {
133                map.data[channel * tokens.rows + pixel] =
134                    tokens.data[pixel * tokens.cols + channel];
135            }
136        }
137        Ok(map)
138    }
139}
140
141/// A linear map applied to every row of a `[tokens, in]` matrix, with the bias
142/// a checkpoint's layer usually carries.
143///
144/// [`crate::param::Linear`] is the trainable, bias-free version. This one holds
145/// plain buffers, because a layer read out of someone else's checkpoint is only
146/// ever run forward.
147#[derive(Clone, Debug)]
148pub struct Dense {
149    pub weight: Matrix,
150    pub bias: Option<Vec<f32>>,
151    /// The same weight at one byte per value, once [`Dense::quantize`] has
152    /// been called. The `f32` copy is dropped then, which is the point.
153    quantized: Option<crate::quantized::Quantized>,
154}
155
156impl Dense {
157    /// Wraps an `[out, in]` weight and its bias.
158    pub fn new(weight: Matrix, bias: Option<Vec<f32>>) -> Result<Self, NetworkError> {
159        if bias.as_ref().is_some_and(|bias| bias.len() != weight.rows) {
160            return Err(NetworkError::InvalidTarget {
161                expected: weight.rows,
162                actual: bias.as_ref().map_or(0, |bias| bias.len()),
163            });
164        }
165        Ok(Self {
166            weight,
167            bias,
168            quantized: None,
169        })
170    }
171
172    /// Output width.
173    pub fn out_dim(&self) -> usize {
174        self.weight.rows
175    }
176
177    /// Input width.
178    pub fn in_dim(&self) -> usize {
179        self.weight.cols
180    }
181
182    /// Stores the weight as one byte per value and drops the `f32` copy.
183    ///
184    /// A quarter of the memory, for about 0.4% relative error per weight. The
185    /// shape is kept, so [`Dense::in_dim`] and [`Dense::out_dim`] still answer.
186    /// There is no way back: the `f32` values are gone.
187    pub fn quantize(&mut self) {
188        if self.quantized.is_none() {
189            self.quantized = Some(crate::quantized::Quantized::from_matrix(&self.weight));
190            self.weight.data = Vec::new();
191        }
192    }
193
194    /// The weight as `f32`, widening it first if it is stored quantized.
195    ///
196    /// Borrowed in the common case, so a float layer costs nothing to ask.
197    pub(crate) fn weight_f32(&self) -> std::borrow::Cow<'_, Matrix> {
198        match &self.quantized {
199            Some(quantized) => std::borrow::Cow::Owned(quantized.dequantize()),
200            None => std::borrow::Cow::Borrowed(&self.weight),
201        }
202    }
203
204    /// Whether the weight is stored quantized.
205    pub fn is_quantized(&self) -> bool {
206        self.quantized.is_some()
207    }
208
209    /// Applies the map to every row.
210    pub fn forward(&self, tokens: &Matrix) -> Result<Matrix, NetworkError> {
211        if tokens.cols != self.weight.cols {
212            return Err(NetworkError::InvalidTarget {
213                expected: self.weight.cols,
214                actual: tokens.cols,
215            });
216        }
217        let mut output = match &self.quantized {
218            Some(weight) => weight.matmul_rhs_transposed(tokens),
219            None => {
220                let mut output = Matrix::new(tokens.rows, self.weight.rows);
221                tokens.dot_rhs_transposed(&self.weight, &mut output);
222                output
223            }
224        };
225        if let Some(bias) = &self.bias {
226            output
227                .data
228                .par_chunks_mut(self.weight.rows)
229                .for_each(|row| {
230                    for (value, bias) in row.iter_mut().zip(bias) {
231                        *value += bias;
232                    }
233                });
234        }
235        Ok(output)
236    }
237
238    /// Applies the map to a single vector.
239    pub fn apply(&self, input: &[f32]) -> Result<Vec<f32>, NetworkError> {
240        let row = Matrix::from_vec(1, input.len(), input.to_vec());
241        Ok(self.forward(&row)?.data)
242    }
243}
244
245/// A two-dimensional convolution with zero padding.
246#[derive(Clone, Debug)]
247pub struct Conv2d {
248    /// `[out_channels, in_channels * kernel * kernel]`, the layout a
249    /// checkpoint's `[out, in, kh, kw]` tensor already has once flattened.
250    pub weight: Matrix,
251    pub bias: Option<Vec<f32>>,
252    pub in_channels: usize,
253    pub kernel: usize,
254    pub stride: usize,
255    pub padding: usize,
256}
257
258impl Conv2d {
259    /// Wraps weights read from a checkpoint.
260    ///
261    /// `weight` is the `[out_channels, in_channels * kernel * kernel]` matrix a
262    /// `[out, in, kh, kw]` tensor flattens to, which is what
263    /// [`crate::safetensors::SafeTensors::tensor`] hands back.
264    pub fn new(
265        weight: Matrix,
266        bias: Option<Vec<f32>>,
267        in_channels: usize,
268        kernel: usize,
269        stride: usize,
270        padding: usize,
271    ) -> Result<Self, NetworkError> {
272        if kernel == 0 || stride == 0 {
273            return Err(NetworkError::InvalidConfig(
274                "a convolution needs a kernel and a stride of at least one".into(),
275            ));
276        }
277        if weight.cols != in_channels * kernel * kernel {
278            return Err(NetworkError::InvalidConfig(format!(
279                "a {kernel}x{kernel} convolution over {in_channels} channels needs {} weights per \
280                 output channel, and this one has {}",
281                in_channels * kernel * kernel,
282                weight.cols
283            )));
284        }
285        if bias.as_ref().is_some_and(|bias| bias.len() != weight.rows) {
286            return Err(NetworkError::InvalidConfig(
287                "a convolution's bias has one value per output channel".into(),
288            ));
289        }
290        Ok(Self {
291            weight,
292            bias,
293            in_channels,
294            kernel,
295            stride,
296            padding,
297        })
298    }
299
300    /// Output channels.
301    pub fn out_channels(&self) -> usize {
302        self.weight.rows
303    }
304
305    /// The output shape for an input of the given size.
306    pub fn output_size(&self, height: usize, width: usize) -> (usize, usize) {
307        // A kernel wider than the padded input fits nowhere, which is zero
308        // outputs rather than one built out of padding.
309        let size = |length: usize| match (length + 2 * self.padding).checked_sub(self.kernel) {
310            Some(span) => span / self.stride + 1,
311            None => 0,
312        };
313        (size(height), size(width))
314    }
315
316    /// Convolves `input`.
317    pub fn forward(&self, input: &FeatureMap) -> Result<FeatureMap, NetworkError> {
318        if input.channels != self.in_channels {
319            return Err(NetworkError::InvalidConfig(format!(
320                "this convolution reads {} channels and was handed {}",
321                self.in_channels, input.channels
322            )));
323        }
324        let (out_height, out_width) = self.output_size(input.height, input.width);
325        if out_height == 0 || out_width == 0 {
326            return Err(NetworkError::InvalidConfig(format!(
327                "a {}x{} kernel over a {}x{} input leaves nothing",
328                self.kernel, self.kernel, input.height, input.width
329            )));
330        }
331
332        let columns = self.im2col(input, out_height, out_width);
333        let mut product = Matrix::new(out_height * out_width, self.out_channels());
334        columns.dot_rhs_transposed(&self.weight, &mut product);
335
336        // Back to channel-major, adding the bias on the way.
337        let mut output = FeatureMap::new(self.out_channels(), out_height, out_width);
338        let pixels = out_height * out_width;
339        output
340            .data
341            .par_chunks_mut(pixels)
342            .enumerate()
343            .for_each(|(channel, plane)| {
344                let bias = self.bias.as_ref().map_or(0.0, |bias| bias[channel]);
345                for (pixel, value) in plane.iter_mut().enumerate() {
346                    *value = product.data[pixel * self.out_channels() + channel] + bias;
347                }
348            });
349        Ok(output)
350    }
351
352    /// One row per output pixel, holding every input value that pixel's kernel
353    /// reads. Padding shows up as the zeros the row was created with.
354    fn im2col(&self, input: &FeatureMap, out_height: usize, out_width: usize) -> Matrix {
355        let patch = self.in_channels * self.kernel * self.kernel;
356        let mut columns = Matrix::new(out_height * out_width, patch);
357        columns
358            .data
359            .par_chunks_mut(patch)
360            .enumerate()
361            .for_each(|(pixel, row)| {
362                let (out_y, out_x) = (pixel / out_width, pixel % out_width);
363                let top = (out_y * self.stride) as isize - self.padding as isize;
364                let left = (out_x * self.stride) as isize - self.padding as isize;
365                for channel in 0..self.in_channels {
366                    let plane = channel * input.height * input.width;
367                    for row_offset in 0..self.kernel {
368                        let y = top + row_offset as isize;
369                        if y < 0 || y >= input.height as isize {
370                            continue;
371                        }
372                        let source = plane + y as usize * input.width;
373                        let target = (channel * self.kernel + row_offset) * self.kernel;
374                        for column_offset in 0..self.kernel {
375                            let x = left + column_offset as isize;
376                            if x >= 0 && x < input.width as isize {
377                                row[target + column_offset] = input.data[source + x as usize];
378                            }
379                        }
380                    }
381                }
382            });
383        columns
384    }
385}
386
387/// A batch of image-shaped activations, one row per pixel.
388///
389/// `[batch * height * width, channels]`, which is what makes a convolution a
390/// matrix multiply and an activation function a pass over a slice. [`Conv2d`]'s
391/// [`FeatureMap`] is the other arrangement, channel-major and unbatched,
392/// because that is the one a published checkpoint's decoder is written against.
393#[derive(Clone, Debug, PartialEq)]
394pub struct ImageBatch {
395    pub batch: usize,
396    pub height: usize,
397    pub width: usize,
398    pub tokens: Matrix,
399}
400
401impl ImageBatch {
402    pub fn new(batch: usize, height: usize, width: usize, channels: usize) -> Self {
403        Self {
404            batch,
405            height,
406            width,
407            tokens: Matrix::new(batch * height * width, channels),
408        }
409    }
410
411    /// Wraps rows that are already one pixel each.
412    pub fn from_tokens(batch: usize, height: usize, width: usize, tokens: Matrix) -> Self {
413        debug_assert_eq!(tokens.rows, batch * height * width);
414        Self {
415            batch,
416            height,
417            width,
418            tokens,
419        }
420    }
421
422    pub fn channels(&self) -> usize {
423        self.tokens.cols
424    }
425
426    /// Pixels in one sample.
427    pub fn pixels(&self) -> usize {
428        self.height * self.width
429    }
430
431    /// A zeroed map of the same size with a different channel count.
432    pub fn like(&self, channels: usize) -> Self {
433        Self::new(self.batch, self.height, self.width, channels)
434    }
435
436    /// One pixel's channels.
437    pub fn pixel(&self, sample: usize, y: usize, x: usize) -> &[f32] {
438        self.tokens.row((sample * self.height + y) * self.width + x)
439    }
440
441    pub fn pixel_mut(&mut self, sample: usize, y: usize, x: usize) -> &mut [f32] {
442        let row = (sample * self.height + y) * self.width + x;
443        self.tokens.row_mut(row)
444    }
445
446    /// Adds another map of the same shape in place.
447    pub fn add(&mut self, other: &ImageBatch) {
448        debug_assert_eq!(self.tokens.data.len(), other.tokens.data.len());
449        for (slot, value) in self.tokens.data.iter_mut().zip(&other.tokens.data) {
450            *slot += value;
451        }
452    }
453}
454
455/// An [`ImageBatch`] held on a device, in the same layout, so a chain of
456/// device layers passes activations without crossing the bus. Made by
457/// [`crate::param::CudaDevice::upload`].
458#[cfg(feature = "cuda")]
459pub struct DeviceImageBatch {
460    pub batch: usize,
461    pub height: usize,
462    pub width: usize,
463    channels: usize,
464    pub(crate) data: cudarc::driver::CudaSlice<f32>,
465    context: std::sync::Arc<crate::gpu_transformer::GpuContext>,
466}
467
468#[cfg(feature = "cuda")]
469impl DeviceImageBatch {
470    pub(crate) fn upload(
471        context: &std::sync::Arc<crate::gpu_transformer::GpuContext>,
472        batch: &ImageBatch,
473    ) -> Result<Self, NetworkError> {
474        Ok(Self {
475            batch: batch.batch,
476            height: batch.height,
477            width: batch.width,
478            channels: batch.channels(),
479            data: context.upload(&batch.tokens)?,
480            context: context.clone(),
481        })
482    }
483
484    /// Copies the batch back to the host.
485    pub fn download(&self) -> Result<ImageBatch, NetworkError> {
486        let mut tokens = Matrix::new(self.batch * self.pixels(), self.channels);
487        self.context.download(&self.data, &mut tokens)?;
488        Ok(ImageBatch::from_tokens(
489            self.batch,
490            self.height,
491            self.width,
492            tokens,
493        ))
494    }
495
496    pub fn channels(&self) -> usize {
497        self.channels
498    }
499
500    pub fn pixels(&self) -> usize {
501        self.height * self.width
502    }
503
504    /// Zeros in this batch's shape with `channels` channels.
505    pub fn like(&self, channels: usize) -> Result<Self, NetworkError> {
506        let data = self.context.zeros(self.batch * self.pixels() * channels)?;
507        Ok(Self {
508            channels,
509            ..self.with_data(data)
510        })
511    }
512
513    /// Another buffer of this shape.
514    fn with_data(&self, data: cudarc::driver::CudaSlice<f32>) -> Self {
515        Self {
516            data,
517            context: self.context.clone(),
518            ..*self
519        }
520    }
521}
522
523#[cfg(feature = "cuda")]
524impl Clone for DeviceImageBatch {
525    fn clone(&self) -> Self {
526        self.with_data(self.data.clone())
527    }
528}
529
530#[cfg(feature = "cuda")]
531impl crate::param::CudaDevice {
532    /// Puts a batch on this device for [`TrainableConv2d::forward_device`].
533    pub fn upload(&self, batch: &ImageBatch) -> Result<DeviceImageBatch, NetworkError> {
534        DeviceImageBatch::upload(self.context(), batch)
535    }
536
537    /// [`DeviceImageBatch::download`].
538    pub fn download(&self, batch: &DeviceImageBatch) -> Result<ImageBatch, NetworkError> {
539        batch.download()
540    }
541}
542
543/// Everything `im2col` and `col2im` need to know about one convolution, which
544/// is what the device kernels take as their arguments.
545#[derive(Clone, Copy, Debug, PartialEq, Eq)]
546pub struct ConvGeometry {
547    pub batch: usize,
548    pub channels: usize,
549    pub height: usize,
550    pub width: usize,
551    pub kernel: usize,
552    pub stride: usize,
553    pub padding: usize,
554    pub out_height: usize,
555    pub out_width: usize,
556}
557
558impl ConvGeometry {
559    /// A 1x1 kernel at stride 1 without padding: each column row is the
560    /// input pixel itself.
561    pub fn is_pointwise(&self) -> bool {
562        self.kernel == 1 && self.stride == 1 && self.padding == 0
563    }
564
565    /// Values in the input map.
566    pub fn input_elements(&self) -> usize {
567        self.batch * self.height * self.width * self.channels
568    }
569
570    /// Values in the column matrix: `kernel * kernel` times the output's
571    /// pixels times the input's channels.
572    pub fn column_elements(&self) -> usize {
573        self.batch * self.out_height * self.out_width * self.patch()
574    }
575
576    /// Values in one column row.
577    pub fn patch(&self) -> usize {
578        self.channels * self.kernel * self.kernel
579    }
580
581    /// Output pixels across the whole batch, which is the column matrix's row
582    /// count and the GEMM's.
583    pub fn rows(&self) -> usize {
584        self.batch * self.out_height * self.out_width
585    }
586}
587
588/// A two-dimensional convolution that trains.
589///
590/// The weight is a [`Linear`] over `im2col` columns, so the matrix multiply,
591/// the weight gradient and the input gradient are the ones the crate already
592/// has, on the host and on a device alike. See the module documentation for why
593/// this is not a mode of [`Conv2d`].
594#[derive(Clone, Debug)]
595pub struct TrainableConv2d {
596    /// `[out_channels, in_channels * kernel * kernel]`.
597    pub weight: Linear,
598    /// `[1, out_channels]`. Zero and unused after
599    /// [`TrainableConv2d::without_bias`].
600    pub bias: Param,
601    has_bias: bool,
602    in_channels: usize,
603    kernel: usize,
604    stride: usize,
605    padding: usize,
606    /// Whether the backward pass rebuilds the columns instead of the forward
607    /// pass keeping them. See [`TrainableConv2d::set_recompute_columns`].
608    recompute_columns: bool,
609}
610
611/// What [`TrainableConv2d::backward`] needs from the forward pass.
612///
613/// The columns are the largest thing a convolution holds, so the cache keeps
614/// them wherever the forward pass built them: on the host, or on the device,
615/// where they never have to cross the bus at all. Under
616/// [`TrainableConv2d::set_recompute_columns`] it keeps the input instead and
617/// the backward pass builds the columns again.
618pub struct ConvCache {
619    columns: Columns,
620    shape: ConvGeometry,
621}
622
623impl ConvCache {
624    /// Whether this cache holds the columns themselves rather than the input
625    /// they were built from, which is the difference between `kernel * kernel`
626    /// values per input value and one.
627    pub fn holds_columns(&self) -> bool {
628        match &self.columns {
629            Columns::Input(_) => false,
630            #[cfg(feature = "cuda")]
631            Columns::DeviceInput(_) | Columns::DeviceElsewhere => false,
632            _ => true,
633        }
634    }
635}
636
637enum Columns {
638    Host(Matrix),
639    #[cfg(feature = "cuda")]
640    Device(cudarc::driver::CudaSlice<f32>),
641    /// The forward input, kept in place of the columns it built.
642    Input(ImageBatch),
643    /// The forward input, left on the device where it was already uploaded, so
644    /// rebuilding the columns costs a kernel and not a second crossing.
645    #[cfg(feature = "cuda")]
646    DeviceInput(cudarc::driver::CudaSlice<f32>),
647    /// Device columns as bf16, from a mixed-precision context.
648    #[cfg(feature = "cuda")]
649    DeviceNarrow(cudarc::driver::CudaSlice<u8>),
650    /// Nothing: the caller keeps the forward input and hands it to
651    /// [`TrainableConv2d::backward_device_from`].
652    #[cfg(feature = "cuda")]
653    DeviceElsewhere,
654}
655
656impl std::fmt::Debug for ConvCache {
657    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
658        f.debug_struct("ConvCache")
659            .field("shape", &self.shape)
660            .finish()
661    }
662}
663
664impl TrainableConv2d {
665    /// A `kernel x kernel` convolution, padded to keep the resolution when
666    /// `stride` is one.
667    pub fn new(
668        in_channels: usize,
669        out_channels: usize,
670        kernel: usize,
671        stride: usize,
672        rng: &mut StdRng,
673    ) -> Result<Self, NetworkError> {
674        if kernel == 0 || stride == 0 || in_channels == 0 || out_channels == 0 {
675            return Err(NetworkError::InvalidConfig(
676                "a convolution needs a non-zero kernel, stride and channel count".into(),
677            ));
678        }
679        let fan_in = in_channels * kernel * kernel;
680        Ok(Self {
681            weight: Linear {
682                weight: Param::he_uniform(out_channels, fan_in, fan_in, rng),
683                lora: None,
684            },
685            bias: Param::zeros(1, out_channels),
686            has_bias: true,
687            in_channels,
688            kernel,
689            stride,
690            padding: kernel / 2,
691            recompute_columns: false,
692        })
693    }
694
695    /// [`TrainableConv2d::new`] with the weight set to zero, which is how a
696    /// residual branch's last convolution starts so the block begins as the
697    /// identity.
698    pub fn zeroed(
699        in_channels: usize,
700        out_channels: usize,
701        kernel: usize,
702        rng: &mut StdRng,
703    ) -> Result<Self, NetworkError> {
704        let mut conv = Self::new(in_channels, out_channels, kernel, 1, rng)?;
705        conv.weight.weight = Param::zeros(out_channels, in_channels * kernel * kernel);
706        Ok(conv)
707    }
708
709    /// Drops the bias: the forward pass adds none, the backward pass gathers
710    /// none, and [`TrainableConv2d::params_mut`] leaves it out. For a
711    /// convolution before a norm, whose mean subtraction would cancel it.
712    pub fn without_bias(mut self) -> Self {
713        self.has_bias = false;
714        self.bias = Param::zeros(1, self.bias.value.cols);
715        self
716    }
717
718    pub fn has_bias(&self) -> bool {
719        self.has_bias
720    }
721
722    pub fn out_channels(&self) -> usize {
723        self.bias.value.cols
724    }
725
726    pub fn in_channels(&self) -> usize {
727        self.in_channels
728    }
729
730    pub fn output_size(&self, height: usize, width: usize) -> (usize, usize) {
731        let size = |length: usize| {
732            (length + 2 * self.padding).saturating_sub(self.kernel) / self.stride + 1
733        };
734        (size(height), size(width))
735    }
736
737    /// The geometry of this convolution over an input of the given size.
738    pub fn geometry(&self, batch: usize, height: usize, width: usize) -> ConvGeometry {
739        let (out_height, out_width) = self.output_size(height, width);
740        ConvGeometry {
741            batch,
742            channels: self.in_channels,
743            height,
744            width,
745            kernel: self.kernel,
746            stride: self.stride,
747            padding: self.padding,
748            out_height,
749            out_width,
750        }
751    }
752
753    /// Trades one `im2col` pass per backward for the memory the columns take.
754    ///
755    /// One convolution's columns are
756    ///
757    /// ```text
758    /// batch * out_height * out_width * in_channels * kernel * kernel * 4 bytes
759    /// ```
760    ///
761    /// which is `kernel * kernel` times its input, and they stay live from the
762    /// forward pass until the backward one. A single convolution should keep
763    /// them: the memory is paid once and the work is saved. A network should
764    /// weigh it layer by layer, because every layer's columns are alive at
765    /// once, and the total is what bounds the batch. Put the batch, the
766    /// resolution and the channel count of the widest few layers through the
767    /// formula above before deciding; in a deep net the sum is usually larger
768    /// than the weights and the optimizer moments together.
769    ///
770    /// What it costs is one extra `im2col` per backward pass. On a device the
771    /// input stays on the card, so nothing crosses the bus twice. Worked
772    /// example, on an RTX 3060: a 3x3 convolution from 192 channels to 96 over
773    /// twelve 64x64 images measures 33.5 ms per forward and backward with the
774    /// columns kept and 41.0 ms with them rebuilt, and the formula gives the
775    /// 324 MiB it stops holding against the 36 MiB of input it holds instead.
776    ///
777    /// The columns are only needed for the weight gradient at all; the input
778    /// gradient needs the weight and the upstream gradient alone.
779    pub fn set_recompute_columns(&mut self, recompute: bool) {
780        self.recompute_columns = recompute;
781    }
782
783    /// Whether the backward pass rebuilds the columns.
784    pub fn recomputes_columns(&self) -> bool {
785        self.recompute_columns
786    }
787
788    /// Moves the weight and the bias onto a device a caller already opened.
789    /// Every later [`TrainableConv2d::forward`] and
790    /// [`TrainableConv2d::backward`] then runs entirely there, and only the
791    /// activation and its gradient cross the bus; `forward_device` and
792    /// `backward_device` skip even that.
793    pub fn to_cuda_on(&mut self, device: &crate::param::CudaDevice) -> Result<(), NetworkError> {
794        self.weight.to_cuda_on(device)?;
795        if self.has_bias {
796            self.bias.to_cuda_on(device)?;
797        }
798        Ok(())
799    }
800
801    /// Brings the weight and the bias back to the host.
802    pub fn to_cpu(&mut self) -> Result<(), NetworkError> {
803        self.weight.to_cpu()?;
804        self.bias.to_cpu()
805    }
806
807    pub fn forward(&self, input: &ImageBatch) -> Result<(ImageBatch, ConvCache), NetworkError> {
808        if input.channels() != self.in_channels {
809            return Err(NetworkError::InvalidConfig(format!(
810                "this convolution reads {} channels and was given {}",
811                self.in_channels,
812                input.channels()
813            )));
814        }
815        let shape = self.geometry(input.batch, input.height, input.width);
816        #[cfg(feature = "cuda")]
817        if let Some(device) = &self.weight.weight.device {
818            return self.forward_on_device(input, device);
819        }
820        let columns = self.im2col(input, &shape);
821        let mut tokens = self.weight.forward(&columns);
822        self.add_bias(&mut tokens);
823        Ok((
824            ImageBatch::from_tokens(input.batch, shape.out_height, shape.out_width, tokens),
825            ConvCache {
826                columns: match self.recompute_columns {
827                    true => Columns::Input(input.clone()),
828                    false => Columns::Host(columns),
829                },
830                shape,
831            },
832        ))
833    }
834
835    /// Accumulates the weight and bias gradients and returns `dL/dinput`.
836    pub fn backward(
837        &mut self,
838        cache: &ConvCache,
839        grad_output: &ImageBatch,
840    ) -> Result<ImageBatch, NetworkError> {
841        match &cache.columns {
842            Columns::Host(columns) => {
843                self.accumulate_bias_grad(&grad_output.tokens);
844                let grad_columns = self.weight.backward(columns, &grad_output.tokens);
845                Ok(self.col2im(&grad_columns, &cache.shape))
846            }
847            #[cfg(feature = "cuda")]
848            Columns::Device(_)
849            | Columns::DeviceInput(_)
850            | Columns::DeviceNarrow(_)
851            | Columns::DeviceElsewhere => self.backward_on_device(cache, grad_output),
852            Columns::Input(input) => {
853                self.accumulate_bias_grad(&grad_output.tokens);
854                let columns = self.im2col(input, &cache.shape);
855                let grad_columns = self.weight.backward(&columns, &grad_output.tokens);
856                Ok(self.col2im(&grad_columns, &cache.shape))
857            }
858        }
859    }
860
861    /// [`TrainableConv2d::forward_device`] for inference: no cache, SiLU under
862    /// `silu` fused into the bias, and a 1x1 stride-1 kernel reads the input
863    /// as its columns instead of copying it into them.
864    #[cfg(feature = "cuda")]
865    pub fn infer_device(
866        &self,
867        input: &DeviceImageBatch,
868        silu: bool,
869    ) -> Result<DeviceImageBatch, NetworkError> {
870        self.infer_device_with(input, silu, None)
871    }
872
873    /// [`TrainableConv2d::infer_device`] with the weight also given as bf16
874    /// bytes, from [`TrainableConv2d::bf16_weight`]: a convolution that needs
875    /// `im2col` then builds its columns as bf16 too, half the bytes, and runs
876    /// a bf16 GEMM with an FP32 accumulator.
877    #[cfg(feature = "cuda")]
878    pub(crate) fn infer_device_with(
879        &self,
880        input: &DeviceImageBatch,
881        silu: bool,
882        narrow_weight: Option<&cudarc::driver::CudaSlice<u8>>,
883    ) -> Result<DeviceImageBatch, NetworkError> {
884        if input.channels != self.in_channels {
885            return Err(NetworkError::InvalidConfig(format!(
886                "this convolution reads {} channels and was given {}",
887                self.in_channels, input.channels
888            )));
889        }
890        let device = self.weight.weight.device.as_ref().ok_or_else(|| {
891            NetworkError::InvalidConfig("infer_device needs the weight on a device".into())
892        })?;
893        let shape = self.geometry(input.batch, input.height, input.width);
894        let context = device.context();
895        let mut product = match ((self.kernel, self.stride, self.padding), narrow_weight) {
896            ((1, 1, 0), _) => device.matmul_rhs_transposed_device(&input.data, shape.rows())?,
897            (_, Some(weight)) => device.matmul_rhs_transposed_bf16(
898                &context.im2col_bf16(&input.data, &shape)?,
899                weight,
900                shape.rows(),
901            )?,
902            _ => device.matmul_rhs_transposed_device(
903                &context.im2col(&input.data, &shape)?,
904                shape.rows(),
905            )?,
906        };
907        let bias = self
908            .bias
909            .device
910            .as_ref()
911            .filter(|_| self.has_bias)
912            .ok_or_else(|| {
913                NetworkError::InvalidConfig("infer_device needs a bias on the device".into())
914            })?;
915        context.add_bias_rows_then(&mut product, bias.value(), self.out_channels(), silu)?;
916        Ok(DeviceImageBatch {
917            batch: input.batch,
918            height: shape.out_height,
919            width: shape.out_width,
920            channels: self.out_channels(),
921            data: product,
922            context: context.clone(),
923        })
924    }
925
926    /// The device weight as bf16 bytes for [`TrainableConv2d::infer_device_with`],
927    /// when its context runs mixed precision and this convolution needs
928    /// `im2col`; `None` otherwise.
929    #[cfg(feature = "cuda")]
930    pub(crate) fn bf16_weight(
931        &self,
932    ) -> Result<Option<cudarc::driver::CudaSlice<u8>>, NetworkError> {
933        let Some(device) = self.weight.weight.device.as_ref() else {
934            return Ok(None);
935        };
936        let context = device.context();
937        match context.mixed_precision && (self.kernel, self.stride, self.padding) != (1, 1, 0) {
938            true => context.bf16_bytes(device.value()).map(Some),
939            false => Ok(None),
940        }
941    }
942
943    /// The forward pass for an input that is already on the device, leaving
944    /// the output there too. Needs [`TrainableConv2d::to_cuda_on`] first.
945    #[cfg(feature = "cuda")]
946    pub fn forward_device(
947        &self,
948        input: &DeviceImageBatch,
949    ) -> Result<(DeviceImageBatch, ConvCache), NetworkError> {
950        self.forward_device_keeping(input, true)
951    }
952
953    /// [`TrainableConv2d::forward_device`], with `keep_input` false for a
954    /// caller that keeps the input itself: a cache that would hold a copy of
955    /// it holds nothing, and the backward pass is
956    /// [`TrainableConv2d::backward_device_from`].
957    #[cfg(feature = "cuda")]
958    pub(crate) fn forward_device_keeping(
959        &self,
960        input: &DeviceImageBatch,
961        keep_input: bool,
962    ) -> Result<(DeviceImageBatch, ConvCache), NetworkError> {
963        if input.channels != self.in_channels {
964            return Err(NetworkError::InvalidConfig(format!(
965                "this convolution reads {} channels and was given {}",
966                self.in_channels, input.channels
967            )));
968        }
969        let device = self.weight.weight.device.as_ref().ok_or_else(|| {
970            NetworkError::InvalidConfig("forward_device needs the weight on a device".into())
971        })?;
972        let shape = self.geometry(input.batch, input.height, input.width);
973        let context = device.context();
974        // A 1x1 stride-1 convolution's columns are its input; a mixed-precision
975        // context builds the others as bf16 for a bf16 GEMM.
976        let (mut product, columns) = match (shape.is_pointwise(), context.mixed_precision) {
977            (true, _) => (
978                device.matmul_rhs_transposed_device(&input.data, shape.rows())?,
979                None,
980            ),
981            (false, true) => {
982                let columns = context.im2col_bf16(&input.data, &shape)?;
983                let weight = context.bf16_bytes(device.value())?;
984                (
985                    device.matmul_rhs_transposed_bf16(&columns, &weight, shape.rows())?,
986                    Some(Columns::DeviceNarrow(columns)),
987                )
988            }
989            (false, false) => {
990                let columns = context.im2col(&input.data, &shape)?;
991                (
992                    device.matmul_rhs_transposed_device(&columns, shape.rows())?,
993                    Some(Columns::Device(columns)),
994                )
995            }
996        };
997        if let Some(bias) = self.bias.device.as_ref().filter(|_| self.has_bias) {
998            context.add_bias_rows(&mut product, bias.value(), self.out_channels())?;
999        }
1000        Ok((
1001            DeviceImageBatch {
1002                batch: input.batch,
1003                height: shape.out_height,
1004                width: shape.out_width,
1005                channels: self.out_channels(),
1006                data: product,
1007                context: context.clone(),
1008            },
1009            ConvCache {
1010                // Dropping the device columns here is the whole point of the
1011                // option: they are freed before the next layer allocates its
1012                // own, so a deep network holds one layer's worth rather than
1013                // every layer's.
1014                columns: match columns.filter(|_| !self.recompute_columns) {
1015                    Some(columns) => columns,
1016                    None if keep_input => Columns::DeviceInput(input.data.clone()),
1017                    None => Columns::DeviceElsewhere,
1018                },
1019                shape,
1020            },
1021        ))
1022    }
1023
1024    /// The backward pass for a gradient that is already on the device, for a
1025    /// cache [`TrainableConv2d::forward_device`] (or a device-resident
1026    /// [`TrainableConv2d::forward`]) built. Accumulates the weight and bias
1027    /// gradients on the device and returns `dL/dinput` there.
1028    #[cfg(feature = "cuda")]
1029    pub fn backward_device(
1030        &mut self,
1031        cache: &ConvCache,
1032        grad_output: &DeviceImageBatch,
1033    ) -> Result<DeviceImageBatch, NetworkError> {
1034        self.backward_device_from(cache, None, grad_output)
1035    }
1036
1037    /// [`TrainableConv2d::backward_device`] for a cache from
1038    /// [`TrainableConv2d::forward_device_keeping`], given the forward input
1039    /// when the cache did not keep it.
1040    #[cfg(feature = "cuda")]
1041    pub(crate) fn backward_device_from(
1042        &mut self,
1043        cache: &ConvCache,
1044        input: Option<&DeviceImageBatch>,
1045        grad_output: &DeviceImageBatch,
1046    ) -> Result<DeviceImageBatch, NetworkError> {
1047        let shape = &cache.shape;
1048        let input = match &cache.columns {
1049            Columns::DeviceInput(input) => Some(input),
1050            Columns::DeviceElsewhere => input.map(|batch| &batch.data),
1051            _ => None,
1052        };
1053        let device = self.weight.weight.device.as_mut().ok_or_else(|| {
1054            NetworkError::InvalidConfig("backward_device needs the weight on a device".into())
1055        })?;
1056        let context = device.context().clone();
1057        if context.mixed_precision && !shape.is_pointwise() {
1058            let rebuilt;
1059            let columns = match (&cache.columns, input) {
1060                (Columns::DeviceNarrow(columns), _) => columns,
1061                (_, Some(input)) => {
1062                    rebuilt = context.im2col_bf16(input, shape)?;
1063                    &rebuilt
1064                }
1065                _ => {
1066                    return Err(NetworkError::InvalidConfig(
1067                        "backward_device needs a cache from a mixed-precision forward pass".into(),
1068                    ));
1069                }
1070            };
1071            let grad = context.bf16_bytes(&grad_output.data)?;
1072            device.accumulate_grad_bf16(&grad, columns, shape.rows())?;
1073            let weight = context.bf16_bytes(device.value())?;
1074            let grad_columns = device.matmul_bf16(&grad, &weight, shape.rows())?;
1075            self.accumulate_bias_device(&context, grad_output, shape.rows())?;
1076            return Ok(DeviceImageBatch {
1077                batch: shape.batch,
1078                height: shape.height,
1079                width: shape.width,
1080                channels: shape.channels,
1081                data: context.col2im_bf16(&grad_columns, shape)?,
1082                context,
1083            });
1084        }
1085        let rebuilt;
1086        let columns = match (&cache.columns, input) {
1087            (Columns::Device(columns), _) => columns,
1088            (_, Some(input)) if shape.is_pointwise() => input,
1089            (_, Some(input)) => {
1090                rebuilt = context.im2col(input, shape)?;
1091                &rebuilt
1092            }
1093            _ => {
1094                return Err(NetworkError::InvalidConfig(
1095                    "backward_device needs a cache from a device forward pass".into(),
1096                ));
1097            }
1098        };
1099        device.accumulate_grad_device(&grad_output.data, columns, shape.rows())?;
1100        let grad_columns = device.matmul_device(&grad_output.data, shape.rows())?;
1101        self.accumulate_bias_device(&context, grad_output, shape.rows())?;
1102        Ok(DeviceImageBatch {
1103            batch: shape.batch,
1104            height: shape.height,
1105            width: shape.width,
1106            channels: shape.channels,
1107            data: match shape.is_pointwise() {
1108                true => grad_columns,
1109                false => context.col2im(&grad_columns, shape)?,
1110            },
1111            context,
1112        })
1113    }
1114
1115    #[cfg(feature = "cuda")]
1116    fn accumulate_bias_device(
1117        &mut self,
1118        context: &std::sync::Arc<crate::gpu_transformer::GpuContext>,
1119        grad_output: &DeviceImageBatch,
1120        rows: usize,
1121    ) -> Result<(), NetworkError> {
1122        if let Some(bias) = self.bias.device.as_mut().filter(|_| self.has_bias) {
1123            // The column sum as a GEMM against ones, which keeps the negated
1124            // accumulate convention in one place.
1125            let ones = context.filled(rows, 1.0)?;
1126            bias.accumulate_grad_device(&ones, &grad_output.data, rows)?;
1127        }
1128        Ok(())
1129    }
1130
1131    /// [`TrainableConv2d::forward_device`] between an upload and a download.
1132    #[cfg(feature = "cuda")]
1133    fn forward_on_device(
1134        &self,
1135        input: &ImageBatch,
1136        device: &crate::gpu_transformer::DeviceParam,
1137    ) -> Result<(ImageBatch, ConvCache), NetworkError> {
1138        let resident = DeviceImageBatch::upload(device.context(), input)?;
1139        let (output, cache) = self.forward_device(&resident)?;
1140        Ok((output.download()?, cache))
1141    }
1142
1143    #[cfg(feature = "cuda")]
1144    fn backward_on_device(
1145        &mut self,
1146        cache: &ConvCache,
1147        grad_output: &ImageBatch,
1148    ) -> Result<ImageBatch, NetworkError> {
1149        let context = self
1150            .weight
1151            .weight
1152            .device
1153            .as_ref()
1154            .expect("a device-resident cache means a device-resident weight")
1155            .context()
1156            .clone();
1157        let resident = DeviceImageBatch::upload(&context, grad_output)?;
1158        self.backward_device(cache, &resident)?.download()
1159    }
1160
1161    fn add_bias(&self, tokens: &mut Matrix) {
1162        if !self.has_bias {
1163            return;
1164        }
1165        let bias = &self.bias.value.data;
1166        tokens.data.par_chunks_mut(tokens.cols).for_each(|row| {
1167            for (value, bias) in row.iter_mut().zip(bias) {
1168                *value += bias;
1169            }
1170        });
1171    }
1172
1173    /// The bias gradient is the upstream gradient summed over every pixel.
1174    ///
1175    /// Always on the host: it is one pass over a buffer that is already here,
1176    /// and the bias is one value per output channel rather than one per weight.
1177    fn accumulate_bias_grad(&mut self, grad_output: &Matrix) {
1178        if self.bias.is_frozen() || !self.has_bias {
1179            return;
1180        }
1181        for row in 0..grad_output.rows {
1182            for (slot, value) in self.bias.grad.data.iter_mut().zip(grad_output.row(row)) {
1183                *slot += value;
1184            }
1185        }
1186    }
1187
1188    pub fn params_mut(&mut self) -> Vec<&mut Param> {
1189        let mut params = self.weight.params_mut();
1190        if self.has_bias {
1191            params.push(&mut self.bias);
1192        }
1193        params
1194    }
1195
1196    /// `[batch * out_height * out_width, in_channels * kernel * kernel]`, one
1197    /// row per output pixel.
1198    fn im2col(&self, input: &ImageBatch, shape: &ConvGeometry) -> Matrix {
1199        let patch = shape.patch();
1200        let mut columns = Matrix::new(shape.rows(), patch);
1201        let (kernel, stride, padding) = (self.kernel, self.stride, self.padding);
1202        let channels = self.in_channels;
1203        columns
1204            .data
1205            .par_chunks_mut(patch)
1206            .enumerate()
1207            .for_each(|(index, row)| {
1208                let plane = shape.out_height * shape.out_width;
1209                let (sample, rest) = (index / plane, index % plane);
1210                let (out_y, out_x) = (rest / shape.out_width, rest % shape.out_width);
1211                for ky in 0..kernel {
1212                    let y = (out_y * stride + ky) as isize - padding as isize;
1213                    if y < 0 || y as usize >= input.height {
1214                        continue;
1215                    }
1216                    for kx in 0..kernel {
1217                        let x = (out_x * stride + kx) as isize - padding as isize;
1218                        if x < 0 || x as usize >= input.width {
1219                            continue;
1220                        }
1221                        let source = input.pixel(sample, y as usize, x as usize);
1222                        let offset = (ky * kernel + kx) * channels;
1223                        row[offset..offset + channels].copy_from_slice(source);
1224                    }
1225                }
1226            });
1227        columns
1228    }
1229
1230    /// The transpose of [`TrainableConv2d::im2col`]: gradients scattered back
1231    /// and summed where the patches overlapped.
1232    fn col2im(&self, columns: &Matrix, shape: &ConvGeometry) -> ImageBatch {
1233        let mut grad = ImageBatch::new(shape.batch, shape.height, shape.width, self.in_channels);
1234        for sample in 0..shape.batch {
1235            for out_y in 0..shape.out_height {
1236                for out_x in 0..shape.out_width {
1237                    let row =
1238                        columns.row((sample * shape.out_height + out_y) * shape.out_width + out_x);
1239                    for ky in 0..self.kernel {
1240                        let y = (out_y * self.stride + ky) as isize - self.padding as isize;
1241                        if y < 0 || y as usize >= shape.height {
1242                            continue;
1243                        }
1244                        for kx in 0..self.kernel {
1245                            let x = (out_x * self.stride + kx) as isize - self.padding as isize;
1246                            if x < 0 || x as usize >= shape.width {
1247                                continue;
1248                            }
1249                            let offset = (ky * self.kernel + kx) * self.in_channels;
1250                            let target = grad.pixel_mut(sample, y as usize, x as usize);
1251                            for channel in 0..self.in_channels {
1252                                target[channel] += row[offset + channel];
1253                            }
1254                        }
1255                    }
1256                }
1257            }
1258        }
1259        grad
1260    }
1261}
1262
1263/// Normalization over groups of channels, which is what an image model uses
1264/// where a language model uses [`crate::norm::RmsNorm`].
1265///
1266/// A batch norm would need statistics collected at training time and a layer
1267/// norm would mix unrelated channels; a group norm normalizes each group of
1268/// channels over its own pixels, which is stable at a batch size of one.
1269#[derive(Clone, Debug)]
1270pub struct GroupNorm {
1271    pub groups: usize,
1272    pub weight: Vec<f32>,
1273    pub bias: Vec<f32>,
1274    pub eps: f32,
1275}
1276
1277impl GroupNorm {
1278    /// Wraps the per-channel scale and shift a checkpoint stores.
1279    pub fn new(
1280        groups: usize,
1281        weight: Vec<f32>,
1282        bias: Vec<f32>,
1283        eps: f32,
1284    ) -> Result<Self, NetworkError> {
1285        if groups == 0 || weight.is_empty() || weight.len() % groups != 0 {
1286            return Err(NetworkError::InvalidConfig(format!(
1287                "{} channels do not divide into {groups} groups",
1288                weight.len()
1289            )));
1290        }
1291        if bias.len() != weight.len() {
1292            return Err(NetworkError::InvalidTarget {
1293                expected: weight.len(),
1294                actual: bias.len(),
1295            });
1296        }
1297        Ok(Self {
1298            groups,
1299            weight,
1300            bias,
1301            eps,
1302        })
1303    }
1304
1305    /// Normalizes in place.
1306    pub fn forward(&self, map: &mut FeatureMap) -> Result<(), NetworkError> {
1307        if map.channels != self.weight.len() {
1308            return Err(NetworkError::InvalidTarget {
1309                expected: self.weight.len(),
1310                actual: map.channels,
1311            });
1312        }
1313        let per_group = map.channels / self.groups;
1314        let pixels = map.pixels();
1315        let span = per_group * pixels;
1316
1317        map.data
1318            .par_chunks_mut(span)
1319            .enumerate()
1320            .for_each(|(group, values)| {
1321                let mean = values.iter().sum::<f32>() / span as f32;
1322                let variance = values
1323                    .iter()
1324                    .map(|value| (value - mean).powi(2))
1325                    .sum::<f32>()
1326                    / span as f32;
1327                let scale = (variance + self.eps).sqrt().recip();
1328                for (index, value) in values.iter_mut().enumerate() {
1329                    let channel = group * per_group + index / pixels;
1330                    *value = (*value - mean) * scale * self.weight[channel] + self.bias[channel];
1331                }
1332            });
1333        Ok(())
1334    }
1335}
1336
1337/// [`GroupNorm`] with a backward pass, over an [`ImageBatch`].
1338///
1339/// Statistics are summed channel by channel in the same order as
1340/// [`GroupNorm::forward`], so the two agree to the bit.
1341#[derive(Clone, Debug)]
1342pub struct TrainableGroupNorm {
1343    pub groups: usize,
1344    pub eps: f32,
1345    pub gamma: Param,
1346    pub beta: Param,
1347}
1348
1349/// What [`TrainableGroupNorm::backward`] needs from the forward pass.
1350#[derive(Clone, Debug)]
1351pub struct GroupNormCache {
1352    normalized: Matrix,
1353    /// `1 / sqrt(variance + eps)` per sample and group.
1354    inv_std: Vec<f32>,
1355}
1356
1357impl TrainableGroupNorm {
1358    /// Scale ones, shift zeros.
1359    pub fn new(groups: usize, channels: usize, eps: f32) -> Result<Self, NetworkError> {
1360        if groups == 0 || channels == 0 || channels % groups != 0 {
1361            return Err(NetworkError::InvalidConfig(format!(
1362                "{channels} channels do not divide into {groups} groups"
1363            )));
1364        }
1365        Ok(Self {
1366            groups,
1367            eps,
1368            gamma: Param::filled(1, channels, 1.0),
1369            beta: Param::zeros(1, channels),
1370        })
1371    }
1372
1373    pub fn forward(
1374        &self,
1375        input: &ImageBatch,
1376    ) -> Result<(ImageBatch, GroupNormCache), NetworkError> {
1377        let channels = self.gamma.value.cols;
1378        if input.channels() != channels {
1379            return Err(NetworkError::InvalidTarget {
1380                expected: channels,
1381                actual: input.channels(),
1382            });
1383        }
1384        let (per_group, pixels) = (channels / self.groups, input.pixels());
1385        let span = (per_group * pixels) as f32;
1386        let mut normalized = Matrix::new(input.tokens.rows, channels);
1387        let mut inv_std = vec![0f32; input.batch * self.groups];
1388        normalized
1389            .data
1390            .par_chunks_mut(pixels * channels)
1391            .zip(input.tokens.data.par_chunks(pixels * channels))
1392            .zip(inv_std.par_chunks_mut(self.groups))
1393            .for_each(|((out, sample), inv_std)| {
1394                for (group, inv_std) in inv_std.iter_mut().enumerate() {
1395                    let range = group * per_group..(group + 1) * per_group;
1396                    let values = || {
1397                        range.clone().flat_map(|channel| {
1398                            (0..pixels).map(move |p| sample[p * channels + channel])
1399                        })
1400                    };
1401                    let mean = values().sum::<f32>() / span;
1402                    let variance = values().map(|value| (value - mean).powi(2)).sum::<f32>() / span;
1403                    *inv_std = (variance + self.eps).sqrt().recip();
1404                    for p in 0..pixels {
1405                        for channel in range.clone() {
1406                            let at = p * channels + channel;
1407                            out[at] = (sample[at] - mean) * *inv_std;
1408                        }
1409                    }
1410                }
1411            });
1412        let (gamma, beta) = (&self.gamma.value.data, &self.beta.value.data);
1413        let mut output = input.like(channels);
1414        for (out, row) in output
1415            .tokens
1416            .data
1417            .chunks_exact_mut(channels)
1418            .zip(normalized.data.chunks_exact(channels))
1419        {
1420            for channel in 0..channels {
1421                out[channel] = row[channel] * gamma[channel] + beta[channel];
1422            }
1423        }
1424        Ok((
1425            output,
1426            GroupNormCache {
1427                normalized,
1428                inv_std,
1429            },
1430        ))
1431    }
1432
1433    /// Adds to `gamma` and `beta`'s gradients and returns the input's.
1434    pub fn backward(&mut self, cache: &GroupNormCache, grad: &ImageBatch) -> ImageBatch {
1435        let channels = grad.channels();
1436        let (per_group, pixels) = (channels / self.groups, grad.pixels());
1437        let span = (per_group * pixels) as f32;
1438        for (g, row) in grad
1439            .tokens
1440            .data
1441            .chunks_exact(channels)
1442            .zip(cache.normalized.data.chunks_exact(channels))
1443        {
1444            for channel in 0..channels {
1445                self.gamma.grad.data[channel] += g[channel] * row[channel];
1446                self.beta.grad.data[channel] += g[channel];
1447            }
1448        }
1449        let gamma = &self.gamma.value.data;
1450        let mut input_grad = grad.like(channels);
1451        input_grad
1452            .tokens
1453            .data
1454            .par_chunks_mut(pixels * channels)
1455            .zip(grad.tokens.data.par_chunks(pixels * channels))
1456            .zip(cache.normalized.data.par_chunks(pixels * channels))
1457            .zip(cache.inv_std.par_chunks(self.groups))
1458            .for_each(|(((out, g), normalized), inv_std)| {
1459                for (group, &inv_std) in inv_std.iter().enumerate() {
1460                    let range = group * per_group..(group + 1) * per_group;
1461                    let (mut mean, mut along) = (0f32, 0f32);
1462                    for p in 0..pixels {
1463                        for channel in range.clone() {
1464                            let at = p * channels + channel;
1465                            let d = g[at] * gamma[channel];
1466                            mean += d;
1467                            along += d * normalized[at];
1468                        }
1469                    }
1470                    let (mean, along) = (mean / span, along / span);
1471                    for p in 0..pixels {
1472                        for channel in range.clone() {
1473                            let at = p * channels + channel;
1474                            out[at] =
1475                                inv_std * (g[at] * gamma[channel] - mean - normalized[at] * along);
1476                        }
1477                    }
1478                }
1479            });
1480        input_grad
1481    }
1482
1483    pub fn params_mut(&mut self) -> Vec<&mut Param> {
1484        vec![&mut self.gamma, &mut self.beta]
1485    }
1486
1487    /// Moves `gamma` and `beta` onto a device for
1488    /// [`TrainableGroupNorm::forward_device`].
1489    pub fn to_cuda_on(&mut self, device: &crate::param::CudaDevice) -> Result<(), NetworkError> {
1490        self.gamma.to_cuda_on(device)?;
1491        self.beta.to_cuda_on(device)
1492    }
1493
1494    pub fn to_cpu(&mut self) -> Result<(), NetworkError> {
1495        self.gamma.to_cpu()?;
1496        self.beta.to_cpu()
1497    }
1498
1499    /// [`TrainableGroupNorm::forward`] for a batch already on the device.
1500    #[cfg(feature = "cuda")]
1501    pub fn forward_device(
1502        &self,
1503        input: &DeviceImageBatch,
1504    ) -> Result<(DeviceImageBatch, DeviceGroupNormCache), NetworkError> {
1505        let channels = self.gamma.value.cols;
1506        if input.channels != channels {
1507            return Err(NetworkError::InvalidTarget {
1508                expected: channels,
1509                actual: input.channels,
1510            });
1511        }
1512        let (gamma, beta) = match (&self.gamma.device, &self.beta.device) {
1513            (Some(gamma), Some(beta)) => (gamma.value(), beta.value()),
1514            _ => {
1515                return Err(NetworkError::InvalidConfig(
1516                    "forward_device needs gamma and beta on a device".into(),
1517                ));
1518            }
1519        };
1520        let context = &input.context;
1521        let err = crate::cuda_training::cuda_err;
1522        let total = input.data.len();
1523        let blocks = input.batch * self.groups;
1524        let (mut mean, mut inv_std) = (context.zeros(blocks)?, context.zeros(blocks)?);
1525        let (mut output, mut normalized) = (context.zeros(total)?, context.zeros(total)?);
1526        let [n, pixels, channels, per_group] =
1527            [total, input.pixels(), channels, channels / self.groups].map(|v| v as i32);
1528        unsafe {
1529            context
1530                .stream
1531                .launch_builder(&context.image.gn_stats)
1532                .arg(&mut mean)
1533                .arg(&mut inv_std)
1534                .arg(&input.data)
1535                .arg(&pixels)
1536                .arg(&channels)
1537                .arg(&per_group)
1538                .arg(&self.eps)
1539                .launch(group_blocks(blocks))
1540                .map_err(err("group norm statistics kernel"))?;
1541            context
1542                .stream
1543                .launch_builder(&context.image.gn_apply)
1544                .arg(&mut output)
1545                .arg(&mut normalized)
1546                .arg(&input.data)
1547                .arg(&mean)
1548                .arg(&inv_std)
1549                .arg(gamma)
1550                .arg(beta)
1551                .arg(&n)
1552                .arg(&pixels)
1553                .arg(&channels)
1554                .arg(&per_group)
1555                .launch(crate::cuda_training::cfg(total))
1556                .map_err(err("group norm kernel"))?;
1557        }
1558        Ok((
1559            input.with_data(output),
1560            DeviceGroupNormCache {
1561                normalized,
1562                inv_std,
1563            },
1564        ))
1565    }
1566
1567    /// [`TrainableGroupNorm::backward`] on the device: adds to `gamma` and
1568    /// `beta`'s device gradients and returns the input's.
1569    #[cfg(feature = "cuda")]
1570    pub fn backward_device(
1571        &mut self,
1572        cache: &DeviceGroupNormCache,
1573        grad: &DeviceImageBatch,
1574    ) -> Result<DeviceImageBatch, NetworkError> {
1575        let context = grad.context.clone();
1576        let err = crate::cuda_training::cuda_err;
1577        let (total, rows) = (grad.data.len(), grad.batch * grad.pixels());
1578        let blocks = grad.batch * self.groups;
1579        let channels = self.gamma.value.cols;
1580        let (Some(gamma), Some(beta)) = (self.gamma.device.as_mut(), self.beta.device.as_mut())
1581        else {
1582            return Err(NetworkError::InvalidConfig(
1583                "backward_device needs gamma and beta on a device".into(),
1584            ));
1585        };
1586        let ones = context.filled(rows, 1.0)?;
1587        let mut product = context.zeros(total)?;
1588        let (mut first, mut second) = (context.zeros(blocks)?, context.zeros(blocks)?);
1589        let mut input_grad = context.zeros(total)?;
1590        let [n, pixels, channels, per_group] =
1591            [total, grad.pixels(), channels, channels / self.groups].map(|v| v as i32);
1592        unsafe {
1593            context
1594                .stream
1595                .launch_builder(&context.image.mul)
1596                .arg(&mut product)
1597                .arg(&grad.data)
1598                .arg(&cache.normalized)
1599                .arg(&n)
1600                .launch(crate::cuda_training::cfg(total))
1601                .map_err(err("multiply kernel"))?;
1602            context
1603                .stream
1604                .launch_builder(&context.image.gn_bwd_stats)
1605                .arg(&mut first)
1606                .arg(&mut second)
1607                .arg(&grad.data)
1608                .arg(&cache.normalized)
1609                .arg(gamma.value())
1610                .arg(&pixels)
1611                .arg(&channels)
1612                .arg(&per_group)
1613                .launch(group_blocks(blocks))
1614                .map_err(err("group norm backward statistics kernel"))?;
1615            context
1616                .stream
1617                .launch_builder(&context.image.gn_bwd_apply)
1618                .arg(&mut input_grad)
1619                .arg(&grad.data)
1620                .arg(&cache.normalized)
1621                .arg(&cache.inv_std)
1622                .arg(&first)
1623                .arg(&second)
1624                .arg(gamma.value())
1625                .arg(&n)
1626                .arg(&pixels)
1627                .arg(&channels)
1628                .arg(&per_group)
1629                .launch(crate::cuda_training::cfg(total))
1630                .map_err(err("group norm backward kernel"))?;
1631        }
1632        gamma.accumulate_grad_device(&ones, &product, rows)?;
1633        beta.accumulate_grad_device(&ones, &grad.data, rows)?;
1634        Ok(grad.with_data(input_grad))
1635    }
1636}
1637
1638/// What [`TrainableGroupNorm::backward_device`] needs from the forward pass.
1639#[cfg(feature = "cuda")]
1640pub struct DeviceGroupNormCache {
1641    normalized: cudarc::driver::CudaSlice<f32>,
1642    inv_std: cudarc::driver::CudaSlice<f32>,
1643}
1644
1645/// One block per sample and group, for the group norm reductions.
1646#[cfg(feature = "cuda")]
1647fn group_blocks(blocks: usize) -> cudarc::driver::LaunchConfig {
1648    cudarc::driver::LaunchConfig {
1649        grid_dim: (blocks as u32, 1, 1),
1650        block_dim: (256, 1, 1),
1651        shared_mem_bytes: 0,
1652    }
1653}
1654
1655/// SiLU over a device-resident batch. Keep the input: it is what
1656/// [`silu_backward_device`] differentiates at.
1657#[cfg(feature = "cuda")]
1658pub fn silu_device(input: &DeviceImageBatch) -> Result<DeviceImageBatch, NetworkError> {
1659    silu_launch(input, None)
1660}
1661
1662/// `grad * silu'(input)`, on the device.
1663#[cfg(feature = "cuda")]
1664pub fn silu_backward_device(
1665    input: &DeviceImageBatch,
1666    grad: &DeviceImageBatch,
1667) -> Result<DeviceImageBatch, NetworkError> {
1668    silu_launch(input, Some(grad))
1669}
1670
1671/// What [`spatial_softmax_backward_device`] needs from the forward pass.
1672#[cfg(feature = "cuda")]
1673pub struct DeviceSpatialSoftmaxCache {
1674    probabilities: DeviceImageBatch,
1675    expected: cudarc::driver::CudaSlice<f32>,
1676}
1677
1678/// [`spatial_softmax`] on the device. The output is a `[batch, 1, 1,
1679/// 2 * channels]` batch, laid out like the host `Matrix`.
1680#[cfg(feature = "cuda")]
1681pub fn spatial_softmax_device(
1682    input: &DeviceImageBatch,
1683) -> Result<(DeviceImageBatch, DeviceSpatialSoftmaxCache), NetworkError> {
1684    let context = &input.context;
1685    let channels = input.channels;
1686    let mut probabilities = context.zeros(input.data.len())?;
1687    let mut expected = context.zeros(input.batch * 2 * channels)?;
1688    let [height, width, channels_arg] = [input.height, input.width, channels].map(|v| v as i32);
1689    unsafe {
1690        context
1691            .stream
1692            .launch_builder(&context.image.ssm_fwd)
1693            .arg(&mut probabilities)
1694            .arg(&mut expected)
1695            .arg(&input.data)
1696            .arg(&height)
1697            .arg(&width)
1698            .arg(&channels_arg)
1699            .launch(group_blocks(input.batch * channels))
1700            .map_err(crate::cuda_training::cuda_err("spatial softmax kernel"))?;
1701    }
1702    let output = DeviceImageBatch {
1703        batch: input.batch,
1704        height: 1,
1705        width: 1,
1706        channels: 2 * channels,
1707        data: expected.clone(),
1708        context: context.clone(),
1709    };
1710    let probabilities = input.with_data(probabilities);
1711    Ok((
1712        output,
1713        DeviceSpatialSoftmaxCache {
1714            probabilities,
1715            expected,
1716        },
1717    ))
1718}
1719
1720/// [`spatial_softmax_backward`] on the device.
1721#[cfg(feature = "cuda")]
1722pub fn spatial_softmax_backward_device(
1723    cache: &DeviceSpatialSoftmaxCache,
1724    grad: &DeviceImageBatch,
1725) -> Result<DeviceImageBatch, NetworkError> {
1726    let probabilities = &cache.probabilities;
1727    let context = &probabilities.context;
1728    let total = probabilities.data.len();
1729    let mut out = context.zeros(total)?;
1730    let [n, height, width, channels] = [
1731        total,
1732        probabilities.height,
1733        probabilities.width,
1734        probabilities.channels,
1735    ]
1736    .map(|v| v as i32);
1737    unsafe {
1738        context
1739            .stream
1740            .launch_builder(&context.image.ssm_bwd)
1741            .arg(&mut out)
1742            .arg(&probabilities.data)
1743            .arg(&grad.data)
1744            .arg(&cache.expected)
1745            .arg(&n)
1746            .arg(&height)
1747            .arg(&width)
1748            .arg(&channels)
1749            .launch(crate::cuda_training::cfg(total))
1750            .map_err(crate::cuda_training::cuda_err(
1751                "spatial softmax backward kernel",
1752            ))?;
1753    }
1754    Ok(probabilities.with_data(out))
1755}
1756
1757/// [`global_avg_pool`] on the device: a `[batch, 1, 1, channels]` batch.
1758#[cfg(feature = "cuda")]
1759pub fn global_avg_pool_device(input: &DeviceImageBatch) -> Result<DeviceImageBatch, NetworkError> {
1760    let context = &input.context;
1761    let mut out = context.zeros(input.batch * input.channels)?;
1762    let [pixels, channels] = [input.pixels(), input.channels].map(|v| v as i32);
1763    unsafe {
1764        context
1765            .stream
1766            .launch_builder(&context.image.gap_fwd)
1767            .arg(&mut out)
1768            .arg(&input.data)
1769            .arg(&pixels)
1770            .arg(&channels)
1771            .launch(group_blocks(input.batch * input.channels))
1772            .map_err(crate::cuda_training::cuda_err("average pool kernel"))?;
1773    }
1774    Ok(DeviceImageBatch {
1775        height: 1,
1776        width: 1,
1777        ..input.with_data(out)
1778    })
1779}
1780
1781/// [`global_avg_pool_backward`] on the device, back to a `height` by `width`
1782/// batch.
1783#[cfg(feature = "cuda")]
1784pub fn global_avg_pool_backward_device(
1785    grad: &DeviceImageBatch,
1786    height: usize,
1787    width: usize,
1788) -> Result<DeviceImageBatch, NetworkError> {
1789    let context = &grad.context;
1790    let total = grad.batch * height * width * grad.channels;
1791    let mut out = context.zeros(total)?;
1792    let [n, pixels, channels] = [total, height * width, grad.channels].map(|v| v as i32);
1793    unsafe {
1794        context
1795            .stream
1796            .launch_builder(&context.image.gap_bwd)
1797            .arg(&mut out)
1798            .arg(&grad.data)
1799            .arg(&n)
1800            .arg(&pixels)
1801            .arg(&channels)
1802            .launch(crate::cuda_training::cfg(total))
1803            .map_err(crate::cuda_training::cuda_err(
1804                "average pool backward kernel",
1805            ))?;
1806    }
1807    Ok(DeviceImageBatch {
1808        height,
1809        width,
1810        ..grad.with_data(out)
1811    })
1812}
1813
1814#[cfg(feature = "cuda")]
1815fn silu_launch(
1816    input: &DeviceImageBatch,
1817    grad: Option<&DeviceImageBatch>,
1818) -> Result<DeviceImageBatch, NetworkError> {
1819    let context = &input.context;
1820    let total = input.data.len();
1821    let mut output = context.uninit(total)?;
1822    let n = total as i32;
1823    let mut launch = match grad {
1824        None => context.stream.launch_builder(&context.image.silu),
1825        Some(_) => context.stream.launch_builder(&context.image.silu_bwd),
1826    };
1827    launch.arg(&mut output);
1828    if let Some(grad) = grad {
1829        launch.arg(&grad.data);
1830    }
1831    launch.arg(&input.data).arg(&n);
1832    unsafe { launch.launch(crate::cuda_training::cfg(total)) }
1833        .map_err(crate::cuda_training::cuda_err("SiLU kernel"))?;
1834    Ok(input.with_data(output))
1835}
1836
1837/// SiLU over a whole map, the activation every one of these decoders uses.
1838pub fn silu(map: &mut FeatureMap) {
1839    map.data
1840        .par_iter_mut()
1841        .for_each(|value| *value = crate::ffn::silu(*value));
1842}
1843
1844/// Doubles each side by repeating pixels, which is how a decoder climbs from a
1845/// latent's resolution to an image's.
1846pub fn upsample_nearest(map: &FeatureMap, factor: usize) -> FeatureMap {
1847    let (height, width) = (map.height * factor, map.width * factor);
1848    let mut output = FeatureMap::new(map.channels, height, width);
1849    let pixels = height * width;
1850    output
1851        .data
1852        .par_chunks_mut(pixels)
1853        .enumerate()
1854        .for_each(|(channel, plane)| {
1855            let source = map.plane(channel);
1856            for (pixel, value) in plane.iter_mut().enumerate() {
1857                let (y, x) = (pixel / width / factor, pixel % width / factor);
1858                *value = source[y * map.width + x];
1859            }
1860        });
1861    output
1862}
1863
1864/// Trades channels for resolution: `[c * factor^2, h, w]` becomes
1865/// `[c, h * factor, w * factor]`.
1866///
1867/// FLUX.2's decoder ends in one of these, and the ordering here is the one
1868/// PyTorch's `pixel_shuffle` uses, so a checkpoint's weights line up.
1869pub fn pixel_shuffle(map: &FeatureMap, factor: usize) -> Result<FeatureMap, NetworkError> {
1870    let square = factor * factor;
1871    if factor == 0 || map.channels % square != 0 {
1872        return Err(NetworkError::InvalidConfig(format!(
1873            "{} channels do not shuffle by {factor}",
1874            map.channels
1875        )));
1876    }
1877    let (channels, height, width) = (
1878        map.channels / square,
1879        map.height * factor,
1880        map.width * factor,
1881    );
1882    let mut output = FeatureMap::new(channels, height, width);
1883    let pixels = height * width;
1884    output
1885        .data
1886        .par_chunks_mut(pixels)
1887        .enumerate()
1888        .for_each(|(channel, plane)| {
1889            for (pixel, value) in plane.iter_mut().enumerate() {
1890                let (y, x) = (pixel / width, pixel % width);
1891                let source = (channel * square + (y % factor) * factor + x % factor)
1892                    * map.height
1893                    * map.width
1894                    + (y / factor) * map.width
1895                    + x / factor;
1896                *value = map.data[source];
1897            }
1898        });
1899    Ok(output)
1900}
1901
1902/// The inverse of [`pixel_shuffle`]: resolution back into channels, which is
1903/// how FLUX.2 packs a latent before the transformer sees it.
1904pub fn pixel_unshuffle(map: &FeatureMap, factor: usize) -> Result<FeatureMap, NetworkError> {
1905    if factor == 0 || map.height % factor != 0 || map.width % factor != 0 {
1906        return Err(NetworkError::InvalidConfig(format!(
1907            "a {}x{} map does not unshuffle by {factor}",
1908            map.height, map.width
1909        )));
1910    }
1911    let square = factor * factor;
1912    let (channels, height, width) = (
1913        map.channels * square,
1914        map.height / factor,
1915        map.width / factor,
1916    );
1917    let mut output = FeatureMap::new(channels, height, width);
1918    let pixels = height * width;
1919    output
1920        .data
1921        .par_chunks_mut(pixels)
1922        .enumerate()
1923        .for_each(|(channel, plane)| {
1924            let (source_channel, offset) = (channel / square, channel % square);
1925            let (row_offset, column_offset) = (offset / factor, offset % factor);
1926            let source = map.plane(source_channel);
1927            for (pixel, value) in plane.iter_mut().enumerate() {
1928                let (y, x) = (pixel / width, pixel % width);
1929                *value = source[(y * factor + row_offset) * map.width + x * factor + column_offset];
1930            }
1931        });
1932    Ok(output)
1933}
1934
1935/// `n` evenly spaced points from -1 to 1; the middle for one.
1936fn unit_coordinates(n: usize) -> Vec<f32> {
1937    match n {
1938        1 => vec![0.0],
1939        _ => (0..n)
1940            .map(|i| -1.0 + 2.0 * i as f32 / (n - 1) as f32)
1941            .collect(),
1942    }
1943}
1944
1945/// What [`spatial_softmax_backward`] needs from the forward pass.
1946pub struct SpatialSoftmaxCache {
1947    /// The softmax over pixels of each sample's channels, laid out like the
1948    /// input.
1949    probabilities: ImageBatch,
1950    /// The forward output.
1951    expected: Matrix,
1952}
1953
1954/// Each channel's softmax over its pixels, reduced to the expected `x` and
1955/// `y` with both axes running from -1 to 1: `[batch, 2 * channels]`, every
1956/// channel's `x` first, then every channel's `y`. A keypoint layer with no
1957/// parameters.
1958pub fn spatial_softmax(input: &ImageBatch) -> (Matrix, SpatialSoftmaxCache) {
1959    let (channels, pixels, width) = (input.channels(), input.pixels(), input.width);
1960    let (xs, ys) = (
1961        unit_coordinates(input.width),
1962        unit_coordinates(input.height),
1963    );
1964    let mut probabilities = input.like(channels);
1965    let mut expected = Matrix::new(input.batch, 2 * channels);
1966    probabilities
1967        .tokens
1968        .data
1969        .par_chunks_mut(pixels * channels)
1970        .zip(input.tokens.data.par_chunks(pixels * channels))
1971        .zip(expected.data.par_chunks_mut(2 * channels))
1972        .for_each(|((probability, logits), expected)| {
1973            for channel in 0..channels {
1974                let logit = |pixel: usize| logits[pixel * channels + channel];
1975                let max = (0..pixels).map(logit).fold(f32::NEG_INFINITY, f32::max);
1976                let mut sum = 0.0;
1977                for pixel in 0..pixels {
1978                    let value = (logit(pixel) - max).exp();
1979                    probability[pixel * channels + channel] = value;
1980                    sum += value;
1981                }
1982                let (mut x, mut y) = (0.0, 0.0);
1983                for pixel in 0..pixels {
1984                    let slot = &mut probability[pixel * channels + channel];
1985                    *slot /= sum;
1986                    x += *slot * xs[pixel % width];
1987                    y += *slot * ys[pixel / width];
1988                }
1989                (expected[channel], expected[channels + channel]) = (x, y);
1990            }
1991        });
1992    let cache = SpatialSoftmaxCache {
1993        probabilities,
1994        expected: expected.clone(),
1995    };
1996    (expected, cache)
1997}
1998
1999/// `dL/d(input)` of [`spatial_softmax`], given `dL/d(output)`.
2000pub fn spatial_softmax_backward(cache: &SpatialSoftmaxCache, grad: &Matrix) -> ImageBatch {
2001    let probabilities = &cache.probabilities;
2002    let (channels, pixels, width) = (
2003        probabilities.channels(),
2004        probabilities.pixels(),
2005        probabilities.width,
2006    );
2007    let (xs, ys) = (
2008        unit_coordinates(probabilities.width),
2009        unit_coordinates(probabilities.height),
2010    );
2011    let mut out = probabilities.like(channels);
2012    out.tokens
2013        .data
2014        .par_chunks_mut(pixels * channels)
2015        .zip(probabilities.tokens.data.par_chunks(pixels * channels))
2016        .zip(
2017            grad.data
2018                .par_chunks(2 * channels)
2019                .zip(cache.expected.data.par_chunks(2 * channels)),
2020        )
2021        .for_each(|((out, probability), (grad, expected))| {
2022            for pixel in 0..pixels {
2023                for channel in 0..channels {
2024                    // The softmax Jacobian: `q_i (dL/dq_i - sum_j q_j dL/dq_j)`,
2025                    // where `dL/dq_i` is the coordinate times the upstream.
2026                    let (gx, gy) = (grad[channel], grad[channels + channel]);
2027                    let along = gx * (xs[pixel % width] - expected[channel])
2028                        + gy * (ys[pixel / width] - expected[channels + channel]);
2029                    let slot = pixel * channels + channel;
2030                    out[slot] = probability[slot] * along;
2031                }
2032            }
2033        });
2034    out
2035}
2036
2037/// The mean of each channel over its pixels: `[batch, channels]`.
2038pub fn global_avg_pool(input: &ImageBatch) -> Matrix {
2039    let (channels, pixels) = (input.channels(), input.pixels());
2040    let mut out = Matrix::new(input.batch, channels);
2041    out.data
2042        .par_chunks_mut(channels)
2043        .zip(input.tokens.data.par_chunks(pixels * channels))
2044        .for_each(|(out, sample)| {
2045            for pixel in sample.chunks_exact(channels) {
2046                out.iter_mut()
2047                    .zip(pixel)
2048                    .for_each(|(sum, value)| *sum += value);
2049            }
2050            out.iter_mut().for_each(|sum| *sum /= pixels as f32);
2051        });
2052    out
2053}
2054
2055/// `dL/d(input)` of [`global_avg_pool`] over a `height` by `width` input.
2056pub fn global_avg_pool_backward(grad: &Matrix, height: usize, width: usize) -> ImageBatch {
2057    let (channels, pixels) = (grad.cols, height * width);
2058    let mut out = ImageBatch::new(grad.rows, height, width, channels);
2059    out.tokens
2060        .data
2061        .par_chunks_mut(pixels * channels)
2062        .zip(grad.data.par_chunks(channels))
2063        .for_each(|(out, grad)| {
2064            for pixel in out.chunks_exact_mut(channels) {
2065                pixel
2066                    .iter_mut()
2067                    .zip(grad)
2068                    .for_each(|(slot, value)| *slot = value / pixels as f32);
2069            }
2070        });
2071    out
2072}
2073
2074/// PyTorch's `BatchNorm2d` over an image batch's channels, training on batch
2075/// statistics and evaluating on running ones. YOLO's convolutions use it with
2076/// [`TrainableBatchNorm2d::yolo`].
2077#[derive(Clone, Debug)]
2078pub struct TrainableBatchNorm2d {
2079    pub eps: f32,
2080    /// Weight of the newest batch in the running statistics.
2081    pub momentum: f32,
2082    pub gamma: Param,
2083    pub beta: Param,
2084    /// `[2, channels]`: the running mean, then the running unbiased variance.
2085    /// A `Param` so it moves to and from a device with the rest; the
2086    /// optimizer never sees it.
2087    pub running: Param,
2088}
2089
2090/// What [`TrainableBatchNorm2d::backward`] needs from the forward pass.
2091#[derive(Clone, Debug)]
2092pub struct BatchNormCache {
2093    normalized: Matrix,
2094    inv_std: Vec<f32>,
2095    training: bool,
2096}
2097
2098impl TrainableBatchNorm2d {
2099    /// Scale ones, shift zeros, running mean zeros and variance ones.
2100    pub fn new(channels: usize, eps: f32, momentum: f32) -> Self {
2101        let mut running = Matrix::new(2, channels);
2102        running.data[channels..].fill(1.0);
2103        Self {
2104            eps,
2105            momentum,
2106            gamma: Param::filled(1, channels, 1.0),
2107            beta: Param::zeros(1, channels),
2108            running: Param::new(running),
2109        }
2110    }
2111
2112    /// ultralytics' settings: eps 1e-3, momentum 0.03.
2113    pub fn yolo(channels: usize) -> Self {
2114        Self::new(channels, 1e-3, 0.03)
2115    }
2116
2117    pub fn channels(&self) -> usize {
2118        self.gamma.value.cols
2119    }
2120
2121    /// Normalizes with this batch's statistics and folds them into the
2122    /// running ones when `training`, else with the running ones.
2123    pub fn forward(
2124        &mut self,
2125        input: &ImageBatch,
2126        training: bool,
2127    ) -> Result<(ImageBatch, BatchNormCache), NetworkError> {
2128        let channels = self.channels();
2129        if input.channels() != channels {
2130            return Err(NetworkError::InvalidTarget {
2131                expected: channels,
2132                actual: input.channels(),
2133            });
2134        }
2135        let rows = input.tokens.rows;
2136        let (mut mean, mut variance) = (vec![0f32; channels], vec![0f32; channels]);
2137        if training {
2138            for row in input.tokens.data.chunks_exact(channels) {
2139                mean.iter_mut().zip(row).for_each(|(m, v)| *m += v);
2140            }
2141            mean.iter_mut().for_each(|m| *m /= rows as f32);
2142            for row in input.tokens.data.chunks_exact(channels) {
2143                for c in 0..channels {
2144                    variance[c] += (row[c] - mean[c]).powi(2);
2145                }
2146            }
2147            variance.iter_mut().for_each(|v| *v /= rows as f32);
2148            let unbias = rows as f32 / (rows as f32 - 1.0).max(1.0);
2149            let m = self.momentum;
2150            let running = &mut self.running.value.data;
2151            for c in 0..channels {
2152                running[c] = (1.0 - m) * running[c] + m * mean[c];
2153                running[channels + c] =
2154                    (1.0 - m) * running[channels + c] + m * variance[c] * unbias;
2155            }
2156        } else {
2157            mean.copy_from_slice(&self.running.value.data[..channels]);
2158            variance.copy_from_slice(&self.running.value.data[channels..]);
2159        }
2160        let inv_std: Vec<f32> = variance
2161            .iter()
2162            .map(|v| 1.0 / (v + self.eps).sqrt())
2163            .collect();
2164        let mut normalized = Matrix::new(rows, channels);
2165        let mut output = input.like(channels);
2166        let (gamma, beta) = (&self.gamma.value.data, &self.beta.value.data);
2167        for ((out, h), row) in output
2168            .tokens
2169            .data
2170            .chunks_exact_mut(channels)
2171            .zip(normalized.data.chunks_exact_mut(channels))
2172            .zip(input.tokens.data.chunks_exact(channels))
2173        {
2174            for c in 0..channels {
2175                h[c] = (row[c] - mean[c]) * inv_std[c];
2176                out[c] = h[c] * gamma[c] + beta[c];
2177            }
2178        }
2179        Ok((
2180            output,
2181            BatchNormCache {
2182                normalized,
2183                inv_std,
2184                training,
2185            },
2186        ))
2187    }
2188
2189    /// Adds to `gamma` and `beta`'s gradients and returns the input's.
2190    pub fn backward(&mut self, cache: &BatchNormCache, grad: &ImageBatch) -> ImageBatch {
2191        let channels = self.channels();
2192        let rows = grad.tokens.rows as f32;
2193        let (mut mean, mut along) = (vec![0f32; channels], vec![0f32; channels]);
2194        for (g, h) in grad
2195            .tokens
2196            .data
2197            .chunks_exact(channels)
2198            .zip(cache.normalized.data.chunks_exact(channels))
2199        {
2200            for c in 0..channels {
2201                mean[c] += g[c];
2202                along[c] += g[c] * h[c];
2203            }
2204        }
2205        for c in 0..channels {
2206            self.gamma.grad.data[c] += along[c];
2207            self.beta.grad.data[c] += mean[c];
2208        }
2209        let gamma = &self.gamma.value.data;
2210        let mut input_grad = grad.like(channels);
2211        for ((out, g), h) in input_grad
2212            .tokens
2213            .data
2214            .chunks_exact_mut(channels)
2215            .zip(grad.tokens.data.chunks_exact(channels))
2216            .zip(cache.normalized.data.chunks_exact(channels))
2217        {
2218            for c in 0..channels {
2219                let centred = match cache.training {
2220                    true => g[c] - (mean[c] + h[c] * along[c]) / rows,
2221                    false => g[c],
2222                };
2223                out[c] = cache.inv_std[c] * gamma[c] * centred;
2224            }
2225        }
2226        input_grad
2227    }
2228
2229    pub fn params_mut(&mut self) -> Vec<&mut Param> {
2230        vec![&mut self.gamma, &mut self.beta]
2231    }
2232
2233    /// Moves `gamma`, `beta` and the running statistics onto a device.
2234    pub fn to_cuda_on(&mut self, device: &crate::param::CudaDevice) -> Result<(), NetworkError> {
2235        self.gamma.to_cuda_on(device)?;
2236        self.beta.to_cuda_on(device)?;
2237        self.running.to_cuda_on(device)
2238    }
2239
2240    pub fn to_cpu(&mut self) -> Result<(), NetworkError> {
2241        self.gamma.to_cpu()?;
2242        self.beta.to_cpu()?;
2243        self.running.to_cpu()
2244    }
2245
2246    /// The convolution that computes this norm, evaluating, after `conv`:
2247    /// each output channel's weights scaled by `gamma / sqrt(var + eps)` and
2248    /// the shift folded into the bias. Both must be on the host.
2249    pub fn fuse_into(&self, conv: &TrainableConv2d) -> TrainableConv2d {
2250        let channels = self.channels();
2251        let mut fused = conv.clone();
2252        let patch = fused.weight.weight.value.cols;
2253        let running = &self.running.value.data;
2254        for c in 0..channels {
2255            let scale = self.gamma.value.data[c] / (running[channels + c] + self.eps).sqrt();
2256            fused.weight.weight.value.data[c * patch..(c + 1) * patch]
2257                .iter_mut()
2258                .for_each(|w| *w *= scale);
2259            let bias = if conv.has_bias {
2260                conv.bias.value.data[c]
2261            } else {
2262                0.0
2263            };
2264            fused.bias.value.data[c] = (bias - running[c]) * scale + self.beta.value.data[c];
2265        }
2266        fused.has_bias = true;
2267        fused
2268    }
2269
2270    /// [`TrainableBatchNorm2d::forward`] for a batch already on the device.
2271    #[cfg(feature = "cuda")]
2272    pub fn forward_device(
2273        &mut self,
2274        input: &DeviceImageBatch,
2275        training: bool,
2276    ) -> Result<(DeviceImageBatch, DeviceBatchNormCache), NetworkError> {
2277        self.forward_device_with(input.clone(), training, false)
2278    }
2279
2280    /// [`TrainableBatchNorm2d::forward_device`] with SiLU over the output
2281    /// under `silu`, in the same kernel; the backward pass differentiates it too.
2282    /// Takes the input to keep it for the backward pass.
2283    #[cfg(feature = "cuda")]
2284    pub(crate) fn forward_device_with(
2285        &mut self,
2286        input: DeviceImageBatch,
2287        training: bool,
2288        silu: bool,
2289    ) -> Result<(DeviceImageBatch, DeviceBatchNormCache), NetworkError> {
2290        let channels = self.channels();
2291        if input.channels != channels {
2292            return Err(NetworkError::InvalidTarget {
2293                expected: channels,
2294                actual: input.channels,
2295            });
2296        }
2297        let (Some(gamma), Some(beta), Some(running)) = (
2298            self.gamma.device.as_ref(),
2299            self.beta.device.as_ref(),
2300            self.running.device.as_mut(),
2301        ) else {
2302            return Err(NetworkError::InvalidConfig(
2303                "forward_device needs the batch norm on a device".into(),
2304            ));
2305        };
2306        let context = &input.context;
2307        let err = crate::cuda_training::cuda_err;
2308        let (total, rows) = (input.data.len(), input.batch * input.pixels());
2309        let (mut mean, mut variance) = (context.zeros(channels)?, context.zeros(channels)?);
2310        let mut inv_std = context.zeros(channels)?;
2311        if training {
2312            let scale = 1.0 / rows as f32;
2313            // Shifted by the first pixel's values, which keeps the variance
2314            // from cancelling away; `bn_finish` takes the shift back out.
2315            let shift = ColumnSum::Shifted(&input.data, &mut variance);
2316            column_sums(
2317                context,
2318                &mut mean,
2319                &input.data,
2320                Some(shift),
2321                rows,
2322                channels,
2323                scale,
2324            )?;
2325        }
2326        let unbias = rows as f32 / (rows as f32 - 1.0).max(1.0);
2327        let [c, train] = [channels as i32, training as i32];
2328        unsafe {
2329            context
2330                .stream
2331                .launch_builder(&context.image.bn_finish)
2332                .arg(&mut mean)
2333                .arg(&mut variance)
2334                .arg(&mut inv_std)
2335                .arg(running.value_mut())
2336                .arg(&input.data)
2337                .arg(&c)
2338                .arg(&self.eps)
2339                .arg(&self.momentum)
2340                .arg(&unbias)
2341                .arg(&train)
2342                .launch(crate::cuda_training::cfg(channels))
2343                .map_err(err("batch norm statistics kernel"))?;
2344        }
2345        let mut output = context.uninit(total)?;
2346        let n = total as i32;
2347        unsafe {
2348            context
2349                .stream
2350                .launch_builder(&context.image.bn_apply)
2351                .arg(&mut output)
2352                .arg(&input.data)
2353                .arg(&mean)
2354                .arg(&inv_std)
2355                .arg(gamma.value())
2356                .arg(beta.value())
2357                .arg(&n)
2358                .arg(&c)
2359                .arg(&(silu as i32))
2360                .launch(crate::cuda_training::cfg(total))
2361                .map_err(err("batch norm kernel"))?;
2362        }
2363        Ok((
2364            input.with_data(output),
2365            DeviceBatchNormCache {
2366                input: input.data,
2367                mean,
2368                inv_std,
2369                training,
2370                silu,
2371            },
2372        ))
2373    }
2374
2375    /// [`TrainableBatchNorm2d::backward`] on the device.
2376    #[cfg(feature = "cuda")]
2377    pub fn backward_device(
2378        &mut self,
2379        cache: &DeviceBatchNormCache,
2380        grad: &DeviceImageBatch,
2381    ) -> Result<DeviceImageBatch, NetworkError> {
2382        let context = grad.context.clone();
2383        let err = crate::cuda_training::cuda_err;
2384        let (total, rows) = (grad.data.len(), grad.batch * grad.pixels());
2385        let channels = self.channels();
2386        let (Some(gamma), Some(beta)) = (self.gamma.device.as_mut(), self.beta.device.as_mut())
2387        else {
2388            return Err(NetworkError::InvalidConfig(
2389                "backward_device needs gamma and beta on a device".into(),
2390            ));
2391        };
2392        // The two column means of the gradient at the norm's output, which
2393        // are also beta's and gamma's gradients over `rows`.
2394        let (mut mean, mut along) = (context.zeros(channels)?, context.zeros(channels)?);
2395        let silu = cache.silu.then_some((gamma.value(), beta.value()));
2396        let times = ColumnSum::Times(
2397            &cache.input,
2398            &mut along,
2399            (&cache.mean, &cache.inv_std),
2400            silu,
2401        );
2402        let scale = 1.0 / rows as f32;
2403        column_sums(
2404            &context,
2405            &mut mean,
2406            &grad.data,
2407            Some(times),
2408            rows,
2409            channels,
2410            scale,
2411        )?;
2412        let unused = context.zeros(channels)?;
2413        let (centre, slope) = match cache.training {
2414            true => (&mean, &along),
2415            false => (&unused, &unused),
2416        };
2417        let mut input_grad = context.uninit(total)?;
2418        let [n, c] = [total, channels].map(|v| v as i32);
2419        unsafe {
2420            context
2421                .stream
2422                .launch_builder(&context.image.bn_bwd_apply)
2423                .arg(&mut input_grad)
2424                .arg(&grad.data)
2425                .arg(&cache.input)
2426                .arg(&cache.mean)
2427                .arg(&cache.inv_std)
2428                .arg(centre)
2429                .arg(slope)
2430                .arg(gamma.value())
2431                .arg(beta.value())
2432                .arg(&n)
2433                .arg(&c)
2434                .arg(&(cache.silu as i32))
2435                .launch(crate::cuda_training::cfg(total))
2436                .map_err(err("batch norm backward kernel"))?;
2437        }
2438        let count = context.filled(1, rows as f32)?;
2439        gamma.accumulate_grad_device(&count, &along, 1)?;
2440        beta.accumulate_grad_device(&count, &mean, 1)?;
2441        Ok(grad.with_data(input_grad))
2442    }
2443}
2444
2445/// What [`TrainableBatchNorm2d::backward_device`] needs from the forward pass.
2446#[cfg(feature = "cuda")]
2447pub struct DeviceBatchNormCache {
2448    /// The input and its statistics, from which the backward pass recomputes
2449    /// the normalized values.
2450    input: cudarc::driver::CudaSlice<f32>,
2451    mean: cudarc::driver::CudaSlice<f32>,
2452    inv_std: cudarc::driver::CudaSlice<f32>,
2453    training: bool,
2454    silu: bool,
2455}
2456
2457/// What [`column_sums`] sums besides plain values.
2458#[cfg(feature = "cuda")]
2459enum ColumnSum<'a> {
2460    /// `x - centre[channel]` in place of `x`, and its square into the second
2461    /// sum.
2462    Shifted(
2463        &'a cudarc::driver::CudaSlice<f32>,
2464        &'a mut cudarc::driver::CudaSlice<f32>,
2465    ),
2466    /// `x` as with none, and `x * xhat` elementwise into the second sum,
2467    /// xhat being a batch norm's input normalized by the mean and inverse
2468    /// standard deviation given. With gamma and beta, `x` is first taken back
2469    /// through the SiLU after that batch norm.
2470    Times(
2471        &'a cudarc::driver::CudaSlice<f32>,
2472        &'a mut cudarc::driver::CudaSlice<f32>,
2473        (
2474            &'a cudarc::driver::CudaSlice<f32>,
2475            &'a cudarc::driver::CudaSlice<f32>,
2476        ),
2477        Option<(
2478            &'a cudarc::driver::CudaSlice<f32>,
2479            &'a cudarc::driver::CudaSlice<f32>,
2480        )>,
2481    ),
2482}
2483
2484/// `scale` times each channel's sum over the rows of `x[rows, channels]`,
2485/// added into `out`, which starts zeroed.
2486#[cfg(feature = "cuda")]
2487fn column_sums(
2488    context: &crate::gpu_transformer::GpuContext,
2489    out: &mut cudarc::driver::CudaSlice<f32>,
2490    x: &cudarc::driver::CudaSlice<f32>,
2491    with: Option<ColumnSum>,
2492    rows: usize,
2493    channels: usize,
2494    scale: f32,
2495) -> Result<(), NetworkError> {
2496    let [r, c] = [rows, channels].map(|v| v as i32);
2497    let mut launch = context.stream.launch_builder(&context.image.col_sums);
2498    launch.arg(out);
2499    // The kernel writes the second sum only under mode 2.
2500    let with_silu = match &with {
2501        Some(ColumnSum::Times(_, _, _, silu)) => *silu,
2502        _ => None,
2503    };
2504    let stats = match &with {
2505        Some(ColumnSum::Times(_, _, stats, _)) => *stats,
2506        _ => (x, x),
2507    };
2508    let mode = match with {
2509        None => {
2510            launch.arg(x).arg(x).arg(x);
2511            0
2512        }
2513        Some(ColumnSum::Shifted(centre, second)) => {
2514            launch.arg(second).arg(x).arg(centre);
2515            1
2516        }
2517        Some(ColumnSum::Times(other, second, _, _)) => {
2518            launch.arg(second).arg(x).arg(other);
2519            2
2520        }
2521    };
2522    launch.arg(&r).arg(&c).arg(&scale).arg(&mode);
2523    let (gamma, beta) = match with_silu {
2524        Some(pair) => pair,
2525        None => (x, x),
2526    };
2527    let act = with_silu.is_some() as i32;
2528    launch
2529        .arg(gamma)
2530        .arg(beta)
2531        .arg(&act)
2532        .arg(stats.0)
2533        .arg(stats.1);
2534    // COL_ROWS rows a thread, as the kernel reads them.
2535    unsafe { launch.launch(crate::cuda_training::cfg(rows.div_ceil(64) * channels)) }
2536        .map_err(crate::cuda_training::cuda_err("column sum kernel"))?;
2537    Ok(())
2538}
2539
2540/// A depthwise convolution without bias, as ultralytics' `DWConv` and C2PSA's
2541/// positional encoding use before a batch norm: each channel convolved with
2542/// its own `kernel x kernel` filter, padded by `kernel / 2`.
2543#[derive(Clone, Debug)]
2544pub struct DepthwiseConv2d {
2545    /// `[1, channels * kernel * kernel]`, PyTorch's `[C, 1, k, k]` flattened.
2546    pub weight: Param,
2547    kernel: usize,
2548    stride: usize,
2549}
2550
2551/// What [`DepthwiseConv2d::backward`] needs: the input.
2552#[derive(Clone, Debug)]
2553pub struct DepthwiseCache {
2554    input: ImageBatch,
2555}
2556
2557impl DepthwiseConv2d {
2558    pub fn new(channels: usize, kernel: usize, stride: usize, rng: &mut StdRng) -> Self {
2559        let fan_in = kernel * kernel;
2560        Self {
2561            weight: Param::he_uniform(1, channels * fan_in, fan_in, rng),
2562            kernel,
2563            stride,
2564        }
2565    }
2566
2567    pub fn channels(&self) -> usize {
2568        self.weight.value.cols / (self.kernel * self.kernel)
2569    }
2570
2571    pub fn output_size(&self, height: usize, width: usize) -> (usize, usize) {
2572        let size = |n: usize| (n + 2 * (self.kernel / 2) - self.kernel) / self.stride + 1;
2573        (size(height), size(width))
2574    }
2575
2576    /// Every `(output index, input index, weight index)` the convolution
2577    /// multiplies, in a fixed order.
2578    fn taps(&self, input: &ImageBatch) -> impl Iterator<Item = (usize, usize, usize)> + use<> {
2579        let (h, w, c) = (input.height, input.width, input.channels());
2580        let (oh, ow) = self.output_size(h, w);
2581        let (k, s, pad) = (self.kernel, self.stride, self.kernel / 2);
2582        let batch = input.batch;
2583        (0..batch * oh * ow).flat_map(move |row| {
2584            let (ox, oy, b) = (row % ow, row / ow % oh, row / ow / oh);
2585            (0..k * k).flat_map(move |t| {
2586                let (sy, sx) = (
2587                    (oy * s + t / k).wrapping_sub(pad),
2588                    (ox * s + t % k).wrapping_sub(pad),
2589                );
2590                let inside = sy < h && sx < w;
2591                (0..c).filter(move |_| inside).map(move |ch| {
2592                    (
2593                        row * c + ch,
2594                        ((b * h + sy) * w + sx) * c + ch,
2595                        ch * k * k + t,
2596                    )
2597                })
2598            })
2599        })
2600    }
2601
2602    pub fn forward(
2603        &self,
2604        input: &ImageBatch,
2605    ) -> Result<(ImageBatch, DepthwiseCache), NetworkError> {
2606        if input.channels() != self.channels() {
2607            return Err(NetworkError::InvalidTarget {
2608                expected: self.channels(),
2609                actual: input.channels(),
2610            });
2611        }
2612        let (oh, ow) = self.output_size(input.height, input.width);
2613        let mut output = ImageBatch::new(input.batch, oh, ow, input.channels());
2614        let weight = &self.weight.value.data;
2615        for (o, i, t) in self.taps(input) {
2616            output.tokens.data[o] += input.tokens.data[i] * weight[t];
2617        }
2618        Ok((
2619            output,
2620            DepthwiseCache {
2621                input: input.clone(),
2622            },
2623        ))
2624    }
2625
2626    /// Adds to the weight's gradient and returns the input's.
2627    pub fn backward(&mut self, cache: &DepthwiseCache, grad: &ImageBatch) -> ImageBatch {
2628        let mut input_grad = cache.input.like(cache.input.channels());
2629        let weight = self.weight.value.data.clone();
2630        for (o, i, t) in self.taps(&cache.input) {
2631            input_grad.tokens.data[i] += grad.tokens.data[o] * weight[t];
2632            self.weight.grad.data[t] += grad.tokens.data[o] * cache.input.tokens.data[i];
2633        }
2634        input_grad
2635    }
2636
2637    pub fn params_mut(&mut self) -> Vec<&mut Param> {
2638        vec![&mut self.weight]
2639    }
2640
2641    pub fn to_cuda_on(&mut self, device: &crate::param::CudaDevice) -> Result<(), NetworkError> {
2642        self.weight.to_cuda_on(device)
2643    }
2644
2645    pub fn to_cpu(&mut self) -> Result<(), NetworkError> {
2646        self.weight.to_cpu()
2647    }
2648
2649    /// [`DepthwiseConv2d::forward`] on the device. The cache is the input,
2650    /// which the caller passes back to [`DepthwiseConv2d::backward_device`].
2651    #[cfg(feature = "cuda")]
2652    pub fn forward_device(
2653        &self,
2654        input: &DeviceImageBatch,
2655    ) -> Result<DeviceImageBatch, NetworkError> {
2656        let Some(weight) = self.weight.device.as_ref() else {
2657            return Err(NetworkError::InvalidConfig(
2658                "forward_device needs the weight on a device".into(),
2659            ));
2660        };
2661        if input.channels != self.channels() {
2662            return Err(NetworkError::InvalidTarget {
2663                expected: self.channels(),
2664                actual: input.channels,
2665            });
2666        }
2667        let (oh, ow) = self.output_size(input.height, input.width);
2668        let context = &input.context;
2669        let total = input.batch * oh * ow * input.channels;
2670        let mut out = context.uninit(total)?;
2671        let dims = self.dims(input);
2672        unsafe {
2673            let mut launch = context.stream.launch_builder(&context.image.dw_fwd);
2674            launch.arg(&mut out).arg(&input.data).arg(weight.value());
2675            dims.iter().for_each(|d| {
2676                launch.arg(d);
2677            });
2678            launch
2679                .launch(crate::cuda_training::cfg(total))
2680                .map_err(crate::cuda_training::cuda_err("depthwise kernel"))?;
2681        }
2682        Ok(DeviceImageBatch {
2683            height: oh,
2684            width: ow,
2685            ..input.with_data(out)
2686        })
2687    }
2688
2689    /// [`DepthwiseConv2d::backward`] on the device, given the forward input.
2690    #[cfg(feature = "cuda")]
2691    pub fn backward_device(
2692        &mut self,
2693        input: &DeviceImageBatch,
2694        grad: &DeviceImageBatch,
2695    ) -> Result<DeviceImageBatch, NetworkError> {
2696        let dims = self.dims(input);
2697        let taps = self.weight.value.cols;
2698        let Some(weight) = self.weight.device.as_mut() else {
2699            return Err(NetworkError::InvalidConfig(
2700                "backward_device needs the weight on a device".into(),
2701            ));
2702        };
2703        let context = input.context.clone();
2704        let err = crate::cuda_training::cuda_err;
2705        let total = input.data.len();
2706        let mut input_grad = context.uninit(total)?;
2707        let mut weight_grad = context.zeros(taps)?;
2708        unsafe {
2709            let mut launch = context.stream.launch_builder(&context.image.dw_bwd_input);
2710            launch
2711                .arg(&mut input_grad)
2712                .arg(&grad.data)
2713                .arg(weight.value());
2714            dims.iter().for_each(|d| {
2715                launch.arg(d);
2716            });
2717            launch
2718                .launch(crate::cuda_training::cfg(total))
2719                .map_err(err("depthwise input gradient kernel"))?;
2720            let mut launch = context.stream.launch_builder(&context.image.dw_bwd_weight);
2721            launch
2722                .arg(&mut weight_grad)
2723                .arg(&grad.data)
2724                .arg(&input.data);
2725            dims.iter().for_each(|d| {
2726                launch.arg(d);
2727            });
2728            // DW_ROWS output pixels a thread, as the kernel reads them.
2729            let pixels = input.batch * dims[6] as usize * dims[7] as usize;
2730            launch
2731                .launch(crate::cuda_training::cfg(pixels.div_ceil(64) * taps))
2732                .map_err(err("depthwise weight gradient kernel"))?;
2733        }
2734        let one = context.filled(1, 1.0)?;
2735        weight.accumulate_grad_device(&one, &weight_grad, 1)?;
2736        Ok(input.with_data(input_grad))
2737    }
2738
2739    /// `[batch, height, width, channels, kernel, stride, out_height, out_width]`.
2740    #[cfg(feature = "cuda")]
2741    fn dims(&self, input: &DeviceImageBatch) -> [i32; 8] {
2742        let (oh, ow) = self.output_size(input.height, input.width);
2743        [
2744            input.batch,
2745            input.height,
2746            input.width,
2747            input.channels,
2748            self.kernel,
2749            self.stride,
2750            oh,
2751            ow,
2752        ]
2753        .map(|v| v as i32)
2754    }
2755}
2756
2757/// The head layout of a `qkv` map for [`pixel_attention`]: each head's
2758/// channels are its `key_dim` queries, `key_dim` keys and `head_dim` values,
2759/// in that order, as ultralytics' `Attention` views them.
2760#[derive(Clone, Copy, Debug, PartialEq, Eq)]
2761pub struct PixelAttentionShape {
2762    pub heads: usize,
2763    pub key_dim: usize,
2764    pub head_dim: usize,
2765}
2766
2767impl PixelAttentionShape {
2768    /// The shape of ultralytics' `Attention(dim, num_heads, attn_ratio=0.5)`.
2769    pub fn yolo(dim: usize, heads: usize) -> Self {
2770        let head_dim = dim / heads;
2771        Self {
2772            heads,
2773            key_dim: head_dim / 2,
2774            head_dim,
2775        }
2776    }
2777
2778    fn per_head(&self) -> usize {
2779        2 * self.key_dim + self.head_dim
2780    }
2781
2782    /// Channels of the attended output and of the values, `heads * head_dim`.
2783    pub fn dim(&self) -> usize {
2784        self.heads * self.head_dim
2785    }
2786
2787    fn scale(&self) -> f32 {
2788        (self.key_dim as f32).powf(-0.5)
2789    }
2790
2791    fn check(&self, channels: usize) -> Result<(), NetworkError> {
2792        match channels == self.heads * self.per_head() {
2793            true => Ok(()),
2794            false => Err(NetworkError::InvalidTarget {
2795                expected: self.heads * self.per_head(),
2796                actual: channels,
2797            }),
2798        }
2799    }
2800}
2801
2802/// What [`pixel_attention_backward`] needs from the forward pass.
2803#[derive(Clone, Debug)]
2804pub struct PixelAttentionCache {
2805    qkv: ImageBatch,
2806    shape: PixelAttentionShape,
2807    /// Per sample and head, `[pixels, pixels]`.
2808    probabilities: Vec<Matrix>,
2809}
2810
2811/// One head's `width` channels from `offset` within it, `[pixels, width]`.
2812fn head_part(map: &ImageBatch, sample: usize, start: usize, width: usize) -> Matrix {
2813    let (pixels, channels) = (map.pixels(), map.channels());
2814    let mut part = Matrix::new(pixels, width);
2815    for p in 0..pixels {
2816        let row = &map.tokens.data[(sample * pixels + p) * channels..][start..start + width];
2817        part.data[p * width..(p + 1) * width].copy_from_slice(row);
2818    }
2819    part
2820}
2821
2822fn add_head_part(map: &mut ImageBatch, sample: usize, start: usize, part: &Matrix) {
2823    let (pixels, channels, width) = (map.pixels(), map.channels(), part.cols);
2824    for p in 0..pixels {
2825        map.tokens.data[(sample * pixels + p) * channels..][start..start + width]
2826            .iter_mut()
2827            .zip(part.row(p))
2828            .for_each(|(slot, v)| *slot += v);
2829    }
2830}
2831
2832/// ultralytics' `Attention` between its `qkv` and `pe`/`proj` convolutions:
2833/// every pixel attends over all pixels of its sample, per head. Returns the
2834/// attended map and the values (the input to the positional encoding), both
2835/// `shape.dim()` channels with head `h` at `h * head_dim`.
2836pub fn pixel_attention(
2837    qkv: &ImageBatch,
2838    shape: PixelAttentionShape,
2839) -> Result<(ImageBatch, ImageBatch, PixelAttentionCache), NetworkError> {
2840    shape.check(qkv.channels())?;
2841    let (pixels, kd, hd) = (qkv.pixels(), shape.key_dim, shape.head_dim);
2842    let mut attended = qkv.like(shape.dim());
2843    let mut values = qkv.like(shape.dim());
2844    let mut probabilities = Vec::new();
2845    for b in 0..qkv.batch {
2846        for h in 0..shape.heads {
2847            let start = h * shape.per_head();
2848            let q = head_part(qkv, b, start, kd);
2849            let k = head_part(qkv, b, start + kd, kd);
2850            let v = head_part(qkv, b, start + 2 * kd, hd);
2851            let mut p = Matrix::new(pixels, pixels);
2852            q.dot_rhs_transposed(&k, &mut p);
2853            for row in p.data.chunks_exact_mut(pixels) {
2854                let max = row.iter().fold(f32::MIN, |m, &s| m.max(s * shape.scale()));
2855                let mut sum = 0.0;
2856                row.iter_mut().for_each(|s| {
2857                    *s = (*s * shape.scale() - max).exp();
2858                    sum += *s;
2859                });
2860                row.iter_mut().for_each(|s| *s /= sum);
2861            }
2862            let mut out = Matrix::new(pixels, hd);
2863            p.dot(&v, &mut out);
2864            add_head_part(&mut attended, b, h * hd, &out);
2865            add_head_part(&mut values, b, h * hd, &v);
2866            probabilities.push(p);
2867        }
2868    }
2869    Ok((
2870        attended,
2871        values,
2872        PixelAttentionCache {
2873            qkv: qkv.clone(),
2874            shape,
2875            probabilities,
2876        },
2877    ))
2878}
2879
2880/// The `qkv` gradient from the attended map's and the values' gradients.
2881pub fn pixel_attention_backward(
2882    cache: &PixelAttentionCache,
2883    grad_attended: &ImageBatch,
2884    grad_values: &ImageBatch,
2885) -> ImageBatch {
2886    let (qkv, shape) = (&cache.qkv, cache.shape);
2887    let (pixels, kd, hd) = (qkv.pixels(), shape.key_dim, shape.head_dim);
2888    let mut grad = qkv.like(qkv.channels());
2889    for b in 0..qkv.batch {
2890        for h in 0..shape.heads {
2891            let start = h * shape.per_head();
2892            let p = &cache.probabilities[b * shape.heads + h];
2893            let q = head_part(qkv, b, start, kd);
2894            let k = head_part(qkv, b, start + kd, kd);
2895            let v = head_part(qkv, b, start + 2 * kd, hd);
2896            let d_out = head_part(grad_attended, b, h * hd, hd);
2897            let mut d_v = head_part(grad_values, b, h * hd, hd);
2898            let mut through = Matrix::new(pixels, hd);
2899            p.dot_self_transposed(&d_out, &mut through);
2900            d_v.data
2901                .iter_mut()
2902                .zip(&through.data)
2903                .for_each(|(a, b)| *a += b);
2904            let mut d_s = Matrix::new(pixels, pixels);
2905            d_out.dot_rhs_transposed(&v, &mut d_s);
2906            for (ds, pr) in d_s
2907                .data
2908                .chunks_exact_mut(pixels)
2909                .zip(p.data.chunks_exact(pixels))
2910            {
2911                let along: f32 = ds.iter().zip(pr).map(|(d, p)| d * p).sum();
2912                ds.iter_mut()
2913                    .zip(pr)
2914                    .for_each(|(d, p)| *d = p * (*d - along) * shape.scale());
2915            }
2916            let (mut d_q, mut d_k) = (Matrix::new(pixels, kd), Matrix::new(pixels, kd));
2917            d_s.dot(&k, &mut d_q);
2918            d_s.dot_self_transposed(&q, &mut d_k);
2919            add_head_part(&mut grad, b, start, &d_q);
2920            add_head_part(&mut grad, b, start + kd, &d_k);
2921            add_head_part(&mut grad, b, start + 2 * kd, &d_v);
2922        }
2923    }
2924    grad
2925}
2926
2927/// What [`pixel_attention_backward_device`] needs from the forward pass.
2928#[cfg(feature = "cuda")]
2929pub struct DevicePixelAttentionCache {
2930    qkv: DeviceImageBatch,
2931    shape: PixelAttentionShape,
2932    /// `[batch, heads, pixels, pixels]`.
2933    probabilities: cudarc::driver::CudaSlice<f32>,
2934}
2935
2936/// [`pixel_attention`] on the device, through cuBLAS batched over the heads.
2937// ponytail: the cache holds every head's `pixels x pixels` probabilities,
2938// 4 x 400 x 400 floats a sample at the P5 map of a 1280 input (2.6 MB);
2939// a flash kernel replaces it if larger maps ever attend.
2940#[cfg(feature = "cuda")]
2941pub fn pixel_attention_device(
2942    qkv: &DeviceImageBatch,
2943    shape: PixelAttentionShape,
2944) -> Result<
2945    (
2946        DeviceImageBatch,
2947        DeviceImageBatch,
2948        DevicePixelAttentionCache,
2949    ),
2950    NetworkError,
2951> {
2952    use crate::gpu_transformer::{gemm_plain_batched, gemm_rhs_transposed_batched};
2953    shape.check(qkv.channels)?;
2954    let context = &qkv.context;
2955    let (n, kd, hd, s, cq, dim) = (
2956        qkv.pixels(),
2957        shape.key_dim,
2958        shape.head_dim,
2959        shape.per_head(),
2960        qkv.channels,
2961        shape.dim(),
2962    );
2963    let (rows, span) = (qkv.batch * n, shape.heads * n * n);
2964    let mut probabilities = context.zeros(qkv.batch * span)?;
2965    let mut lse = context.zeros(qkv.batch * shape.heads * n)?;
2966    let mut attended = context.zeros(rows * dim)?;
2967    let mut values = context.zeros(rows * dim)?;
2968    for b in 0..qkv.batch {
2969        let base = b * n * cq;
2970        gemm_rhs_transposed_batched(
2971            context,
2972            &qkv.data.slice(base..),
2973            cq,
2974            &qkv.data.slice(base + kd..),
2975            cq,
2976            &mut probabilities.slice_mut(b * span..),
2977            n,
2978            n,
2979            n,
2980            kd,
2981            shape.scale(),
2982            0.0,
2983            shape.heads,
2984            s,
2985            s,
2986            n * n,
2987        )?;
2988    }
2989    let gpu = crate::gpu_model::Gpu { context };
2990    gpu.attention_softmax(
2991        &mut probabilities,
2992        &mut lse,
2993        qkv.batch * shape.heads * n,
2994        n,
2995        false,
2996        n,
2997        0,
2998    )?;
2999    for b in 0..qkv.batch {
3000        gemm_plain_batched(
3001            context,
3002            &probabilities.slice(b * span..),
3003            n,
3004            &qkv.data.slice(b * n * cq + 2 * kd..),
3005            cq,
3006            &mut attended.slice_mut(b * n * dim..),
3007            dim,
3008            n,
3009            hd,
3010            n,
3011            1.0,
3012            0.0,
3013            shape.heads,
3014            n * n,
3015            s,
3016            hd,
3017        )?;
3018    }
3019    for h in 0..shape.heads {
3020        copy_channels(
3021            context,
3022            &mut values,
3023            dim,
3024            h * hd,
3025            &qkv.data,
3026            cq,
3027            h * s + 2 * kd,
3028            hd,
3029            rows,
3030            false,
3031        )?;
3032    }
3033    Ok((
3034        DeviceImageBatch {
3035            channels: dim,
3036            ..qkv.with_data(attended)
3037        },
3038        DeviceImageBatch {
3039            channels: dim,
3040            ..qkv.with_data(values)
3041        },
3042        DevicePixelAttentionCache {
3043            qkv: qkv.clone(),
3044            shape,
3045            probabilities,
3046        },
3047    ))
3048}
3049
3050/// [`pixel_attention_backward`] on the device.
3051#[cfg(feature = "cuda")]
3052pub fn pixel_attention_backward_device(
3053    cache: &DevicePixelAttentionCache,
3054    grad_attended: &DeviceImageBatch,
3055    grad_values: &DeviceImageBatch,
3056) -> Result<DeviceImageBatch, NetworkError> {
3057    use crate::gpu_transformer::{
3058        gemm_lhs_transposed_batched, gemm_plain_batched, gemm_rhs_transposed_batched,
3059    };
3060    let (qkv, shape) = (&cache.qkv, cache.shape);
3061    let context = &qkv.context;
3062    let (n, kd, hd, s, cq, dim) = (
3063        qkv.pixels(),
3064        shape.key_dim,
3065        shape.head_dim,
3066        shape.per_head(),
3067        qkv.channels,
3068        shape.dim(),
3069    );
3070    let (rows, span) = (qkv.batch * n, shape.heads * n * n);
3071    let mut grad = context.zeros(rows * cq)?;
3072    for h in 0..shape.heads {
3073        copy_channels(
3074            context,
3075            &mut grad,
3076            cq,
3077            h * s + 2 * kd,
3078            &grad_values.data,
3079            dim,
3080            h * hd,
3081            hd,
3082            rows,
3083            false,
3084        )?;
3085    }
3086    let mut d_s = context.zeros(qkv.batch * span)?;
3087    for b in 0..qkv.batch {
3088        let base = b * n * cq;
3089        let d_out = grad_attended.data.slice(b * n * dim..);
3090        // dV += P^T dO
3091        gemm_lhs_transposed_batched(
3092            context,
3093            &cache.probabilities.slice(b * span..),
3094            n,
3095            &d_out,
3096            dim,
3097            &mut grad.slice_mut(base + 2 * kd..),
3098            cq,
3099            n,
3100            n,
3101            hd,
3102            1.0,
3103            1.0,
3104            shape.heads,
3105            n * n,
3106            hd,
3107            s,
3108        )?;
3109        // dP = dO V^T
3110        gemm_rhs_transposed_batched(
3111            context,
3112            &d_out,
3113            dim,
3114            &qkv.data.slice(base + 2 * kd..),
3115            cq,
3116            &mut d_s.slice_mut(b * span..),
3117            n,
3118            n,
3119            n,
3120            hd,
3121            1.0,
3122            0.0,
3123            shape.heads,
3124            hd,
3125            s,
3126            n * n,
3127        )?;
3128    }
3129    let gpu = crate::gpu_model::Gpu { context };
3130    gpu.softmax_backward(
3131        &mut d_s,
3132        &cache.probabilities,
3133        qkv.batch * shape.heads * n,
3134        n,
3135        false,
3136    )?;
3137    for b in 0..qkv.batch {
3138        let base = b * n * cq;
3139        // dQ = scale dS K, dK = scale dS^T Q
3140        gemm_plain_batched(
3141            context,
3142            &d_s.slice(b * span..),
3143            n,
3144            &qkv.data.slice(base + kd..),
3145            cq,
3146            &mut grad.slice_mut(base..),
3147            cq,
3148            n,
3149            kd,
3150            n,
3151            shape.scale(),
3152            0.0,
3153            shape.heads,
3154            n * n,
3155            s,
3156            s,
3157        )?;
3158        gemm_lhs_transposed_batched(
3159            context,
3160            &d_s.slice(b * span..),
3161            n,
3162            &qkv.data.slice(base..),
3163            cq,
3164            &mut grad.slice_mut(base + kd..),
3165            cq,
3166            n,
3167            n,
3168            kd,
3169            shape.scale(),
3170            0.0,
3171            shape.heads,
3172            n * n,
3173            s,
3174            s,
3175        )?;
3176    }
3177    Ok(qkv.with_data(grad))
3178}
3179
3180/// Nearest-neighbour upsampling by `factor` along both axes.
3181pub fn upsample_nearest_batch(input: &ImageBatch, factor: usize) -> ImageBatch {
3182    let mut output = ImageBatch::new(
3183        input.batch,
3184        input.height * factor,
3185        input.width * factor,
3186        input.channels(),
3187    );
3188    for b in 0..input.batch {
3189        for y in 0..output.height {
3190            for x in 0..output.width {
3191                let source = input.pixel(b, y / factor, x / factor).to_vec();
3192                output.pixel_mut(b, y, x).copy_from_slice(&source);
3193            }
3194        }
3195    }
3196    output
3197}
3198
3199/// The gradient of [`upsample_nearest_batch`]: each input pixel sums the
3200/// `factor x factor` block it was copied to.
3201pub fn upsample_nearest_backward(grad: &ImageBatch, factor: usize) -> ImageBatch {
3202    let mut input_grad = ImageBatch::new(
3203        grad.batch,
3204        grad.height / factor,
3205        grad.width / factor,
3206        grad.channels(),
3207    );
3208    for b in 0..grad.batch {
3209        for y in 0..grad.height {
3210            for x in 0..grad.width {
3211                let g = grad.pixel(b, y, x).to_vec();
3212                input_grad
3213                    .pixel_mut(b, y / factor, x / factor)
3214                    .iter_mut()
3215                    .zip(g)
3216                    .for_each(|(s, g)| *s += g);
3217            }
3218        }
3219    }
3220    input_grad
3221}
3222
3223/// The maps side by side along the channels, which must share their size.
3224/// Its gradient is [`split_channels`].
3225pub fn concat_channels(parts: &[&ImageBatch]) -> ImageBatch {
3226    let channels = parts.iter().map(|p| p.channels()).sum();
3227    let mut output = parts[0].like(channels);
3228    let mut offset = 0;
3229    for part in parts {
3230        let width = part.channels();
3231        for (out, row) in output
3232            .tokens
3233            .data
3234            .chunks_exact_mut(channels)
3235            .zip(part.tokens.data.chunks_exact(width))
3236        {
3237            out[offset..offset + width].copy_from_slice(row);
3238        }
3239        offset += width;
3240    }
3241    output
3242}
3243
3244/// The map cut along the channels into parts of the given widths, which must
3245/// add up to its channels. Its gradient is [`concat_channels`].
3246pub fn split_channels(input: &ImageBatch, widths: &[usize]) -> Vec<ImageBatch> {
3247    let mut offset = 0;
3248    widths
3249        .iter()
3250        .map(|&width| {
3251            let mut part = input.like(width);
3252            for (out, row) in part
3253                .tokens
3254                .data
3255                .chunks_exact_mut(width)
3256                .zip(input.tokens.data.chunks_exact(input.channels()))
3257            {
3258                out.copy_from_slice(&row[offset..offset + width]);
3259            }
3260            offset += width;
3261            part
3262        })
3263        .collect()
3264}
3265
3266/// Which window position won each output of [`max_pool_same`].
3267#[derive(Clone, Debug)]
3268pub struct MaxPoolCache {
3269    kernel: usize,
3270    argmax: Vec<u32>,
3271}
3272
3273/// A `kernel x kernel` max pool with stride one, padded so the size stays,
3274/// as SPPF's `nn.MaxPool2d(5, 1, 2)`. The first maximum in the window wins a
3275/// tie.
3276pub fn max_pool_same(input: &ImageBatch, kernel: usize) -> (ImageBatch, MaxPoolCache) {
3277    let (height, width, channels, pad) = (input.height, input.width, input.channels(), kernel / 2);
3278    let mut output = input.like(channels);
3279    let mut argmax = vec![0u32; input.tokens.data.len()];
3280    for b in 0..input.batch {
3281        for y in 0..height {
3282            for x in 0..width {
3283                let at = ((b * height + y) * width + x) * channels;
3284                for c in 0..channels {
3285                    let (mut best, mut arg) = (f32::MIN, 0);
3286                    for ky in 0..kernel {
3287                        for kx in 0..kernel {
3288                            let (sy, sx) = ((y + ky).wrapping_sub(pad), (x + kx).wrapping_sub(pad));
3289                            if sy < height && sx < width {
3290                                let v = input.pixel(b, sy, sx)[c];
3291                                if v > best {
3292                                    (best, arg) = (v, ky * kernel + kx);
3293                                }
3294                            }
3295                        }
3296                    }
3297                    output.tokens.data[at + c] = best;
3298                    argmax[at + c] = arg as u32;
3299                }
3300            }
3301        }
3302    }
3303    (output, MaxPoolCache { kernel, argmax })
3304}
3305
3306/// Routes each output's gradient to the input that won its window.
3307pub fn max_pool_same_backward(cache: &MaxPoolCache, grad: &ImageBatch) -> ImageBatch {
3308    let (height, width, channels, kernel) =
3309        (grad.height, grad.width, grad.channels(), cache.kernel);
3310    let pad = kernel / 2;
3311    let mut input_grad = grad.like(channels);
3312    for (index, (&g, &arg)) in grad.tokens.data.iter().zip(&cache.argmax).enumerate() {
3313        let (c, pixel) = (index % channels, index / channels);
3314        let (x, y, b) = (
3315            pixel % width,
3316            pixel / width % height,
3317            pixel / width / height,
3318        );
3319        let (ky, kx) = (arg as usize / kernel, arg as usize % kernel);
3320        let (sy, sx) = (y + ky - pad, x + kx - pad);
3321        input_grad.tokens.data[((b * height + sy) * width + sx) * channels + c] += g;
3322    }
3323    input_grad
3324}
3325
3326/// [`upsample_nearest_batch`] on the device.
3327#[cfg(feature = "cuda")]
3328pub fn upsample_nearest_device(
3329    input: &DeviceImageBatch,
3330    factor: usize,
3331) -> Result<DeviceImageBatch, NetworkError> {
3332    let total = input.data.len() * factor * factor;
3333    let out = image_launch(
3334        input,
3335        total,
3336        "upsample kernel",
3337        |context| &context.image.upsample,
3338        &[input.height, input.width, input.channels, factor],
3339    )?;
3340    Ok(DeviceImageBatch {
3341        height: input.height * factor,
3342        width: input.width * factor,
3343        ..input.with_data(out)
3344    })
3345}
3346
3347/// [`upsample_nearest_backward`] on the device.
3348#[cfg(feature = "cuda")]
3349pub fn upsample_nearest_backward_device(
3350    grad: &DeviceImageBatch,
3351    factor: usize,
3352) -> Result<DeviceImageBatch, NetworkError> {
3353    let (height, width) = (grad.height / factor, grad.width / factor);
3354    let total = grad.data.len() / (factor * factor);
3355    let out = image_launch(
3356        grad,
3357        total,
3358        "upsample backward kernel",
3359        |context| &context.image.upsample_bwd,
3360        &[height, width, grad.channels, factor],
3361    )?;
3362    Ok(DeviceImageBatch {
3363        height,
3364        width,
3365        ..grad.with_data(out)
3366    })
3367}
3368
3369/// `a + b` on the device, for residual connections.
3370#[cfg(feature = "cuda")]
3371pub fn add_device(
3372    a: &DeviceImageBatch,
3373    b: &DeviceImageBatch,
3374) -> Result<DeviceImageBatch, NetworkError> {
3375    let context = &a.context;
3376    let total = a.data.len();
3377    if b.data.len() != total {
3378        return Err(NetworkError::InvalidTarget {
3379            expected: total,
3380            actual: b.data.len(),
3381        });
3382    }
3383    let mut out = context.uninit(total)?;
3384    let n = total as i32;
3385    unsafe {
3386        context
3387            .stream
3388            .launch_builder(&context.image.add)
3389            .arg(&mut out)
3390            .arg(&a.data)
3391            .arg(&b.data)
3392            .arg(&n)
3393            .launch(crate::cuda_training::cfg(total))
3394            .map_err(crate::cuda_training::cuda_err("add kernel"))?;
3395    }
3396    Ok(a.with_data(out))
3397}
3398
3399/// [`concat_channels`] on the device.
3400#[cfg(feature = "cuda")]
3401pub fn concat_channels_device(
3402    parts: &[&DeviceImageBatch],
3403) -> Result<DeviceImageBatch, NetworkError> {
3404    let first = parts[0];
3405    let channels: usize = parts.iter().map(|p| p.channels).sum();
3406    let rows = first.batch * first.pixels();
3407    let mut out = first.context.uninit(rows * channels)?;
3408    let mut offset = 0;
3409    for part in parts {
3410        copy_channels(
3411            &first.context,
3412            &mut out,
3413            channels,
3414            offset,
3415            &part.data,
3416            part.channels,
3417            0,
3418            part.channels,
3419            rows,
3420            false,
3421        )?;
3422        offset += part.channels;
3423    }
3424    Ok(DeviceImageBatch {
3425        channels,
3426        ..first.with_data(out)
3427    })
3428}
3429
3430/// [`split_channels`] on the device.
3431#[cfg(feature = "cuda")]
3432pub fn split_channels_device(
3433    input: &DeviceImageBatch,
3434    widths: &[usize],
3435) -> Result<Vec<DeviceImageBatch>, NetworkError> {
3436    let mut offset = 0;
3437    widths
3438        .iter()
3439        .map(|&width| {
3440            offset += width;
3441            slice_channels_device(input, offset - width, width)
3442        })
3443        .collect()
3444}
3445
3446/// Adds `src` into `dst`'s channels from `offset` on, in place.
3447#[cfg(feature = "cuda")]
3448pub(crate) fn add_channels_device(
3449    dst: &mut DeviceImageBatch,
3450    offset: usize,
3451    src: &DeviceImageBatch,
3452) -> Result<(), NetworkError> {
3453    let rows = src.batch * src.pixels();
3454    let context = dst.context.clone();
3455    copy_channels(
3456        &context,
3457        &mut dst.data,
3458        dst.channels,
3459        offset,
3460        &src.data,
3461        src.channels,
3462        0,
3463        src.channels,
3464        rows,
3465        true,
3466    )
3467}
3468
3469/// `width` channels from `offset`, copied out on the device.
3470#[cfg(feature = "cuda")]
3471pub fn slice_channels_device(
3472    input: &DeviceImageBatch,
3473    offset: usize,
3474    width: usize,
3475) -> Result<DeviceImageBatch, NetworkError> {
3476    let rows = input.batch * input.pixels();
3477    let mut out = input.context.uninit(rows * width)?;
3478    copy_channels(
3479        &input.context,
3480        &mut out,
3481        width,
3482        0,
3483        &input.data,
3484        input.channels,
3485        offset,
3486        width,
3487        rows,
3488        false,
3489    )?;
3490    Ok(DeviceImageBatch {
3491        channels: width,
3492        ..input.with_data(out)
3493    })
3494}
3495
3496#[cfg(feature = "cuda")]
3497#[allow(clippy::too_many_arguments)]
3498fn copy_channels(
3499    context: &crate::gpu_transformer::GpuContext,
3500    dst: &mut cudarc::driver::CudaSlice<f32>,
3501    dst_channels: usize,
3502    dst_offset: usize,
3503    src: &cudarc::driver::CudaSlice<f32>,
3504    src_channels: usize,
3505    src_offset: usize,
3506    width: usize,
3507    rows: usize,
3508    add: bool,
3509) -> Result<(), NetworkError> {
3510    let [rows_arg, width_arg, dc, doff, sc, soff, add] = [
3511        rows,
3512        width,
3513        dst_channels,
3514        dst_offset,
3515        src_channels,
3516        src_offset,
3517        add as usize,
3518    ]
3519    .map(|v| v as i32);
3520    unsafe {
3521        context
3522            .stream
3523            .launch_builder(&context.image.copy_channels)
3524            .arg(dst)
3525            .arg(src)
3526            .arg(&rows_arg)
3527            .arg(&width_arg)
3528            .arg(&dc)
3529            .arg(&doff)
3530            .arg(&sc)
3531            .arg(&soff)
3532            .arg(&add)
3533            .launch(crate::cuda_training::cfg(rows * width))
3534            .map_err(crate::cuda_training::cuda_err("channel copy kernel"))?;
3535    }
3536    Ok(())
3537}
3538
3539/// What [`max_pool_same_backward_device`] needs from the forward pass.
3540#[cfg(feature = "cuda")]
3541pub struct DeviceMaxPoolCache {
3542    kernel: usize,
3543    argmax: cudarc::driver::CudaSlice<i32>,
3544}
3545
3546/// [`max_pool_same`] on the device.
3547#[cfg(feature = "cuda")]
3548pub fn max_pool_same_device(
3549    input: &DeviceImageBatch,
3550    kernel: usize,
3551) -> Result<(DeviceImageBatch, DeviceMaxPoolCache), NetworkError> {
3552    let context = &input.context;
3553    let total = input.data.len();
3554    let mut out = context.uninit(total)?;
3555    let mut argmax = context
3556        .stream
3557        .alloc_zeros::<i32>(total)
3558        .map_err(crate::cuda_training::cuda_err("max pool allocation"))?;
3559    let [n, h, w, c, k] =
3560        [total, input.height, input.width, input.channels, kernel].map(|v| v as i32);
3561    unsafe {
3562        context
3563            .stream
3564            .launch_builder(&context.image.maxpool)
3565            .arg(&mut out)
3566            .arg(&mut argmax)
3567            .arg(&input.data)
3568            .arg(&n)
3569            .arg(&h)
3570            .arg(&w)
3571            .arg(&c)
3572            .arg(&k)
3573            .launch(crate::cuda_training::cfg(total))
3574            .map_err(crate::cuda_training::cuda_err("max pool kernel"))?;
3575    }
3576    Ok((input.with_data(out), DeviceMaxPoolCache { kernel, argmax }))
3577}
3578
3579/// [`max_pool_same_backward`] on the device, gathering rather than
3580/// scattering so it is deterministic.
3581#[cfg(feature = "cuda")]
3582pub fn max_pool_same_backward_device(
3583    cache: &DeviceMaxPoolCache,
3584    grad: &DeviceImageBatch,
3585) -> Result<DeviceImageBatch, NetworkError> {
3586    let context = &grad.context;
3587    let total = grad.data.len();
3588    let mut out = context.zeros(total)?;
3589    let [n, h, w, c, k] =
3590        [total, grad.height, grad.width, grad.channels, cache.kernel].map(|v| v as i32);
3591    unsafe {
3592        context
3593            .stream
3594            .launch_builder(&context.image.maxpool_bwd)
3595            .arg(&mut out)
3596            .arg(&grad.data)
3597            .arg(&cache.argmax)
3598            .arg(&n)
3599            .arg(&h)
3600            .arg(&w)
3601            .arg(&c)
3602            .arg(&k)
3603            .launch(crate::cuda_training::cfg(total))
3604            .map_err(crate::cuda_training::cuda_err("max pool backward kernel"))?;
3605    }
3606    Ok(grad.with_data(out))
3607}
3608
3609/// One thread per output element of a kernel taking `(out, in, total, dims...)`.
3610#[cfg(feature = "cuda")]
3611fn image_launch(
3612    input: &DeviceImageBatch,
3613    total: usize,
3614    what: &'static str,
3615    kernel: impl Fn(&crate::gpu_transformer::GpuContext) -> &cudarc::driver::CudaFunction,
3616    dims: &[usize],
3617) -> Result<cudarc::driver::CudaSlice<f32>, NetworkError> {
3618    let context = &input.context;
3619    let mut out = context.uninit(total)?;
3620    let n = total as i32;
3621    let dims: Vec<i32> = dims.iter().map(|&d| d as i32).collect();
3622    let mut launch = context.stream.launch_builder(kernel(context));
3623    launch.arg(&mut out).arg(&input.data).arg(&n);
3624    for d in &dims {
3625        launch.arg(d);
3626    }
3627    unsafe { launch.launch(crate::cuda_training::cfg(total)) }
3628        .map_err(crate::cuda_training::cuda_err(what))?;
3629    Ok(out)
3630}
3631
3632#[cfg(test)]
3633mod tests {
3634    use super::*;
3635    use crate::optimizers::Optimizer;
3636    use rand::SeedableRng;
3637
3638    /// A deterministic batch of activations, spread either side of zero.
3639    fn noise(
3640        batch: usize,
3641        height: usize,
3642        width: usize,
3643        channels: usize,
3644        seed: usize,
3645    ) -> ImageBatch {
3646        let data = (0..batch * height * width * channels)
3647            .map(|index| (((index * 37 + seed * 11) % 197) as f32 / 98.0) - 1.0)
3648            .collect();
3649        ImageBatch::from_tokens(
3650            batch,
3651            height,
3652            width,
3653            Matrix::from_vec(batch * height * width, channels, data),
3654        )
3655    }
3656
3657    /// Mean squared error against a fixed target, and its gradient.
3658    fn mse(prediction: &ImageBatch, target: &ImageBatch) -> (f32, ImageBatch) {
3659        let count = prediction.tokens.data.len() as f32;
3660        let mut grad = prediction.like(prediction.channels());
3661        let mut loss = 0.0;
3662        for (index, (value, wanted)) in prediction
3663            .tokens
3664            .data
3665            .iter()
3666            .zip(&target.tokens.data)
3667            .enumerate()
3668        {
3669            let difference = value - wanted;
3670            loss += difference * difference / count;
3671            grad.tokens.data[index] = 2.0 * difference / count;
3672        }
3673        (loss, grad)
3674    }
3675
3676    /// One hundred Adam steps, returning the loss before each one.
3677    fn loss_curve(
3678        conv: &mut TrainableConv2d,
3679        input: &ImageBatch,
3680        target: &ImageBatch,
3681        steps: usize,
3682    ) -> Result<Vec<f32>, NetworkError> {
3683        let optimizer = Optimizer::adam(1e-2);
3684        let mut curve = Vec::with_capacity(steps);
3685        for step in 1..=steps {
3686            let (prediction, cache) = conv.forward(input)?;
3687            let (loss, grad_output) = mse(&prediction, target);
3688            curve.push(loss);
3689            for param in conv.params_mut() {
3690                param.zero_grad();
3691            }
3692            conv.backward(&cache, &grad_output)?;
3693            for param in conv.params_mut() {
3694                param.step(&optimizer, step, 1.0);
3695            }
3696        }
3697        Ok(curve)
3698    }
3699
3700    /// `sum(f(input) * weights)`'s gradient by central differences, against
3701    /// `analytic`, relative to the largest analytic value.
3702    fn pooled_gap(
3703        f: impl Fn(&ImageBatch) -> Matrix,
3704        input: &ImageBatch,
3705        weights: &[f32],
3706        analytic: &[f32],
3707    ) -> f32 {
3708        let loss = |input: &ImageBatch| -> f32 {
3709            f(input).data.iter().zip(weights).map(|(a, b)| a * b).sum()
3710        };
3711        let scale = analytic.iter().fold(0f32, |m, g| m.max(g.abs()));
3712        let mut moved = input.clone();
3713        let mut worst = 0f32;
3714        for (index, analytic) in analytic.iter().enumerate() {
3715            let h = 1e-2;
3716            moved.tokens.data[index] += h;
3717            let up = loss(&moved);
3718            moved.tokens.data[index] -= 2.0 * h;
3719            let down = loss(&moved);
3720            moved.tokens.data[index] += h;
3721            worst = worst.max(((up - down) / (2.0 * h) - analytic).abs() / scale);
3722        }
3723        worst
3724    }
3725
3726    #[test]
3727    fn spatial_softmax_matches_finite_differences_and_finds_a_peak() {
3728        let mut input = noise(2, 3, 4, 5, 3);
3729        input.tokens.data.iter_mut().for_each(|v| *v *= 3.0);
3730        let weights = noise(1, 1, 20, 1, 4).tokens.data;
3731        let (_, cache) = spatial_softmax(&input);
3732        let grad = spatial_softmax_backward(&cache, &Matrix::from_vec(2, 10, weights.clone()));
3733        let gap = pooled_gap(
3734            |x| spatial_softmax(x).0,
3735            &input,
3736            &weights,
3737            &grad.tokens.data,
3738        );
3739        assert!(gap < 1e-2, "{gap}");
3740
3741        // A spike at sample 0, row 0, column 3, channel 2 is the top right.
3742        let mut spike = ImageBatch::new(2, 3, 4, 5);
3743        spike.tokens.data[3 * 5 + 2] = 50.0;
3744        let (expected, _) = spatial_softmax(&spike);
3745        assert!((expected.data[2] - 1.0).abs() < 1e-4);
3746        assert!((expected.data[5 + 2] + 1.0).abs() < 1e-4);
3747    }
3748
3749    #[test]
3750    fn trainable_group_norm_matches_the_inference_one_and_finite_differences() {
3751        let input = noise(2, 3, 4, 8, 3);
3752        let mut norm = TrainableGroupNorm::new(4, 8, 1e-5).unwrap();
3753        norm.gamma.value.data = noise(1, 1, 8, 1, 4)
3754            .tokens
3755            .data
3756            .iter()
3757            .map(|v| v + 1.5)
3758            .collect();
3759        norm.beta.value.data = noise(1, 1, 8, 1, 5).tokens.data;
3760        let fixed = GroupNorm::new(
3761            4,
3762            norm.gamma.value.data.clone(),
3763            norm.beta.value.data.clone(),
3764            1e-5,
3765        )
3766        .unwrap();
3767        let (output, cache) = norm.forward(&input).unwrap();
3768        for sample in 0..2 {
3769            let rows = 12 * sample..12 * (sample + 1);
3770            let tokens = Matrix::from_vec(
3771                12,
3772                8,
3773                input.tokens.data[rows.start * 8..rows.end * 8].to_vec(),
3774            );
3775            let mut map = FeatureMap::from_tokens(&tokens, 3, 4).unwrap();
3776            fixed.forward(&mut map).unwrap();
3777            assert_eq!(
3778                map.to_tokens().data,
3779                output.tokens.data[rows.start * 8..rows.end * 8]
3780            );
3781        }
3782
3783        let weights = noise(2, 3, 4, 8, 6);
3784        let grad = norm.backward(&cache, &weights);
3785        let through = |input: &ImageBatch| norm.forward(input).unwrap().0.tokens;
3786        let gap = pooled_gap(through, &input, &weights.tokens.data, &grad.tokens.data);
3787        assert!(gap < 1e-3, "input {gap}");
3788        for (which, analytic) in [(0, &norm.gamma.grad.data), (1, &norm.beta.grad.data)] {
3789            let params = ImageBatch::from_tokens(
3790                1,
3791                1,
3792                8,
3793                Matrix::from_vec(
3794                    8,
3795                    1,
3796                    if which == 0 {
3797                        norm.gamma.value.data.clone()
3798                    } else {
3799                        norm.beta.value.data.clone()
3800                    },
3801                ),
3802            );
3803            let through = |params: &ImageBatch| {
3804                let mut moved = norm.clone();
3805                let target = if which == 0 {
3806                    &mut moved.gamma
3807                } else {
3808                    &mut moved.beta
3809                };
3810                target.value.data.copy_from_slice(&params.tokens.data);
3811                moved.forward(&input).unwrap().0.tokens
3812            };
3813            let gap = pooled_gap(through, &params, &weights.tokens.data, analytic);
3814            assert!(gap < 1e-3, "param {which}: {gap}");
3815        }
3816        assert!(TrainableGroupNorm::new(3, 8, 1e-5).is_err());
3817    }
3818
3819    #[test]
3820    fn global_avg_pool_matches_finite_differences() {
3821        let input = noise(2, 3, 4, 5, 5);
3822        let weights = noise(1, 1, 10, 1, 6).tokens.data;
3823        let grad = global_avg_pool_backward(&Matrix::from_vec(2, 5, weights.clone()), 3, 4);
3824        let gap = pooled_gap(global_avg_pool, &input, &weights, &grad.tokens.data);
3825        assert!(gap < 1e-2, "{gap}");
3826        assert!((global_avg_pool(&ImageBatch::new(1, 2, 2, 3)).data[0]).abs() < 1e-9);
3827    }
3828
3829    #[test]
3830    fn a_convolution_without_bias_adds_and_trains_none() {
3831        let mut rng = rand::rngs::StdRng::seed_from_u64(12);
3832        let mut biased = TrainableConv2d::new(3, 4, 3, 2, &mut rng).unwrap();
3833        biased.bias.value.data.fill(1.0);
3834        let mut plain = biased.clone().without_bias();
3835        assert!(!plain.has_bias());
3836        assert_eq!(plain.params_mut().len() + 1, biased.params_mut().len());
3837
3838        let (input, target) = (noise(2, 5, 6, 3, 1), noise(2, 3, 3, 4, 2));
3839        let (with, with_cache) = biased.forward(&input).unwrap();
3840        let (without, cache) = plain.forward(&input).unwrap();
3841        for (with, without) in with.tokens.data.iter().zip(&without.tokens.data) {
3842            assert!((with - without - 1.0).abs() < 1e-6);
3843        }
3844        let (_, grad) = mse(&without, &target);
3845        biased.backward(&with_cache, &grad).unwrap();
3846        plain.backward(&cache, &grad).unwrap();
3847        assert_eq!(
3848            plain.weight.weight.grad.data,
3849            biased.weight.weight.grad.data
3850        );
3851        assert!(plain.bias.grad.data.iter().all(|g| *g == 0.0));
3852        assert!(plain.bias.value.data.iter().all(|b| *b == 0.0));
3853    }
3854
3855    #[test]
3856    fn a_trainable_convolution_matches_finite_differences() {
3857        let mut rng = rand::rngs::StdRng::seed_from_u64(11);
3858        let mut conv = TrainableConv2d::new(3, 4, 3, 1, &mut rng).unwrap();
3859        let input = noise(2, 5, 6, 3, 1);
3860        let target = noise(2, 5, 6, 4, 2);
3861
3862        let (prediction, cache) = conv.forward(&input).unwrap();
3863        let (_, grad_output) = mse(&prediction, &target);
3864        let grad_input = conv.backward(&cache, &grad_output).unwrap();
3865
3866        let epsilon = 1e-3;
3867        for index in [0usize, 17, 43, 88] {
3868            let mut moved = input.clone();
3869            moved.tokens.data[index] += epsilon;
3870            let (up, _) = conv.forward(&moved).unwrap();
3871            moved.tokens.data[index] -= 2.0 * epsilon;
3872            let (down, _) = conv.forward(&moved).unwrap();
3873            let numerical = (mse(&up, &target).0 - mse(&down, &target).0) / (2.0 * epsilon);
3874            let analytic = grad_input.tokens.data[index];
3875            assert!(
3876                (numerical - analytic).abs() < 1e-3,
3877                "dL/dinput[{index}]: {analytic} against {numerical} by finite difference"
3878            );
3879        }
3880
3881        for index in [0usize, 7, 26] {
3882            let mut moved = conv.clone();
3883            moved.weight.weight.value.data[index] += epsilon;
3884            let up = mse(&moved.forward(&input).unwrap().0, &target).0;
3885            moved.weight.weight.value.data[index] -= 2.0 * epsilon;
3886            let down = mse(&moved.forward(&input).unwrap().0, &target).0;
3887            let numerical = (up - down) / (2.0 * epsilon);
3888            let analytic = conv.weight.weight.grad.data[index];
3889            assert!(
3890                (numerical - analytic).abs() < 1e-3,
3891                "dL/dweight[{index}]: {analytic} against {numerical} by finite difference"
3892            );
3893        }
3894    }
3895
3896    #[test]
3897    fn training_a_convolution_drives_its_loss_down() {
3898        let mut rng = rand::rngs::StdRng::seed_from_u64(17);
3899        let mut conv = TrainableConv2d::new(8, 8, 3, 1, &mut rng).unwrap();
3900        let input = noise(1, 8, 8, 8, 9);
3901        let target = noise(1, 8, 8, 8, 10);
3902        let curve = loss_curve(&mut conv, &input, &target, 20).unwrap();
3903        assert!(
3904            curve[19] < curve[0] / 4.0,
3905            "twenty Adam steps went from {} to {}",
3906            curve[0],
3907            curve[19]
3908        );
3909    }
3910
3911    #[test]
3912    fn a_strided_convolution_shrinks_the_map_and_still_reaches_every_pixel() {
3913        let mut rng = rand::rngs::StdRng::seed_from_u64(5);
3914        let mut conv = TrainableConv2d::new(2, 3, 3, 2, &mut rng).unwrap();
3915        let input = noise(1, 8, 8, 2, 3);
3916        let (output, cache) = conv.forward(&input).unwrap();
3917        assert_eq!((output.height, output.width), (4, 4));
3918
3919        let grad_output = noise(1, 4, 4, 3, 4);
3920        let grad_input = conv.backward(&cache, &grad_output).unwrap();
3921        assert_eq!(grad_input.tokens.rows, input.tokens.rows);
3922        assert!(
3923            grad_input.tokens.data.iter().any(|value| *value != 0.0),
3924            "a stride of two still reads most of the input"
3925        );
3926    }
3927
3928    #[test]
3929    fn rebuilding_the_columns_reaches_the_same_gradients_as_keeping_them() {
3930        let mut rng = rand::rngs::StdRng::seed_from_u64(31);
3931        let mut keeping = TrainableConv2d::new(6, 8, 3, 2, &mut rng).unwrap();
3932        let mut rebuilding = keeping.clone();
3933        rebuilding.set_recompute_columns(true);
3934        let input = noise(2, 11, 9, 6, 3);
3935
3936        let (kept, kept_cache) = keeping.forward(&input).unwrap();
3937        let (rebuilt, rebuilt_cache) = rebuilding.forward(&input).unwrap();
3938        assert_eq!(
3939            kept.tokens.data, rebuilt.tokens.data,
3940            "the forward pass is the same either way"
3941        );
3942        assert!(kept_cache.holds_columns());
3943        assert!(
3944            !rebuilt_cache.holds_columns(),
3945            "the whole point is that the columns are not held"
3946        );
3947
3948        let grad_output = noise(2, kept.height, kept.width, 8, 4);
3949        let kept_input = keeping.backward(&kept_cache, &grad_output).unwrap();
3950        let rebuilt_input = rebuilding.backward(&rebuilt_cache, &grad_output).unwrap();
3951
3952        assert_eq!(kept_input.tokens.data, rebuilt_input.tokens.data);
3953        assert_eq!(
3954            keeping.weight.weight.grad.data,
3955            rebuilding.weight.weight.grad.data
3956        );
3957        assert_eq!(keeping.bias.grad.data, rebuilding.bias.grad.data);
3958    }
3959
3960    /// The option has to survive a whole run, not one step: the cache it builds
3961    /// is read by a backward pass whose weight has already moved twice.
3962    #[test]
3963    fn a_convolution_that_rebuilds_its_columns_trains_to_the_same_place() {
3964        let mut rng = rand::rngs::StdRng::seed_from_u64(37);
3965        let mut keeping = TrainableConv2d::new(8, 8, 3, 1, &mut rng).unwrap();
3966        let mut rebuilding = keeping.clone();
3967        rebuilding.set_recompute_columns(true);
3968        let input = noise(1, 12, 12, 8, 5);
3969        let target = noise(1, 12, 12, 8, 6);
3970
3971        let kept = loss_curve(&mut keeping, &input, &target, 40).unwrap();
3972        let rebuilt = loss_curve(&mut rebuilding, &input, &target, 40).unwrap();
3973        assert!(
3974            kept[39] < kept[0] / 2.0,
3975            "the reference run has to be learning"
3976        );
3977        assert_eq!(kept, rebuilt);
3978    }
3979
3980    #[cfg(feature = "cuda")]
3981    fn cuda_or_skip() -> Option<crate::param::CudaDevice> {
3982        match crate::cuda_training::cuda_doctor(0, 4096) {
3983            Ok(_) => Some(crate::param::CudaDevice::new(0, 4096).expect("the CUDA doctor passed")),
3984            Err(NetworkError::Cuda(message))
3985                if message.contains("NO_DEVICE") || message.contains("no CUDA-capable device") =>
3986            {
3987                crate::cuda_training::skip_without_cuda();
3988                None
3989            }
3990            Err(error) => panic!("CUDA is present but the CUDA doctor failed: {error}"),
3991        }
3992    }
3993
3994    /// The acceptance test for the device path: a 96-channel 3x3 convolution
3995    /// over a 64x64 map trains on the device and its loss curve follows the
3996    /// host's for a hundred steps.
3997    #[cfg(feature = "cuda")]
3998    #[test]
3999    fn a_convolution_trained_on_the_device_follows_the_hosts_loss_curve() {
4000        let Some(device) = cuda_or_skip() else {
4001            return;
4002        };
4003        let mut rng = rand::rngs::StdRng::seed_from_u64(29);
4004        let mut host = TrainableConv2d::new(96, 96, 3, 1, &mut rng).unwrap();
4005        let mut resident = host.clone();
4006        let input = noise(1, 64, 64, 96, 7);
4007        let target = noise(1, 64, 64, 96, 8);
4008
4009        let on_host = loss_curve(&mut host, &input, &target, 100).unwrap();
4010        resident.to_cuda_on(&device).unwrap();
4011        let on_device = loss_curve(&mut resident, &input, &target, 100).unwrap();
4012
4013        assert!(
4014            on_host[99] < on_host[0] / 2.0,
4015            "the host path has to be learning"
4016        );
4017        for (step, (a, b)) in on_host.iter().zip(&on_device).enumerate() {
4018            assert!(
4019                (a - b).abs() <= 1e-4 + 1e-3 * a.abs(),
4020                "step {step}: {a} on the host against {b} on the device"
4021            );
4022        }
4023
4024        // And the weights themselves agree, not only the losses they produced.
4025        resident.to_cpu().unwrap();
4026        for (index, (a, b)) in host
4027            .weight
4028            .weight
4029            .value
4030            .data
4031            .iter()
4032            .zip(&resident.weight.weight.value.data)
4033            .enumerate()
4034        {
4035            assert!(
4036                (a - b).abs() < 1e-3,
4037                "weight[{index}]: {a} on the host against {b} on the device"
4038            );
4039        }
4040    }
4041
4042    /// A 3x3 convolution on a mixed-precision device, where `im2col` builds
4043    /// bf16 columns and all three GEMMs run in bf16, stays within bf16
4044    /// rounding of the host.
4045    #[cfg(feature = "cuda")]
4046    #[test]
4047    fn a_mixed_precision_convolution_tracks_the_host() {
4048        let Some(_) = cuda_or_skip() else {
4049            return;
4050        };
4051        let device = crate::param::CudaDevice::with_mixed_precision(0, 1024).unwrap();
4052        // Columns kept and rebuilt, with channels in fours and not.
4053        for (channels, recompute) in [(8, false), (6, true)] {
4054            let mut rng = rand::rngs::StdRng::seed_from_u64(32);
4055            let mut conv = TrainableConv2d::new(channels, 16, 3, 1, &mut rng).unwrap();
4056            conv.bias.value.data = noise(1, 1, 16, 1, 3).tokens.data;
4057            conv.set_recompute_columns(recompute);
4058            let mut host = conv.clone();
4059            conv.to_cuda_on(&device).unwrap();
4060            let input = noise(2, 10, 9, channels, 1);
4061            let grad = noise(2, 10, 9, 16, 2);
4062            let (expected, cache) = host.forward(&input).unwrap();
4063            let expected_grad = host.backward(&cache, &grad).unwrap();
4064            let (output, cache) = conv
4065                .forward_device(&device.upload(&input).unwrap())
4066                .unwrap();
4067            let input_grad = conv
4068                .backward_device(&cache, &device.upload(&grad).unwrap())
4069                .unwrap();
4070            // bf16 keeps 8 bits of mantissa: about 0.4% per operand.
4071            let near = |a: &[f32], b: &[f32], what: &str| {
4072                let gap: f32 = a.iter().zip(b).map(|(a, b)| (a - b).powi(2)).sum();
4073                let scale: f32 = b.iter().map(|b| b * b).sum();
4074                assert!(
4075                    gap.sqrt() < 1e-2 * scale.sqrt(),
4076                    "{what}: {gap} against {scale}"
4077                );
4078            };
4079            near(
4080                &output.download().unwrap().tokens.data,
4081                &expected.tokens.data,
4082                "output",
4083            );
4084            near(
4085                &input_grad.download().unwrap().tokens.data,
4086                &expected_grad.tokens.data,
4087                "input grad",
4088            );
4089            conv.to_cpu().unwrap();
4090            near(
4091                &conv.weight.weight.grad.data,
4092                &host.weight.weight.grad.data,
4093                "weight grad",
4094            );
4095            near(&conv.bias.grad.data, &host.bias.grad.data, "bias grad");
4096        }
4097    }
4098
4099    /// Two biased convolutions chained on the device: activations and
4100    /// gradients cross the bus only at the two ends, and every output and
4101    /// gradient matches the host chain.
4102    #[cfg(feature = "cuda")]
4103    #[test]
4104    fn a_device_chain_stays_on_the_card_and_matches_the_host() {
4105        let Some(device) = cuda_or_skip() else {
4106            return;
4107        };
4108        let mut rng = rand::rngs::StdRng::seed_from_u64(31);
4109        let mut first = TrainableConv2d::new(6, 8, 3, 2, &mut rng).unwrap();
4110        // A 1x1 convolution, whose columns are its input.
4111        let mut second = TrainableConv2d::new(8, 5, 1, 1, &mut rng).unwrap();
4112        first.bias.value.data = noise(1, 1, 8, 1, 3).tokens.data;
4113        second.bias.value.data = noise(1, 1, 5, 1, 4).tokens.data;
4114        second.set_recompute_columns(true);
4115        let (mut host_first, mut host_second) = (first.clone(), second.clone());
4116        first.to_cuda_on(&device).unwrap();
4117        second.to_cuda_on(&device).unwrap();
4118
4119        let input = noise(3, 11, 9, 6, 1);
4120        let (middle, first_cache) = host_first.forward(&input).unwrap();
4121        let (expected, second_cache) = host_second.forward(&middle).unwrap();
4122        let grad = noise(3, expected.height, expected.width, 5, 2);
4123        let expected_grad = host_first
4124            .backward(
4125                &first_cache,
4126                &host_second.backward(&second_cache, &grad).unwrap(),
4127            )
4128            .unwrap();
4129
4130        let context = device.context();
4131        let resident = device.upload(&input).unwrap();
4132        let resident_grad = device.upload(&grad).unwrap();
4133        let before = context.transfers();
4134        let (middle, first_cache) = first.forward_device(&resident).unwrap();
4135        let (output, second_cache) = second.forward_device(&middle).unwrap();
4136        let back = second
4137            .backward_device(&second_cache, &resident_grad)
4138            .unwrap();
4139        let input_grad = first.backward_device(&first_cache, &back).unwrap();
4140        assert_eq!(
4141            context.transfers(),
4142            before,
4143            "nothing crosses inside the chain"
4144        );
4145
4146        let close = |a: &[f32], b: &[f32], what: &str| {
4147            assert_eq!(a.len(), b.len(), "{what}");
4148            for (index, (a, b)) in a.iter().zip(b).enumerate() {
4149                assert!(
4150                    (a - b).abs() < 1e-4 + 1e-4 * b.abs(),
4151                    "{what}[{index}]: {a} against {b}"
4152                );
4153            }
4154        };
4155        close(
4156            &device.download(&output).unwrap().tokens.data,
4157            &expected.tokens.data,
4158            "output",
4159        );
4160        close(
4161            &input_grad.download().unwrap().tokens.data,
4162            &expected_grad.tokens.data,
4163            "input grad",
4164        );
4165        first.to_cpu().unwrap();
4166        second.to_cpu().unwrap();
4167        for (device_side, host_side, what) in [
4168            (&first, &host_first, "first"),
4169            (&second, &host_second, "second"),
4170        ] {
4171            close(
4172                &device_side.weight.weight.grad.data,
4173                &host_side.weight.weight.grad.data,
4174                what,
4175            );
4176            close(&device_side.bias.grad.data, &host_side.bias.grad.data, what);
4177        }
4178    }
4179
4180    /// Group norm then SiLU on the device, forward and back, against the host
4181    /// layer and the host SiLU, with nothing crossing the bus in between.
4182    #[cfg(feature = "cuda")]
4183    #[test]
4184    fn device_group_norm_and_silu_match_the_host() {
4185        let Some(device) = cuda_or_skip() else {
4186            return;
4187        };
4188        let input = noise(2, 16, 17, 12, 1);
4189        let grad = noise(2, 16, 17, 12, 2);
4190        let mut host = TrainableGroupNorm::new(4, 12, 1e-5).unwrap();
4191        host.gamma.value.data = noise(1, 1, 12, 1, 3)
4192            .tokens
4193            .data
4194            .iter()
4195            .map(|v| v + 1.2)
4196            .collect();
4197        host.beta.value.data = noise(1, 1, 12, 1, 4).tokens.data;
4198        let mut resident = host.clone();
4199        resident.to_cuda_on(&device).unwrap();
4200
4201        let (normed, cache) = host.forward(&input).unwrap();
4202        let mut activated = normed.clone();
4203        let mut back = grad.clone();
4204        for ((value, grad), pre) in activated
4205            .tokens
4206            .data
4207            .iter_mut()
4208            .zip(&mut back.tokens.data)
4209            .zip(&normed.tokens.data)
4210        {
4211            *value = crate::ffn::silu(*pre);
4212            *grad *= crate::ffn::silu_derivative(*pre);
4213        }
4214        let expected_grad = host.backward(&cache, &back);
4215
4216        let context = device.context();
4217        let (on_card, on_card_grad) = (
4218            device.upload(&input).unwrap(),
4219            device.upload(&grad).unwrap(),
4220        );
4221        let before = context.transfers();
4222        let (normed, cache) = resident.forward_device(&on_card).unwrap();
4223        let output = silu_device(&normed).unwrap();
4224        let back = silu_backward_device(&normed, &on_card_grad).unwrap();
4225        let input_grad = resident.backward_device(&cache, &back).unwrap();
4226        assert_eq!(
4227            context.transfers(),
4228            before,
4229            "nothing crosses inside the chain"
4230        );
4231
4232        let close = |a: &[f32], b: &[f32], what: &str| {
4233            assert_eq!(a.len(), b.len(), "{what}");
4234            for (index, (a, b)) in a.iter().zip(b).enumerate() {
4235                assert!(
4236                    (a - b).abs() < 1e-4 + 1e-4 * b.abs(),
4237                    "{what}[{index}]: {a} against {b}"
4238                );
4239            }
4240        };
4241        close(
4242            &output.download().unwrap().tokens.data,
4243            &activated.tokens.data,
4244            "output",
4245        );
4246        close(
4247            &input_grad.download().unwrap().tokens.data,
4248            &expected_grad.tokens.data,
4249            "input grad",
4250        );
4251        resident.to_cpu().unwrap();
4252        close(
4253            &resident.gamma.grad.data,
4254            &host.gamma.grad.data,
4255            "gamma grad",
4256        );
4257        close(&resident.beta.grad.data, &host.beta.grad.data, "beta grad");
4258    }
4259
4260    /// Spatial softmax and average pooling on the device against the host.
4261    #[cfg(feature = "cuda")]
4262    #[test]
4263    fn device_spatial_softmax_and_pooling_match_the_host() {
4264        let Some(device) = cuda_or_skip() else {
4265            return;
4266        };
4267        let input = noise(3, 17, 19, 5, 1);
4268        let (expected, cache) = spatial_softmax(&input);
4269        let grad = noise(3, 1, 1, 10, 2).tokens;
4270        let expected_grad = spatial_softmax_backward(&cache, &grad);
4271        let pooled = global_avg_pool(&input);
4272        let pool_grad = noise(3, 1, 1, 5, 3).tokens;
4273        let expected_pool_grad = global_avg_pool_backward(&pool_grad, 17, 19);
4274
4275        let context = device.context();
4276        let on_card = device.upload(&input).unwrap();
4277        let on_card_grad = device
4278            .upload(&ImageBatch::from_tokens(3, 1, 1, grad))
4279            .unwrap();
4280        let on_card_pool_grad = device
4281            .upload(&ImageBatch::from_tokens(3, 1, 1, pool_grad))
4282            .unwrap();
4283        let before = context.transfers();
4284        let (output, cache) = spatial_softmax_device(&on_card).unwrap();
4285        let back = spatial_softmax_backward_device(&cache, &on_card_grad).unwrap();
4286        let mean = global_avg_pool_device(&on_card).unwrap();
4287        let spread = global_avg_pool_backward_device(&on_card_pool_grad, 17, 19).unwrap();
4288        assert_eq!(
4289            context.transfers(),
4290            before,
4291            "nothing crosses inside the chain"
4292        );
4293
4294        let close = |a: &[f32], b: &[f32], what: &str| {
4295            assert_eq!(a.len(), b.len(), "{what}");
4296            for (index, (a, b)) in a.iter().zip(b).enumerate() {
4297                assert!(
4298                    (a - b).abs() < 1e-5 + 1e-4 * b.abs(),
4299                    "{what}[{index}]: {a} against {b}"
4300                );
4301            }
4302        };
4303        close(
4304            &output.download().unwrap().tokens.data,
4305            &expected.data,
4306            "keypoints",
4307        );
4308        close(
4309            &back.download().unwrap().tokens.data,
4310            &expected_grad.tokens.data,
4311            "keypoint grad",
4312        );
4313        close(&mean.download().unwrap().tokens.data, &pooled.data, "pool");
4314        close(
4315            &spread.download().unwrap().tokens.data,
4316            &expected_pool_grad.tokens.data,
4317            "pool grad",
4318        );
4319    }
4320
4321    /// The host path is unchanged by a device being present, which is what
4322    /// makes the parity test above meaningful.
4323    #[cfg(feature = "cuda")]
4324    #[test]
4325    fn im2col_on_the_device_builds_the_same_columns_as_the_host() {
4326        let Some(device) = cuda_or_skip() else {
4327            return;
4328        };
4329        let mut rng = rand::rngs::StdRng::seed_from_u64(13);
4330        let conv = TrainableConv2d::new(5, 7, 3, 2, &mut rng).unwrap();
4331        let input = noise(2, 9, 11, 5, 6);
4332        let shape = conv.geometry(input.batch, input.height, input.width);
4333        let expected = conv.im2col(&input, &shape);
4334
4335        let context = device.context();
4336        let resident = context.upload(&input.tokens).unwrap();
4337        let columns = context.im2col(&resident, &shape).unwrap();
4338        let mut actual = Matrix::new(expected.rows, expected.cols);
4339        context.download(&columns, &mut actual).unwrap();
4340        assert_eq!(actual.data, expected.data);
4341
4342        // col2im is its transpose, so scattering the columns back has to match
4343        // the host's scatter exactly too.
4344        let scattered = context.col2im(&columns, &shape).unwrap();
4345        let mut actual = Matrix::new(input.tokens.rows, input.tokens.cols);
4346        context.download(&scattered, &mut actual).unwrap();
4347        let expected = conv.col2im(&expected, &shape);
4348        for (index, (a, b)) in actual.data.iter().zip(&expected.tokens.data).enumerate() {
4349            assert!(
4350                (a - b).abs() < 1e-5,
4351                "col2im[{index}]: {a} on the device against {b} on the host"
4352            );
4353        }
4354    }
4355
4356    /// The device path has its own cache, so it needs its own parity check:
4357    /// rebuilding the columns from an uploaded input has to reach the same
4358    /// gradients as reading the ones the forward pass left on the card.
4359    #[cfg(feature = "cuda")]
4360    #[test]
4361    fn rebuilding_the_columns_on_the_device_reaches_the_same_gradients() {
4362        let Some(device) = cuda_or_skip() else {
4363            return;
4364        };
4365        let mut rng = rand::rngs::StdRng::seed_from_u64(41);
4366        let mut keeping = TrainableConv2d::new(16, 24, 3, 1, &mut rng).unwrap();
4367        let mut rebuilding = keeping.clone();
4368        rebuilding.set_recompute_columns(true);
4369        keeping.to_cuda_on(&device).unwrap();
4370        rebuilding.to_cuda_on(&device).unwrap();
4371        let input = noise(2, 16, 16, 16, 9);
4372
4373        let (kept, kept_cache) = keeping.forward(&input).unwrap();
4374        let (rebuilt, rebuilt_cache) = rebuilding.forward(&input).unwrap();
4375        assert_eq!(kept.tokens.data, rebuilt.tokens.data);
4376        assert!(!rebuilt_cache.holds_columns());
4377
4378        let grad_output = noise(2, kept.height, kept.width, 24, 10);
4379        let kept_input = keeping.backward(&kept_cache, &grad_output).unwrap();
4380        let rebuilt_input = rebuilding.backward(&rebuilt_cache, &grad_output).unwrap();
4381        assert_eq!(kept_input.tokens.data, rebuilt_input.tokens.data);
4382
4383        keeping.to_cpu().unwrap();
4384        rebuilding.to_cpu().unwrap();
4385        assert_eq!(
4386            keeping.weight.weight.grad.data,
4387            rebuilding.weight.weight.grad.data
4388        );
4389        assert_eq!(keeping.bias.grad.data, rebuilding.bias.grad.data);
4390    }
4391
4392    fn ramp(channels: usize, height: usize, width: usize) -> FeatureMap {
4393        let data = (0..channels * height * width)
4394            .map(|value| value as f32)
4395            .collect();
4396        FeatureMap::from_vec(channels, height, width, data).unwrap()
4397    }
4398
4399    #[test]
4400    fn quantizing_a_dense_keeps_its_answer_and_drops_its_floats() {
4401        let weight = Matrix::from_vec(
4402            3,
4403            4,
4404            (0..12).map(|index| (index as f32 * 0.7).sin()).collect(),
4405        );
4406        let tokens = Matrix::from_vec(2, 4, (0..8).map(|index| index as f32 * 0.25).collect());
4407        let mut layer = Dense::new(weight, Some(vec![0.5, -0.5, 0.25])).unwrap();
4408        let expected = layer.forward(&tokens).unwrap();
4409
4410        layer.quantize();
4411        assert!(layer.is_quantized());
4412        assert!(layer.weight.data.is_empty());
4413        assert_eq!((layer.out_dim(), layer.in_dim()), (3, 4));
4414
4415        let actual = layer.forward(&tokens).unwrap();
4416        for (actual, expected) in actual.data.iter().zip(&expected.data) {
4417            assert!(
4418                (actual - expected).abs() < 0.02,
4419                "{actual} is not {expected}"
4420            );
4421        }
4422    }
4423
4424    #[test]
4425    fn a_one_by_one_convolution_is_a_per_pixel_linear_layer() {
4426        // Two output channels: the first sums the inputs, the second negates
4427        // the second input.
4428        let weight = Matrix::from_vec(2, 2, vec![1.0, 1.0, 0.0, -1.0]);
4429        let conv = Conv2d::new(weight, Some(vec![0.5, 0.0]), 2, 1, 1, 0).unwrap();
4430        let input = ramp(2, 2, 2);
4431
4432        let output = conv.forward(&input).unwrap();
4433
4434        assert_eq!(output.channels, 2);
4435        assert_eq!((output.height, output.width), (2, 2));
4436        // Channel zero: input[0] + input[1] + 0.5, over pixels 0..4 where the
4437        // second plane starts at 4.
4438        assert_eq!(output.plane(0), &[4.5, 6.5, 8.5, 10.5]);
4439        assert_eq!(output.plane(1), &[-4.0, -5.0, -6.0, -7.0]);
4440    }
4441
4442    #[test]
4443    fn a_padded_three_by_three_convolution_keeps_the_size_and_reads_zeros_outside() {
4444        // A kernel that picks the pixel above, so the top row reads padding.
4445        let mut weight = vec![0.0; 9];
4446        weight[1] = 1.0;
4447        let conv = Conv2d::new(Matrix::from_vec(1, 9, weight), None, 1, 3, 1, 1).unwrap();
4448        let input = ramp(1, 3, 3);
4449
4450        let output = conv.forward(&input).unwrap();
4451
4452        assert_eq!((output.height, output.width), (3, 3));
4453        assert_eq!(
4454            output.data,
4455            vec![0.0, 0.0, 0.0, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0]
4456        );
4457    }
4458
4459    #[test]
4460    fn a_strided_convolution_halves_the_resolution() {
4461        let conv = Conv2d::new(Matrix::from_vec(1, 4, vec![0.25; 4]), None, 1, 2, 2, 0).unwrap();
4462        let output = conv.forward(&ramp(1, 4, 4)).unwrap();
4463
4464        assert_eq!((output.height, output.width), (2, 2));
4465        // Each output is the mean of its 2x2 block.
4466        assert_eq!(output.data, vec![2.5, 4.5, 10.5, 12.5]);
4467    }
4468
4469    #[test]
4470    fn a_convolution_refuses_shapes_it_cannot_read() {
4471        let conv = Conv2d::new(Matrix::from_vec(1, 4, vec![1.0; 4]), None, 1, 2, 1, 0).unwrap();
4472        assert!(conv.forward(&ramp(2, 4, 4)).is_err());
4473        assert!(conv.forward(&ramp(1, 1, 1)).is_err());
4474        assert!(Conv2d::new(Matrix::from_vec(1, 3, vec![1.0; 3]), None, 1, 2, 1, 0).is_err());
4475        assert!(
4476            Conv2d::new(
4477                Matrix::from_vec(1, 4, vec![1.0; 4]),
4478                Some(vec![0.0; 2]),
4479                1,
4480                2,
4481                1,
4482                0
4483            )
4484            .is_err()
4485        );
4486    }
4487
4488    #[test]
4489    fn group_norm_standardizes_each_group_over_its_own_pixels() {
4490        let mut map = ramp(4, 2, 2);
4491        let norm = GroupNorm::new(2, vec![1.0; 4], vec![0.0; 4], 1e-5).unwrap();
4492        norm.forward(&mut map).unwrap();
4493
4494        for group in 0..2 {
4495            let values = &map.data[group * 8..(group + 1) * 8];
4496            let mean = values.iter().sum::<f32>() / 8.0;
4497            let variance = values
4498                .iter()
4499                .map(|value| (value - mean).powi(2))
4500                .sum::<f32>()
4501                / 8.0;
4502            assert!(mean.abs() < 1e-5, "{mean}");
4503            assert!((variance - 1.0).abs() < 1e-3, "{variance}");
4504        }
4505
4506        // The scale and shift are per channel, not per group.
4507        let mut map = ramp(2, 1, 2);
4508        let norm = GroupNorm::new(1, vec![2.0, 0.5], vec![1.0, -1.0], 1e-5).unwrap();
4509        norm.forward(&mut map).unwrap();
4510        assert!(
4511            (map.data[0] - (-1.341_640_8 * 2.0 + 1.0)).abs() < 1e-4,
4512            "{:?}",
4513            map.data
4514        );
4515
4516        assert!(GroupNorm::new(3, vec![1.0; 4], vec![0.0; 4], 1e-5).is_err());
4517    }
4518
4519    #[test]
4520    fn upsampling_repeats_pixels_and_shuffling_round_trips() {
4521        let map = ramp(1, 2, 2);
4522        let large = upsample_nearest(&map, 2);
4523        assert_eq!((large.height, large.width), (4, 4));
4524        assert_eq!(&large.data[..4], &[0.0, 0.0, 1.0, 1.0]);
4525        assert_eq!(&large.data[4..8], &[0.0, 0.0, 1.0, 1.0]);
4526
4527        let packed = ramp(8, 2, 3);
4528        let shuffled = pixel_shuffle(&packed, 2).unwrap();
4529        assert_eq!(
4530            (shuffled.channels, shuffled.height, shuffled.width),
4531            (2, 4, 6)
4532        );
4533        assert_eq!(pixel_unshuffle(&shuffled, 2).unwrap(), packed);
4534
4535        assert!(pixel_shuffle(&packed, 3).is_err());
4536        assert!(pixel_unshuffle(&ramp(1, 3, 3), 2).is_err());
4537    }
4538
4539    #[test]
4540    fn a_map_survives_the_trip_through_token_shape() {
4541        let map = ramp(3, 2, 4);
4542        let tokens = map.to_tokens();
4543        assert_eq!((tokens.rows, tokens.cols), (8, 3));
4544        assert_eq!(tokens.row(1), &[1.0, 9.0, 17.0]);
4545        assert_eq!(FeatureMap::from_tokens(&tokens, 2, 4).unwrap(), map);
4546        assert!(FeatureMap::from_tokens(&tokens, 3, 4).is_err());
4547    }
4548    fn dot(a: &[f32], b: &[f32]) -> f32 {
4549        a.iter().zip(b).map(|(a, b)| a * b).sum()
4550    }
4551
4552    fn batch_norm_for_tests() -> TrainableBatchNorm2d {
4553        let mut norm = TrainableBatchNorm2d::yolo(6);
4554        norm.gamma.value.data = noise(1, 1, 6, 1, 4)
4555            .tokens
4556            .data
4557            .iter()
4558            .map(|v| v + 1.5)
4559            .collect();
4560        norm.beta.value.data = noise(1, 1, 6, 1, 5).tokens.data;
4561        norm.running.value.data = noise(1, 2, 6, 1, 9)
4562            .tokens
4563            .data
4564            .iter()
4565            .enumerate()
4566            .map(|(i, v)| if i < 6 { *v } else { v.abs() + 0.5 })
4567            .collect();
4568        norm
4569    }
4570
4571    #[test]
4572    fn batch_norm_matches_finite_differences_in_both_modes_and_tracks_running_statistics() {
4573        let input = noise(2, 3, 5, 6, 3);
4574        let weights = noise(2, 3, 5, 6, 6);
4575        for training in [true, false] {
4576            let mut norm = batch_norm_for_tests();
4577            let before = norm.running.value.data.clone();
4578            let (output, cache) = norm.forward(&input, training).unwrap();
4579            let grad = norm.backward(&cache, &weights);
4580            let through = |input: &ImageBatch| {
4581                batch_norm_for_tests()
4582                    .forward(input, training)
4583                    .unwrap()
4584                    .0
4585                    .tokens
4586            };
4587            let gap = pooled_gap(through, &input, &weights.tokens.data, &grad.tokens.data);
4588            assert!(gap < 1e-3, "training {training}: input {gap}");
4589            // gamma and beta enter linearly, so their gradients are exact sums.
4590            let normalized: Vec<f32> = output
4591                .tokens
4592                .data
4593                .iter()
4594                .enumerate()
4595                .map(|(i, y)| (y - norm.beta.value.data[i % 6]) / norm.gamma.value.data[i % 6])
4596                .collect();
4597            for c in 0..6 {
4598                let (mut dg, mut db) = (0.0, 0.0);
4599                for r in 0..30 {
4600                    dg += weights.tokens.data[r * 6 + c] * normalized[r * 6 + c];
4601                    db += weights.tokens.data[r * 6 + c];
4602                }
4603                assert!((norm.gamma.grad.data[c] - dg).abs() < 1e-4);
4604                assert!((norm.beta.grad.data[c] - db).abs() < 1e-4);
4605            }
4606            if training {
4607                let c = 2;
4608                let column: Vec<f32> = (0..30).map(|r| input.tokens.data[r * 6 + c]).collect();
4609                let mean = column.iter().sum::<f32>() / 30.0;
4610                let unbiased = column.iter().map(|v| (v - mean).powi(2)).sum::<f32>() / 29.0;
4611                let running = &norm.running.value.data;
4612                assert!((running[c] - (0.97 * before[c] + 0.03 * mean)).abs() < 1e-6);
4613                assert!((running[6 + c] - (0.97 * before[6 + c] + 0.03 * unbiased)).abs() < 1e-6);
4614            } else {
4615                assert_eq!(norm.running.value.data, before);
4616            }
4617        }
4618    }
4619
4620    #[test]
4621    fn a_fused_convolution_equals_the_convolution_then_the_evaluating_norm() {
4622        let mut rng = rand::rngs::StdRng::seed_from_u64(3);
4623        let conv = TrainableConv2d::new(4, 6, 3, 2, &mut rng)
4624            .unwrap()
4625            .without_bias();
4626        let mut norm = batch_norm_for_tests();
4627        let input = noise(2, 7, 9, 4, 1);
4628        let (expected, _) = norm
4629            .forward(&conv.forward(&input).unwrap().0, false)
4630            .unwrap();
4631        let fused = norm.fuse_into(&conv).forward(&input).unwrap().0;
4632        for (a, b) in fused.tokens.data.iter().zip(&expected.tokens.data) {
4633            assert!((a - b).abs() < 1e-5, "{a} against {b}");
4634        }
4635    }
4636
4637    #[test]
4638    fn upsample_concat_split_and_max_pool_backwards_are_their_adjoints() {
4639        let input = noise(2, 3, 4, 5, 1);
4640        let up = upsample_nearest_batch(&input, 2);
4641        assert_eq!((up.height, up.width), (6, 8));
4642        assert_eq!(up.pixel(1, 5, 7), input.pixel(1, 2, 3));
4643        let grad = noise(2, 6, 8, 5, 2);
4644        let back = upsample_nearest_backward(&grad, 2);
4645        let gap =
4646            dot(&up.tokens.data, &grad.tokens.data) - dot(&input.tokens.data, &back.tokens.data);
4647        assert!(gap.abs() < 1e-3, "upsample {gap}");
4648
4649        let other = noise(2, 3, 4, 2, 7);
4650        let joined = concat_channels(&[&input, &other]);
4651        assert_eq!(joined.pixel(1, 2, 3)[5..], *other.pixel(1, 2, 3));
4652        let parts = split_channels(&joined, &[5, 2]);
4653        assert_eq!(parts[0].tokens.data, input.tokens.data);
4654        assert_eq!(parts[1].tokens.data, other.tokens.data);
4655
4656        // Distinct values keep every window's winner clear of a tie.
4657        let mut distinct = noise(2, 6, 7, 3, 4);
4658        distinct
4659            .tokens
4660            .data
4661            .iter_mut()
4662            .enumerate()
4663            .for_each(|(i, v)| *v += i as f32 * 1e-3);
4664        let (pooled, cache) = max_pool_same(&distinct, 5);
4665        assert_eq!(
4666            pooled.pixel(0, 0, 0)[0],
4667            (0..3)
4668                .flat_map(|y| (0..3).map(move |x| (y, x)))
4669                .map(|(y, x)| distinct.pixel(0, y, x)[0])
4670                .fold(f32::MIN, f32::max)
4671        );
4672        let weights = noise(2, 6, 7, 3, 8);
4673        let back = max_pool_same_backward(&cache, &weights);
4674        let through = |input: &ImageBatch| max_pool_same(input, 5).0.tokens;
4675        let gap = pooled_gap(through, &distinct, &weights.tokens.data, &back.tokens.data);
4676        assert!(gap < 1e-3, "max pool {gap}");
4677    }
4678
4679    /// The device batch norm sums a mean and variance in one pass, shifted by
4680    /// the running mean; a fresh norm (running mean zero) over values far
4681    /// from zero is where that shift does the least, and it still has to
4682    /// match the host's two-pass statistics.
4683    #[cfg(feature = "cuda")]
4684    #[test]
4685    fn a_device_batch_norm_over_offset_values_matches_the_host() {
4686        let Some(device) = cuda_or_skip() else {
4687            return;
4688        };
4689        let mut input = noise(4, 48, 48, 6, 3);
4690        input.tokens.data.iter_mut().for_each(|v| *v += 30.0);
4691        let mut host = batch_norm_for_tests();
4692        let mut resident = batch_norm_for_tests();
4693        resident.to_cuda_on(&device).unwrap();
4694        let (expected, _) = host.forward(&input, true).unwrap();
4695        let (output, _) = resident
4696            .forward_device(&device.upload(&input).unwrap(), true)
4697            .unwrap();
4698        let output = output.download().unwrap().tokens.data;
4699        let gap = output
4700            .iter()
4701            .zip(&expected.tokens.data)
4702            .map(|(a, b)| (a - b).abs())
4703            .fold(0.0f32, f32::max);
4704        assert!(gap < 5e-4, "normalized values {gap} apart");
4705    }
4706
4707    /// The detection layers on the device against the host.
4708    #[cfg(feature = "cuda")]
4709    #[test]
4710    fn device_detection_layers_match_the_host() {
4711        let Some(device) = cuda_or_skip() else {
4712            return;
4713        };
4714        let close = |a: &[f32], b: &[f32], what: &str| {
4715            assert_eq!(a.len(), b.len(), "{what}");
4716            for (index, (a, b)) in a.iter().zip(b).enumerate() {
4717                assert!(
4718                    (a - b).abs() < 1e-4 + 1e-4 * b.abs(),
4719                    "{what}[{index}]: {a} against {b}"
4720                );
4721            }
4722        };
4723        let input = noise(3, 9, 11, 6, 1);
4724        let weights = noise(3, 9, 11, 6, 2);
4725        let on_card = device.upload(&input).unwrap();
4726        let on_card_weights = device.upload(&weights).unwrap();
4727        for training in [true, false] {
4728            let mut host = batch_norm_for_tests();
4729            let mut resident = batch_norm_for_tests();
4730            resident.to_cuda_on(&device).unwrap();
4731            let (expected, cache) = host.forward(&input, training).unwrap();
4732            let expected_grad = host.backward(&cache, &weights);
4733            let (output, cache) = resident.forward_device(&on_card, training).unwrap();
4734            let grad = resident.backward_device(&cache, &on_card_weights).unwrap();
4735            close(
4736                &output.download().unwrap().tokens.data,
4737                &expected.tokens.data,
4738                "norm",
4739            );
4740            close(
4741                &grad.download().unwrap().tokens.data,
4742                &expected_grad.tokens.data,
4743                "norm grad",
4744            );
4745            resident.to_cpu().unwrap();
4746            close(
4747                &resident.gamma.grad.data,
4748                &host.gamma.grad.data,
4749                "gamma grad",
4750            );
4751            close(&resident.beta.grad.data, &host.beta.grad.data, "beta grad");
4752            close(
4753                &resident.running.value.data,
4754                &host.running.value.data,
4755                "running",
4756            );
4757        }
4758
4759        let up = upsample_nearest_device(&on_card, 2).unwrap();
4760        close(
4761            &up.download().unwrap().tokens.data,
4762            &upsample_nearest_batch(&input, 2).tokens.data,
4763            "upsample",
4764        );
4765        let back = upsample_nearest_backward_device(&up, 2).unwrap();
4766        close(
4767            &back.download().unwrap().tokens.data,
4768            &upsample_nearest_backward(&upsample_nearest_batch(&input, 2), 2)
4769                .tokens
4770                .data,
4771            "upsample grad",
4772        );
4773
4774        let joined = concat_channels_device(&[&on_card, &on_card_weights]).unwrap();
4775        close(
4776            &joined.download().unwrap().tokens.data,
4777            &concat_channels(&[&input, &weights]).tokens.data,
4778            "concat",
4779        );
4780        let parts = split_channels_device(&joined, &[4, 8]).unwrap();
4781        for (part, expected) in parts.iter().zip(split_channels(
4782            &concat_channels(&[&input, &weights]),
4783            &[4, 8],
4784        )) {
4785            close(
4786                &part.download().unwrap().tokens.data,
4787                &expected.tokens.data,
4788                "split",
4789            );
4790        }
4791        let sum = add_device(&on_card, &on_card_weights).unwrap();
4792        let mut expected = input.clone();
4793        expected.add(&weights);
4794        close(
4795            &sum.download().unwrap().tokens.data,
4796            &expected.tokens.data,
4797            "add",
4798        );
4799
4800        let (pooled, cache) = max_pool_same(&input, 5);
4801        let (on_card_pooled, device_cache) = max_pool_same_device(&on_card, 5).unwrap();
4802        close(
4803            &on_card_pooled.download().unwrap().tokens.data,
4804            &pooled.tokens.data,
4805            "max pool",
4806        );
4807        let back = max_pool_same_backward_device(&device_cache, &on_card_weights).unwrap();
4808        close(
4809            &back.download().unwrap().tokens.data,
4810            &max_pool_same_backward(&cache, &weights).tokens.data,
4811            "max pool grad",
4812        );
4813    }
4814    #[test]
4815    fn depthwise_convolution_matches_finite_differences() {
4816        let mut rng = rand::rngs::StdRng::seed_from_u64(5);
4817        for stride in [1, 2] {
4818            let mut conv = DepthwiseConv2d::new(4, 3, stride, &mut rng);
4819            let input = noise(2, 7, 6, 4, 1);
4820            let (output, cache) = conv.forward(&input).unwrap();
4821            assert_eq!((output.height, output.width), conv.output_size(7, 6));
4822            // Channel 1's centre pixel against a direct sum.
4823            let w = &conv.weight.value.data[9..18];
4824            let (y, x) = (1, 2 / stride);
4825            let direct: f32 = (0..9)
4826                .map(|t| input.pixel(1, y * stride + t / 3 - 1, x * stride + t % 3 - 1)[1] * w[t])
4827                .sum();
4828            assert!((output.pixel(1, y, x)[1] - direct).abs() < 1e-5);
4829            let weights = noise(2, output.height, output.width, 4, 6);
4830            let grad = conv.backward(&cache, &weights);
4831            let through = |input: &ImageBatch| conv.forward(input).unwrap().0.tokens;
4832            let gap = pooled_gap(through, &input, &weights.tokens.data, &grad.tokens.data);
4833            assert!(gap < 1e-3, "stride {stride}: input {gap}");
4834            let as_batch =
4835                |data: Vec<f32>| ImageBatch::from_tokens(1, 1, 36, Matrix::from_vec(36, 1, data));
4836            let through = |params: &ImageBatch| {
4837                let mut moved = conv.clone();
4838                moved.weight.value.data.copy_from_slice(&params.tokens.data);
4839                moved.forward(&input).unwrap().0.tokens
4840            };
4841            let gap = pooled_gap(
4842                through,
4843                &as_batch(conv.weight.value.data.clone()),
4844                &weights.tokens.data,
4845                &conv.weight.grad.data,
4846            );
4847            assert!(gap < 1e-3, "stride {stride}: weight {gap}");
4848        }
4849    }
4850
4851    #[cfg(feature = "cuda")]
4852    #[test]
4853    fn device_depthwise_convolution_matches_the_host() {
4854        let Some(device) = cuda_or_skip() else {
4855            return;
4856        };
4857        let mut rng = rand::rngs::StdRng::seed_from_u64(5);
4858        for stride in [1, 2] {
4859            let mut host = DepthwiseConv2d::new(6, 3, stride, &mut rng);
4860            let mut resident = host.clone();
4861            resident.to_cuda_on(&device).unwrap();
4862            let input = noise(3, 13, 10, 6, 1);
4863            let (expected, cache) = host.forward(&input).unwrap();
4864            let weights = noise(3, expected.height, expected.width, 6, 2);
4865            let expected_grad = host.backward(&cache, &weights);
4866            let on_card = device.upload(&input).unwrap();
4867            let output = resident.forward_device(&on_card).unwrap();
4868            let grad = resident
4869                .backward_device(&on_card, &device.upload(&weights).unwrap())
4870                .unwrap();
4871            let close = |a: &[f32], b: &[f32], what: &str| {
4872                assert_eq!(a.len(), b.len(), "{what}");
4873                for (index, (a, b)) in a.iter().zip(b).enumerate() {
4874                    assert!(
4875                        (a - b).abs() < 1e-4 + 1e-4 * b.abs(),
4876                        "{what}[{index}]: {a} against {b}"
4877                    );
4878                }
4879            };
4880            close(
4881                &output.download().unwrap().tokens.data,
4882                &expected.tokens.data,
4883                "output",
4884            );
4885            close(
4886                &grad.download().unwrap().tokens.data,
4887                &expected_grad.tokens.data,
4888                "input grad",
4889            );
4890            resident.to_cpu().unwrap();
4891            close(
4892                &resident.weight.grad.data,
4893                &host.weight.grad.data,
4894                "weight grad",
4895            );
4896        }
4897    }
4898
4899    #[test]
4900    fn pixel_attention_matches_ultralytics() {
4901        let fixture = std::path::Path::new("/home/vasylt/Code/games/cs2/fixtures/k13a_attention.json");
4902        let Ok(text) = std::fs::read_to_string(fixture) else {
4903            eprintln!("skipped: no {}", fixture.display());
4904            return;
4905        };
4906        let json: serde_json::Value = serde_json::from_str(&text).unwrap();
4907        let size = |key: &str| json[key].as_u64().unwrap() as usize;
4908        let map = |key: &str| {
4909            let data: Vec<f32> = serde_json::from_value(json[key].clone()).unwrap();
4910            let rows = size("batch") * size("height") * size("width");
4911            let tokens = Matrix::from_vec(rows, data.len() / rows, data);
4912            ImageBatch::from_tokens(size("batch"), size("height"), size("width"), tokens)
4913        };
4914        let shape = PixelAttentionShape::yolo(size("dim"), size("heads"));
4915        let (attended, values, cache) = pixel_attention(&map("qkv"), shape).unwrap();
4916        let grad = pixel_attention_backward(&cache, &map("grad_attended"), &map("grad_values"));
4917        for (got, key) in [
4918            (attended, "attended"),
4919            (values, "values"),
4920            (grad, "grad_qkv"),
4921        ] {
4922            let gap = got
4923                .tokens
4924                .data
4925                .iter()
4926                .zip(&map(key).tokens.data)
4927                .fold(0.0f32, |m, (a, b)| m.max((a - b).abs()));
4928            assert!(gap < 1e-5, "{key}: {gap}");
4929        }
4930    }
4931
4932    #[cfg(feature = "cuda")]
4933    #[test]
4934    fn device_pixel_attention_matches_the_host() {
4935        let Some(device) = cuda_or_skip() else {
4936            return;
4937        };
4938        let shape = PixelAttentionShape::yolo(16, 2);
4939        let qkv = noise(3, 5, 4, 32, 1);
4940        let (attended, values, cache) = pixel_attention(&qkv, shape).unwrap();
4941        let (grad_attended, grad_values) = (noise(3, 5, 4, 16, 2), noise(3, 5, 4, 16, 3));
4942        let grad = pixel_attention_backward(&cache, &grad_attended, &grad_values);
4943        let (on_attended, on_values, on_cache) =
4944            pixel_attention_device(&device.upload(&qkv).unwrap(), shape).unwrap();
4945        let on_grad = pixel_attention_backward_device(
4946            &on_cache,
4947            &device.upload(&grad_attended).unwrap(),
4948            &device.upload(&grad_values).unwrap(),
4949        )
4950        .unwrap();
4951        for (got, expected, what) in [
4952            (on_attended, attended, "attended"),
4953            (on_values, values, "values"),
4954            (on_grad, grad, "qkv grad"),
4955        ] {
4956            let got = got.download().unwrap();
4957            for (index, (a, b)) in got
4958                .tokens
4959                .data
4960                .iter()
4961                .zip(&expected.tokens.data)
4962                .enumerate()
4963            {
4964                assert!(
4965                    (a - b).abs() < 1e-4 + 1e-4 * b.abs(),
4966                    "{what}[{index}]: {a} against {b}"
4967                );
4968            }
4969        }
4970    }
4971}